sPhenix code displayed by LXR master/​JETSCAPE/​external_packages/​googletest/​googlemock/​scripts/​generator/​cpp/​ast.py	Source navigation Diff markup Identifier search General search
	Version: master Revert   Change

File indexing completed on 2025-08-06 08:19:53

 #!/usr/bin/env python
 #
 # Copyright 2007 Neal Norwitz
 # Portions Copyright 2007 Google Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #      http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 """Generate an Abstract Syntax Tree (AST) for C++."""
 
 __author__ = 'nnorwitz@google.com (Neal Norwitz)'
 
 
 # TODO:
 #  * Tokens should never be exported, need to convert to Nodes
 #    (return types, parameters, etc.)
 #  * Handle static class data for templatized classes
 #  * Handle casts (both C++ and C-style)
 #  * Handle conditions and loops (if/else, switch, for, while/do)
 #
 # TODO much, much later:
 #  * Handle #define
 #  * exceptions
 
 
 try:
     # Python 3.x
     import builtins
 except ImportError:
     # Python 2.x
     import __builtin__ as builtins
 
 import sys
 import traceback
 
 from cpp import keywords
 from cpp import tokenize
 from cpp import utils
 
 
 if not hasattr(builtins, 'reversed'):
     # Support Python 2.3 and earlier.
     def reversed(seq):
         for i in range(len(seq)-1, -1, -1):
             yield seq[i]
 
 if not hasattr(builtins, 'next'):
     # Support Python 2.5 and earlier.
     def next(obj):
         return obj.next()
 
 
 VISIBILITY_PUBLIC, VISIBILITY_PROTECTED, VISIBILITY_PRIVATE = range(3)
 
 FUNCTION_NONE = 0x00
 FUNCTION_CONST = 0x01
 FUNCTION_VIRTUAL = 0x02
 FUNCTION_PURE_VIRTUAL = 0x04
 FUNCTION_CTOR = 0x08
 FUNCTION_DTOR = 0x10
 FUNCTION_ATTRIBUTE = 0x20
 FUNCTION_UNKNOWN_ANNOTATION = 0x40
 FUNCTION_THROW = 0x80
 FUNCTION_OVERRIDE = 0x100
 
 """
 These are currently unused.  Should really handle these properly at some point.
 
 TYPE_MODIFIER_INLINE   = 0x010000
 TYPE_MODIFIER_EXTERN   = 0x020000
 TYPE_MODIFIER_STATIC   = 0x040000
 TYPE_MODIFIER_CONST    = 0x080000
 TYPE_MODIFIER_REGISTER = 0x100000
 TYPE_MODIFIER_VOLATILE = 0x200000
 TYPE_MODIFIER_MUTABLE  = 0x400000
 
 TYPE_MODIFIER_MAP = {
     'inline': TYPE_MODIFIER_INLINE,
     'extern': TYPE_MODIFIER_EXTERN,
     'static': TYPE_MODIFIER_STATIC,
     'const': TYPE_MODIFIER_CONST,
     'register': TYPE_MODIFIER_REGISTER,
     'volatile': TYPE_MODIFIER_VOLATILE,
     'mutable': TYPE_MODIFIER_MUTABLE,
     }
 """
 
 _INTERNAL_TOKEN = 'internal'
 _NAMESPACE_POP = 'ns-pop'
 
 
 # TODO(nnorwitz): use this as a singleton for templated_types, etc
 # where we don't want to create a new empty dict each time.  It is also const.
 class _NullDict(object):
     __contains__ = lambda self: False
     keys = values = items = iterkeys = itervalues = iteritems = lambda self: ()
 
 
 # TODO(nnorwitz): move AST nodes into a separate module.
 class Node(object):
     """Base AST node."""
 
     def __init__(self, start, end):
         self.start = start
         self.end = end
 
     def IsDeclaration(self):
         """Returns bool if this node is a declaration."""
         return False
 
     def IsDefinition(self):
         """Returns bool if this node is a definition."""
         return False
 
     def IsExportable(self):
         """Returns bool if this node exportable from a header file."""
         return False
 
     def Requires(self, node):
         """Does this AST node require the definition of the node passed in?"""
         return False
 
     def XXX__str__(self):
         return self._StringHelper(self.__class__.__name__, '')
 
     def _StringHelper(self, name, suffix):
         if not utils.DEBUG:
             return '%s(%s)' % (name, suffix)
         return '%s(%d, %d, %s)' % (name, self.start, self.end, suffix)
 
     def __repr__(self):
         return str(self)
 
 
 class Define(Node):
     def __init__(self, start, end, name, definition):
         Node.__init__(self, start, end)
         self.name = name
         self.definition = definition
 
     def __str__(self):
         value = '%s %s' % (self.name, self.definition)
         return self._StringHelper(self.__class__.__name__, value)
 
 
 class Include(Node):
     def __init__(self, start, end, filename, system):
         Node.__init__(self, start, end)
         self.filename = filename
         self.system = system
 
     def __str__(self):
         fmt = '"%s"'
         if self.system:
             fmt = '<%s>'
         return self._StringHelper(self.__class__.__name__, fmt % self.filename)
 
 
 class Goto(Node):
     def __init__(self, start, end, label):
         Node.__init__(self, start, end)
         self.label = label
 
     def __str__(self):
         return self._StringHelper(self.__class__.__name__, str(self.label))
 
 
 class Expr(Node):
     def __init__(self, start, end, expr):
         Node.__init__(self, start, end)
         self.expr = expr
 
     def Requires(self, node):
         # TODO(nnorwitz): impl.
         return False
 
     def __str__(self):
         return self._StringHelper(self.__class__.__name__, str(self.expr))
 
 
 class Return(Expr):
     pass
 
 
 class Delete(Expr):
     pass
 
 
 class Friend(Expr):
     def __init__(self, start, end, expr, namespace):
         Expr.__init__(self, start, end, expr)
         self.namespace = namespace[:]
 
 
 class Using(Node):
     def __init__(self, start, end, names):
         Node.__init__(self, start, end)
         self.names = names
 
     def __str__(self):
         return self._StringHelper(self.__class__.__name__, str(self.names))
 
 
 class Parameter(Node):
     def __init__(self, start, end, name, parameter_type, default):
         Node.__init__(self, start, end)
         self.name = name
         self.type = parameter_type
         self.default = default
 
     def Requires(self, node):
         # TODO(nnorwitz): handle namespaces, etc.
         return self.type.name == node.name
 
     def __str__(self):
         name = str(self.type)
         suffix = '%s %s' % (name, self.name)
         if self.default:
             suffix += ' = ' + ''.join([d.name for d in self.default])
         return self._StringHelper(self.__class__.__name__, suffix)
 
 
 class _GenericDeclaration(Node):
     def __init__(self, start, end, name, namespace):
         Node.__init__(self, start, end)
         self.name = name
         self.namespace = namespace[:]
 
     def FullName(self):
         prefix = ''
         if self.namespace and self.namespace[-1]:
             prefix = '::'.join(self.namespace) + '::'
         return prefix + self.name
 
     def _TypeStringHelper(self, suffix):
         if self.namespace:
             names = [n or '<anonymous>' for n in self.namespace]
             suffix += ' in ' + '::'.join(names)
         return self._StringHelper(self.__class__.__name__, suffix)
 
 
 # TODO(nnorwitz): merge with Parameter in some way?
 class VariableDeclaration(_GenericDeclaration):
     def __init__(self, start, end, name, var_type, initial_value, namespace):
         _GenericDeclaration.__init__(self, start, end, name, namespace)
         self.type = var_type
         self.initial_value = initial_value
 
     def Requires(self, node):
         # TODO(nnorwitz): handle namespaces, etc.
         return self.type.name == node.name
 
     def ToString(self):
         """Return a string that tries to reconstitute the variable decl."""
         suffix = '%s %s' % (self.type, self.name)
         if self.initial_value:
             suffix += ' = ' + self.initial_value
         return suffix
 
     def __str__(self):
         return self._StringHelper(self.__class__.__name__, self.ToString())
 
 
 class Typedef(_GenericDeclaration):
     def __init__(self, start, end, name, alias, namespace):
         _GenericDeclaration.__init__(self, start, end, name, namespace)
         self.alias = alias
 
     def IsDefinition(self):
         return True
 
     def IsExportable(self):
         return True
 
     def Requires(self, node):
         # TODO(nnorwitz): handle namespaces, etc.
         name = node.name
         for token in self.alias:
             if token is not None and name == token.name:
                 return True
         return False
 
     def __str__(self):
         suffix = '%s, %s' % (self.name, self.alias)
         return self._TypeStringHelper(suffix)
 
 
 class _NestedType(_GenericDeclaration):
     def __init__(self, start, end, name, fields, namespace):
         _GenericDeclaration.__init__(self, start, end, name, namespace)
         self.fields = fields
 
     def IsDefinition(self):
         return True
 
     def IsExportable(self):
         return True
 
     def __str__(self):
         suffix = '%s, {%s}' % (self.name, self.fields)
         return self._TypeStringHelper(suffix)
 
 
 class Union(_NestedType):
     pass
 
 
 class Enum(_NestedType):
     pass
 
 
 class Class(_GenericDeclaration):
     def __init__(self, start, end, name, bases, templated_types, body, namespace):
         _GenericDeclaration.__init__(self, start, end, name, namespace)
         self.bases = bases
         self.body = body
         self.templated_types = templated_types
 
     def IsDeclaration(self):
         return self.bases is None and self.body is None
 
     def IsDefinition(self):
         return not self.IsDeclaration()
 
     def IsExportable(self):
         return not self.IsDeclaration()
 
     def Requires(self, node):
         # TODO(nnorwitz): handle namespaces, etc.
         if self.bases:
             for token_list in self.bases:
                 # TODO(nnorwitz): bases are tokens, do name comparision.
                 for token in token_list:
                     if token.name == node.name:
                         return True
         # TODO(nnorwitz): search in body too.
         return False
 
     def __str__(self):
         name = self.name
         if self.templated_types:
             name += '<%s>' % self.templated_types
         suffix = '%s, %s, %s' % (name, self.bases, self.body)
         return self._TypeStringHelper(suffix)
 
 
 class Struct(Class):
     pass
 
 
 class Function(_GenericDeclaration):
     def __init__(self, start, end, name, return_type, parameters,
                  modifiers, templated_types, body, namespace):
         _GenericDeclaration.__init__(self, start, end, name, namespace)
         converter = TypeConverter(namespace)
         self.return_type = converter.CreateReturnType(return_type)
         self.parameters = converter.ToParameters(parameters)
         self.modifiers = modifiers
         self.body = body
         self.templated_types = templated_types
 
     def IsDeclaration(self):
         return self.body is None
 
     def IsDefinition(self):
         return self.body is not None
 
     def IsExportable(self):
         if self.return_type and 'static' in self.return_type.modifiers:
             return False
         return None not in self.namespace
 
     def Requires(self, node):
         if self.parameters:
             # TODO(nnorwitz): parameters are tokens, do name comparision.
             for p in self.parameters:
                 if p.name == node.name:
                     return True
         # TODO(nnorwitz): search in body too.
         return False
 
     def __str__(self):
         # TODO(nnorwitz): add templated_types.
         suffix = ('%s %s(%s), 0x%02x, %s' %
                   (self.return_type, self.name, self.parameters,
                    self.modifiers, self.body))
         return self._TypeStringHelper(suffix)
 
 
 class Method(Function):
     def __init__(self, start, end, name, in_class, return_type, parameters,
                  modifiers, templated_types, body, namespace):
         Function.__init__(self, start, end, name, return_type, parameters,
                           modifiers, templated_types, body, namespace)
         # TODO(nnorwitz): in_class could also be a namespace which can
         # mess up finding functions properly.
         self.in_class = in_class
 
 
 class Type(_GenericDeclaration):
     """Type used for any variable (eg class, primitive, struct, etc)."""
 
     def __init__(self, start, end, name, templated_types, modifiers,
                  reference, pointer, array):
         """
         Args:
           name: str name of main type
           templated_types: [Class (Type?)] template type info between <>
           modifiers: [str] type modifiers (keywords) eg, const, mutable, etc.
           reference, pointer, array: bools
         """
         _GenericDeclaration.__init__(self, start, end, name, [])
         self.templated_types = templated_types
         if not name and modifiers:
             self.name = modifiers.pop()
         self.modifiers = modifiers
         self.reference = reference
         self.pointer = pointer
         self.array = array
 
     def __str__(self):
         prefix = ''
         if self.modifiers:
             prefix = ' '.join(self.modifiers) + ' '
         name = str(self.name)
         if self.templated_types:
             name += '<%s>' % self.templated_types
         suffix = prefix + name
         if self.reference:
             suffix += '&'
         if self.pointer:
             suffix += '*'
         if self.array:
             suffix += '[]'
         return self._TypeStringHelper(suffix)
 
     # By definition, Is* are always False.  A Type can only exist in
     # some sort of variable declaration, parameter, or return value.
     def IsDeclaration(self):
         return False
 
     def IsDefinition(self):
         return False
 
     def IsExportable(self):
         return False
 
 
 class TypeConverter(object):
 
     def __init__(self, namespace_stack):
         self.namespace_stack = namespace_stack
 
     def _GetTemplateEnd(self, tokens, start):
         count = 1
         end = start
         while 1:
             token = tokens[end]
             end += 1
             if token.name == '<':
                 count += 1
             elif token.name == '>':
                 count -= 1
                 if count == 0:
                     break
         return tokens[start:end-1], end
 
     def ToType(self, tokens):
         """Convert [Token,...] to [Class(...), ] useful for base classes.
         For example, code like class Foo : public Bar<x, y> { ... };
         the "Bar<x, y>" portion gets converted to an AST.
 
         Returns:
           [Class(...), ...]
         """
         result = []
         name_tokens = []
         reference = pointer = array = False
 
         def AddType(templated_types):
             # Partition tokens into name and modifier tokens.
             names = []
             modifiers = []
             for t in name_tokens:
                 if keywords.IsKeyword(t.name):
                     modifiers.append(t.name)
                 else:
                     names.append(t.name)
             name = ''.join(names)
             if name_tokens:
                 result.append(Type(name_tokens[0].start, name_tokens[-1].end,
                                    name, templated_types, modifiers,
                                    reference, pointer, array))
             del name_tokens[:]
 
         i = 0
         end = len(tokens)
         while i < end:
             token = tokens[i]
             if token.name == '<':
                 new_tokens, new_end = self._GetTemplateEnd(tokens, i+1)
                 AddType(self.ToType(new_tokens))
                 # If there is a comma after the template, we need to consume
                 # that here otherwise it becomes part of the name.
                 i = new_end
                 reference = pointer = array = False
             elif token.name == ',':
                 AddType([])
                 reference = pointer = array = False
             elif token.name == '*':
                 pointer = True
             elif token.name == '&':
                 reference = True
             elif token.name == '[':
                pointer = True
             elif token.name == ']':
                 pass
             else:
                 name_tokens.append(token)
             i += 1
 
         if name_tokens:
             # No '<' in the tokens, just a simple name and no template.
             AddType([])
         return result
 
     def DeclarationToParts(self, parts, needs_name_removed):
         name = None
         default = []
         if needs_name_removed:
             # Handle default (initial) values properly.
             for i, t in enumerate(parts):
                 if t.name == '=':
                     default = parts[i+1:]
                     name = parts[i-1].name
                     if name == ']' and parts[i-2].name == '[':
                         name = parts[i-3].name
                         i -= 1
                     parts = parts[:i-1]
                     break
             else:
                 if parts[-1].token_type == tokenize.NAME:
                     name = parts.pop().name
                 else:
                     # TODO(nnorwitz): this is a hack that happens for code like
                     # Register(Foo<T>); where it thinks this is a function call
                     # but it's actually a declaration.
                     name = '???'
         modifiers = []
         type_name = []
         other_tokens = []
         templated_types = []
         i = 0
         end = len(parts)
         while i < end:
             p = parts[i]
             if keywords.IsKeyword(p.name):
                 modifiers.append(p.name)
             elif p.name == '<':
                 templated_tokens, new_end = self._GetTemplateEnd(parts, i+1)
                 templated_types = self.ToType(templated_tokens)
                 i = new_end - 1
                 # Don't add a spurious :: to data members being initialized.
                 next_index = i + 1
                 if next_index < end and parts[next_index].name == '::':
                     i += 1
             elif p.name in ('[', ']', '='):
                 # These are handled elsewhere.
                 other_tokens.append(p)
             elif p.name not in ('*', '&', '>'):
                 # Ensure that names have a space between them.
                 if (type_name and type_name[-1].token_type == tokenize.NAME and
                     p.token_type == tokenize.NAME):
                     type_name.append(tokenize.Token(tokenize.SYNTAX, ' ', 0, 0))
                 type_name.append(p)
             else:
                 other_tokens.append(p)
             i += 1
         type_name = ''.join([t.name for t in type_name])
         return name, type_name, templated_types, modifiers, default, other_tokens
 
     def ToParameters(self, tokens):
         if not tokens:
             return []
 
         result = []
         name = type_name = ''
         type_modifiers = []
         pointer = reference = array = False
         first_token = None
         default = []
 
         def AddParameter(end):
             if default:
                 del default[0]  # Remove flag.
             parts = self.DeclarationToParts(type_modifiers, True)
             (name, type_name, templated_types, modifiers,
              unused_default, unused_other_tokens) = parts
             parameter_type = Type(first_token.start, first_token.end,
                                   type_name, templated_types, modifiers,
                                   reference, pointer, array)
             p = Parameter(first_token.start, end, name,
                           parameter_type, default)
             result.append(p)
 
         template_count = 0
         for s in tokens:
             if not first_token:
                 first_token = s
             if s.name == '<':
                 template_count += 1
             elif s.name == '>':
                 template_count -= 1
             if template_count > 0:
                 type_modifiers.append(s)
                 continue
 
             if s.name == ',':
                 AddParameter(s.start)
                 name = type_name = ''
                 type_modifiers = []
                 pointer = reference = array = False
                 first_token = None
                 default = []
             elif s.name == '*':
                 pointer = True
             elif s.name == '&':
                 reference = True
             elif s.name == '[':
                 array = True
             elif s.name == ']':
                 pass  # Just don't add to type_modifiers.
             elif s.name == '=':
                 # Got a default value.  Add any value (None) as a flag.
                 default.append(None)
             elif default:
                 default.append(s)
             else:
                 type_modifiers.append(s)
         AddParameter(tokens[-1].end)
         return result
 
     def CreateReturnType(self, return_type_seq):
         if not return_type_seq:
             return None
         start = return_type_seq[0].start
         end = return_type_seq[-1].end
         _, name, templated_types, modifiers, default, other_tokens = \
            self.DeclarationToParts(return_type_seq, False)
         names = [n.name for n in other_tokens]
         reference = '&' in names
         pointer = '*' in names
         array = '[' in names
         return Type(start, end, name, templated_types, modifiers,
                     reference, pointer, array)
 
     def GetTemplateIndices(self, names):
         # names is a list of strings.
         start = names.index('<')
         end = len(names) - 1
         while end > 0:
             if names[end] == '>':
                 break
             end -= 1
         return start, end+1
 
 class AstBuilder(object):
     def __init__(self, token_stream, filename, in_class='', visibility=None,
                  namespace_stack=[]):
         self.tokens = token_stream
         self.filename = filename
         # TODO(nnorwitz): use a better data structure (deque) for the queue.
         # Switching directions of the "queue" improved perf by about 25%.
         # Using a deque should be even better since we access from both sides.
         self.token_queue = []
         self.namespace_stack = namespace_stack[:]
         self.in_class = in_class
         if in_class is None:
             self.in_class_name_only = None
         else:
             self.in_class_name_only = in_class.split('::')[-1]
         self.visibility = visibility
         self.in_function = False
         self.current_token = None
         # Keep the state whether we are currently handling a typedef or not.
         self._handling_typedef = False
 
         self.converter = TypeConverter(self.namespace_stack)
 
     def HandleError(self, msg, token):
         printable_queue = list(reversed(self.token_queue[-20:]))
         sys.stderr.write('Got %s in %s @ %s %s\n' %
                          (msg, self.filename, token, printable_queue))
 
     def Generate(self):
         while 1:
             token = self._GetNextToken()
             if not token:
                 break
 
             # Get the next token.
             self.current_token = token
 
             # Dispatch on the next token type.
             if token.token_type == _INTERNAL_TOKEN:
                 if token.name == _NAMESPACE_POP:
                     self.namespace_stack.pop()
                 continue
 
             try:
                 result = self._GenerateOne(token)
                 if result is not None:
                     yield result
             except:
                 self.HandleError('exception', token)
                 raise
 
     def _CreateVariable(self, pos_token, name, type_name, type_modifiers,
                         ref_pointer_name_seq, templated_types, value=None):
         reference = '&' in ref_pointer_name_seq
         pointer = '*' in ref_pointer_name_seq
         array = '[' in ref_pointer_name_seq
         var_type = Type(pos_token.start, pos_token.end, type_name,
                         templated_types, type_modifiers,
                         reference, pointer, array)
         return VariableDeclaration(pos_token.start, pos_token.end,
                                    name, var_type, value, self.namespace_stack)
 
     def _GenerateOne(self, token):
         if token.token_type == tokenize.NAME:
             if (keywords.IsKeyword(token.name) and
                 not keywords.IsBuiltinType(token.name)):
                 method = getattr(self, 'handle_' + token.name)
                 return method()
             elif token.name == self.in_class_name_only:
                 # The token name is the same as the class, must be a ctor if
                 # there is a paren.  Otherwise, it's the return type.
                 # Peek ahead to get the next token to figure out which.
                 next = self._GetNextToken()
                 self._AddBackToken(next)
                 if next.token_type == tokenize.SYNTAX and next.name == '(':
                     return self._GetMethod([token], FUNCTION_CTOR, None, True)
                 # Fall through--handle like any other method.
 
             # Handle data or function declaration/definition.
             syntax = tokenize.SYNTAX
             temp_tokens, last_token = \
                 self._GetVarTokensUpTo(syntax, '(', ';', '{', '[')
             temp_tokens.insert(0, token)
             if last_token.name == '(':
                 # If there is an assignment before the paren,
                 # this is an expression, not a method.
                 expr = bool([e for e in temp_tokens if e.name == '='])
                 if expr:
                     new_temp = self._GetTokensUpTo(tokenize.SYNTAX, ';')
                     temp_tokens.append(last_token)
                     temp_tokens.extend(new_temp)
                     last_token = tokenize.Token(tokenize.SYNTAX, ';', 0, 0)
 
             if last_token.name == '[':
                 # Handle array, this isn't a method, unless it's an operator.
                 # TODO(nnorwitz): keep the size somewhere.
                 # unused_size = self._GetTokensUpTo(tokenize.SYNTAX, ']')
                 temp_tokens.append(last_token)
                 if temp_tokens[-2].name == 'operator':
                     temp_tokens.append(self._GetNextToken())
                 else:
                     temp_tokens2, last_token = \
                         self._GetVarTokensUpTo(tokenize.SYNTAX, ';')
                     temp_tokens.extend(temp_tokens2)
 
             if last_token.name == ';':
                 # Handle data, this isn't a method.
                 parts = self.converter.DeclarationToParts(temp_tokens, True)
                 (name, type_name, templated_types, modifiers, default,
                  unused_other_tokens) = parts
 
                 t0 = temp_tokens[0]
                 names = [t.name for t in temp_tokens]
                 if templated_types:
                     start, end = self.converter.GetTemplateIndices(names)
                     names = names[:start] + names[end:]
                 default = ''.join([t.name for t in default])
                 return self._CreateVariable(t0, name, type_name, modifiers,
                                             names, templated_types, default)
             if last_token.name == '{':
                 self._AddBackTokens(temp_tokens[1:])
                 self._AddBackToken(last_token)
                 method_name = temp_tokens[0].name
                 method = getattr(self, 'handle_' + method_name, None)
                 if not method:
                     # Must be declaring a variable.
                     # TODO(nnorwitz): handle the declaration.
                     return None
                 return method()
             return self._GetMethod(temp_tokens, 0, None, False)
         elif token.token_type == tokenize.SYNTAX:
             if token.name == '~' and self.in_class:
                 # Must be a dtor (probably not in method body).
                 token = self._GetNextToken()
                 # self.in_class can contain A::Name, but the dtor will only
                 # be Name.  Make sure to compare against the right value.
                 if (token.token_type == tokenize.NAME and
                     token.name == self.in_class_name_only):
                     return self._GetMethod([token], FUNCTION_DTOR, None, True)
             # TODO(nnorwitz): handle a lot more syntax.
         elif token.token_type == tokenize.PREPROCESSOR:
             # TODO(nnorwitz): handle more preprocessor directives.
             # token starts with a #, so remove it and strip whitespace.
             name = token.name[1:].lstrip()
             if name.startswith('include'):
                 # Remove "include".
                 name = name[7:].strip()
                 assert name
                 # Handle #include \<newline> "header-on-second-line.h".
                 if name.startswith('\\'):
                     name = name[1:].strip()
                 assert name[0] in '<"', token
                 assert name[-1] in '>"', token
                 system = name[0] == '<'
                 filename = name[1:-1]
                 return Include(token.start, token.end, filename, system)
             if name.startswith('define'):
                 # Remove "define".
                 name = name[6:].strip()
                 assert name
                 value = ''
                 for i, c in enumerate(name):
                     if c.isspace():
                         value = name[i:].lstrip()
                         name = name[:i]
                         break
                 return Define(token.start, token.end, name, value)
             if name.startswith('if') and name[2:3].isspace():
                 condition = name[3:].strip()
                 if condition.startswith('0') or condition.startswith('(0)'):
                     self._SkipIf0Blocks()
         return None
 
     def _GetTokensUpTo(self, expected_token_type, expected_token):
         return self._GetVarTokensUpTo(expected_token_type, expected_token)[0]
 
     def _GetVarTokensUpTo(self, expected_token_type, *expected_tokens):
         last_token = self._GetNextToken()
         tokens = []
         while (last_token.token_type != expected_token_type or
                last_token.name not in expected_tokens):
             tokens.append(last_token)
             last_token = self._GetNextToken()
         return tokens, last_token
 
     # TODO(nnorwitz): remove _IgnoreUpTo() it shouldn't be necesary.
     def _IgnoreUpTo(self, token_type, token):
         unused_tokens = self._GetTokensUpTo(token_type, token)
 
     def _SkipIf0Blocks(self):
         count = 1
         while 1:
             token = self._GetNextToken()
             if token.token_type != tokenize.PREPROCESSOR:
                 continue
 
             name = token.name[1:].lstrip()
             if name.startswith('endif'):
                 count -= 1
                 if count == 0:
                     break
             elif name.startswith('if'):
                 count += 1
 
     def _GetMatchingChar(self, open_paren, close_paren, GetNextToken=None):
         if GetNextToken is None:
             GetNextToken = self._GetNextToken
         # Assumes the current token is open_paren and we will consume
         # and return up to the close_paren.
         count = 1
         token = GetNextToken()
         while 1:
             if token.token_type == tokenize.SYNTAX:
                 if token.name == open_paren:
                     count += 1
                 elif token.name == close_paren:
                     count -= 1
                     if count == 0:
                         break
             yield token
             token = GetNextToken()
         yield token
 
     def _GetParameters(self):
         return self._GetMatchingChar('(', ')')
 
     def GetScope(self):
         return self._GetMatchingChar('{', '}')
 
     def _GetNextToken(self):
         if self.token_queue:
             return self.token_queue.pop()
         return next(self.tokens)
 
     def _AddBackToken(self, token):
         if token.whence == tokenize.WHENCE_STREAM:
             token.whence = tokenize.WHENCE_QUEUE
             self.token_queue.insert(0, token)
         else:
             assert token.whence == tokenize.WHENCE_QUEUE, token
             self.token_queue.append(token)
 
     def _AddBackTokens(self, tokens):
         if tokens:
             if tokens[-1].whence == tokenize.WHENCE_STREAM:
                 for token in tokens:
                     token.whence = tokenize.WHENCE_QUEUE
                 self.token_queue[:0] = reversed(tokens)
             else:
                 assert tokens[-1].whence == tokenize.WHENCE_QUEUE, tokens
                 self.token_queue.extend(reversed(tokens))
 
     def GetName(self, seq=None):
         """Returns ([tokens], next_token_info)."""
         GetNextToken = self._GetNextToken
         if seq is not None:
             it = iter(seq)
             GetNextToken = lambda: next(it)
         next_token = GetNextToken()
         tokens = []
         last_token_was_name = False
         while (next_token.token_type == tokenize.NAME or
                (next_token.token_type == tokenize.SYNTAX and
                 next_token.name in ('::', '<'))):
             # Two NAMEs in a row means the identifier should terminate.
             # It's probably some sort of variable declaration.
             if last_token_was_name and next_token.token_type == tokenize.NAME:
                 break
             last_token_was_name = next_token.token_type == tokenize.NAME
             tokens.append(next_token)
             # Handle templated names.
             if next_token.name == '<':
                 tokens.extend(self._GetMatchingChar('<', '>', GetNextToken))
                 last_token_was_name = True
             next_token = GetNextToken()
         return tokens, next_token
 
     def GetMethod(self, modifiers, templated_types):
         return_type_and_name = self._GetTokensUpTo(tokenize.SYNTAX, '(')
         assert len(return_type_and_name) >= 1
         return self._GetMethod(return_type_and_name, modifiers, templated_types,
                                False)
 
     def _GetMethod(self, return_type_and_name, modifiers, templated_types,
                    get_paren):
         template_portion = None
         if get_paren:
             token = self._GetNextToken()
             assert token.token_type == tokenize.SYNTAX, token
             if token.name == '<':
                 # Handle templatized dtors.
                 template_portion = [token]
                 template_portion.extend(self._GetMatchingChar('<', '>'))
                 token = self._GetNextToken()
             assert token.token_type == tokenize.SYNTAX, token
             assert token.name == '(', token
 
         name = return_type_and_name.pop()
         # Handle templatized ctors.
         if name.name == '>':
             index = 1
             while return_type_and_name[index].name != '<':
                 index += 1
             template_portion = return_type_and_name[index:] + [name]
             del return_type_and_name[index:]
             name = return_type_and_name.pop()
         elif name.name == ']':
             rt = return_type_and_name
             assert rt[-1].name == '[', return_type_and_name
             assert rt[-2].name == 'operator', return_type_and_name
             name_seq = return_type_and_name[-2:]
             del return_type_and_name[-2:]
             name = tokenize.Token(tokenize.NAME, 'operator[]',
                                   name_seq[0].start, name.end)
             # Get the open paren so _GetParameters() below works.
             unused_open_paren = self._GetNextToken()
 
         # TODO(nnorwitz): store template_portion.
         return_type = return_type_and_name
         indices = name
         if return_type:
             indices = return_type[0]
 
         # Force ctor for templatized ctors.
         if name.name == self.in_class and not modifiers:
             modifiers |= FUNCTION_CTOR
         parameters = list(self._GetParameters())
         del parameters[-1]              # Remove trailing ')'.
 
         # Handling operator() is especially weird.
         if name.name == 'operator' and not parameters:
             token = self._GetNextToken()
             assert token.name == '(', token
             parameters = list(self._GetParameters())
             del parameters[-1]          # Remove trailing ')'.
 
         token = self._GetNextToken()
         while token.token_type == tokenize.NAME:
             modifier_token = token
             token = self._GetNextToken()
             if modifier_token.name == 'const':
                 modifiers |= FUNCTION_CONST
             elif modifier_token.name == '__attribute__':
                 # TODO(nnorwitz): handle more __attribute__ details.
                 modifiers |= FUNCTION_ATTRIBUTE
                 assert token.name == '(', token
                 # Consume everything between the (parens).
                 unused_tokens = list(self._GetMatchingChar('(', ')'))
                 token = self._GetNextToken()
             elif modifier_token.name == 'throw':
                 modifiers |= FUNCTION_THROW
                 assert token.name == '(', token
                 # Consume everything between the (parens).
                 unused_tokens = list(self._GetMatchingChar('(', ')'))
                 token = self._GetNextToken()
             elif modifier_token.name == 'override':
                 modifiers |= FUNCTION_OVERRIDE
             elif modifier_token.name == modifier_token.name.upper():
                 # HACK(nnorwitz):  assume that all upper-case names
                 # are some macro we aren't expanding.
                 modifiers |= FUNCTION_UNKNOWN_ANNOTATION
             else:
                 self.HandleError('unexpected token', modifier_token)
 
         assert token.token_type == tokenize.SYNTAX, token
         # Handle ctor initializers.
         if token.name == ':':
             # TODO(nnorwitz): anything else to handle for initializer list?
             while token.name != ';' and token.name != '{':
                 token = self._GetNextToken()
 
         # Handle pointer to functions that are really data but look
         # like method declarations.
         if token.name == '(':
             if parameters[0].name == '*':
                 # name contains the return type.
                 name = parameters.pop()
                 # parameters contains the name of the data.
                 modifiers = [p.name for p in parameters]
                 # Already at the ( to open the parameter list.
                 function_parameters = list(self._GetMatchingChar('(', ')'))
                 del function_parameters[-1]  # Remove trailing ')'.
                 # TODO(nnorwitz): store the function_parameters.
                 token = self._GetNextToken()
                 assert token.token_type == tokenize.SYNTAX, token
                 assert token.name == ';', token
                 return self._CreateVariable(indices, name.name, indices.name,
                                             modifiers, '', None)
             # At this point, we got something like:
             #  return_type (type::*name_)(params);
             # This is a data member called name_ that is a function pointer.
             # With this code: void (sq_type::*field_)(string&);
             # We get: name=void return_type=[] parameters=sq_type ... field_
             # TODO(nnorwitz): is return_type always empty?
             # TODO(nnorwitz): this isn't even close to being correct.
             # Just put in something so we don't crash and can move on.
             real_name = parameters[-1]
             modifiers = [p.name for p in self._GetParameters()]
             del modifiers[-1]           # Remove trailing ')'.
             return self._CreateVariable(indices, real_name.name, indices.name,
                                         modifiers, '', None)
 
         if token.name == '{':
             body = list(self.GetScope())
             del body[-1]                # Remove trailing '}'.
         else:
             body = None
             if token.name == '=':
                 token = self._GetNextToken()
 
                 if token.name == 'default' or token.name == 'delete':
                     # Ignore explicitly defaulted and deleted special members
                     # in C++11.
                     token = self._GetNextToken()
                 else:
                     # Handle pure-virtual declarations.
                     assert token.token_type == tokenize.CONSTANT, token
                     assert token.name == '0', token
                     modifiers |= FUNCTION_PURE_VIRTUAL
                     token = self._GetNextToken()
 
             if token.name == '[':
                 # TODO(nnorwitz): store tokens and improve parsing.
                 # template <typename T, size_t N> char (&ASH(T (&seq)[N]))[N];
                 tokens = list(self._GetMatchingChar('[', ']'))
                 token = self._GetNextToken()
 
             assert token.name == ';', (token, return_type_and_name, parameters)
 
         # Looks like we got a method, not a function.
         if len(return_type) > 2 and return_type[-1].name == '::':
             return_type, in_class = \
                          self._GetReturnTypeAndClassName(return_type)
             return Method(indices.start, indices.end, name.name, in_class,
                           return_type, parameters, modifiers, templated_types,
                           body, self.namespace_stack)
         return Function(indices.start, indices.end, name.name, return_type,
                         parameters, modifiers, templated_types, body,
                         self.namespace_stack)
 
     def _GetReturnTypeAndClassName(self, token_seq):
         # Splitting the return type from the class name in a method
         # can be tricky.  For example, Return::Type::Is::Hard::To::Find().
         # Where is the return type and where is the class name?
         # The heuristic used is to pull the last name as the class name.
         # This includes all the templated type info.
         # TODO(nnorwitz): if there is only One name like in the
         # example above, punt and assume the last bit is the class name.
 
         # Ignore a :: prefix, if exists so we can find the first real name.
         i = 0
         if token_seq[0].name == '::':
             i = 1
         # Ignore a :: suffix, if exists.
         end = len(token_seq) - 1
         if token_seq[end-1].name == '::':
             end -= 1
 
         # Make a copy of the sequence so we can append a sentinel
         # value. This is required for GetName will has to have some
         # terminating condition beyond the last name.
         seq_copy = token_seq[i:end]
         seq_copy.append(tokenize.Token(tokenize.SYNTAX, '', 0, 0))
         names = []
         while i < end:
             # Iterate through the sequence parsing out each name.
             new_name, next = self.GetName(seq_copy[i:])
             assert new_name, 'Got empty new_name, next=%s' % next
             # We got a pointer or ref.  Add it to the name.
             if next and next.token_type == tokenize.SYNTAX:
                 new_name.append(next)
             names.append(new_name)
             i += len(new_name)
 
         # Now that we have the names, it's time to undo what we did.
 
         # Remove the sentinel value.
         names[-1].pop()
         # Flatten the token sequence for the return type.
         return_type = [e for seq in names[:-1] for e in seq]
         # The class name is the last name.
         class_name = names[-1]
         return return_type, class_name
 
     def handle_bool(self):
         pass
 
     def handle_char(self):
         pass
 
     def handle_int(self):
         pass
 
     def handle_long(self):
         pass
 
     def handle_short(self):
         pass
 
     def handle_double(self):
         pass
 
     def handle_float(self):
         pass
 
     def handle_void(self):
         pass
 
     def handle_wchar_t(self):
         pass
 
     def handle_unsigned(self):
         pass
 
     def handle_signed(self):
         pass
 
     def _GetNestedType(self, ctor):
         name = None
         name_tokens, token = self.GetName()
         if name_tokens:
             name = ''.join([t.name for t in name_tokens])
 
         # Handle forward declarations.
         if token.token_type == tokenize.SYNTAX and token.name == ';':
             return ctor(token.start, token.end, name, None,
                         self.namespace_stack)
 
         if token.token_type == tokenize.NAME and self._handling_typedef:
             self._AddBackToken(token)
             return ctor(token.start, token.end, name, None,
                         self.namespace_stack)
 
         # Must be the type declaration.
         fields = list(self._GetMatchingChar('{', '}'))
         del fields[-1]                  # Remove trailing '}'.
         if token.token_type == tokenize.SYNTAX and token.name == '{':
             next = self._GetNextToken()
             new_type = ctor(token.start, token.end, name, fields,
                             self.namespace_stack)
             # A name means this is an anonymous type and the name
             # is the variable declaration.
             if next.token_type != tokenize.NAME:
                 return new_type
             name = new_type
             token = next
 
         # Must be variable declaration using the type prefixed with keyword.
         assert token.token_type == tokenize.NAME, token
         return self._CreateVariable(token, token.name, name, [], '', None)
 
     def handle_struct(self):
         # Special case the handling typedef/aliasing of structs here.
         # It would be a pain to handle in the class code.
         name_tokens, var_token = self.GetName()
         if name_tokens:
             next_token = self._GetNextToken()
             is_syntax = (var_token.token_type == tokenize.SYNTAX and
                          var_token.name[0] in '*&')
             is_variable = (var_token.token_type == tokenize.NAME and
                            next_token.name == ';')
             variable = var_token
             if is_syntax and not is_variable:
                 variable = next_token
                 temp = self._GetNextToken()
                 if temp.token_type == tokenize.SYNTAX and temp.name == '(':
                     # Handle methods declared to return a struct.
                     t0 = name_tokens[0]
                     struct = tokenize.Token(tokenize.NAME, 'struct',
                                             t0.start-7, t0.start-2)
                     type_and_name = [struct]
                     type_and_name.extend(name_tokens)
                     type_and_name.extend((var_token, next_token))
                     return self._GetMethod(type_and_name, 0, None, False)
                 assert temp.name == ';', (temp, name_tokens, var_token)
             if is_syntax or (is_variable and not self._handling_typedef):
                 modifiers = ['struct']
                 type_name = ''.join([t.name for t in name_tokens])
                 position = name_tokens[0]
                 return self._CreateVariable(position, variable.name, type_name,
                                             modifiers, var_token.name, None)
             name_tokens.extend((var_token, next_token))
             self._AddBackTokens(name_tokens)
         else:
             self._AddBackToken(var_token)
         return self._GetClass(Struct, VISIBILITY_PUBLIC, None)
 
     def handle_union(self):
         return self._GetNestedType(Union)
 
     def handle_enum(self):
         return self._GetNestedType(Enum)
 
     def handle_auto(self):
         # TODO(nnorwitz): warn about using auto?  Probably not since it
         # will be reclaimed and useful for C++0x.
         pass
 
     def handle_register(self):
         pass
 
     def handle_const(self):
         pass
 
     def handle_inline(self):
         pass
 
     def handle_extern(self):
         pass
 
     def handle_static(self):
         pass
 
     def handle_virtual(self):
         # What follows must be a method.
         token = token2 = self._GetNextToken()
         if token.name == 'inline':
             # HACK(nnorwitz): handle inline dtors by ignoring 'inline'.
             token2 = self._GetNextToken()
         if token2.token_type == tokenize.SYNTAX and token2.name == '~':
             return self.GetMethod(FUNCTION_VIRTUAL + FUNCTION_DTOR, None)
         assert token.token_type == tokenize.NAME or token.name == '::', token
         return_type_and_name = self._GetTokensUpTo(tokenize.SYNTAX, '(')  # )
         return_type_and_name.insert(0, token)
         if token2 is not token:
             return_type_and_name.insert(1, token2)
         return self._GetMethod(return_type_and_name, FUNCTION_VIRTUAL,
                                None, False)
 
     def handle_volatile(self):
         pass
 
     def handle_mutable(self):
         pass
 
     def handle_public(self):
         assert self.in_class
         self.visibility = VISIBILITY_PUBLIC
 
     def handle_protected(self):
         assert self.in_class
         self.visibility = VISIBILITY_PROTECTED
 
     def handle_private(self):
         assert self.in_class
         self.visibility = VISIBILITY_PRIVATE
 
     def handle_friend(self):
         tokens = self._GetTokensUpTo(tokenize.SYNTAX, ';')
         assert tokens
         t0 = tokens[0]
         return Friend(t0.start, t0.end, tokens, self.namespace_stack)
 
     def handle_static_cast(self):
         pass
 
     def handle_const_cast(self):
         pass
 
     def handle_dynamic_cast(self):
         pass
 
     def handle_reinterpret_cast(self):
         pass
 
     def handle_new(self):
         pass
 
     def handle_delete(self):
         tokens = self._GetTokensUpTo(tokenize.SYNTAX, ';')
         assert tokens
         return Delete(tokens[0].start, tokens[0].end, tokens)
 
     def handle_typedef(self):
         token = self._GetNextToken()
         if (token.token_type == tokenize.NAME and
             keywords.IsKeyword(token.name)):
             # Token must be struct/enum/union/class.
             method = getattr(self, 'handle_' + token.name)
             self._handling_typedef = True
             tokens = [method()]
             self._handling_typedef = False
         else:
             tokens = [token]
 
         # Get the remainder of the typedef up to the semi-colon.
         tokens.extend(self._GetTokensUpTo(tokenize.SYNTAX, ';'))
 
         # TODO(nnorwitz): clean all this up.
         assert tokens
         name = tokens.pop()
         indices = name
         if tokens:
             indices = tokens[0]
         if not indices:
             indices = token
         if name.name == ')':
             # HACK(nnorwitz): Handle pointers to functions "properly".
             if (len(tokens) >= 4 and
                 tokens[1].name == '(' and tokens[2].name == '*'):
                 tokens.append(name)
                 name = tokens[3]
         elif name.name == ']':
             # HACK(nnorwitz): Handle arrays properly.
             if len(tokens) >= 2:
                 tokens.append(name)
                 name = tokens[1]
         new_type = tokens
         if tokens and isinstance(tokens[0], tokenize.Token):
             new_type = self.converter.ToType(tokens)[0]
         return Typedef(indices.start, indices.end, name.name,
                        new_type, self.namespace_stack)
 
     def handle_typeid(self):
         pass  # Not needed yet.
 
     def handle_typename(self):
         pass  # Not needed yet.
 
     def _GetTemplatedTypes(self):
         result = {}
         tokens = list(self._GetMatchingChar('<', '>'))
         len_tokens = len(tokens) - 1    # Ignore trailing '>'.
         i = 0
         while i < len_tokens:
             key = tokens[i].name
             i += 1
             if keywords.IsKeyword(key) or key == ',':
                 continue
             type_name = default = None
             if i < len_tokens:
                 i += 1
                 if tokens[i-1].name == '=':
                     assert i < len_tokens, '%s %s' % (i, tokens)
                     default, unused_next_token = self.GetName(tokens[i:])
                     i += len(default)
                 else:
                     if tokens[i-1].name != ',':
                         # We got something like: Type variable.
                         # Re-adjust the key (variable) and type_name (Type).
                         key = tokens[i-1].name
                         type_name = tokens[i-2]
 
             result[key] = (type_name, default)
         return result
 
     def handle_template(self):
         token = self._GetNextToken()
         assert token.token_type == tokenize.SYNTAX, token
         assert token.name == '<', token
         templated_types = self._GetTemplatedTypes()
         # TODO(nnorwitz): for now, just ignore the template params.
         token = self._GetNextToken()
         if token.token_type == tokenize.NAME:
             if token.name == 'class':
                 return self._GetClass(Class, VISIBILITY_PRIVATE, templated_types)
             elif token.name == 'struct':
                 return self._GetClass(Struct, VISIBILITY_PUBLIC, templated_types)
             elif token.name == 'friend':
                 return self.handle_friend()
         self._AddBackToken(token)
         tokens, last = self._GetVarTokensUpTo(tokenize.SYNTAX, '(', ';')
         tokens.append(last)
         self._AddBackTokens(tokens)
         if last.name == '(':
             return self.GetMethod(FUNCTION_NONE, templated_types)
         # Must be a variable definition.
         return None
 
     def handle_true(self):
         pass  # Nothing to do.
 
     def handle_false(self):
         pass  # Nothing to do.
 
     def handle_asm(self):
         pass  # Not needed yet.
 
     def handle_class(self):
         return self._GetClass(Class, VISIBILITY_PRIVATE, None)
 
     def _GetBases(self):
         # Get base classes.
         bases = []
         while 1:
             token = self._GetNextToken()
             assert token.token_type == tokenize.NAME, token
             # TODO(nnorwitz): store kind of inheritance...maybe.
             if token.name not in ('public', 'protected', 'private'):
                 # If inheritance type is not specified, it is private.
                 # Just put the token back so we can form a name.
                 # TODO(nnorwitz): it would be good to warn about this.
                 self._AddBackToken(token)
             else:
                 # Check for virtual inheritance.
                 token = self._GetNextToken()
                 if token.name != 'virtual':
                     self._AddBackToken(token)
                 else:
                     # TODO(nnorwitz): store that we got virtual for this base.
                     pass
             base, next_token = self.GetName()
             bases_ast = self.converter.ToType(base)
             assert len(bases_ast) == 1, bases_ast
             bases.append(bases_ast[0])
             assert next_token.token_type == tokenize.SYNTAX, next_token
             if next_token.name == '{':
                 token = next_token
                 break
             # Support multiple inheritance.
             assert next_token.name == ',', next_token
         return bases, token
 
     def _GetClass(self, class_type, visibility, templated_types):
         class_name = None
         class_token = self._GetNextToken()
         if class_token.token_type != tokenize.NAME:
             assert class_token.token_type == tokenize.SYNTAX, class_token
             token = class_token
         else:
             # Skip any macro (e.g. storage class specifiers) after the
             # 'class' keyword.
             next_token = self._GetNextToken()
             if next_token.token_type == tokenize.NAME:
                 self._AddBackToken(next_token)
             else:
                 self._AddBackTokens([class_token, next_token])
             name_tokens, token = self.GetName()
             class_name = ''.join([t.name for t in name_tokens])
         bases = None
         if token.token_type == tokenize.SYNTAX:
             if token.name == ';':
                 # Forward declaration.
                 return class_type(class_token.start, class_token.end,
                                   class_name, None, templated_types, None,
                                   self.namespace_stack)
             if token.name in '*&':
                 # Inline forward declaration.  Could be method or data.
                 name_token = self._GetNextToken()
                 next_token = self._GetNextToken()
                 if next_token.name == ';':
                     # Handle data
                     modifiers = ['class']
                     return self._CreateVariable(class_token, name_token.name,
                                                 class_name,
                                                 modifiers, token.name, None)
                 else:
                     # Assume this is a method.
                     tokens = (class_token, token, name_token, next_token)
                     self._AddBackTokens(tokens)
                     return self.GetMethod(FUNCTION_NONE, None)
             if token.name == ':':
                 bases, token = self._GetBases()
 
         body = None
         if token.token_type == tokenize.SYNTAX and token.name == '{':
             assert token.token_type == tokenize.SYNTAX, token
             assert token.name == '{', token
 
             ast = AstBuilder(self.GetScope(), self.filename, class_name,
                              visibility, self.namespace_stack)
             body = list(ast.Generate())
 
             if not self._handling_typedef:
                 token = self._GetNextToken()
                 if token.token_type != tokenize.NAME:
                     assert token.token_type == tokenize.SYNTAX, token
                     assert token.name == ';', token
                 else:
                     new_class = class_type(class_token.start, class_token.end,
                                            class_name, bases, None,
                                            body, self.namespace_stack)
 
                     modifiers = []
                     return self._CreateVariable(class_token,
                                                 token.name, new_class,
                                                 modifiers, token.name, None)
         else:
             if not self._handling_typedef:
                 self.HandleError('non-typedef token', token)
             self._AddBackToken(token)
 
         return class_type(class_token.start, class_token.end, class_name,
                           bases, templated_types, body, self.namespace_stack)
 
     def handle_namespace(self):
         token = self._GetNextToken()
         # Support anonymous namespaces.
         name = None
         if token.token_type == tokenize.NAME:
             name = token.name
             token = self._GetNextToken()
         self.namespace_stack.append(name)
         assert token.token_type == tokenize.SYNTAX, token
         # Create an internal token that denotes when the namespace is complete.
         internal_token = tokenize.Token(_INTERNAL_TOKEN, _NAMESPACE_POP,
                                         None, None)
         internal_token.whence = token.whence
         if token.name == '=':
             # TODO(nnorwitz): handle aliasing namespaces.
             name, next_token = self.GetName()
             assert next_token.name == ';', next_token
             self._AddBackToken(internal_token)
         else:
             assert token.name == '{', token
             tokens = list(self.GetScope())
             # Replace the trailing } with the internal namespace pop token.
             tokens[-1] = internal_token
             # Handle namespace with nothing in it.
             self._AddBackTokens(tokens)
         return None
 
     def handle_using(self):
         tokens = self._GetTokensUpTo(tokenize.SYNTAX, ';')
         assert tokens
         return Using(tokens[0].start, tokens[0].end, tokens)
 
     def handle_explicit(self):
         assert self.in_class
         # Nothing much to do.
         # TODO(nnorwitz): maybe verify the method name == class name.
         # This must be a ctor.
         return self.GetMethod(FUNCTION_CTOR, None)
 
     def handle_this(self):
         pass  # Nothing to do.
 
     def handle_operator(self):
         # Pull off the next token(s?) and make that part of the method name.
         pass
 
     def handle_sizeof(self):
         pass
 
     def handle_case(self):
         pass
 
     def handle_switch(self):
         pass
 
     def handle_default(self):
         token = self._GetNextToken()
         assert token.token_type == tokenize.SYNTAX
         assert token.name == ':'
 
     def handle_if(self):
         pass
 
     def handle_else(self):
         pass
 
     def handle_return(self):
         tokens = self._GetTokensUpTo(tokenize.SYNTAX, ';')
         if not tokens:
             return Return(self.current_token.start, self.current_token.end, None)
         return Return(tokens[0].start, tokens[0].end, tokens)
 
     def handle_goto(self):
         tokens = self._GetTokensUpTo(tokenize.SYNTAX, ';')
         assert len(tokens) == 1, str(tokens)
         return Goto(tokens[0].start, tokens[0].end, tokens[0].name)
 
     def handle_try(self):
         pass  # Not needed yet.
 
     def handle_catch(self):
         pass  # Not needed yet.
 
     def handle_throw(self):
         pass  # Not needed yet.
 
     def handle_while(self):
         pass
 
     def handle_do(self):
         pass
 
     def handle_for(self):
         pass
 
     def handle_break(self):
         self._IgnoreUpTo(tokenize.SYNTAX, ';')
 
     def handle_continue(self):
         self._IgnoreUpTo(tokenize.SYNTAX, ';')
 
 
 def BuilderFromSource(source, filename):
     """Utility method that returns an AstBuilder from source code.
 
     Args:
       source: 'C++ source code'
       filename: 'file1'
 
     Returns:
       AstBuilder
     """
     return AstBuilder(tokenize.GetTokens(source), filename)
 
 
 def PrintIndentifiers(filename, should_print):
     """Prints all identifiers for a C++ source file.
 
     Args:
       filename: 'file1'
       should_print: predicate with signature: bool Function(token)
     """
     source = utils.ReadFile(filename, False)
     if source is None:
         sys.stderr.write('Unable to find: %s\n' % filename)
         return
 
     #print('Processing %s' % actual_filename)
     builder = BuilderFromSource(source, filename)
     try:
         for node in builder.Generate():
             if should_print(node):
                 print(node.name)
     except KeyboardInterrupt:
         return
     except:
         pass
 
 
 def PrintAllIndentifiers(filenames, should_print):
     """Prints all identifiers for each C++ source file in filenames.
 
     Args:
       filenames: ['file1', 'file2', ...]
       should_print: predicate with signature: bool Function(token)
     """
     for path in filenames:
         PrintIndentifiers(path, should_print)
 
 
 def main(argv):
     for filename in argv[1:]:
         source = utils.ReadFile(filename)
         if source is None:
             continue
 
         print('Processing %s' % filename)
         builder = BuilderFromSource(source, filename)
         try:
             entire_ast = filter(None, builder.Generate())
         except KeyboardInterrupt:
             return
         except:
             # Already printed a warning, print the traceback and continue.
             traceback.print_exc()
         else:
             if utils.DEBUG:
                 for ast in entire_ast:
                     print(ast)
 
 
 if __name__ == '__main__':
     main(sys.argv)

This page was automatically generated by the 2.3.7 LXR engine. The LXR team