"""A parser for SGML, using the derived class as a static DTD.""" # XXX This only supports those SGML features used by HTML. # XXX There should be a way to distinguish between PCDATA (parsed # character data -- the normal case), RCDATA (replaceable character # data -- only char and entity references and end tags are special) # and CDATA (character data -- only end tags are special). import re import string __all__ = ["SGMLParser"] # Regular expressions used for parsing interesting = re.compile('[&<]') incomplete = re.compile('&([a-zA-Z][a-zA-Z0-9]*|#[0-9]*)?|' '<([a-zA-Z][^<>]*|' '/([a-zA-Z][^<>]*)?|' '![^<>]*)?') entityref = re.compile('&([a-zA-Z][-.a-zA-Z0-9]*)[^a-zA-Z0-9]') charref = re.compile('&#([0-9]+)[^0-9]') starttagopen = re.compile('<[>a-zA-Z]') shorttagopen = re.compile('<[a-zA-Z][-.a-zA-Z0-9]*/') shorttag = re.compile('<([a-zA-Z][-.a-zA-Z0-9]*)/([^/]*)/') piopen = re.compile('<\?') piclose = re.compile('>') endtagopen = re.compile('a-zA-Z]') endbracket = re.compile('[<>]') special = re.compile(']*>') commentopen = re.compile(' lf.rawdata match = endbracket.search(rawdata, i+1) if not match: return -1 j = match.start(0) tag = rawdata[i+2:j].strip().lower() if rawdata[j] == '>': j = j+1 self.finish_endtag(tag) return j # Internal -- finish parsing of data) def finish_shorttag(self, tag, data): self.finish_starttag(tag, []) self.handle_data(data) self.finish_endtag(tag) # Internal -- finish processing of start tag # Return -1 for unknown tag, 0 for open-only tag, 1 for balanced tag def finish_starttag(self, tag, attrs): try: method = getattr(self, 'start_' + tag) except AttributeError: try: method = getattr(self, 'do_' + tag) except AttributeError: self.unknown_starttag(tag, attrs) return -1 else: self.handle_starttag(tag, method, attrs) return 0 else: self.stack.append(tag) self.handle_starttag(tag, method, attrs) return 1 # Internal -- finish processing of end tag def finish_endtag(self, tag): if not tag: found = len(self.stack) - 1 if found < 0: self.unknown_endtag(tag) return else: if tag not in self.stack: try: method = getattr(self, 'end_' + tag) except AttributeError: self.unknown_endtag(tag) else: self.report_unbalanced(tag) return found = len(self.stack) for i in range(found): if self.stack[i] == tag: found = i while len(self.stack) > found: tag = self.stack[-1] try: method = getattr(self, 'end_' + tag) except AttributeError: method = None if method: self.handle_endtag(tag, method) else: self.unknown_endtag(tag) del self.stack[-1] # Overridable -- handle start tag def handle_starttag(self, tag, method, attrs): method(attrs) # Overridable -- handle end tag def handle_endtag(self, tag, method): method() # Example -- report an unbalanced tag. def report_unbalanced(self, tag): if self.verbose: print '*** Unbalanced ' print '*** Stack:', self.stack # Example -- handle character reference, no need to override def handle_charref(self, name): try: n = int(name) except ValueError: self.unknown_charref(name) return if not 0 <= n <= 255: self.unknown_charref(name) return self.handle_data(chr(n)) # Definition of entities -- derived classes may override entitydefs = \ {'lt': '<', 'gt': '>', 'amp': '&', 'quot': '"', 'apos': '\''} # Example -- handle entity reference, no need to override def handle_entityref(self, name): table = self.entitydefs if table.has_key(name): self.handle_data(table[name]) else: self.unknown_entityref(name) return # Example -- handle data, should be overridden def handle_data(self, data): pass # Example -- handle comment, could be overridden def handle_comment(self, data): pass # Example -- handle declaration, could be overridden def handle_decl(self, decl): pass # Example -- handle processing instruction, could be overridden def handle_pi(self, data): pass # To be overridden -- handlers for unknown objects def unknown_starttag(self, tag, attrs): pass def unknown_endtag(self, tag): pass def unknown_charref(self, ref): pass def unknown_entityref(self, ref): pass class TestSGMLParser(SGMLParser): def __init__(self, verbose=0): self.testdata = "" SGMLParser.__init__(self, verbose) def handle_data(self, data): self.testdata = self.testdata + data if len(`self.testdata`) >= 70: self.flush() def flush(self): data = self.testdata if data: self.testdata = "" print 'data:', `data` def handle_comment(self, data): self.flush() r = `data` if len(r) > 68: r = r[:32] + '...' + r[-32:] print 'comment:', r def unknown_starttag(self, tag, attrs): self.flush() if not attrs: print 'start tag: <' + tag + '>' else: print 'start tag: <' + tag, for name, value in attrs: print name + '=' + '"' + value + '"', print '>' def unknown_endtag(self, tag): self.flush() print 'end tag: ' def unknown_entityref(self, ref): self.flush() print '*** unknown entity ref: &' + ref + ';' def unknown_charref(self, ref): self.flush() print '*** unknown char ref: &#' + ref + ';' def close(self): SGMLParser.close(self) self.flush() def test(args = None): import sys if not args: args = sys.argv[1:] if args and args[0] == '-s': args = args[1:] klass = SGMLParser else: klass = TestSGMLParser if args: file = args[0] else: file = 'test.html' if file == '-': f = sys.stdin else: try: f = open(file, 'r') except IOError, msg: print file, ":", msg sys.exit(1) data = f.read() if f is not sys.stdin: f.close() x = klass() for c in data: x.feed(c) x.close() if __name__ == '__main__': test()