Viewing file:  TreeCompare.py (17.37 KB) -rw-r--r--
Select action/file-type:
 (+) |  (+) |  (+) | Code (+) | Session (+) |  (+) | SDB (+) |  (+) |  (+) |  (+) |  (+) |  (+) |
########################################################################
# $Header: /var/local/cvsroot/4Suite/Ft/Xml/Lib/TreeCompare.py,v 1.26 2005/03/18 23:47:16 jkloth Exp $
"""
Comparison functions for XML and HTML documents
(mainly used in the test suites)
Copyright 2005 Fourthought, Inc. (USA).
Detailed license and copyright information: http://4suite.org/COPYRIGHT
Project home, documentation, distributions: http://4suite.org/
"""
import re, sgmllib
from sgmllib import SGMLParser
from xml.dom import Node
# adds colon to regex patterns for names
sgmllib.tagfind = re.compile('[a-zA-Z][-:_.a-zA-Z0-9]*')
sgmllib.attrfind = re.compile(r'\s*([a-zA-Z_][-:_.a-zA-Z0-9]*)(\s*=\s*'
r'(\'[^\']*\'|"[^"]*"|\S*))?')
from Ft.Lib.Uri import BASIC_RESOLVER, OsPathToUri
from Ft.Xml import Domlette, InputSource, XMLNS_NAMESPACE
from Ft.Xml.Lib.XmlString import XmlStrStrip, IsXmlSpace
import HtmlPrinter
FORBIDDEN_END_ELEMENTS = HtmlPrinter.HtmlPrinter.forbiddenEndElements.keys()
del HtmlPrinter
_S = "[\x20\x09\x0D\x0A]"
_OptionalS = _S + "?"
_VersionNum = "[a-zA-Z0-9_.:-]+"
_Eq = "%s?=%s?" % (_S, _S)
_VersionInfo = _S + "version" + _Eq + \
"(?:(?:'" + _VersionNum + "')|" + '(?:"' + _VersionNum + '"))'
_EncName = "[A-Za-z][A-Za-z0-9._-]*"
_EncodingDecl = _S + "encoding" + _Eq + \
"(?:(?:'" + _EncName + "')|" + '(?:"' + _EncName + '"))'
_SDDecl = _S + "standalone" + _Eq + \
"(?:(?:'(?:yes|no)')|" + '(?:"(?:yes|no)"))'
g_xmlTest = re.compile(r"<\?xml" +
r"(?P<VersionInfo>%s)" % _VersionInfo +
r"(?P<EncodingDecl>%s)?" % _EncodingDecl +
r"(?P<SDDecl>%s)?" % _SDDecl +
r"%s?\?>" % _S)
g_doctypeTest = re.compile("(<!DOCTYPE[\x20\x09\x0D\x0A])")
g_htmlTest = re.compile("(<!doctype html)|(<html)",re.IGNORECASE)
def HtmlTreeCompare(expected, compared):
"""
Compare two HTML strings. The result is similar to the builtin cmp()
function such that non-zero indicates non equal and zero means equal.
"""
return not CompareHTML(expected, compared)
def XmlTreeCompare(expected, compared):
# External Parsed Entities cannot have a standalone declaration or
# DOCTYPE declaration.
# See XML 1.0 2nd, 4.3.2, Well-Formed Parsed Entities
match = g_xmlTest.match(expected)
if match and match.groupdict().get('SDDecl'):
asEntity = False
else:
asEntity = not g_doctypeTest.search(expected)
return TreeCompare(expected, compared, asEntity=asEntity)
def NoWsTreeCompare(expected, compared):
"""
Equivalent to calling TreeCompare() with ignoreWhitespace=1.
"""
return TreeCompare(expected, compared, ignoreWhitespace=1)
from Ft.Xml import READ_EXTERNAL_DTD
def TreeCompare(expected, compared, ignoreWhitespace=0, baseUri=None,
readExtDtd=READ_EXTERNAL_DTD, ignoreNsDecls=0, asEntity=False):
"""
A cmp()-like function that compares two XML or HTML strings and
has the side effect of reporting differences to stdout. Returns
false if the nodes compare equal.
XML strings are parsed into a Domlette and compared node-by-node.
HTML strings are parsed with an SGML parser and are compared
event-by-event. The markup type is guessed based on clues in the
expected string.
ignoreWhitespace controls whether whitespace differences in text
nodes are ignored.
'file:' URIs based on the current working directory are generated
for each document. The baseUri argument is an optional absolute URI
to use as the basis of the generated URIs, if a 'file' URI is
undesirable.
readExtDtd controls whether the external DTD subset is read
when parsing XML. It does not affect the reading of external
entities declared in the internal DTD subset.
ignoreNsDecls controls whether namespace declarations are ignored
when comparing XML documents.
"""
# See if we need to use XML or HTML
if not g_xmlTest.match(expected) and g_htmlTest.search(expected):
return not CompareHTML(expected, compared, ignoreWhitespace)
if not baseUri:
uri = OsPathToUri('expected', attemptAbsolute=True)
else:
uri = BASIC_RESOLVER.normalize('expected', baseUri)
try:
if asEntity:
reader = Domlette.EntityReader
elif readExtDtd:
reader = Domlette.NonvalidatingReader
else:
reader = Domlette.NoExtDtdReader
doc1 = reader.parseString(str(expected), uri)
except:
print '--- Expected ---'
print expected
raise
if not baseUri:
uri = OsPathToUri('expected', attemptAbsolute=True)
else:
uri = BASIC_RESOLVER.normalize('compared', baseUri)
try:
doc2 = reader.parseString(compared, uri)
except:
print '--- Compared ---'
print compared
raise
# If an entity is also a well-formed document entity, remove any
# top-level whitespace-only text nodes.
if asEntity:
_TryEntityAsDocumentEntity(doc1)
_TryEntityAsDocumentEntity(doc2)
result = NodeCompare(doc1, doc2,
ignoreWhitespace=ignoreWhitespace,
ignoreNsDecls=ignoreNsDecls)
return not result
def _TryEntityAsDocumentEntity(entity):
# If the entity only has 1 top-level element, strip top-level whitespace
# only text nodes to match how a document entity would have been parsed.
elements = 0
for node in entity.childNodes:
elements += int(node.nodeType == Node.ELEMENT_NODE)
if elements == 1:
# OK to strip WS-only text nodes
nodes = [ x for x in entity.childNodes
if x.nodeType == Node.TEXT_NODE and IsXmlSpace(x.data) ]
for node in nodes:
entity.removeChild(node)
return
def NodeCompare(node1, node2, ignoreWhitespace=0, ignoreComments=0,
ignoreNsDecls=0):
"""
A function that compares two XML DOM nodes by traversing their
attributes and descendants recursively until a mismatch is found.
It has the side effect of reporting differences to stdout. Returns
true if the nodes compare equal.
ignoreWhitespace controls whether whitespace differences in text
nodes are ignored.
ignoreComments controls whether comment nodes are ignored.
ignoreNsDecls controls whether namespace declarations are ignored.
"""
if node1.nodeType != node2.nodeType:
return __ReportError(node1, node2, 'nodeType')
# -- Document Nodes --------------------------------------
if node1.nodeType in (Node.DOCUMENT_NODE, Node.DOCUMENT_FRAGMENT_NODE):
if ignoreComments:
children1 = filter(lambda n: n.nodeType != Node.COMMENT_NODE, node1.childNodes)
children2 = filter(lambda n: n.nodeType != Node.COMMENT_NODE, node2.childNodes)
else:
children1 = node1.childNodes
children2 = node2.childNodes
for child1, child2 in map(None, children1, children2):
if not NodeCompare(child1, child2, ignoreWhitespace,
ignoreComments, ignoreNsDecls):
return 0
# -- Document Type Nodes ---------------------------------
elif node1.nodeType == Node.DOCUMENT_TYPE_NODE:
if node1.name != node2.name:
return __ReportError(node1, node2, 'name')
if node1.publicId != node2.publicId:
return __ReportError(node1, node2, 'publicId')
if node1.systemId != node2.systemId:
return __ReportError(node1, node2, 'systemId')
if node1.internalSubset != node2.internalSubset:
return __ReportError(node1, node2, 'internalSubset')
if len(node1.entities) != len(node2.entities):
return __ReportError(node1, node2, 'entities')
if len(node1.notations) != len(node2.notations):
return __ReportError(node1, node2, 'notations')
# -- Element Nodes ---------------------------------------
elif node1.nodeType == Node.ELEMENT_NODE:
if node1.localName != node2.localName:
return __ReportError(node1, node2, 'localName')
if node1.namespaceURI != node2.namespaceURI:
return __ReportError(node1, node2, 'namespaceURI')
# Compare attributes
attrs1 = node1.attributes.values()
attrs2 = node2.attributes.values()
if ignoreNsDecls:
# Remove XML Namespace declarations
attrs1 = [ a for a in attrs1 if a.namespaceURI != XMLNS_NAMESPACE ]
attrs2 = [ a for a in attrs2 if a.namespaceURI != XMLNS_NAMESPACE ]
if len(attrs1) != len(attrs2):
return __ReportError(node1, node2, 'attributes')
# Sort the attributes by qualified name
attrs1.sort(lambda a, b: cmp(a.name, b.name))
attrs2.sort(lambda a, b: cmp(a.name, b.name))
for attr1, attr2 in zip(attrs1, attrs2):
if attr1.localName != attr2.localName:
print node1.attributes.keys()
print node2.attributes.keys()
return __ReportError(attr1, attr2, 'localName')
if attr1.namespaceURI != attr2.namespaceURI:
return __ReportError(attr1, attr2, 'namespaceURI')
# Compare children
if ignoreComments:
# Remove comment nodes
children1 = [ c for c in node1.childNodes
if c.nodeType != Node.COMMENT_NODE ]
children2 = [ c for c in node2.childNodes
if c.nodeType != Node.COMMENT_NODE ]
else:
children1 = node1.childNodes
children2 = node2.childNodes
if len(children1) != len(children2):
return __ReportError(node1, node2, 'childNodes')
for child1, child2 in zip(children1, children2):
if not NodeCompare(child1, child2, ignoreWhitespace,
ignoreComments, ignoreNsDecls):
return 0
# -- Text Nodes ------------------------------------------
elif node1.nodeType == Node.TEXT_NODE:
text1 = node1.data
text2 = node2.data
if ignoreWhitespace:
if IsXmlSpace(text1):
text1 = None
if IsXmlSpace(text2):
text2 = None
if cmp(text1, text2):
return __ReportError(node1, node2, 'data')
# -- Comment Nodes ---------------------------------------
elif node1.nodeType == Node.COMMENT_NODE:
if node1.data != node2.data:
return __ReportError(node1, node2, 'data')
# -- Processing Instruction Nodes ------------------------
elif node1.nodeType == Node.PROCESSING_INSTRUCTION_NODE:
if node1.target != node2.target:
return __ReportError(node1, node2, 'target')
if node1.data != node2.data:
return __ReportError(node1, node2, 'data')
# All tests pass, they are the same
return 1
def __ReportError(node1, node2, attribute):
import pprint
print '--- expected ---'
__PrintParentage(node1)
print 'node: %s' % repr(node1)
print 'node.%s:' % attribute
pprint.pprint(getattr(node1, attribute))
print '--- compared ---'
__PrintParentage(node2)
print 'node: %s' % repr(node2)
print 'node.%s:' % attribute
pprint.pprint(getattr(node2, attribute))
return 0
def __PrintParentage(node):
nodes = [node]
if node.nodeType == Node.ATTRIBUTE_NODE:
parent = node.ownerElement
else:
parent = node.parentNode
while parent:
nodes.insert(0, parent)
parent = parent.parentNode
indent = ''
for node in nodes:
print '%s%s' % (indent, node.nodeName)
indent = indent + ' '
class SGMLParserEventGenerator(SGMLParser):
"""
An HTML parser that meets our needs better than Python's
htmllib.HTMLParser, and that works with Python 2.1.
Used by CompareHTML().
"""
TEXT_EVENT = 1
COMMENT_EVENT = 2
START_TAG_EVENT = 3
END_TAG_EVENT = 4
ENTITYREF_EVENT = 6
CHARREF_EVENT = 7
def __init__(self, verbose=0):
self.testdata = ""
SGMLParser.__init__(self, verbose)
self.events = []
self.ignoreable_ws = 1
def handle_data(self, data):
self.testdata = self.testdata + data
def flush(self):
data = self.testdata
if data:
self.testdata = ""
if not self.ignoreable_ws:
self.events.append((self.TEXT_EVENT,data))
def handle_comment(self, data):
self.flush()
self.events.append((self.COMMENT_EVENT,data))
def unknown_starttag(self, tagname, attrs):
self.flush()
if self.ignoreable_ws:
self.ignoreable_ws = 0
# Convert list of tuples to dictionary
dict = {}
for name, value in attrs:
dict[name] = value
self.events.append((self.START_TAG_EVENT, tagname, dict))
if tagname.lower() in FORBIDDEN_END_ELEMENTS:
self.events.append((self.END_TAG_EVENT,tagname))
return
def unknown_endtag(self, tag):
self.flush()
self.events.append((self.END_TAG_EVENT,tag))
def unknown_entityref(self, ref):
self.flush()
self.events.append((self.ENTITYREF_EVENT,ref))
def unknown_charref(self, ref):
self.flush()
self.events.append((self.CHARREF_EVENT,ref))
def close(self):
SGMLParser.close(self)
self.flush()
g_xmlEmptyTagPattern = re.compile(
'<([a-zA-Z][-:_.a-zA-Z0-9]*\s*([a-zA-Z_][-:_.a-zA-Z0-9]*)(\s*=\s*'
r'(\'[^\']*\'|"[^"]*"|\S*))?)/>')
def CompareHTML(html1, html2, ignoreWhitespace=0):
"""
A cmp()-like function that compares two HTML strings by parsing
with sgmllib.SGMLParser and comparing events until a mismatch is
found. It has the side effect of reporting differences to stdout.
ignoreWhitespace controls whether whitespace differences in text
events are ignored.
"""
# SGMLParser goes awry when <emptyelement/> tags are encountered.
# Obviously this is a very kludgy 'solution'
html1 = g_xmlEmptyTagPattern.sub(r'<\1>',html1)
html2 = g_xmlEmptyTagPattern.sub(r'<\1>',html2)
p1 = SGMLParserEventGenerator()
p1.feed(html1)
p1.close()
p2 = SGMLParserEventGenerator()
p2.feed(html2)
p2.close()
stack = [] #Not 100% accurate, but close enough
for cur1, cur2 in zip(p1.events, p2.events):
if cur1[0] != cur2[0]:
#different events
return __ReportEventError(cur1,cur2,stack,'different events')
event = cur1[0]
if event == SGMLParserEventGenerator.TEXT_EVENT:
# Compare the text of each
d1 = cur1[1]
d2 = cur2[1]
if ignoreWhitespace and XmlStrStrip(d1) != XmlStrStrip(d2):
return __ReportEventError(cur1,cur2,stack,'data')
elif event == SGMLParserEventGenerator.COMMENT_EVENT:
d1 = cur1[1]
d2 = cur2[1]
if d1.strip() != d2.strip():
return __ReportEventError(cur1,cur2,stack,'comment data')
elif event == SGMLParserEventGenerator.START_TAG_EVENT:
if cur1[1] != cur2[1]:
return __ReportEventError(cur1,cur2,stack,'start tag name')
stack.append(cur1[1]) #Save for a nice print out
att1 = cur1[2]
att2 = cur2[2]
if len(att1) != len(att2):
return __ReportEventError(cur1,cur2,stack,'number of attributes')
for name, value in att1.items():
if att2.get(name, -1) != value:
return __ReportEventError(cur1,cur2,stack,'attribute value %s' % name)
elif event == SGMLParserEventGenerator.END_TAG_EVENT:
if cur1[1] != cur2[1]:
return __ReportEventError(cur1,cur2,stack,'end tag name')
while stack and stack[-1] != cur1[1]: #Remove it
del stack[-1]
del stack[-1]
elif event == SGMLParserEventGenerator.ENTITYREF_EVENT:
if cur1[1] != cur2[1]:
return __ReportEventError(cur1,cur2,stack,'entity ref')
elif event == SGMLParserEventGenerator.CHARREF_EVENT:
if cur1[1] != cur2[1]:
return __ReportEventError(cur1,cur2,stack,'char ref')
else:
raise cur1
return 1
def __ReportEventError(event1, event2, stack, attribute):
__PrintStack(stack)
print '--- Expected ---'
print attribute, repr(event1[1:])
print '--- Compared ---'
print attribute, repr(event2[1:])
return 0
def __PrintStack(stack):
indent = ''
for name in stack:
print "%s%s" % (indent,name)
indent += ' '
## if isHtml:
## # HTML DOM should already capitalize all tagNames
## if node1.tagName != node2.tagName:
## return __ReportError(node1, node2, 'tagName')
## # Elements where whitespace is significant
## if node1.tagName in ['SCRIPT', 'STYLE', 'PRE', 'TEXTAREA']:
## ignoreWhitespace = 0
## if isHtml:
## # HTML DOMs should force upper case already
## # FIXME: PyXML 0.7 changed HTML attributes to be NS so they
## # are no longer forced to uppercase!
## if attr1.name.upper() != attr2.name.upper():
## return __ReportError(attr1, attr2, 'name')
## if ignoreWhitespace or isHtml:
## if not XmlStrStrip(text1):
## text1 = None
## if not XmlStrStrip(text2):
## text2 = None
|