#!/usr/bin/env python
#
# macroscope -- generate reports on WML macro and resource usage
#
# By Eric S. Raymond April 2007.
# (Yes, this *is* named after an ancient Piers Anthony novel.)
#
# This tool cross-references macro definitions with macro calls, and
# resource (sound or image) files with uses of the resources in WML.
# and generates various useful reports from such cross-references.
#
#
# It takes a list of directories as arguments; if none is given, it
# behaves as though the current directory had been specified as a
# single argument.  Each directory is treated as a separate domain for
# macro and resource visibility purposes, except that macros and resources
# under the first directory are made visible in all later ones.  (Typically
# the first directory should point at a copy of mainline and all later
# ones at UMC.)
#
# The checking done by this tool has a couple of flaws:
#
# (1) It doesn't actually evaluate file inclusions.  Instead, any
# macro definition satisfies any macro call made under the same
# directory.  Exception: when an #undef is detected, the macro is
# tagged local and not visible outside the span of lines where it's
# defined.
#
# (2) It doesn't read [binary_path] tags, as this would require
# implementing a WML parser.  Instead, it assumes that a resource-file
# reference can be satisfied by any matching image file from anywhere
# in the same directory it came from.  The resources under the *first*
# directory argument (only) are visible everywhere.
#
# (3) A reference with embedded {}s in a macro will have the macro's
# formal args substituted in at WML evaluation time.  Instead, this
# tool treats each {} as a .* wildcard and considers the reference to
# match *every* resource filename that matches that pattern.  Under
# appropriate circumstances this might report a resource filename
# statically matching the pattern as having been referenced even
# though none of the actual macro calls would actually generate it.
#
# (4) There are some implicit references.  Notably, if an attack name
# is specified but no icon is given, the attack icon will default to
# a name generated from the attack name,
#
# Problems (1) and (2) imply that this tool might conceivably report
# that a reference has been satisfied when under actual
# WML-interpreter rules it has not.  Problem (4) means the reverse
# can also occur.
#
# The reporting format is compatible with GNU Emacs compile mode.

import sys, os, time, re, getopt, sre_constants, md5

resource_extensions = ("png", "jpg", "ogg", "wav")

def htmlize(line):
    "HTML-escape a text line"
    return line.replace("<", "&lt;").replace(">", "&gt;").replace("&", "&amp;")

def interpret(lines, css):
    "Interpret the ! convention for .cfg comments."
    inlisting = False
    outstr = '<p class="%s">' % css
    for line in lines:
        line = line.rstrip()
        if not inlisting and not line:
            outstr += "</p><p>"
            continue
        if not inlisting and line[0] == '!':
            outstr += "</p>\n<pre class='listing'>"
            inlisting = True
            bracketdepth = curlydepth = 0
        line = htmlize(line)
        if inlisting:
            outstr += line[1:] + "\n"
        else:
            outstr += line + "\n"
        if inlisting:
            if line and line[0] != '!':
                outstr += "</pre>\n<p>"
                inlisting = False
    if not inlisting:
        outstr += "</p>\n"
    else:
        outstr += "</pre>\n"
    outstr = outstr.replace("<p></p>", "")
    outstr = outstr.replace("\n\n</pre>", "\n</pre>")
    return outstr

class Forest:
    "Return an iterable directory forest object."
    def __init__(self, dirpath, exclude=None):
        "Get the names of all files under dirpath, ignoring .svn directories."
        self.forest = []
        self.dirpath = dirpath
        for dir in dirpath:
            subtree = []
            if os.path.isdir(dir):	# So we skip .cfgs in a UMC mirror
                os.path.walk(dir,
                         lambda arg, dir, names: subtree.extend(map(lambda x: os.path.normpath(os.path.join(dir, x)), names)),
                         None)
            self.forest.append(subtree)
        for i in range(len(self.forest)):
            self.forest[i] = filter(lambda x: ".svn" not in x, self.forest[i])
            self.forest[i] = filter(lambda x: not os.path.isdir(x), self.forest[i])
            if exclude:
                self.forest[i] = filter(lambda x: not re.search(exclude, x), self.forest[i])
            self.forest[i] = filter(lambda x: not x.endswith("-bak"), self.forest[i])
        # Compute cliques (will be used later for visibility checks) 
        self.clique = {}
        counter = 0
        for tree in self.forest:
            for filename in tree:
                self.clique[filename] = counter
            counter += 1
    def parent(self, filename):
        "Return the directory root that caused this path to be included."
        return self.dirpath[self.clique[filename]]
    def neighbors(self, fn1, fn2):
        "Are two files from the same tree?"
        return self.clique[fn1] == self.clique[fn2]
    def flatten(self):
        allfiles = []
        for tree in self.forest:
            allfiles += tree
        return allfiles
    def generator(self):
        "Return a generator that walks through all files."
        for tree in self.forest:
            for filename in tree:
                yield filename

def iswml(filename):
    "Is the specified filename WML?"
    return filename.endswith(".cfg")

def isresource(filename):
    "Is the specifired name a resource?"
    (root, ext) = os.path.splitext(filename)
    return ext and ext[1:] in resource_extensions

class reference:
    "Describes a location by file and line."
    def __init__(self, filename, lineno=None, docstring=None):
        self.filename = filename
        self.lineno = lineno
        self.references = {}
        self.docstring = docstring
        self.undef = None
    def append(self, fn, n):
        if fn not in self.references:
            self.references[fn] = []
        self.references[fn].append(n+1)
    def dump_references(self):
        for (file, linenumbers) in self.references.items():
            print "    %s: %s" % (file, `linenumbers`[1:-1])
    def __cmp__(self, other):
        "Compare two documentation objects for place in the sort order."
        # Major sort by file, minor by line number.  This presumes that the
        # files correspond to coherent topics and gives us control of the
        # sequence.
        byfile = cmp(self.filename, other.filename)
        if byfile:
            return byfile
        else:
            return cmp(self.lineno, other.lineno)
    def __str__(self):
        if self.lineno:
            return '"%s", line %d' % (self.filename, self.lineno)
        else:
            return self.filename

class CrossRef:
    macro_reference = re.compile(r"\{([A-Z_][A-Z0-9_:]*[A-Za-z0-9_])\b")
    file_reference =  re.compile(r"[A-Za-z0-9{}.][A-Za-z0-9_/+{}.-]*\.(" + "|".join(resource_extensions) + ")")
    def mark_matching_resources(self, pattern, fn, n):
        "Mark all definitions matching a specified pattern with a reference."
        pattern = pattern.replace("+", r"\+")
        try:
            pattern = re.compile(os.sep + pattern + "$")
        except sre_constants.error:
            print >>sys.stderr, "macroscope: confused by %s" % pattern
            return None
        key = None
        for trial in self.fileref:
            if pattern.search(trial) and self.visible_from(trial, fn, n):
                key = trial
                self.fileref[key].append(fn, n)
        return key
    def visible_from(self, defn, fn, n):
        "Is specified definition visible from the specified file and line?"
        if type(defn) == type(""):
            defn = self.fileref[defn]
        if defn.undef != None:
            # Local macros are only visible in the file where they were defined
            # FIXME: we should check line spans here.
            return defn.filename == fn
        elif defn.filename in self.filelist.forest[0]:
            # Macros in the first subtree are visible everywhere.
            return True
        elif not self.filelist.neighbors(defn.filename, fn):
            # Otherwise, must be in the same subtree.
            return False
        else:
            # If the two files are in the same subtree, assume visibility.
            # This doesn't match the actual preprocessor semantics.
            # It means any macro without an undef is visible anywhere in the
            # same argument directory.
            #
            # We can't do better than this without a lot of hairy graph-
            # coloring logic to simulate include path interpretation.
            # If that logic ever gets built, it will go here.
            return True
    def __init__(self, dirpath, exclude="", warnlevel=0):
        "Build cross-reference object from the specified filelist."
        self.dirpath = dirpath
        self.filelist = Forest(dirpath, exclude)
        self.xref = {}
        self.fileref = {}
        self.noxref = False
        for filename in self.filelist.generator():
            if warnlevel > 1:
                print filename + ":"
            if isresource(filename):
                self.fileref[filename] = reference(filename) 
            elif iswml(filename):
                # It's a WML file, scan for macro defitions
                dfp = open(filename)
                state = "outside"
                for (n, line) in enumerate(dfp):
                    if warnlevel > 1:
                        print `line`[1:-1]
                    if line.strip().startswith("#define"):
                        tokens = line.split()
                        name = tokens[1]
                        here = reference(filename, n+1, line)
                        here.hash = md5.new()
                        here.docstring = line.lstrip()[8:]	# Strip off #define_
                        state = "macro_header"
                        continue
                    elif state != 'outside' and line.strip().endswith("#enddef"):
                        here.hash.update(line)
                        here.hash = here.hash.digest()
                        if name in self.xref:
                            for defn in self.xref[name]:
                                if not self.visible_from(defn, filename, n):
                                    continue
                                elif defn.hash != here.hash:
                                    print >>sys.stderr, \
                                            "%s: overrides different %s definition at %s" \
                                            % (here, name, defn)
                                elif warnlevel > 0:
                                    print >>sys.stderr, \
                                            "%s: duplicates %s definition at %s" \
                                            % (here, name, defn)
                        if name not in self.xref:
                            self.xref[name] = []
                        self.xref[name].append(here)
                        state = "outside"
                    elif state == "macro_header" and line and line[0] != "#":
                        state = "macro_body"
                    if state == "macro_header":
                        here.docstring += line[1:]
                    if state in ("macro_header", "macro_body"):
                        here.hash.update(line)
                    elif line.strip().startswith("#undef"):
                        tokens = line.split()
                        name = tokens[1]
                        if name in self.xref and self.xref[name]:
                            self.xref[name][-1].undef = n
                        else:
                            print "%s: unbalanced #undef on %s" \
                                  % (reference(filename, n), name)
                dfp.close()
            elif filename.endswith(".def"):
                # It's a list of names to be considered defined
                self.noxref = True
                dfp = open(filename)
                for line in dfp:
                    self.xref[line.strip()] = True
                dfp.close()
            
        # Next, decorate definitions with all references from the filelist.
        self.unresolved = []
        self.missing = []
        formals = []
        for fn in self.filelist.generator():
            if iswml(fn):
                rfp = open(fn)
                for (n, line) in enumerate(rfp):
                    if line.startswith("#define"):
                        formals = line.split()[2:]
                    elif line.startswith("#enddef"):
                        formals = []
                    if '#' in line:
                        line = line.split('#')[0]
                    if not line:
                        continue
                    # Find references to macros
                    for match in re.finditer(CrossRef.macro_reference, line):
                        name = match.group(1)
                        candidates = 0
                        if name in formals:
                            continue
                        elif name in self.xref:
                            for defn in self.xref[name]:
                                if self.visible_from(defn, fn, n+1):
                                    candidates += 1
                                    defn.append(fn, n+1)
                            if candidates > 1:
                                print "%s: more than one definition of %s is visible here." % (reference(fn, n), name)
                            if candidates == 0:
                                 self.unresolved.append((name, reference(fn,n+1)))
                    # Find references to resource files
                    for match in re.finditer(CrossRef.file_reference, line):
                        name = match.group(0)
                        key = None
                        # If name is already in our resource list, it's easy.
                        if name in self.fileref and self.visible_from(name, fn, n):
                            self.fileref[trial].append(fn, n+1)
                            continue
                        # If the name contains subtitutable parts, count
                        # it as a reference to everything the substitutions
                        # could potentially match.
                        elif '{' in name:
                            pattern = re.sub(r"\{[^}]*\}", '.*', name)
                            key = self.mark_matching_resources(pattern, fn,n+1)
                            if key:
                                self.fileref[key].append(fn, n+1)
                        else:
                            candidates = []
                            for trial in self.fileref:
                                if trial.endswith(os.sep + name) and self.visible_from(trial, fn, n):
                                    key = trial
                                    self.fileref[trial].append(fn, n+1)
                                    candidates.append(trial)
                            if len(candidates) > 1:
                                print "%s: more than one definition of %s is visible here (%s)." % (reference(fn, n), name, ", ".join(candidates))
                        if not key:
                            self.missing.append((name, reference(fn,n+1)))
                rfp.close()
    def xrefdump(self, pred=None):
        "Report resolved macro references."
        for name in self.xref:
            for defn in self.xref[name]:
                if pred and not pred(name, defn):
                    continue
                defn.dump_references()
                if defn.undef:
                    type = "local"
                else:
                    type = "global"
                nrefs = len(defn.references)
                if nrefs == 0:
                    print "%s: %s macro %s is unused" % (defn, type, name)
                else:
                    print "%s: %s macro %s is used in %d files:" % (defn, type, name, nrefs)
        for (name, defloc) in self.fileref.items():
            if pred and not pred(name, defloc):
                continue
            nrefs = len(defloc.references)
            if nrefs == 0:
                print "Resource %s is unused" % defloc
            else:
                print "Resource %s is used in %d files:" % (defloc, nrefs)
            defloc.dump_references()
    def unresdump(self):
        "Report unresolved references."
        if len(self.unresolved) == 0 and len(self.missing) == 0:
            print "# No unresolved references"
        else:
            #print self.fileref.keys()
            print "# Unresolved references:"
            for (name, reference) in self.unresolved + self.missing:
                print "%s -> %s" % (reference, name)
    def deflist(self, pred=None):
        "List all resource definitions."
        for name in self.xref:
            for defn in self.xref[name]:
                if not pred or pred(name, defn):
                    print name
        for (name, defloc) in self.fileref.items():
            if not pred or pred(name, defloc):
                print name
    def extracthelp(self, pref, fp):
        "Deliver all macro help comments in HTML form."
        # Bug: finds only the first definition of each macro in scope.
        doclist = self.xref.keys()
        doclist = filter(lambda x: self.xref[x][0].docstring.count("\n") > 1, doclist)
        doclist.sort(lambda x, y: cmp(self.xref[x][0], self.xref[y]))
        outstr = ""
        filename = None
        counted = 0
        for name in doclist:
            entry = self.xref[name][0]
            if entry.filename != filename:
                if counted:
                    outstr += "</dl>\n"
                counted += 1
                filename = entry.filename
                if filename.startswith(pref):
                    displayname = filename[len(pref):]
                else:
                    displayname = filename
                outstr += "<h1 class='file_header'>From file: " + displayname + "</h1>\n"
                hdr = []
                dfp = open(filename)
                for line in dfp:
                    if line[0] == '#':
                        hdr.append(line[1:])
                    else:
                        break
                dfp.close()
                if hdr:
                    outstr += interpret(hdr, "file_explanation")
                outstr += "<dl>\n"
            if entry.docstring:
                lines = entry.docstring.split("\n")
                header = lines.pop(0).split()
                if lines and not lines[-1]:	# Ignore trailing blank lines
                    lines.pop()
                if not lines:	# Ignore definitions without a docstring
                    continue
                outstr += "\n<dt>\n"
                outstr += "<em class='macro_name'>" + header[0] + "</em>"
                if header[1:]:
                    outstr += " <em class='macro_formals'>"+" ".join(header[1:])+"</em>"
                outstr += "\n</dt>\n"
                outstr += "<dd>\n"
                outstr += interpret(lines, "macro_explanation")
                outstr += "</dd>\n"
        outstr += "</dl>\n"
        fp.write(outstr)

if __name__ == "__main__":
    def help():
        sys.stderr.write("""\
Usage: macroscope [options] dirpath
    Options may be any of these:
    -h, --help                   Emit this help message and quit
    -c, --crossreference         Report resolved macro references (implies -w 1)
    -C, --collisions             Report duplicate resource files   
    -d, --deflist                Make definition list
    -e regexp, --exclude regexp  Ignore files matching the specified regular expression
    -f dir, --from dir           Report only on macros defined under dir
    -l, --listfiles              List files that will be processed
    -r ddd, --refcount=ddd       Report only on macros w/references in ddd files
    -u, --unresolved             Report unresolved macro references
    -w, --warnlevel              Set to 1 to warn of duplicate macro definitions
    --forced-used regexp         Ignore refcount 0 on names matching regexp
    --extracthelp                Extract help from macro definition comments.
   Options may be followed by any number of directiories to check.  If no
   directories are given, all files under the current directory are checked.
""")

    # Process options
    (options, arguments) = getopt.getopt(sys.argv[1:], "cCdhe:f:lr:uw:",
                                         [
                                          'crossreference',
                                          'collisions',
                                          'definitions',
                                          'exclude=',
                                          'extracthelp',
                                          'force-used=',
                                          'from=',
        				  'help',
                                          'listfiles',
                                          'refcount=',
                                          'unresolved',
                                          'warnlevel=',
                                          ])
    crossreference = definitions = listfiles = unresolved = extracthelp = False
    from_restrict = None
    refcount_restrict = None
    forceused = None
    exclude = []
    warnlevel = 0
    collisions = False
    for (switch, val) in options:
        if switch in ('-h', '--help'):
            help()
            sys.exit(0)
        if switch in ('-f', '--from'):
            from_restrict = val
        elif switch in ('-c', '--crossreference'):
            crossreference = True
            warnlevel = 1
        elif switch in ('-C', '--collisions'):
            collisions = True
        elif switch in ('-d', '--definitions'):
            definitions = True
        elif switch in ('-e', '--exclude'):
            exclude.append(val)
        elif switch == '--extracthelp':
            extracthelp = True
        elif switch == '--force-used':
            forceused = val
        elif switch in ('-l', '--listfiles'):
            listfiles = True
        elif switch in ('-r', '--refcount'):
            refcount_restrict = int(val)
        elif switch in ('-u', '--unresolved'):
            unresolved = True
        elif switch in ('-w', '--warnlevel'):
            warnlevel = int(val)

    if len(arguments):
        dirpath = arguments
    else:
        dirpath = ['.']
    if not extracthelp:
        print "# Macroscope reporting on %s" % time.ctime()
        print "# Invocation: %s" % " ".join(sys.argv)
        print "# Working directory: %s" % os.getcwd()
    xref = CrossRef(dirpath, "|".join(exclude), warnlevel)
    if extracthelp:
        xref.extracthelp(dirpath[0], sys.stdout)
    elif listfiles:
        for filename in xref.filelist.generator():
            print filename
    if collisions:
        collisions = []
        for filename in xref.filelist.generator():
            ifp = open(filename)
            collisions.append(md5.new(ifp.read()).digest())
            ifp.close()
        collisions = zip(xref.filelist.flatten(), collisions)
        hashcounts = {}
        for (n, h) in collisions:
            hashcounts[h] = hashcounts.get(h, 0) + 1
        collisions = filter(lambda (n, h): hashcounts[h] > 1, collisions)
        collisions.sort(lambda (n1, h1), (n2, h2): cmp(h1, h2))
        lasthash = None
        for (n, h) in collisions:
            if h != lasthash:
                print "%%"
                lasthash = h
            print n
    elif crossreference or definitions or listfiles or unresolved:
        def predicate(name, defloc):
            if from_restrict and not defloc.filename.startswith(from_restrict):
                return False
            if refcount_restrict!=None \
                   and len(defloc.references) != refcount_restrict \
                   or (refcount_restrict == 0 and forceused and re.search(forceused, name)):
                return False
            return True
        if crossreference:
            if xref.noxref:
                print >>sys.stderr, "macroscope: can't make cross-reference, input included a definitions file."
            else:
                xref.xrefdump(predicate)
        if definitions:
            xref.deflist(predicate)
        if unresolved:
            xref.unresdump()
    
# macroscope ends here