#!/usr/bin/env python
#
# macroscope -- generate reports on WML macro and resource usage
#
# By Eric S. Raymond April 2007.
# (Yes, this *is* named after an ancient Piers Anthony novel.)
#
# This tool cross-references macro definitions with macro calls, and
# resource (sound or image) files with uses of the resources in WML.
# and generates various useful reports from such cross-references.
# The checking done by this tool has a couple of flaws:
#
# (1) It doesn't actually evaluate file inclusions.  Instead, any
# macro definition from anywhere in the set of input trees can be used
# to satisfy a macro call anywhere else.
#
# (2) It doesn't read [binary_path] tags, as this would require
# implementing a WML parser.  Instead, it assumes that a resource-file
# reference can be satisfied by any matching image file from anywhere
# in the set of input trees.
#
# (3) A reference with embedded {}s in a macro will have the macro's
# formal args substituted in at WML evaluation time.  Instead, this
# tool treats each {} as a .* wildcard and considers the reference to
# match *every* resource filename that matches that pattern.  Under
# appropriate circumstances this might report a resource filename
# statically matching the pattern as having been referenced even
# though none of the actual macro calls would actually generate it.
#
# Problems (1) and (2) imply that this tool might conceivably report
# that a reference has been satisfied when under actual
# WML-interpreter rules it has not.  The reverse will not occur.
#
# The reporting format is compatible with GNU Emacs compile mode.

import sys, os, time, re, getopt

resource_extensions = ("png", "jpg", "ogg", "wav")

def allfiles(dirpath, exclude):
    "Get the names of all files under dirpath, ignoring .svn directories."
    datafiles = []
    for dir in dirpath:
        os.path.walk(dir,
                     lambda arg, dir, names: datafiles.extend(map(lambda x: os.path.normpath(os.path.join(dir,x)), names)),
                     None)
    datafiles = filter(lambda x: ".svn" not in x, datafiles)
    datafiles = filter(lambda x: not os.path.isdir(x), datafiles)
    if exclude:
        datafiles = filter(lambda x: not re.search(exclude, x), datafiles)
    return datafiles

def iswml(filename):
    "Is the specified filename WML?"
    return filename.endswith(".cfg")

class reference:
    "Describes a location by file and line."
    def __init__(self, filename, line=None):
        self.filename = filename
        self.line = line
        self.references = {}
    def append(self, fn, n):
        if fn not in self.references:
            self.references[fn] = []
        self.references[fn].append(n+1)
    def dump_references(self):
        for (file, linenumbers) in self.references.items():
            print "    %s: %s" % (file, `linenumbers`[1:-1])
    def __str__(self):
        if self.line:
            return '"%s", line %d' % (self.filename, self.line)
        else:
            return self.filename

class CrossRef:
    macro_reference = re.compile(r"\{([A-Z_][A-Z0-9_:]*[A-Za-z0-9_])\b")
    file_reference =  re.compile(r"[A-Za-z0-9{}][A-Za-z0-9_/+{}-]*\.(" + "|".join(resource_extensions) + ")")
    def imagesearch(self, name):
        # Here is where we implement the funky rules
        # for image resolution.  If we can't identify
        # a reference to the image name under an image
        # directory, look for it under an image/units
        # or image/terrain directory.
        for super in ("units", "terrain", "portraits", "items"):
            trial = os.path.join(super, name)
            if trial in self.fileref:
                return trial
        return None
    def mark_matching_resources(self, pattern, fn, n):
        "Mark all definitions matching a specified pattern with a reference."
        pattern = re.compile(re.escape(pattern) + "$")
        key = None
        for trial in self.fileref:
            if pattern.match(trial):
                key = trial
                self.fileref[key].append(fn, n)
        return key
    def __init__(self, filelist):
        # First, collect macro definitions from the specified filelist."
        self.xref = {}
        self.fileref = {}
        for filename in filelist:
            if filter(lambda x: x, map(lambda x: filename.endswith("." + x), resource_extensions)):
                # The rule we're applying here is:
                # 1) If it's a sound file, its name is the part of
                #    the path after "sounds/" or "music/".
                # 2) If it's an image file, its name is the part of
                #    the path after "images/".
                (root, ext) = os.path.splitext(filename)
                if ext in (".ogg", ".wav"):
                    for superdir in ("music", "sounds"):
                        foundit = filename.find(superdir)
                        if foundit > -1:
                            name = filename[foundit:]
                            name = name[len(superdir)+1:]
                elif ext in (".png", ".jpg"):
                    foundit = filename.find("images")
                    if foundit > -1:
                        name = filename[foundit:]
                        name = name[len("images")+1:]                    
                self.fileref[name] = reference(filename) 
            elif iswml(filename):
                dfp = open(filename)
                for (n, line) in enumerate(dfp):
                    if line.startswith("#define"):
                        tokens = line.split()
                        name = tokens[1]
                        here = reference(filename, n+1)
                        if name in self.xref:
                            print >>sys.stderr, "*** Warning: duplicate definition of %s from %s, at %s" \
                                  % (name, self.xref[name], here)
                        self.xref[name] = here
                dfp.close()
        # Next, decorate definitions with all references from the filelist.
        self.unresolved = []
        self.missing = []
        formals = []
        for fn in filelist:
            if iswml(fn):
                rfp = open(fn)
                for (n, line) in enumerate(rfp):
                    if line.startswith("#define"):
                        formals = line.split()[2:]
                    elif line.startswith("#enddef"):
                        formals = []
                    if '#' in line:
                        line = line.split('#')[0]
                    if not line:
                        continue
                    # Find references to macros
                    for match in re.finditer(CrossRef.macro_reference, line):
                        name = match.group(1)
                        if name in formals:
                            continue
                        elif name in self.xref:
                            self.xref[name].append(fn, n+1)
                        else:
                            self.unresolved.append((name, reference(fn,n+1)))
                    # Find references to resource files
                    for match in re.finditer(CrossRef.file_reference, line):
                        name = match.group(0)
                        # If name is already in our resource list, it's easy.
                        if name in self.fileref:
                            self.fileref[name].append(fn, n+1)
                            continue
                        # If the name contains subtitutable parts, count
                        # it as a reference to everything the substitutions
                        # could potentially match.
                        elif '{' in name:
                            pattern = "^" + re.sub(r"\{[^}]*\}", '.*', name)
                            key = self.mark_matching_resources(pattern, fn,n+1)
                            if key:
                                self.fileref[key].append(fn, n+1)
                        else:
                            key = self.imagesearch(name)
                            if key:
                                self.fileref[key].append(fn, n+1)
                        if not key:
                            self.missing.append((name, reference(fn,n+1)))
                rfp.close()
    def xrefdump(self, pred=None):
        "Report resolved macro references."
        for (name, defloc) in self.xref.items():
            if pred and not pred(name, defloc):
                continue
            nrefs = len(defloc.references)
            if nrefs == 0:
                print "%s: macro %s is unused" % (defloc, name)
            else:
                print "%s: macro %s is used in %d files:" % (defloc, name, nrefs)
            defloc.dump_references()
        for (name, defloc) in self.fileref.items():
            if pred and not pred(name, defloc):
                continue
            nrefs = len(defloc.references)
            if nrefs == 0:
                print "Resource %s is unused" % defloc
            else:
                print "Resource %s is used in %d files:" % (defloc, nrefs)
            defloc.dump_references()
    def unresdump(self):
        "Report unresolved references."
        if len(self.unresolved) == 0 and len(self.missing) == 0:
            print "# No unresolved references"
        else:
            #print self.fileref.keys()
            print "# Unresolved references:"
            for (name, reference) in self.unresolved + self.missing:
                print "%s -> %s" % (reference, name)

if __name__ == "__main__":
    def help():
        sys.stderr.write("""\
Usage: macroscope [options] dirpath
    Options may be any of these:
    -h, --help                 Emit this help message and quit
    -c, --crossreference       Report resolved macro references
    -e reg, --exclude reg      Ignore files matching 
    -u, --unresolved           Report unresolved macro references
    -f dir, --from dir         Report only on macros defined under dir
    -r ddd, --refcount=ddd     Report only on macros w/references in ddd files
    --forced-used reg          Ignore refcount 0 on names matching regexp
   The required dirpath argument may be a colon-separated directory list.
""")

    # Process options
    (options, arguments) = getopt.getopt(sys.argv[1:], "che:f:r:u",
                                         ['help', 'force-used=', 'exclude=',
                                          'crossreference', 'unresolved',
                                          'from=', 'refcount='])
    crossreference = unresolved = False
    from_restrict = None
    refcount_restrict = None
    forceused = None
    exclude = None
    for (switch, val) in options:
        if switch in ('-h', '--help'):
            help()
            sys.exit(0)
        if switch in ('-f', '--from'):
            from_restrict = val
        elif switch in ('-c', '--crossreference'):
            crossreference = True
        elif switch in ('-e', '--exclude'):
            exclude = val
        elif switch == '--force-used':
            forceused = val
        elif switch in ('-u', '--unresolved'):
            unresolved = True
        elif switch in ('-r', '--refcount'):
            refcount_restrict = int(val)

    if len(arguments):
        dirpath = arguments[0].split(":")
    else:
        dirpath = ['.']
    print "# Macroscope reporting on %s" % time.ctime()
    print "# Invocation: %s" % " ".join(sys.argv)
    print "# Working directory: %s" % os.getcwd()
    if crossreference or unresolved:
        xref = CrossRef(allfiles(dirpath, exclude))
        def predicate(name, defloc):
            if from_restrict and not defloc.filename.startswith(from_restrict):
                return False
            if refcount_restrict!=None \
                   and len(defloc.references) != refcount_restrict \
                   or (refcount_restrict == 0 and forceused and re.search(forceused, name)):
                return False
            return True
        if crossreference:
            xref.xrefdump(predicate)
        if unresolved:
            xref.unresdump()