Third step in automated documentation extraction for macros.

We're making web pages that pass XHTML validation now; 
the rest of the work is cleanup and polishing.

data/abilities.cfg

@@ -436,7 +436,7 @@ Enemy units cannot see or attack this unit when it is in deep water, except for
 #weapons specials
 
 #define WEAPON_SPECIAL_BERSERK
-# Canned definition of the Berserk ability to be included in an 
+# Canned definition of the Berserk ability to be included in a 
 # [specials] clause.
 		[berserk]
 			id=berserk
@@ -448,7 +448,7 @@ Whether used offensively or defensively, this attack presses the engagement unti
 #enddef
 
 #define WEAPON_SPECIAL_BACKSTAB
-# Canned definition of the Backstab ability to be included in an 
+# Canned definition of the Backstab ability to be included in a 
 # [specials] clause.
 		[damage]
 			id=backstab
@@ -462,7 +462,7 @@ This attack deals double damage if there is an enemy of the target on the opposi
 #enddef
 
 #define WEAPON_SPECIAL_PLAGUE_TYPE TYPE
-# Canned definition of the Plague ability to be included in an 
+# Canned definition of the Plague ability to be included in a 
 # [specials] clause (with type specifier).
 		[plague]
 			id=plague({TYPE})
@@ -474,7 +474,7 @@ When a unit is killed by a Plague attack, that unit is replaced with a unit iden
 #enddef
 
 #define WEAPON_SPECIAL_PLAGUE
-# Canned definition of the Plague ability to be included in an 
+# Canned definition of the Plague ability to be included in a 
 # [specials] clause (without type specifier).
 		[plague]
 			id=plague
@@ -485,7 +485,7 @@ When a unit is killed by a Plague attack, that unit is replaced with a unit iden
 #enddef
 
 #define WEAPON_SPECIAL_SLOW
-# Canned definition of the Slow ability to be included in an 
+# Canned definition of the Slow ability to be included in a 
 # [specials] clause.
 		[slow]
 			id=slow
@@ -496,7 +496,7 @@ This attack slows the target until it ends a turn. Slow halves the damage caused
 #enddef
 
 #define WEAPON_SPECIAL_STONE
-# Canned definition of the Stone ability to be included in an 
+# Canned definition of the Stone ability to be included in a 
 # [specials] clause.
 		[stones]
 			id=stones
@@ -507,7 +507,7 @@ This attack turns the target to stone. Units that have been turned to stone may
 #enddef
 
 #define WEAPON_SPECIAL_MARKSMAN
-# Canned definition of the Marksman ability to be included in an 
+# Canned definition of the Marksman ability to be included in a 
 # [specials] clause.
 		[chance_to_hit]
 			id=marksman
@@ -521,7 +521,7 @@ When used offensively, this attack always has at least a 60% chance to hit."
 #enddef
 
 #define WEAPON_SPECIAL_MAGICAL
-# Canned definition of the Magical (targeting) ability to be included in an 
+# Canned definition of the Magical (targeting) ability to be included in a 
 # [specials] clause.
 		[chance_to_hit]
 			id=magical
@@ -534,7 +534,7 @@ This attack always has a 70% chance to hit."
 #enddef
 
 #define WEAPON_SPECIAL_SWARM
-# Canned definition of the Swarm ability to be included in an 
+# Canned definition of the Swarm ability to be included in a 
 # [specials] clause.
 		[attacks]
 			id=swarm
@@ -545,7 +545,7 @@ The number of strikes of this attack decreases when the unit is wounded. The num
 #enddef
 
 #define WEAPON_SPECIAL_CHARGE
-# Canned definition of the Charge ability to be included in an 
+# Canned definition of the Charge ability to be included in a 
 # [specials] clause.
 		[damage]
 			id=charge
@@ -559,7 +559,7 @@ This attack deals double damage to the target. It also causes this unit to take
 #enddef
 
 #define WEAPON_SPECIAL_DRAIN
-# Canned definition of the Drain ability to be included in an 
+# Canned definition of the Drain ability to be included in a 
 # [specials] clause.
 		[drains]
 			id=drains
@@ -570,7 +570,7 @@ This unit drains health from living units, healing itself for half the amount of
 #enddef
 
 #define WEAPON_SPECIAL_FIRSTSTRIKE
-# Canned definition of the First-strike ability to be included in an 
+# Canned definition of the First-strike ability to be included in a 
 # [specials] clause.
 		[firststrike]
 			id=firststrike
@@ -581,7 +581,7 @@ This unit always strikes first with this attack, even if they are defending."
 #enddef
 
 #define WEAPON_SPECIAL_POISON
-# Canned definition of the Poison ability to be included in an 
+# Canned definition of the Poison ability to be included in a 
 # [specials] clause.
 		[poison]
 			id=poison

data/tools/Makefile

@@ -34,5 +34,8 @@ utils-macros:
 definitions:
 	@./macroscope --definitions --exclude data/scenarios --exclude data/campaigns $(EXCLUDE) $(TOPDIR)
 
-help:
+macro-reference.xhtml:
-	@./macroscope --extracthelp --exclude data/scenarios --exclude data/campaigns $(EXCLUDE) $(TOPDIR)
+	@cat helpheader.xhtml >macro-reference.xhtml
+	@./macroscope --extracthelp --exclude data/scenarios --exclude data/campaigns $(EXCLUDE) $(TOPDIR) >>macro-reference.xhtml
+	@cat helptrailer.xhtml >>macro-reference.xhtml
+

data/tools/helpheader.xhtml

@@ -0,0 +1,21 @@
+<?xml version="1.0" encoding="ISO-8859-1"?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+    "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+
+<head>
+<title>Utility macro reference</title>
+<link rev="made" href="mailto:esr@snark.thyrsus.com" />
+<style type="text/css">
+/*<![CDATA[*/
+.listing {font-family:monospace;}
+.bold    {font-style:bold;}
+/*]]>*/
+</style>
+</head>
+<body>
+
+<p>This page is an automatically-generated reference for all the 
+utility macros with documentation strings in the Wesnoth source
+distribution.</p>
+

data/tools/helptrailer.xhtml

@@ -0,0 +1,3 @@
+
+</body>
+</html>

data/tools/macroscope

@@ -42,6 +42,40 @@ import sys, os, time, re, getopt, sre_constants
 
 resource_extensions = ("png", "jpg", "ogg", "wav")
 
+def htmlize(line):
+    "HTML-escape a text line"
+    return line.replace("<", "&lt;").replace(">", "&gt;").replace("&", "&amp;")
+
+def interpret(lines):
+    "Interpret the ! convention for .cfg comments."
+    inlisting = False
+    outstr = "<p>"
+    for line in lines:
+        line = line.rstrip()
+        if not inlisting and not line:
+            outstr += "</p><p>"
+            continue
+        if not inlisting and line[0] == '!':
+            outstr += "</p>\n<pre class='listing'>"
+            inlisting = True
+            bracketdepth = curlydepth = 0
+        line = htmlize(line)
+        if inlisting:
+            outstr += line[1:] + "\n"
+        else:
+            outstr += line + "\n"
+        if inlisting:
+            if line and line[0] != '!':
+                outstr += "</pre>\n<p>"
+                inlisting = False
+    if not inlisting:
+        outstr += "</p>\n"
+    else:
+        outstr += "</pre>\n"
+    outstr = outstr.replace("<p></p>", "")
+    outstr = outstr.replace("\n\n</pre>", "\n</pre>")
+    return outstr
+
 def allfiles(dirpath, exclude):
     "Get the names of all files under dirpath, ignoring .svn directories."
     datafiles = []
@@ -73,6 +107,16 @@ class reference:
     def dump_references(self):
         for (file, linenumbers) in self.references.items():
             print "    %s: %s" % (file, `linenumbers`[1:-1])
+    def __cmp__(self, other):
+        "Compare two documentation objects for place in the sort order."
+        # Major sort by file, minor by line number.  This presumes that the
+        # files correspond to coherent topics and gives us control of the
+        # sequence.
+        byfile = cmp(self.filename, other.filename)
+        if byfile:
+            return byfile
+        else:
+            return cmp(self.line, other.line)
     def __str__(self):
         if self.line:
             return '"%s", line %d' % (self.filename, self.line)
@@ -249,20 +293,32 @@ class CrossRef:
     def extracthelp(self, fp):
         "Deliver all macro help comments in HTML form."
         doclist = self.xref.keys()
-        def defcmp(s, t):
+        doclist = filter(lambda x: self.xref[x].docstring, doclist)
-            "Compare two documentation objects for place in the sort order."
+        doclist.sort(lambda x, y: cmp(self.xref[x], self.xref[y]))
-            # Major sort by file, minor by name.  This presumes that the
-            # files correspond to coherent topics.
-            byfile = cmp(self.xref[s].filename, self.xref[t].filename)
-            if byfile:
-                return byfile
-            else:
-                return cmp(s, t)
-        doclist.sort(defcmp)
         outstr = ""
+        filename = None
+        counted = 0
         for name in doclist:
-            if self.xref[name].docstring:
+            entry = self.xref[name]
-                lines = self.xref[name].docstring.split("\n")
+            if entry.filename != filename:
+                if counted:
+                    outstr += "</dl>\n"
+                counted += 1
+                filename = entry.filename
+                outstr += "<h1>From file: " + filename + "</h1>\n"
+                hdr = []
+                dfp = open(filename)
+                for line in dfp:
+                    if line[0] == '#':
+                        hdr.append(line[1:])
+                    else:
+                        break
+                dfp.close()
+                if hdr:
+                    outstr += interpret(hdr)
+                outstr += "<dl>\n"
+            if entry.docstring:
+                lines = entry.docstring.split("\n")
                 header = lines.pop(0).split()
                 if lines and not lines[-1]:	# Ignore trailing blank lines
                     lines.pop()
@@ -273,32 +329,10 @@ class CrossRef:
                 if header[1:]:
                     outstr += " <emphasis>"+" ".join(header[1:])+"</emphasis>"
                 outstr += "\n</dt>\n"
-                outstr += "<dd>\n<p>"
+                outstr += "<dd>\n"
-                inlisting = False
+                outstr += interpret(lines)
-                for line in lines:
-                    line = line.rstrip()
-                    if not inlisting and not line:
-                        outstr += "</p><p>"
-                        continue
-                    if not inlisting and line[0] == '!':
-                        outstr += "</p>\n<listing>\n"
-                        inlisting = True
-                        bracketdepth = curlydepth = 0
-                    if inlisting:
-                        outstr += line[1:] + "\n"
-                    else:
-                        outstr += line + "\n"
-                    if inlisting:
-                        if line and line[0] != '!':
-                            outstr += "</listing>\n<p>"
-                            inlisting = False
-                if not inlisting:
-                    outstr += "</p>\n"
-                else:
-                    outstr += "</listing>\n"
                 outstr += "</dd>\n"
-        outstr = outstr.replace("<p></p>", "")
+        outstr += "</dl>\n"
-        outstr = outstr.replace("\n\n</listing>", "\n</listing>")
         fp.write(outstr)
 
 if __name__ == "__main__":
@@ -365,15 +399,17 @@ Usage: macroscope [options] dirpath
         dirpath = arguments[0].split(":")
     else:
         dirpath = ['.']
+    filelist = allfiles(dirpath, "|".join(exclude))
+    xref = CrossRef(filelist)
+    if extracthelp:
+        xref.extracthelp(sys.stdout)
+    elif listfiles:
+        for filename in filelist:
+            print filename
+    elif crossreference or definitions or listfiles or unresolved:
         print "# Macroscope reporting on %s" % time.ctime()
         print "# Invocation: %s" % " ".join(sys.argv)
         print "# Working directory: %s" % os.getcwd()
-    if crossreference or definitions or listfiles or unresolved or extracthelp:
-        filelist = allfiles(dirpath, "|".join(exclude))
-        if listfiles:
-            for filename in filelist:
-                print filename
-        xref = CrossRef(filelist)
         def predicate(name, defloc):
             if from_restrict and not defloc.filename.startswith(from_restrict):
                 return False
@@ -391,7 +427,5 @@ Usage: macroscope [options] dirpath
             xref.deflist(predicate)
         if unresolved:
             xref.unresdump()
-        if extracthelp:
-            xref.extracthelp(sys.stdout)
     
 

data/utils/sound-utils.cfg

@@ -1,9 +1,9 @@
 # Music control macros, and declarations of sound resource lists.
 #
 # As of 1.1.3, music is parsed as follows:
-# 1. the [scenario]-level [music] tag
+#! 1. the [scenario]-level [music] tag
-# 2. the [story]-level music key
+#! 2. the [story]-level music key
-# 3. any [event]-level [music] tags
+#! 3. any [event]-level [music] tags
 #
 # If you change the music at a lower level, the tags above it will NOT
 # be re-parsed and your scenario will sound wrong.  For example, if
@@ -91,7 +91,7 @@
 # It should be positioned at the top of the scenario file
 # so it can be overridden by other prestart or start events.
 #
-# It also allows for the convenient use of a standardized
+# It also allows for the convenient use of standardized
 # intra-scenario music, should we decide to use one.
 [music]
 	name="wesnoth-2.ogg"
@@ -108,7 +108,7 @@
 # randomly picked every time, instead of a single sound. Here the most commonly
 # used lists are wrapped inside macros.
 #
-# These are used in unit .cfg's for example like this:
+# These are used in unit .cfg files, for example like this:
 #
 #!     [animation]
 #!         hits=no