wesnoth/data/tools/upconvert

#!/usr/bin/env python
#
# Up-convert WML and maps between versions.
#
# By Eric S. Raymond April 2007.
#
# All conversion logic for lifting WML and maps from older versions of the
# markup to newer ones should live here.  This includes resource path changes
# and renames, also map format conversions.
#
# Takes any number of directories as arguments.  Each directory is converted.
# If no directiories are specified, acts on the current directory.
#
# The recommended procedure is this:
# 1. Run it with --dryrun first to see what it will do.
# 2. If the messages look good, run without --dryrun; the old content
#    will be left in backup files with a -bak extension.
# 3. Eyeball the changes with the --diff option.
# 4. Use macroscope, with a directory path including the Wesnoth mainline WML,
#    to check that you have no unresolved references.
# 5. Test the conversion.
# 6. Use either --clean to remove the -bak files or --revert to
#    undo the conversion.
#
# This script will barf on maps with custom terrains.

import sys, os, re, getopt, curses.ascii

filemoves = {
    # Older includes all previous to 1.3.1.
    "older" : (
        # File naming error made repeatedly in NR and elsewhere.
        ("human-loyalists/human-", "human-loyalists/"),
        # These are picked to cover as many as possible of the broken
        # references in UMC on the campaign server.  Some things we
        # don't try to fix include:
        # - attack/staff.png may map to one of several staves.
        # - magic.wav may map to one of several sounds depending on the init.
        # Some other assumption sound in current UMC as of April 2007
        # but theoretically dubious are marked with *.
        ("../music/defeat.ogg",     "defeat.ogg"),
        ("../music/victory.ogg",    "victory.ogg"),
        ("AMLA_TOUGH_2",            "AMLA_TOUGH 2"),
        ("AMLA_TOUGH_3",            "AMLA_TOUGH 3"),
        ("SOUND_LIST:DAGGER_SWISH", "SOUND_LIST:SWORD_SWISH"),
        ("arrow-hit.wav",           "bow.ogg"),
        ("arrow-miss.wav",          "bow-miss.ogg"),
        ("attacks/animal-fangs.png","attacks/fangs-animal.png"),
        ("attacks/crossbow.png",    "attacks/human-crossbow.png"),        #*
        ("attacks/dagger.png",      "attacks/human-dagger.png"),          #*
        ("attacks/darkstaff.png",   "attacks/staff-necromantic.png"),
        ("attacks/human-fist.png",  "attacks/fist-human.png"),
        ("attacks/human-mace.png",  "attacks/mace.png"),
        ("attacks/human-sabre.png", "attacks/sabre-human.png"),
        ("attacks/icebolt.png",     "attacks/iceball.png"),    # Is this right?
        ("attacks/lightingbolt.png","attacks/lightning.png"),
        ("attacks/missile.png",     "attacks/magic-missile.png"),
        ("attacks/morning_star.png","attacks/morning-star.png"),
        ("attacks/plaguestaff.png", "attacks/staff-plague.png"),
        ("attacks/slam.png",        "attacks/slam-drake.png"),
        ("attacks/staff-magical.png","attacks/staff-magic.png"),
        ("attacks/sword-paladin.png","attacks/sword-holy.png"),
        ("attacks/sword.png",       "attacks/human-sword.png"),                #*
        ("attacks/sword_holy.png",  "attacks/sword-holy.png"),
        ("attacks/throwing-dagger-human.png", "attacks/dagger-thrown-human.png"),
        ("bow-hit.ogg",             "bow.ogg"),
        ("bow-hit.wav",             "bow.ogg"),
        ("bowman-attack-sword.png", "bowman-sword-1.png"),
        ("bowman-attack1.png",      "bowman-ranged-1.png"),
        ("bowman-attack2.png",      "bowman-ranged-2.png"),
        ("creepy.ogg",              "underground.ogg"),
        ("dwarves/warrior.png",     "dwarves/fighter.png"),
        ("eagle.wav",               "gryphon-shriek-1.ogg"),
        ("elfland.ogg",             "elf-land.ogg"),
        ("elvish-fighter.png",      "elves-wood/fighter.png"),
        ("elvish-hero.png",         "elves-wood/hero.png"),
        ("fist.wav",                "fist.ogg"),
        ("flame-miss.ogg",          "flame-big-miss.ogg"),
        ("flame.ogg",               "flame-big.ogg"),
        ("gameplay2.ogg",           "gameplay02.ogg"),                    # Changes in 1.3.2
        ("goblin-hit2.ogg",         "goblin-hit-2.ogg"),
        ("hatchet-miss-1.ogg",      "hatchet-miss.wav"),
        ("heal.ogg",                "heal.wav"),
        ("hiss-big.ogg",            "hiss-big.wav"),
        ("human-dagger.png",        "dagger-human.png"),
        ("human-male-die.ogg",      "human-die-1.ogg"),
        ("human-male-hit.ogg",      "human-hit-1.ogg"),
        ("human-male-weak-die.ogg", "human-old-die-1.ogg"),
        ("human-male-weak-hit.ogg", "human-old-hit-1.ogg"),
        ("human-sword.png",         "sword-human.png"),
        ("items/castle-ruins.png",  "scenery/castle-ruins.png"),
        ("items/fire.png",          "scenery/fire1.png"),
        ("items/fire1.png",         "scenery/fire1.png"),
        ("items/fire2.png",         "scenery/fire2.png"),
        ("items/fire3.png",         "scenery/fire3.png"),
        ("items/fire4.png",         "scenery/fire4.png"),
        ("items/hero-icon.png",     "misc/hero-icon.png"),
        ("items/leanto.png",        "scenery/leanto.png"),
        ("items/lighthouse.png",    "scenery/lighthouse.png"),
        ("items/monolith1.png",     "scenery/monolith1.png"),
        ("items/monolith2.png",     "scenery/monolith2.png"),
        ("items/monolith3.png",     "scenery/monolith3.png"),
        ("items/monolith4.png",     "scenery/monolith4.png"),
        ("items/ring1.png",         "items/ring-silver.png"),  # Is this right?
        ("items/ring2.png",         "items/ring-gold.png"),    # Is this right?
        ("items/rock1.png",         "scenery/rock1.png"),
        ("items/rock2.png",         "scenery/rock2.png"),
        ("items/rock3.png",         "scenery/rock3.png"),
        ("items/rock4.png",         "scenery/rock4.png"),
        ("items/signpost.png",      "scenery/signpost.png"),
        ("items/slab.png",          "scenery/slab-1.png"),
        ("items/well.png",          "scenery/well.png"),
        ("knife.ogg",               "dagger-swish.wav"),       # Is this right?
        ("knife.wav",               "dagger-swish.wav"),       # Is this right?
        ("lightning.wav",           "lightning.ogg"),
        ("longbowman-ranged-1.png", "longbowman-bow-attack1.png"),
        ("longbowman-ranged-2.png", "longbowman-bow-attack2.png"),
        ("longbowman-ranged-3.png", "longbowman-bow-attack3.png"),
        ("longbowman-ranged-4.png", "longbowman-bow-attack4.png"),
        ("misc/chest.png",          "items/chest.png"),
        ("misc/dwarven-doors.png",  "scenery/dwarven-doors-closed.png"),
        ("misc/mine.png",           "scenery/mine-abandoned.png"),
        ("misc/nest-empty.png",     "scenery/nest-empty.png"),
        ("misc/rocks.png",          "scenery/rubble.png"),
        ("misc/snowbits.png",       "scenery/snowbits.png"),
        ("misc/temple.png",         "scenery/temple1.png"),
        ("miss.wav",                "miss-1.ogg"),
        ("orc-die.wav",             "orc-die-1.ogg"),
        ("orc-hit.wav",             "orc-hit-1.ogg"),
        ("ork-die-2.ogg",           "orc-die-2.ogg"),
        ("pistol.wav",              "gunshot.wav"),
        ("spear-miss-1.ogg",        "spear-miss.ogg"),
        ("spearman-attack-south-1.png", "spearman-attack-s-1.png"),
        ("spearman-attack-south-2.png", "spearman-attack-s-2.png"),
        ("spearman-attack-south-3.png", "spearman-attack-s-3.png"),
        ("squishy-miss-1.ogg",      "squishy-miss.wav"),
        ("sword-swish.wav",         "sword-1.ogg"),
        ("sword.wav",               "sword-1.ogg"),
        ("terrain/flag-1.png",      "flags/flag-1.png"),
        ("terrain/flag-2.png",      "flags/flag-2.png"),
        ("terrain/flag-3.png",      "flags/flag-3.png"),
        ("terrain/flag-4.png",      "flags/flag-4.png"),
        ("terrain/rocks.png",       "scenery/rock2.png"),
        ("terrain/signpost.png",    "scenery/signpost.png"),
        ("terrain/village-cave-tile.png","terrain/village/cave-tile.png"),
        ("terrain/village-dwarven-tile.png","terrain/village/dwarven-tile.png"),
        ("terrain/village-elven4.png","terrain/village/elven4.png"),
        ("terrain/village-human-snow.png", "terrain/village/human-snow.png"),
        ("terrain/village-human.png","terrain/village/human.png"),
        ("terrain/village-human4.png","terrain/village/human4.png"),
        ("throwing-dagger-swish.wav","dagger-swish.wav"),      # Is this right?
        ("units/undead/ghost-attack.png", "units/undead/ghost-attack-2.png"),
        ("units/undead/ghost-attack1.png", "units/undead/ghost-attack-1.png"),
        ("wolf-attack.wav",         "wolf-bite.ogg"),
        ("wolf-cry.wav",            "wolf-die.wav"),
        ("wose-attack.wav",            "wose.attack.ogg"),
        ("wose.attack.ogg",            "wose.attack.ogg"),
    ),
    "1.3.1" : (
        # Peasant images moved to a new directory
        ("human-loyalists/peasant.png", "human-peasants/peasant.png"),
        ("human-loyalists/peasant-attack.png", "human-peasants/peasant-attack.png"),
        ("human-loyalists/peasant-attack2.png", "human-peasants/peasant-attack2.png"),
        ("human-loyalists/peasant-ranged.png", "human-peasants/peasant-ranged.png"),
        ("human-loyalists/peasant-idle-1.png", "human-peasants/peasant-idle-1.png"),
        ("human-loyalists/peasant-idle-2.png", "human-peasants/peasant-idle-2.png"),
        ("human-loyalists/peasant-idle-3.png", "human-peasants/peasant-idle-3.png"),
        ("human-loyalists/peasant-idle-4.png", "human-peasants/peasant-idle-4.png"),
        ("human-loyalists/peasant-idle-5.png", "human-peasants/peasant-idle-5.png"),
        ("human-loyalists/peasant-idle-6.png", "human-peasants/peasant-idle-6.png"),
        ("human-loyalists/peasant-idle-7.png", "human-peasants/peasant-idle-7.png"),
        # All Great Mage attacks were renamed
        ("great-mage-attack-magic1.png", "great-mage-attack-magic-1.png"),
        ("great-mage-attack-magic2.png", "great-mage-attack-magic-2.png"),
        ("great-mage+female-attack-magic1.png", "great-mage+female-attack-magic-1.png"),
        ("great-mage+female-attack-magic2.png", "great-mage+female-attack-magic-2.png"),
        ("great-mage-attack-staff1.png", "great-mage-attack-staff-1.png"),
        ("great-mage-attack-staff2.png", "great-mage-attack-staff-2.png"),
        ("great-mage+female-attack-staff1.png", "great-mage+female-attack-staff-1.png"),
        ("great-mage+female-attack-staff2.png", "great-mage+female-attack-staff-2.png"),
        # All Arch Mage attacks were renamed
        ("arch-mage-attack-magic1.png", "arch-mage-attack-magic-1.png"),
        ("arch-mage-attack-magic2.png", "arch-mage-attack-magic-2.png"),
        ("arch-mage+female-attack-magic1.png", "arch-mage+female-attack-magic-1.png"),
        ("arch-mage+female-attack-magic2.png", "arch-mage+female-attack-magic-2.png"),
        ("arch-mage-attack-staff1.png", "arch-mage-attack-staff-1.png"),
        ("arch-mage-attack-staff2.png", "arch-mage-attack-staff-2.png"),
        ("arch-mage+female-attack-staff1.png", "arch-mage+female-attack-staff-1.png"),
        ("arch-mage+female-attack-staff2.png", "arch-mage+female-attack-staff-2.png"),
        # All Red Mage attacks were renamed
        ("red-mage-attack-magic1.png", "red-mage-attack-magic-1.png"),
        ("red-mage-attack-magic2.png", "red-mage-attack-magic-2.png"),
        ("red-mage+female-attack-magic1.png", "red-mage+female-attack-magic-1.png"),
        ("red-mage+female-attack-magic2.png", "red-mage+female-attack-magic-2.png"),
        ("red-mage-attack-staff1.png", "red-mage-attack-staff-1.png"),
        ("red-mage-attack-staff2.png", "red-mage-attack-staff-2.png"),
        ("red-mage+female-attack-staff1.png", "red-mage+female-attack-staff-1.png"),
        ("red-mage+female-attack-staff2.png", "red-mage+female-attack-staff-2.png"),
        # Timothy Pinkham supplied titles for two of his music files.
        # Zhaytee supplied a title for wesnoth-1.ogg
        # gameplay03.ogg, and and wesnoth-[25].ogg already had titles.
        ("gameplay01.ogg", "knolls.ogg"),
        ("gameplay02.ogg", "wanderer.ogg"),
        ("gameplay03.ogg", "battle.ogg"),
        ("wesnoth-1.ogg", "revelation.ogg"),
        ("wesnoth-2.ogg", "loyalists.ogg"),
        ("wesnoth-5.ogg", "northerners.ogg"),
        # And the holy->arcane change
        ("type=holy", "type=arcane"),
        ("holy=", "arcane=")
    ),
    # An empty sentinel value at end is required.
    # Always have the current version here.
    "1.3.2" : (),
}

# 1.2.x to 1.3.2 terrain conversion
conversion1 = {
    " " : "_s",
    "&" : "Mm^Xm",
    "'" : "Uu^Ii",
    "/" : "Ww^Bw/",
    "1" : "1 _K",
    "2" : "2 _K",
    "3" : "3 _K",
    "4" : "4 _K",
    "5" : "5 _K",
    "6" : "6 _K",
    "7" : "7 _K",
    "8" : "8 _K",
    "9" : "9 _K",
    "?" : "Gg^Fet",
    "A" : "Ha^Vhha",
    "B" : "Dd^Vda",
    "C" : "Ch",
    "D" : "Uu^Vu",
    "E" : "Rd",
    "F" : "Aa^Fpa",
    "G" : "Gs",
    "H" : "Ha",
    "I" : "Dd",
    "J" : "Hd",
    "K" : "_K",
    "L" : "Gs^Vht",
    "M" : "Md",
    "N" : "Chr",
    "P" : "Dd^Do",
    "Q" : "Chw",
    "R" : "Rr",
    "S" : "Aa",
    "T" : "Gs^Ft",
    "U" : "Dd^Vdt",
    "V" : "Aa^Vha",
    "W" : "Xu",
    "X" : "Qxu",
    "Y" : "Ss^Vhs",
    "Z" : "Ww^Vm",
    "[" : "Uh",
    "\\": "Ww^Bw\\",
    "]" : "Uu^Uf",
    "a" : "Hh^Vhh",
    "b" : "Mm^Vhh",
    "c" : "Ww",
    "d" : "Ds",
    "e" : "Aa^Vea",
    "f" : "Gs^Fp",
    "g" : "Gg",
    "h" : "Hh",
    "i" : "Ai",
    "k" : "Wwf",
    "l" : "Ql",
    "m" : "Mm",
    "n" : "Ce",
    "o" : "Cud",
    "p" : "Uu^Vud",
    "q" : "Chs",
    "r" : "Re",
    "s" : "Wo",
    "t" : "Gg^Ve",
    "u" : "Uu",
    "v" : "Gg^Vh",
    "w" : "Ss",
    "|" : "Ww^Bw|",
    "~" : "_f",
}
max_len = max(*map(len, conversion1.values()))

def get_adjacent(x, y, map):
    "Returns string of original location+adjacent locations on hex 1-char map"
    odd = (x) % 2
    adj = map[y][x];
    if x > 0:
	adj += map[y][x-1]
    if x < len(map[y])-1:
	adj += map[y][x+1]
    if y > 0:
	adj += map[y-1][x]
    if y < len(map)-1:
	adj += map[y+1][x]
    if x > 0 and y > 0 and not odd:
	adj += map[y-1][x-1]
    if x < len(map[y])-1 and y > 0 and not odd:
	adj += map[y-1][x+1];
    if x > 0 and y < len(map)-1 and odd:
	adj += map[y+1][x-1]
    if x < len(map[y])-1 and y < len(map)-1 and odd:
	adj += map[y+1][x+1]
    adj = adj.replace("\n", "").replace("\r", "")
    return adj

width = max_len+2

def maptransform1(input, baseline, inmap, y):
    "Transform a map line from 1.2.x to 1.3.x format."
    format = "%%%d.%ds" % (width, max_len)
    x = 0
    if "," in inmap[y]:
        raise maptransform_error(2, input, baseline, None,
                                 "map file appears to be converted already")
    line = ''
    for char in inmap[y]:
        ohex = ''
        if char in ('\n', '\r'):
            ohex += char
        elif char in conversion1:
            ohex = format % conversion1[char] + ','
        else:
            raise maptransform_error(0, input, baseline+y+1, (x, y),
                                     "unrecognized character %s (%d)" % (`char`, ord(char)))
            # ohex = format % char
            sys.exit(1)
        if "_K" in ohex:
            # Convert keeps according to adjacent hexes
            adj = get_adjacent(x, y, inmap)
            # print "adjacent: %s" % adj
            hexcount = {}
            for i in range(1, len(adj)):
                # Intentionally skipping 0 as it is original hex
                a = adj[i];
                if not a in conversion1:
                    raise maptransform_error(0, input, baseline, (x, y),
                                 "error in adjacent hexes")
                    sys.exit(1)
                ca = conversion1[a]
                if ca.startswith("C"): #this is a castle hex	
                     hexcount[ca] = hexcount.get(ca, 0) + 1
            maxc = 0;
            maxk = "Ch";
            # Next line is a hack to make this code pass
            # regression testing against the Perl
            # original. Without the sort, when there are
            # two terrain types that occur in equal
            # numbers greater than any others, which one
            # gets picked will be randomly dependent on
            # Python's dictionary hash function.
            sorted = hexcount.keys()
            sorted.sort()
            for k in sorted:
                if hexcount[k] > maxc:
                    maxc = hexcount[k]
                    maxk = k
            #print "Dominated by %s" % maxk
            maxk = re.sub("^C", "K", maxk)
            ohex = ohex.replace("_K", maxk)
        line += ohex
        x += 1
    return line.replace(",\n", "\n")

# 1.3.1 -> 1.3.2 terrain conversions
conversion2 = {
    re.compile(r"(?<!\^)Bww([|/\\])\b") : "Ww^Bw\\1",
    re.compile(r"(?<!\^)Bwo([|/\\])\b") : "Wo^Bw\\1",
    re.compile(r"(?<!\^)Bss([|/\\])\b") : "Ss^Bw\\1",
    re.compile(r"(?<!\^)Dc\b") : "Dd^Dc",
    re.compile(r"(?<!\^)Dr\b") : "Dd^Dr",
    re.compile(r"(?<!\^)Do\b") : "Dd^Do",
    re.compile(r"(?<!\^)Fa\b") : "Aa^Fpa",
    re.compile(r"(?<!\^)Fet\b") : "Gg^Fet",
    re.compile(r"(?<!\^)Ff\b") : "Gs^Fp",
    re.compile(r"(?<!\^)Ft\b") : "Gs^Ft",
    re.compile(r"(?<!\^)Rfvs\b") : "Re^Gvs",
    re.compile(r"(?<!\^)Uf\b") : "Uu^Uf",
    re.compile(r"(?<!\^)Uui\b") : "Uu^Ii",
    re.compile(r"(?<!\^)Uhi\b") : "Uh^Ii",
    re.compile(r"(?<!\^)Vda\b") : "Dd^Vda",
    re.compile(r"(?<!\^)Vdt\b") : "Dd^Vdt",
    re.compile(r"(?<!\^)Vea\b") : "Aa^Vea",
    re.compile(r"(?<!\^)Veg\b") : "Gg^Ve",
    re.compile(r"(?<!\^)Vha\b") : "Aa^Vha",
    re.compile(r"(?<!\^)Vhg\b") : "Gg^Vh",
    re.compile(r"(?<!\^)Vhh\b") : "Hh^Vhh",
    re.compile(r"(?<!\^)Vhha\b") : "Ha^Vhha",
    re.compile(r"(?<!\^)Vhm\b") : "Mm^Vhh",
    re.compile(r"(?<!\^)Vht\b") : "Gs^Vht",
    re.compile(r"(?<!\^)Vu\b") : "Uu^Vu",
    re.compile(r"(?<!\^)Vud\b") : "Uu^Vud",
    re.compile(r"(?<!\^)Vwm\b") : "Ww^Vm",
    re.compile(r"(?<!\^)Vs\b") : "Ss^Vhs",
    re.compile(r"(?<!\^)Vsm\b") : "Ss^Vm",
    re.compile(r"(?<!\^)Xm\b") : "Mm^Xm",
    }

def maptransform2(input, baseline, inmap, y):
    "Convert a map line from 1.3.1 multiletter format to 1.3.2 format."
    mapline = inmap[y]
    for (old, new) in conversion2.items():
        mapline = old.sub(new, mapline)
    return mapline

# Generic machinery starts here

class maptransform_error:
    "Error object to be thrown by maptransform."
    def __init__(self, level, infile, inline, loc, type):
        self.level = level
        self.infile = infile
        self.inline = inline
        self.loc = loc
        self.type = type
    def __repr__(self):
        errmsg = '"%s", line %d: %s' % (self.infile, self.inline, self.type)
        if self.loc:
            errmsg += " at %s" % (self.loc,)
        return errmsg

def translator(filename, mapxform, textxform):
    "Apply mapxform to map lines and textxform to non-map lines."
    # This hairy regexp excludes map_data lines that contain {} file
    # references, also lines that are empty or hold just one keep
    # character (somewhat pathological, but not handling these will
    # make the regression tests break).
    mapdata = re.compile(r'map_data="[A-Za-z0-9\/|\\&_~?\[\]\']{2,}') 
    modified = False
    mfile = []
    map_only = not filename.endswith(".cfg")
    for line in open(filename):
        mfile.append(line);
        if mapdata.search(line):
            map_only = False
    cont = False
    outmap = []
    newdata = []
    lineno = baseline = 0
    while mfile:
        line = mfile.pop(0)
        if verbose >= 4:
            sys.stdout.write(line)
        lineno += 1
        # Exclude map_data= lines that are just 1 line without
        # continuation, or which contain {}.  The former are
        # pathological and the parse won't handle them, the latter
        # refer to map files which will be checked separately.
        if map_only or ("map_data=" in line
                        and line.count('"') == 1
                        and line.count("{") == 0
                        and  line.count("}") == 0):
            baseline = 0
            cont = True
            if verbose >= 4:
                print "*** Entering map mode."
            # Assumes map is more than 1 line long.
            if not map_only:
                line = line.split('"')[1]
            if line.strip():
                outmap.append(line)
            while cont and mfile:
                line = mfile.pop(0)
                if verbose >= 4:
                    sys.stdout.write(line)
                lineno += 1
                if line and line[0] == '#':
                    newdata.append(line)
                    continue
                if '"' in line:
                    cont = False
                    if verbose >= 4:
                        print "*** Exiting map mode."
                    line = line.split('"')[0]
                if line and not line.endswith("\n"):
                    line += "\n"
                if line:
                    outmap.append(line)
            if not map_only: 
                line="map_data=\"\n";
                newdata.append(line)
            for y in range(len(outmap)):
                newline = mapxform(filename, baseline, outmap, y)
                newdata.append(newline)
                if newline != outmap[y]:
                    modified = True
            # All lines of the map are processed, add the appropriate trailer
            if map_only:
                newline="\n"
            else:
                newline="\"\n"
            newdata.append(newline)
        elif "map_data=" in line and (line.count("{") or line.count("}")):
            newdata.append(line)
        elif "map_data=" in line and line.count('"') > 1:
            print >>sys.stderr, 'upconvert: "%s", line %d: one-line map.' % (filename, lineno)
            newdata.append(line)
        else:
            # Handle text (non-map) lines
            newline = textxform(filename, lineno, line)
            newdata.append(newline)
            if newline != line:
                modified = True
    # Return None if the transformation functions made no changes.
    if modified:
        return "".join(newdata)
    else:
        return None

ignore = (".tgz", ".png", ".jpg")

def allcfgfiles(dir):
    "Get the names of all .cfg and map files under dir, ignoring .svn directories."
    datafiles = []
    os.path.walk(dir,
                 lambda arg, dir, names: datafiles.extend(map(lambda x: os.path.normpath(os.path.join(dir, x)), names)),
                 None)
    datafiles = filter(lambda x: ".svn" not in x, datafiles)
    datafiles = filter(lambda x: x.endswith(".cfg") or ('maps' in x and os.path.isfile(x) and x[-4:] not in ignore), datafiles)
    datafiles = filter(lambda x: not x.endswith("-bak"), datafiles)
    datafiles.sort()	# So changes and diffs for the same campaigns will cluster in the report
    return datafiles

def help():
        sys.stderr.write("""\
Usage: upconvert [options]
    Convert Battle of Wesnoth WML from older versions to newer ones.
    Options may be any of these:
    -h, --help                 Emit this help message and quit.
    -d, --dryrun               List changes but don't perform them.
    -o, --oldversion           Specify version to begin with.
    -v, --verbose              -v          lists changes.
                               -v -v       warns of maps already converted.
                               -v -v -v    names each file before it's processed.
                               -v -v -v -v shows verbose parse details.
    -c, --clean                Clean up -bak files.
    -D, --diff                 Display diffs between unconverted and unconverted files.
    -r, --revert               Revert the conversion from the -bak files.
""")

if __name__ == '__main__':
    (options, arguments) = getopt.getopt(sys.argv[1:], "cdDho:rv", [
	"help",
        "oldversion=",
	"dryrun",
        "verbose",
        "clean",
        "revert",
        "diffs",
        ])
    oldversion = 'older'
    dryrun = False
    verbose = 0
    clean = False
    diffs = False
    revert = False
    for (switch, val) in options:
        if switch in ('-h', '--help'):
            help()
            sys.exit(0)
        elif switch in ('-o', '--oldversion'):
            oldversion = val
        elif switch in ('-v', '--verbose'):
            verbose += 1
        elif switch in ('-d', '--dryrun'):
            dryrun = True
            verbose = max(1, verbose)
        elif switch in ('-c', '--clean'):
            clean = True
        elif switch in ('-d', '--diffs'):
            diffs = True
        elif switch in ('-r', '--revert'):
            revert = True

    if clean and revert:
        sys.stderr.write("upconvert: can't do clean and revert together.\n")
        sys.exit(1)

    # Compute the series of version upgrades to perform, and describe it.
    versions = filemoves.keys()
    versions.sort()
    versions = [versions[-1]] + versions[:-1]	# Move 'older' to front
    if oldversion in versions:
        versions = versions[versions.index(oldversion):]
    else:
        print >>sys.stderr, "upconvert: unrecognized version."
        sys.exit(1)
    explain = "Performing upgrades for:"
    for i in range(len(versions)-1):
        explain += " %s -> %s," % (versions[i],  versions[i+1])
    sys.stdout.write(explain[:-1] + ".\n")
    fileconversions = map(lambda x: filemoves[x], versions[:-1])

    def hasdigit(str):
        for c in str:
            if c in "0123456789":
                return True
        return False

    def parse_attribute(str):
        "Parse a WML key-value pair from a line."
        if '=' not in str:
            return None
        m = re.match(r"(^\s*[a-z0-9_]+\s*=\s*)(\S+)(\s*#?.*\s*)", str)
        if not m:
            return None
        # Four fields: stripped key, part of line before value,
        # value, trailing whitespace and comments
        return (m.group(1).replace("=", "").strip(),) + m.groups()

    def texttransform(filename, lineno, line):
        "Resource-name transformation on text lines."
        transformed = line
        # First, do resource-file moves
        for step in fileconversions:
            for (old, new) in step:
                transformed = transformed.replace(old, new)
        # Handle terrain_liked=, terrain=, valid_terrain=
        spaceless = transformed.replace(" ", "")
        if "terrain_liked=" in spaceless or "terrain=" in spaceless:
            (key, pre, value, post) = parse_attribute(transformed)
            # We have to cope with the following cases...
            # Old style:
            #    terrain_liked=ghM
            #    terrain_liked=BEITU
            #    valid_terrain=gfh
            #    terrain=AaBbDeLptUVvYZ
            #    terrain=r
            #    terrain={LETTERS}
            #    terrain=""
            #    terrain=s,c,w,k
            # New style:
            #    terrain=Mm
            #    terrain=Gs^Fp
            #    terrain=Hh, Gg^Vh, Mm
            # The sticky part is that, while it never happens in the current
            # corpus, terrain=Mm (capital letter followed by small) could be
            # interpreted either way.
            newstyle = len(value) > 1 \
                       and value[0].isupper() and value[1].islower() \
                       and (',' in value or len(value) == 2)
            if newstyle:
                if len(value) == 2:
                    print "%s, line %d: ambiguous terrain value %s." \
                          % (filename, lineno+1, value)
                # 1.3.1 to 1.3.2 conversion
                for (old, new) in conversion2.items():
                    transformed = old.sub(new, transformed)
            else:
                # 1.2.x to 1.3.2 conversions
                newterrains = ""
                inmacro = False
                for c in value:
                    if not inmacro:
                        if c == '{':
                            inmacro = True
                            newterrains += c                            
                        elif c == ',':
                            pass
                        elif c.isspace():
                            newterrains += c
                        elif c in conversion1:
                            newterrains += conversion1[c] + ","
                        else:
                            print "%s, line %d: custom terrain %s ignored." \
                              % (filename, lineno+1, c)
                    else: # inmacro == True
                        if c == '}':
                            inmacro = False
                        newterrains += c
                transformed = pre + newterrains[:-1] + post
        # Report the changes
        if verbose > 0 and transformed != line:
            msg = "%s, line %d: %s -> %s" % \
                  (filename, lineno+1, line.strip(), transformed.strip())
            if not hasdigit(os.path.basename(old)) and hasdigit(new):
                msg += " (check manually, alternatives may be better)"
            print msg
        return transformed

    if "1.3.1" in versions and "older" not in versions:
        maptransform = maptransform2
    else:
        maptransform = maptransform1

    if not arguments:
        arguments = ["."]

    for dir in arguments:
        ofp = None
        for fn in allcfgfiles(dir):
            if verbose >= 3:
                print fn + ":"
            backup = fn + "-bak"
            if clean or revert:
                # Do housekeeping
                if os.path.exists(backup):
                    if clean:
                        print "upconvert: removing %s" % backup
                        if not dryrun:
                            os.remove(backup)
                    elif revert:
                        print "upconvert: reverting %s" % backup
                        if not dryrun:
                            os.rename(backup, fn)
            elif diffs:
                # Display diffs
                if os.path.exists(backup):
                    print fn
                    os.system("diff -u %s %s" % (backup, fn))
            else:
                # Do file conversions
                try:
                    changed = translator(fn, maptransform, texttransform)
                    if changed:
                        print "upconvert: converting", fn
                        if not dryrun:
                            os.rename(fn, backup)
                            ofp = open(fn, "w")
                            ofp.write(changed)
                            ofp.close()
                except maptransform_error, e:
                    if e.level <= verbose:
                        sys.stderr.write("upconvert: " + `e` + "\n")
                except:
                    sys.stderr.write("upconvert: internal error on %s\n" % fn)
                    (exc_type, exc_value, exc_traceback) = sys.exc_info()
                    raise exc_type, exc_value, exc_traceback

# upconvert ends here