wesnoth/data/tools/upconvert

#!/usr/bin/env python
#
# Up-convert WML and maps between versions.
#
# By Eric S. Raymond April 2007.
#
# All conversion logic for lifting WML and maps from older versions of the
# markup to newer ones should live here.  This includes resource path changes
# and renames, also map format conversions.
#
# Takes any number of directories as arguments.  Each directory is converted.
# If no directiories are specified, acts on the current directory.
#
# The recommended procedure is this:
# 1. Run it with --dryrun first to see what it will do.
# 2. If the messages look good, run without --dryrun; the old content
#    will be left in backup files with a -bak extension.
# 3. Eyeball the changes with the --diff option.
# 4. Use macroscope, with a directory list including the Wesnoth mainline WML
#    as first argument, to check that you have no unresolved references.
# 5. Test the conversion.
# 6. Use either --clean to remove the -bak files or --revert to
#    undo the conversion.
#
# This script will barf on maps with custom terrains.  Also, if you a single
# subdirectory that mixes old-style and new-style terrain coding it might
# get confused.

import sys, os, re, getopt, curses.ascii, string, copy

## Version-control hooks begin here.
#
# Change these if we move away from  Subversion

vcdir = ".svn"

def vcmove(src, dst):
    "Move a file under version control. Only applied to unmodified files."
    return "svn mv %s %s" % (src, dst)

## Version-control hooks end here

filemoves = {
    # Older includes all previous to 1.3.1.
    "older" : (
        # File naming error made repeatedly in NR and elsewhere.
        ("human-loyalists/human-", "human-loyalists/"),
        # These are picked to cover as many as possible of the broken
        # references in UMC on the campaign server.  Some things we
        # don't try to fix include:
        # - attack/staff.png may map to one of several staves.
        # - magic.wav may map to one of several sounds depending on the unit.
        # Some other assumption sound in current UMC as of April 2007
        # but theoretically dubious are marked with *.
        ("../music/defeat.ogg",     "defeat.ogg"),
        ("../music/victory.ogg",    "victory.ogg"),
        ("AMLA_TOUGH_2",            "AMLA_TOUGH 2"),
        ("AMLA_TOUGH_3",            "AMLA_TOUGH 3"),
        ("SOUND_LIST:DAGGER_SWISH", "SOUND_LIST:SWORD_SWISH"),
        ("arrow-hit.wav",           "bow.ogg"),
        ("arrow-miss.wav",          "bow-miss.ogg"),
        ("attacks/animal-fangs.png","attacks/fangs-animal.png"),
        ("attacks/crossbow.png",    "attacks/human-crossbow.png"),        #*
        ("attacks/dagger.png",      "attacks/human-dagger.png"),          #*
        ("attacks/darkstaff.png",   "attacks/staff-necromantic.png"),
        ("attacks/human-fist.png",  "attacks/fist-human.png"),
        ("attacks/human-mace.png",  "attacks/mace.png"),
        ("attacks/human-sabre.png", "attacks/sabre-human.png"),
        ("attacks/icebolt.png",     "attacks/iceball.png"),    # Is this right?
        ("attacks/lightingbolt.png","attacks/lightning.png"),
        ("attacks/missile.png",     "attacks/magic-missile.png"),
        ("attacks/morning_star.png","attacks/morning-star.png"),
        ("attacks/plaguestaff.png", "attacks/staff-plague.png"),
        ("attacks/slam.png",        "attacks/slam-drake.png"),
        ("attacks/staff-magical.png","attacks/staff-magic.png"),
        ("attacks/sword-paladin.png","attacks/sword-holy.png"),
        ("attacks/sword.png",       "attacks/human-sword.png"),                #*
        ("attacks/sword_holy.png",  "attacks/sword-holy.png"),
        ("attacks/throwing-dagger-human.png", "attacks/dagger-thrown-human.png"),
        ("bow-hit.ogg",             "bow.ogg"),
        ("bow-hit.wav",             "bow.ogg"),
        ("bowman-attack-sword.png", "bowman-sword-1.png"),
        ("bowman-attack1.png",      "bowman-ranged-1.png"),
        ("bowman-attack2.png",      "bowman-ranged-2.png"),
        ("creepy.ogg",              "underground.ogg"),
        ("dwarves/warrior.png",     "dwarves/fighter.png"),
        ("eagle.wav",               "gryphon-shriek-1.ogg"),
        ("elfland.ogg",             "elf-land.ogg"),
        ("elvish-fighter.png",      "elves-wood/fighter.png"),
        ("elvish-hero.png",         "elves-wood/hero.png"),
        ("fist.wav",                "fist.ogg"),
        ("flame-miss.ogg",          "flame-big-miss.ogg"),
        ("flame.ogg",               "flame-big.ogg"),
        ("gameplay2.ogg",           "gameplay02.ogg"),                    # Changes in 1.3.2
        ("goblin-hit2.ogg",         "goblin-hit-2.ogg"),
        ("hatchet-miss-1.ogg",      "hatchet-miss.wav"),
        ("heal.ogg",                "heal.wav"),
        ("hiss-big.ogg",            "hiss-big.wav"),
        ("human-dagger.png",        "dagger-human.png"),
        ("human-male-die.ogg",      "human-die-1.ogg"),
        ("human-male-hit.ogg",      "human-hit-1.ogg"),
        ("human-male-weak-die.ogg", "human-old-die-1.ogg"),
        ("human-male-weak-hit.ogg", "human-old-hit-1.ogg"),
        ("human-sword.png",         "sword-human.png"),
        ("items/castle-ruins.png",  "scenery/castle-ruins.png"),
        ("items/fire.png",          "scenery/fire1.png"),
        ("items/fire1.png",         "scenery/fire1.png"),
        ("items/fire2.png",         "scenery/fire2.png"),
        ("items/fire3.png",         "scenery/fire3.png"),
        ("items/fire4.png",         "scenery/fire4.png"),
        ("items/hero-icon.png",     "misc/hero-icon.png"),
        ("items/leanto.png",        "scenery/leanto.png"),
        ("items/lighthouse.png",    "scenery/lighthouse.png"),
        ("items/monolith1.png",     "scenery/monolith1.png"),
        ("items/monolith2.png",     "scenery/monolith2.png"),
        ("items/monolith3.png",     "scenery/monolith3.png"),
        ("items/monolith4.png",     "scenery/monolith4.png"),
        ("items/ring1.png",         "items/ring-silver.png"),  # Is this right?
        ("items/ring2.png",         "items/ring-gold.png"),    # Is this right?
        ("items/rock1.png",         "scenery/rock1.png"),
        ("items/rock2.png",         "scenery/rock2.png"),
        ("items/rock3.png",         "scenery/rock3.png"),
        ("items/rock4.png",         "scenery/rock4.png"),
        ("items/signpost.png",      "scenery/signpost.png"),
        ("items/slab.png",          "scenery/slab-1.png"),
        ("items/well.png",          "scenery/well.png"),
        ("knife.ogg",               "dagger-swish.wav"),       # Is this right?
        ("knife.wav",               "dagger-swish.wav"),       # Is this right?
        ("lightning.wav",           "lightning.ogg"),
        ("longbowman-ranged-1.png", "longbowman-bow-attack1.png"),
        ("longbowman-ranged-2.png", "longbowman-bow-attack2.png"),
        ("longbowman-ranged-3.png", "longbowman-bow-attack3.png"),
        ("longbowman-ranged-4.png", "longbowman-bow-attack4.png"),
        ("misc/chest.png",          "items/chest.png"),
        ("misc/dwarven-doors.png",  "scenery/dwarven-doors-closed.png"),
        ("misc/mine.png",           "scenery/mine-abandoned.png"),
        ("misc/nest-empty.png",     "scenery/nest-empty.png"),
        ("misc/rocks.png",          "scenery/rubble.png"),
        ("misc/snowbits.png",       "scenery/snowbits.png"),
        ("misc/temple.png",         "scenery/temple1.png"),
        ("miss.wav",                "miss-1.ogg"),
        ("orc-die.wav",             "orc-die-1.ogg"),
        ("orc-hit.wav",             "orc-hit-1.ogg"),
        ("ork-die-2.ogg",           "orc-die-2.ogg"),
        ("pistol.wav",              "gunshot.wav"),
        ("spear-miss-1.ogg",        "spear-miss.ogg"),
        ("spearman-attack-south-1.png", "spearman-attack-s-1.png"),
        ("spearman-attack-south-2.png", "spearman-attack-s-2.png"),
        ("spearman-attack-south-3.png", "spearman-attack-s-3.png"),
        ("squishy-miss-1.ogg",      "squishy-miss.wav"),
        ("sword-swish.wav",         "sword-1.ogg"),
        ("sword.wav",               "sword-1.ogg"),
        ("terrain/flag-1.png",      "flags/flag-1.png"),
        ("terrain/flag-2.png",      "flags/flag-2.png"),
        ("terrain/flag-3.png",      "flags/flag-3.png"),
        ("terrain/flag-4.png",      "flags/flag-4.png"),
        ("terrain/rocks.png",       "scenery/rock2.png"),
        ("terrain/signpost.png",    "scenery/signpost.png"),
        ("terrain/village-cave-tile.png","terrain/village/cave-tile.png"),
        ("terrain/village-dwarven-tile.png","terrain/village/dwarven-tile.png"),
        ("terrain/village-elven4.png","terrain/village/elven4.png"),
        ("terrain/village-human-snow.png", "terrain/village/human-snow.png"),
        ("terrain/village-human.png","terrain/village/human.png"),
        ("terrain/village-human4.png","terrain/village/human4.png"),
        ("throwing-dagger-swish.wav","dagger-swish.wav"),      # Is this right?
        ("units/undead/ghost-attack.png", "units/undead/ghost-attack-2.png"),
        ("units/undead/ghost-attack1.png", "units/undead/ghost-attack-1.png"),
        ("wolf-attack.wav",         "wolf-bite.ogg"),
        ("wolf-cry.wav",            "wolf-die.wav"),
        ("wose-attack.wav",         "wose-attack.ogg"),
        (r"wose\.attack.ogg",       "wose-attack.ogg"),
    ),
    "1.3.1" : (
        # Peasant images moved to a new directory
        ("human-loyalists/peasant.png", "human-peasants/peasant.png"),
        ("human-loyalists/peasant-attack.png", "human-peasants/peasant-attack.png"),
        ("human-loyalists/peasant-attack2.png", "human-peasants/peasant-attack2.png"),
        ("human-loyalists/peasant-ranged.png", "human-peasants/peasant-ranged.png"),
        ("human-loyalists/peasant-idle-1.png", "human-peasants/peasant-idle-1.png"),
        ("human-loyalists/peasant-idle-2.png", "human-peasants/peasant-idle-2.png"),
        ("human-loyalists/peasant-idle-3.png", "human-peasants/peasant-idle-3.png"),
        ("human-loyalists/peasant-idle-4.png", "human-peasants/peasant-idle-4.png"),
        ("human-loyalists/peasant-idle-5.png", "human-peasants/peasant-idle-5.png"),
        ("human-loyalists/peasant-idle-6.png", "human-peasants/peasant-idle-6.png"),
        ("human-loyalists/peasant-idle-7.png", "human-peasants/peasant-idle-7.png"),
        # All Great Mage attacks were renamed
        ("great-mage-attack-magic1.png", "great-mage-attack-magic-1.png"),
        ("great-mage-attack-magic2.png", "great-mage-attack-magic-2.png"),
        ("great-mage+female-attack-magic1.png", "great-mage+female-attack-magic-1.png"),
        ("great-mage+female-attack-magic2.png", "great-mage+female-attack-magic-2.png"),
        ("great-mage-attack-staff1.png", "great-mage-attack-staff-1.png"),
        ("great-mage-attack-staff2.png", "great-mage-attack-staff-2.png"),
        ("great-mage+female-attack-staff1.png", "great-mage+female-attack-staff-1.png"),
        ("great-mage+female-attack-staff2.png", "great-mage+female-attack-staff-2.png"),
        # All Arch Mage attacks were renamed
        ("arch-mage-attack-magic1.png", "arch-mage-attack-magic-1.png"),
        ("arch-mage-attack-magic2.png", "arch-mage-attack-magic-2.png"),
        ("arch-mage+female-attack-magic1.png", "arch-mage+female-attack-magic-1.png"),
        ("arch-mage+female-attack-magic2.png", "arch-mage+female-attack-magic-2.png"),
        ("arch-mage-attack-staff1.png", "arch-mage-attack-staff-1.png"),
        ("arch-mage-attack-staff2.png", "arch-mage-attack-staff-2.png"),
        ("arch-mage+female-attack-staff1.png", "arch-mage+female-attack-staff-1.png"),
        ("arch-mage+female-attack-staff2.png", "arch-mage+female-attack-staff-2.png"),
        # All Red Mage attacks were renamed
        ("red-mage-attack-magic1.png", "red-mage-attack-magic-1.png"),
        ("red-mage-attack-magic2.png", "red-mage-attack-magic-2.png"),
        ("red-mage+female-attack-magic1.png", "red-mage+female-attack-magic-1.png"),
        ("red-mage+female-attack-magic2.png", "red-mage+female-attack-magic-2.png"),
        ("red-mage-attack-staff1.png", "red-mage-attack-staff-1.png"),
        ("red-mage-attack-staff2.png", "red-mage-attack-staff-2.png"),
        ("red-mage+female-attack-staff1.png", "red-mage+female-attack-staff-1.png"),
        ("red-mage+female-attack-staff2.png", "red-mage+female-attack-staff-2.png"),
        # Timothy Pinkham supplied titles for two of his music files.
        # Zhaytee supplied a title for wesnoth-1.ogg
        # gameplay03.ogg, and and wesnoth-[25].ogg already had titles.
        ("gameplay01.ogg", "knolls.ogg"),
        ("gameplay02.ogg", "wanderer.ogg"),
        ("gameplay03.ogg", "battle.ogg"),
        ("wesnoth-1.ogg", "revelation.ogg"),
        ("wesnoth-2.ogg", "loyalists.ogg"),
        ("wesnoth-5.ogg", "northerners.ogg"),
        # And the holy->arcane change
        ("type=holy", "type=arcane"),
        ("holy=", "arcane=")
    ),
    "1.3.2" : (
        ("misc/item-holywater.png", "items/holywater.png"),
        ("orc-small-hit.wav",       "orc-small-hit-1.ogg"),
    ),
    # An empty sentinel value at end is required.
    # Always have the current version here.
    "1.3.3" : ()
}

# Turn all the filemove string substition pairs into nearly equivalent
# regexp-substitution pairs, forbidding the match from being preceded
# by a dash.  This prevents, e.g., "miss.ogg" false-matching on "big-miss.ogg".
for (key, value) in filemoves.items():
    filemoves[key] = map(lambda (old, new): (re.compile("(?<!-)"+old), new), value)

# 1.2.x to 1.3.2 terrain conversion
conversion1 = {
    " " : "_s",
    "&" : "Mm^Xm",
    "'" : "Uu^Ii",
    "/" : "Ww^Bw/",
    "1" : "1 _K",
    "2" : "2 _K",
    "3" : "3 _K",
    "4" : "4 _K",
    "5" : "5 _K",
    "6" : "6 _K",
    "7" : "7 _K",
    "8" : "8 _K",
    "9" : "9 _K",
    "?" : "Gg^Fet",
    "A" : "Ha^Vhha",
    "B" : "Dd^Vda",
    "C" : "Ch",
    "D" : "Uu^Vu",
    "E" : "Rd",
    "F" : "Aa^Fpa",
    "G" : "Gs",
    "H" : "Ha",
    "I" : "Dd",
    "J" : "Hd",
    "K" : "_K",
    "L" : "Gs^Vht",
    "M" : "Md",
    "N" : "Chr",
    "P" : "Dd^Do",
    "Q" : "Chw",
    "R" : "Rr",
    "S" : "Aa",
    "T" : "Gs^Ft",
    "U" : "Dd^Vdt",
    "V" : "Aa^Vha",
    "W" : "Xu",
    "X" : "Qxu",
    "Y" : "Ss^Vhs",
    "Z" : "Ww^Vm",
    "[" : "Uh",
    "\\": "Ww^Bw\\",
    "]" : "Uu^Uf",
    "a" : "Hh^Vhh",
    "b" : "Mm^Vhh",
    "c" : "Ww",
    "d" : "Ds",
    "e" : "Aa^Vea",
    "f" : "Gs^Fp",
    "g" : "Gg",
    "h" : "Hh",
    "i" : "Ai",
    "k" : "Wwf",
    "l" : "Ql",
    "m" : "Mm",
    "n" : "Ce",
    "o" : "Cud",
    "p" : "Uu^Vud",
    "q" : "Chs",
    "r" : "Re",
    "s" : "Wo",
    "t" : "Gg^Ve",
    "u" : "Uu",
    "v" : "Gg^Vh",
    "w" : "Ss",
    "|" : "Ww^Bw|",
    "~" : "_f",
}
max_len = max(*map(len, conversion1.values()))
width = max_len+2

def neighborhood(x, y, map):
    "Returns list of original location+adjacent locations from a hex map"
    odd = (x) % 2
    adj = [map[y][x]];
    if x > 0:
	adj.append(map[y][x-1])
    if x < len(map[y])-1:
	adj.append(map[y][x+1])
    if y > 0:
	adj.append(map[y-1][x])
    if y < len(map)-1:
	adj.append(map[y+1][x])
    if x > 0 and y > 0 and not odd:
	adj.append(map[y-1][x-1])
    if x < len(map[y])-1 and y > 0 and not odd:
	adj.append(map[y-1][x+1])
    if x > 0 and y < len(map)-1 and odd:
	adj.append(map[y+1][x-1])
    if x < len(map[y])-1 and y < len(map)-1 and odd:
	adj.append(map[y+1][x+1])
    return adj

def maptransform1(filename, baseline, inmap, y):
    "Transform a map line from 1.2.x to 1.3.x format."
    global lock_terrain_coding
    # The one truly ugly piece of implementation.
    # We're relying here on maps being seen before scenario files.
    # We notice whether the maps are oldstyle (single-letter codes)
    # or newstyle (multiletter comma-seeparated fields) and retain that
    # information to help with ambiguous cases later on.  We're also relying
    # on terrain coding to be consistent within a single subdirectory. 
    if len(inmap[y][0]) > 1:
        lock_terrain_coding = "newstyle"
    else:
        format = "%%%d.%ds" % (width, max_len)
        for (x, field) in enumerate(inmap[y]):
            if field in conversion1:
                lock_terrain_coding = "oldstyle"
                inmap[y][x] = format % conversion1[field]
            else:
                raise maptransform_error(filename, baseline+y+1,
                                         "unrecognized map element %s at (%s, %s)" % (`field`, x, y))

# 1.3.1 -> 1.3.2 terrain conversions
conversion2 = {
    re.compile(r"(?<!\^)Bww([|/\\])") : "Ww^Bw\\1",
    re.compile(r"(?<!\^)Bwo([|/\\])") : "Wo^Bw\\1",
    re.compile(r"(?<!\^)Bss([|/\\])") : "Ss^Bw\\1",
    re.compile(r"(?<!\^)Dc\b") : "Dd^Dc",
    re.compile(r"(?<!\^)Dr\b") : "Dd^Dr",
    re.compile(r"(?<!\^)Do\b") : "Dd^Do",
    re.compile(r"(?<!\^)Fa\b") : "Aa^Fpa",
    re.compile(r"(?<!\^)Fet\b") : "Gg^Fet",
    re.compile(r"(?<!\^)Ff\b") : "Gs^Fp",
    re.compile(r"(?<!\^)Ft\b") : "Gs^Ft",
    re.compile(r"(?<!\^)Rfvs\b") : "Re^Gvs",
    re.compile(r"(?<!\^)Uf\b") : "Uu^Uf",
    re.compile(r"(?<!\^)Uui\b") : "Uu^Ii",
    re.compile(r"(?<!\^)Uhi\b") : "Uh^Ii",
    re.compile(r"(?<!\^)Vda\b") : "Dd^Vda",
    re.compile(r"(?<!\^)Vdt\b") : "Dd^Vdt",
    re.compile(r"(?<!\^)Vea\b") : "Aa^Vea",
    re.compile(r"(?<!\^)Veg\b") : "Gg^Ve",
    re.compile(r"(?<!\^)Vha\b") : "Aa^Vha",
    re.compile(r"(?<!\^)Vhg\b") : "Gg^Vh",
    re.compile(r"(?<!\^)Vhh\b") : "Hh^Vhh",
    re.compile(r"(?<!\^)Vhha\b") : "Ha^Vhha",
    re.compile(r"(?<!\^)Vhm\b") : "Mm^Vhh",
    re.compile(r"(?<!\^)Vht\b") : "Gs^Vht",
    re.compile(r"(?<!\^)Vu\b") : "Uu^Vu",
    re.compile(r"(?<!\^)Vud\b") : "Uu^Vud",
    re.compile(r"(?<!\^)Vwm\b") : "Ww^Vm",
    re.compile(r"(?<!\^)Vs\b") : "Ss^Vhs",
    re.compile(r"(?<!\^)Vsm\b") : "Ss^Vm",
    re.compile(r"(?<!\^)Xm\b") : "Mm^Xm",
    }

def maptransform2(filename, baseline, inmap, y):
    "Convert a map line from 1.3.1 multiletter format to 1.3.2 format."
    for x in range(len(inmap[y])):
        # General conversions
        for (old, new) in conversion2.items():
            inmap[y][x] = old.sub(new, inmap[y][x])
        # Convert keeps according to adjacent hexes
        if "_K" in inmap[y][x]:
            adj = map(string.strip, neighborhood(x, y, inmap))

            # print "adjacent: %s" % adj
            hexcount = {}
            # Intentionally skipping 0 as it is original hex
            for i in range(1, len(adj)):
                if adj[i].startswith("C"): # this is a castle hex
                    # Magic: extract second character of each adjacent castle,
                    # which is its base type.  Count occurrences of each type.
                    basetype = adj[i][1]
                    hexcount[basetype] = hexcount.get(basetype, 0) + 1
            maxc = 0;
            maxk = "h";
            # Note: if two kinds of basetype tie for most instances adjacent,
            # which one dominates will be a pseudorandom artifact of
            # Python's hash function.
            for k in hexcount.keys():
                if hexcount[k] > maxc:
                    maxc = hexcount[k]
                    maxk = k
            #print "Dominated by %s" % maxk
            inmap[y][x] = inmap[y][x].replace("_K", "K" + maxk)
            # There's only one kind of underground keep at present.
            inmap[y][x] = inmap[y][x].replace("Ku", "Kud")

# Generic machinery starts here

class maptransform_error:
    "Error object to be thrown by maptransform."
    def __init__(self, infile, inline, type):
        self.infile = infile
        self.inline = inline
        self.type = type
    def __repr__(self):
        return '"%s", line %d: %s' % (self.infile, self.inline, self.type)

def translator(filename, mapxforms, textxform):
    "Apply mapxform to map lines and textxform to non-map lines."
    modified = False
    mfile = []
    map_only = not filename.endswith(".cfg")
    terminator = "\n"
    for line in open(filename):
        if line.endswith("\n"):
            line = line[:-1]
        if line.endswith("\r"):
            line = line[:-1]
            terminator = '\r\n'
        mfile.append(line)
        if "map_data" in line:
            map_only = False
    cont = False
    outmap = []
    newdata = []
    lineno = baseline = 0
    while mfile:
        line = mfile.pop(0)
        if verbose >= 3:
            sys.stdout.write(line + terminator)
        lineno += 1
        # Exclude map_data= lines that are just 1 line without
        # continuation, or which contain {}.  The former are
        # pathological and the parse won't handle them, the latter
        # refer to map files which will be checked separately.
        if map_only or ("map_data=" in line
                        and line.count('"') in (1, 2)
                        and line.count("{") == 0
                        and  line.count("}") == 0):
            baseline = 0
            cont = True
            if verbose >= 3:
                print "*** Entering map mode."
            if not map_only:
                fields = line.split('"')
                if fields[1].strip():
                    mfile.insert(0, fields[1])
                if len(fields) == 3:
                    mfile.insert(1, '"')
            while cont and mfile:
                line = mfile.pop(0)
                if verbose >= 3:
                    sys.stdout.write(line + terminator)
                lineno += 1
                if len(line) == 0 or line[0] == '#':
                    newdata.append(line + terminator)
                    continue
                if '"' in line:
                    cont = False
                    if verbose >= 3:
                        print "*** Exiting map mode."
                    line = line.split('"')[0]
                if line:
                    if ',' in line:
                        fields = line.split(",")
                    else:
                        fields = map(lambda x: x, line)
                    outmap.append(fields)
            if not map_only: 
                newdata.append("map_data=\"" + terminator)
            original = copy.deepcopy(outmap)
            for transform in mapxforms:
                for y in range(len(outmap)):
                    transform(filename, baseline, outmap, y)
            for y in range(len(outmap)):
                newdata.append(",".join(outmap[y]) + terminator)
                if original[y] != outmap[y]:
                    modified = True
            # All lines of the map are processed, add the appropriate trailer
            if not map_only:
                newdata.append("\"" + terminator)
        elif "map_data=" in line and (line.count("{") or line.count("}")):
            newline = line
            refre = re.compile(r"\{@?([^A-Z].*)\}").search(line)
            if refre:
                mapfile = refre.group(1)
                if not mapfile.endswith(".map") and modified_maps.get(mapfile)==False:
                    newline = newline.replace(mapfile, mapfile + ".map") 
            newdata.append(newline + terminator)
            if newline != line:
                modified = True
                if verbose > 0:
                    print >>sys.stderr, 'upconvert: "%s", line %d: %s -> %s.' % (filename, lineno, line, newline)
        elif "map_data=" in line and line.count('"') > 1:
            print >>sys.stderr, 'upconvert: "%s", line %d: one-line map.' % (filename, lineno)
            newdata.append(line + terminator)
        else:
            # Handle text (non-map) lines
            newline = textxform(filename, lineno, line)
            newdata.append(newline + terminator)
            if newline != line:
                modified = True
    # Track which maps are modified, we'll use this later for determining
    # which files get a .map extension.
    if "maps" in filename:
        modified_maps[filename] = modified
    # Return None if the transformation functions made no changes.
    if modified:
        return "".join(newdata)
    else:
        return None

ignore = (".tgz", ".png", ".jpg", "-bak")

def interesting(fn):
    "Is a file interesting for conversion purposes?"
    return fn.endswith(".cfg") or fn.endswith(".map") \
        or ("maps" in fn and fn[-4:] not in ignore)

def allcfgfiles(dir):
    "Get the names of all interesting files under dir."
    datafiles = []
    if not os.path.isdir(dir):
        if interesting(dir):
            datafiles.append(dir)
    else:
        for root, dirs, files in os.walk(dir):
            if vcdir in dirs:
                dirs.remove(vcdir)
            for name in files:
                if interesting(os.path.join(root, name)):
                    datafiles.append(os.path.join(root, name))
    datafiles.sort()	# So diffs for same campaigns will cluster in reports
    return map(os.path.normpath, datafiles)

def help():
        sys.stderr.write("""\
Usage: upconvert [options]
    Convert Battle of Wesnoth WML from older versions to newer ones.
    Options may be any of these:
    -h, --help                 Emit this help message and quit.
    -d, --dryrun               List changes but don't perform them.
    -o, --oldversion           Specify version to begin with.
    -v, --verbose              -v        lists changes.
                               -v -v     names each file before it's processed.
                               -v -v -v  shows verbose parse details.
    -c, --clean                Clean up -bak files.
    -D, --diff                 Display diffs between unconverted and unconverted files.
    -r, --revert               Revert the conversion from the -bak files.
""")

if __name__ == '__main__':
    (options, arguments) = getopt.getopt(sys.argv[1:], "cdDho:rv", [
	"help",
        "oldversion=",
	"dryrun",
        "verbose",
        "clean",
        "revert",
        "diffs",
        ])
    oldversion = 'older'
    dryrun = False
    verbose = 0
    clean = False
    diffs = False
    revert = False
    for (switch, val) in options:
        if switch in ('-h', '--help'):
            help()
            sys.exit(0)
        elif switch in ('-o', '--oldversion'):
            oldversion = val
        elif switch in ('-v', '--verbose'):
            verbose += 1
        elif switch in ('-d', '--dryrun'):
            dryrun = True
            verbose = max(1, verbose)
        elif switch in ('-c', '--clean'):
            clean = True
        elif switch in ('-d', '--diffs'):
            diffs = True
        elif switch in ('-r', '--revert'):
            revert = True

    if clean and revert:
        sys.stderr.write("upconvert: can't do clean and revert together.\n")
        sys.exit(1)

    # Compute the series of version upgrades to perform, and describe it.
    versions = filemoves.keys()
    versions.sort()
    versions = [versions[-1]] + versions[:-1]	# Move 'older' to front
    if oldversion in versions:
        versions = versions[versions.index(oldversion):]
    else:
        print >>sys.stderr, "upconvert: unrecognized version."
        sys.exit(1)
    explain = "Performing upgrades for:"
    for i in range(len(versions)-1):
        explain += " %s -> %s," % (versions[i],  versions[i+1])
    sys.stdout.write(explain[:-1] + ".\n")
    fileconversions = map(lambda x: filemoves[x], versions[:-1])

    def hasdigit(str):
        for c in str:
            if c in "0123456789":
                return True
        return False

    def parse_attribute(str):
        "Parse a WML key-value pair from a line."
        if '=' not in str:
            return None
        m = re.match(r"(^\s*[a-z0-9_]+\s*=\s*)(\S+)(\s*#?.*\s*)", str)
        if not m:
            return None
        # Four fields: stripped key, part of line before value,
        # value, trailing whitespace and comments
        return (m.group(1).replace("=", "").strip(),) + m.groups()

    def texttransform(filename, lineno, line):
        "Resource-name transformation on text lines."
        transformed = line
        # First, do resource-file moves
        for step in fileconversions:
            for (old, new) in step:
                transformed = old.sub(new, transformed)
        # Handle terrain_liked=, terrain=, valid_terrain=, letter=
        spaceless = transformed.replace(" ", "").replace("\t", "")
        if spaceless and spaceless[0] != "#" and ("terrain_liked=" in spaceless or "terrain=" in spaceless or 'letter=' in spaceless):
            (key, pre, value, post) = parse_attribute(transformed)
            # We have to cope with the following cases...
            # Old style:
            #    terrain_liked=ghM
            #    terrain_liked=BEITU
            #    valid_terrain=gfh
            #    terrain=AaBbDeLptUVvYZ
            #    terrain=r
            #    terrain={LETTERS}
            #    terrain=""
            #    terrain=s,c,w,k
            # New style:
            #    terrain=Mm
            #    terrain=Gs^Fp
            #    terrain=Hh, Gg^Vh, Mm
            # The sticky part is that, while it never happens in the current
            # corpus, terrain=Mm (capital letter followed by small) could be
            # interpreted either way.
            #
            # There are some unambiguous tests: 
            oldstyle = (len(value) == 1 or len(value) > 6) and not ',' in value
            newstyle = len(value) > 1 \
                       and value[0].isupper() and value[1].islower() \
                       and (',' in value \
                            or len(value) == 2 \
                            or (len(value) >= 3 and value[2] == "^"))
            # See maptransform1() for explanation of this ugly hack.
            oldstyle = oldstyle or lock_terrain_coding == "oldstyle"
            newstyle = newstyle or lock_terrain_coding == "newstyle"
            # Maybe we lose...
            if not oldstyle and not newstyle:
                print "%s, line %d: leaving ambiguous terrain value %s alone." \
                      % (filename, lineno+1, value)
            else:
                if oldstyle:
                    # 1.2.x to 1.3.2 conversions
                    newterrains = ""
                    inmacro = False
                    for c in value:
                        if not inmacro:
                            if c == '{':
                                inmacro = True
                                newterrains += c                            
                            elif c == ',':
                                pass
                            elif c.isspace():
                                newterrains += c
                            elif c in conversion1:
                                newterrains += conversion1[c] + ","
                            else:
                                print "%s, line %d: custom terrain %s ignored." \
                                  % (filename, lineno+1, c)
                        else: # inmacro == True
                            if c == '}':
                                inmacro = False
                            newterrains += c
                    transformed = pre + newterrains[:-1] + post
                if newstyle:
                    if len(value) == 2:
                        # 1.3.1 to 1.3.2 conversion
                        for (old, new) in conversion2.items():
                            transformed = old.sub(new, transformed)
        # Report the changes
        if verbose > 0 and transformed != line:
            msg = "%s, line %d: %s -> %s" % \
                  (filename, lineno+1, line.strip(), transformed.strip())
            print msg
        return transformed

    if "1.3.1" in versions and "older" not in versions:
        maptransforms = [maptransform2]
    else:
        maptransforms = [maptransform1, maptransform2]

    if not arguments:
        arguments = ["."]

    for dir in arguments:
        ofp = None
        modified_maps = {}
        if "older" in versions:
            lock_terrain_coding = None
        else:
            lock_terrain_coding = "newstyle"
        for fn in allcfgfiles(dir):
            if verbose >= 2:
                print fn + ":"
            backup = fn + "-bak"
            if clean or revert:
                # Do housekeeping
                if os.path.exists(backup):
                    if clean:
                        print "upconvert: removing %s" % backup
                        if not dryrun:
                            os.remove(backup)
                    elif revert:
                        print "upconvert: reverting %s" % backup
                        if not dryrun:
                            os.rename(backup, fn)
            elif diffs:
                # Display diffs
                if os.path.exists(backup):
                    print fn
                    os.system("diff -u %s %s" % (backup, fn))
            else:
                # Do file conversions
                try:
                    changed = translator(fn, maptransforms, texttransform)
                    if changed:
                        print "upconvert: converting", fn
                        if not dryrun:
                            os.rename(fn, backup)
                            ofp = open(fn, "w")
                            ofp.write(changed)
                            ofp.close()
                except maptransform_error, e:
                    sys.stderr.write("upconvert: " + `e` + "\n")
                except:
                    sys.stderr.write("upconvert: internal error on %s\n" % fn)
                    (exc_type, exc_value, exc_traceback) = sys.exc_info()
                    raise exc_type, exc_value, exc_traceback
            # Time for map file renames
            if not fn.endswith(".map") and modified_maps.get(fn) == False:
                mover = vcmove(fn, fn + ".map")
                print mover
                if not dryrun:
                    os.system(mover)

# upconvert ends here