wesnoth/data/tools/wmlxgettext

#!/usr/bin/env python
#
# wmlxgettext - extract .po files from Wesnoth WML
#
# WARNING! This file is basically a "proof of concept" and was meant as
# replacement for the perl script "utils/wmlxgettext". This script you
# are looking at is not used for the extraction of strings in Wesnoth mainline.
# During tests it has shown to be significantly slower than the existing
# solution so (at least for the moment) that older implementation will be used.
# This also means that this script can (partly) be broken and/or not lead to
# the results you might expect.

import sys, os, time, re, getopt
from wesnoth.wmltools import *
from wesnoth.wmliterator import *

# Code swiped from wmllint: probably should live in the wesnoth module

vctypes = (".svn", ".git")

def cfgfile(fn):
    "Is a file interesting for translation purposes?"
    return fn.endswith(".cfg")

def allcfgfiles(dir):
    "Get the names of all cfgfile files under dir."
    datafiles = []
    if not os.path.isdir(dir):
        if cfgfile(dir):
            if not os.path.exists(dir):
                sys.stderr.write("wmllint: %s does not exist\n" % dir)
            else:
                datafiles.append(dir)
    else:
        for root, dirs, files in os.walk(dir):
            for vcsubdir in vctypes:
                if vcsubdir in dirs:
                    dirs.remove(vcsubdir)
            for name in files:
                if cfgfile(os.path.join(root, name)):
                    datafiles.append(os.path.join(root, name))
    return map(os.path.normpath, datafiles)

class WmllintIterator(WmlIterator):
    "Fold an Emacs-compatible error reporter into WmlIterator."
    def printError(self, *misc):
        """Emit an error locator compatible with Emacs compilation mode."""
        if not hasattr(self, 'lineno') or self.lineno == -1:
            print >>sys.stderr, '"%s":' % self.fname
        else:
            print >>sys.stderr, '"%s", line %d:' % (self.fname, self.lineno+1),
        for item in misc:
            print >>sys.stderr, item,
        print >>sys.stderr #terminate line

# Swiped code ends here

def interesting(text):
    "Is the given text line interesting to render as part of a context?"
    # Ignore translatables, those are guaranteed to be nearby
    if re.search('_ *"', text):
        return False
    return True

if __name__ == "__main__":
    def help():
        sys.stderr.write("""\
Usage: wmlxgettext [options] dirpath
    Options may be any of these:
    -h, --help                   Emit this help message and quit
    -d dir, --directory=dir      operate on specified directory
    -s dn, --domain=dn           Generate only for specified domain
    -v val. --verbose=val        Set warning level
   Options may be followed by any number of directiories to check.  If no
   directories are given, all files under the current directory are checked.
""")

    directory = os.getcwd()

    # Deduce package and version
    pop_to_top("wmlxgettext")    
    m = re.search(r"#define VERSION\s+(.*)", open("src/wesconfig.h").read())
    if not m:
        print >>sys.stderr, "wmlgettext: can't get Wesnoth version."
        sys.exit(1)
    else:
        version = string_strip(m.group(1))

    os.chdir(directory)

    print '''\
msgid ""
msgstr ""
"Project-Id-Version: wesnoth %s\\n"
"Report-Msgid-Bugs-To: http://bugs.wesnoth.org/\\n"
"POT-Creation-Date: %s\\n"
"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\\n"
"Last-Translator: FULL NAME <EMAIL\@ADDRESS>\\n"
"Language-Team: LANGUAGE <LL\@li.org>\\n"
"MIME-Version: 1.0\\n"
"Content-Type: text/plain; charset=UTF-8\\n"
"Content-Transfer-Encoding: 8bit\\n"
''' % (version, time.strftime("%Y-%m-%d, %H:%M+0000", time.gmtime()))

    try:
        domain = None
        verbose = 0
        # Process options
        (options, arguments) = getopt.getopt(sys.argv[1:], "d:h:s:v:",
                                             [
                                              'directory=',
                                              'help',
                                              'domain=',
                                              'verbose=',
                                              ])
        for (switch, val) in options:
            if switch in ('-d', '--directory'):
                directory = val
            elif switch in ('-h', '--help'):
                help()
                sys.exit(0)
            elif switch in ('-s', '--domain'):
                domain = val
            elif switch in ('-v', '--verbose'):
                verbose = int(val)

        if not arguments:
            arguments = '.'

        os.chdir(directory)

        opener_stack = []
        attributes_stack = []
        translatables = []
        contexts = {}
        find_translatable = re.compile('_ *"([^"]*)"')

        def get_translatables(nav, fn):
            "Mine translatable strings "
            itor = find_translatable.finditer(nav.text)
            for match in itor:
                opener_stack.append((nav.element, fn, nav.lineno))
                translatables.append((match.group(1), opener_stack[:]))
                opener_stack.pop()

        def inMacroContinuation(nav):
            return opener_stack and type(opener_stack[-1][0]) == type("") \
                   and opener_stack[-1][0].startswith("{")

        def isInlineMacro(nav):
            return type(nav.element) == type([]) and nav.element and nav.element[0].startswith("{")

        def handle_element(nav, fn):
            if isAttribute(nav):
                if verbose > 1:
                    print "Attribute", nav, "with ancestors", nav.ancestors()
                attributes_stack[-1].append(nav.text.strip())
                get_translatables(nav, fn)
            elif isCloser(nav):
                if verbose > 1:
                    print "Closing scope", nav
                contexts[opener_stack.pop()] = attributes_stack.pop()
            elif isOpener(nav):
                if verbose > 1:
                    print "Opening scope", nav
                attributes_stack.append([])
                opener_stack.append((nav.element, fn, nav.lineno))
            elif isMacroOpener(nav):
                if verbose > 1:
                    print "Opening macro scope", nav
                opener_stack.append((nav.element, fn, nav.lineno))
                get_translatables(nav, fn)
            elif isMacroCloser(nav):
                if verbose > 1:
                    print "Closing macro scope", nav
                opener_stack.pop()
            elif inMacroContinuation(nav):
                if verbose > 1:
                    print "In macro continuation", repr(nav.text)
                nav.element = "argument"
                get_translatables(nav, fn)
            elif isInlineMacro(nav):
                if verbose > 1:
                    print "Inline macro", nav
                nav.element = nav.element[0]
                get_translatables(nav, fn)
            elif nav.element == "#po":
                if verbose > 1:
                    print "Passthrough for", nav
                opener_stack.append((nav.element, fn, nav.lineno))
                translatables.append((nav.text.lstrip(), opener_stack[:]))
                opener_stack.pop()
            elif verbose > 1:
                print "Unhandled", nav
        # Gather a list describing the context of every
        # translatable string.
        for dir in arguments:
            seqno = 0
            for fn in allcfgfiles(dir):
                if verbose >= 2:
                    print fn + ":"
                lines = file(fn).readlines()
                if domain is not None:
                    if lines[0].startswith("#textdomain"):
                        belongs_to = lines[0].split()[1]
                        if belongs_to != domain:
                            if verbose:
                                print "wmlxgettext: skipping %s, wrong domain" % fn
                            continue
                opener_stack.append(("<toplevel>", fn, 0))
                attributes_stack.append([])
                for nav in WmllintIterator(lines, fn):
                    handle_element(nav, fn)
                opener_stack.pop()

        # Debugging output
        if verbose:
            print "Translatables:"
            for (translatable, context) in translatables:
                print "%s: %s" % (translatable, context)
            print "Contexts:"
            for (key, value) in contexts.items():
                print key, "->", value
        # Generate a report from the translatables
        notes = ""
        for (translatable, context) in translatables:
            if translatable.startswith("#po"):
                notes += translatable.replace("po:", "")
                continue
            attribs = ""
            for (i, (tag, file, line)) in enumerate(context):
                if i == len(context)-1 or tag.startswith("{"):
                    print "# %s, line %d: %s" % (file, line, tag)
                else:
                    for (key, value) in contexts.items():
                        value = filter(interesting, value)
                        if key[0] == tag and key[1] == file and key[2] == line:
                            attribs = " has " + ", ".join(value)
                    print "# %s, line %d: %s%s" % (file, line, tag, attribs)
            if notes:
                print notes,
            notes = ""
            print 'msgid "%s"' % translatable
            print 'msgstr ""'
            print ""
    except KeyboardInterrupt:
        print >>sys.stderr, "wmlxgettext: aborted."