Another large update,

the code now starts to settle and slowly gets into a usable state.
This commit is contained in:
Mark de Wever 2007-11-10 14:54:21 +00:00
parent f9ceae7ee5
commit ab6f8fed56
2 changed files with 233 additions and 225 deletions

View File

@ -19,24 +19,15 @@ the command line svn tool.
import os, shutil, logging;
class result:
"""V Result object for most svn functions
class error(Exception):
"""Base class for exceptions in this module."""
pass
status -1 error occured err has the message
0 nothing to do (command might be executed)
1 success
out output of the svn command
err error of the svn command
"""
def __init__(self, status, out = "", err = ""):
self.status = status
self.out = out
self.err = err
class SVN:
"""\ Initializes a SVN object
"""Initializes a SVN object.
checkout the root of the local checkout eg /src/wesnoth
Checkout the root of the local checkout eg /src/wesnoth
do not include a trailing slash!
"""
def __init__(self, checkout):
@ -57,39 +48,35 @@ class SVN:
self.STATUS_FLAG_NON_SVN = 0x10
self.STATUS_FLAG_NON_EXISTANT = 0x20
"""V Makes a new checkout.
"""Makes a new checkout.
repo The repo to checkout eg
http://svn.gna.org/svn/wesnoth/trunk
returns A result object (note if an checkout was already there
it always returns 1 no indication whether something is
updated).
http://svn.gna.org/svn/wesnoth/trunk.
returns Nothing.
"""
def svn_checkout(self, repo):
logging.debug("checkout " + repo)
logging.debug("checkout repo = '%s'", repo)
out, err = self.execute("svn co --non-interactive " + repo + " " +
self.checkout_path)
if(err != ""):
return result(-1, out, err)
raise error("checkout failed with message:" + err)
return result(1, out)
logging.debug("checkout output:" + out)
"""V Commits the changes
"""Commits the changes.
After deleting a local file and committing that change the file remains.
msg The commit message.
files Optional list with files/directories to check in if
files Optional list with files/directories to commit if
ommitted all modifications are send.
returns A result object.
returns True if committed, False if nothing to commit.
"""
def svn_commit(self, msg, files = None):
logging.debug("commit msg " + msg)
logging.debug("commit msg = '%s' files = '%s' ", msg, files)
command = "svn commit --non-interactive -m " + '"' + msg + '"'
if(files != None):
@ -99,21 +86,26 @@ class SVN:
out, err = self.execute(command + " " + self.checkout_path)
if(err != ""):
return result(-1, out, err)
raise error("commit failed with message:" +err)
elif(out != ""):
return result(1, out)
logging.debug("commit output:" + out)
return True
else:
# no output nothing committed
return result(0, out)
logging.debug("commit has no output")
return False
"""V updates the local checkout
"""Updates the local checkout.
rev Revision to update to, if ommitted updates to HEAD.
files Optional list of files to update, if ommitted the
checkout is updated.
returns A result object, returns 0 if no changes were made.
returns True if update changed files, False otherwise.
"""
def svn_update(self, rev = None, files = None):
logging.debug("update rev = '%s' files = '%s'", rev, files)
command = "svn up --non-interactive "
if(rev != None):
command += "-r " + rev + " "
@ -124,63 +116,58 @@ class SVN:
out, err = self.execute(command + self.checkout_path)
if(err != ""):
return result(-1, out, err)
raise error("update failed with message:" +err)
if(out.count('\n') == 1):
return result(0, out)
logging.debug("update didn't change anything")
return False
return result(1, out)
logging.debug("update output:" + out)
return True
"""T Copies local files to an svn checkout.
"""Copies local files to an svn checkout.
src Directory with the source files.
exclude List with names to ignore.
returns A result object, returns 0 if no changes are made after
the copy operation.
returns True if the copy resulted in a modified checkout,
False otherwise.
"""
def copy_to_svn(self, src, exclude):# = None):
def copy_to_svn(self, src, exclude):
logging.debug("copy_to_svn :\n\tsvn = "
+ self.checkout_path + "\n\tsrc = " + src)
logging.debug("copy to svn src = '%s' exclude = '%s'",
src, exclude)
# check whether the status of the repo is clean
# Check whether the status of the repo is clean.
out, err = self.execute("svn st " + self.checkout_path)
# if not clean or an error bail out
# If not clean or an error bail out.
if(err != ""):
return result(-1, out, err)
raise error("status failed with message:" + err)
elif(out != ""):
return result(-1, out, "checkout not clean:\n" + out)
raise error("checout is not clean:" + out)
# update
res = self.sync_dir(src, self.checkout_path, False, exclude)
# Update.
self.sync_dir(src, self.checkout_path, False, exclude)
# Only if if the status is 1 it might change to 0
if(res.status != 1) :
return res
# no error, test whether clean or not, if clean we need to set the
# status to 0
# Test whether the checkout is clean if not something is modified.
# An error shouldn't occur, since we tested that before.
out, err = self.execute("svn st " + self.checkout_path)
if(out == ""):
res.status = 0
return res
return (out != "")
"""T Syncronizes two directories.
"""Syncronizes two directories.
src The source directory.
dest The destination directory.
src_svn Either the source or the target is a svn checkout
if True the source is, else the destination.
exclude List with names to ignore.
returns A result object, 0 if nothing has been done, if
something is copied it returns 1 and doesn't check
whether source and target file are different.
returns Nothing.
"""
def sync_dir(self, src, dest, src_svn, exclude ):#= None):
def sync_dir(self, src, dest, src_svn, exclude ):
logging.debug("sync_dir :\n\tsrc = " + src + "\n\tdest = " + dest)
logging.debug("sync_dir src = '%s' dest = '%s'", src, dest)
# Get the contents of src and dest
src_dirs, src_files = self.get_dir_contents(src, exclude)
dest_dirs, dest_files = self.get_dir_contents(dest, exclude)
@ -195,21 +182,15 @@ class SVN:
for dir in src_dirs:
if(os.path.isdir(dest + "/" + dir) == False):
# src only
res = self.dir_add(src + "/" + dir, dest + "/" + dir, src_svn, exclude)
if(res.status == -1):
return res
self.dir_add(src + "/" + dir, dest + "/" + dir, src_svn, exclude)
else:
# in both
res = self.sync_dir(src + "/" + dir, dest + "/" + dir, src_svn, exclude)
if(res.status == -1):
return res
self.sync_dir(src + "/" + dir, dest + "/" + dir, src_svn, exclude)
for dir in dest_dirs:
if(os.path.isdir(src + "/" + dir) == False):
# dest only
res = self.dir_remove(dest + "/" + dir, not(src_svn))
if(res.status == -1):
return res
self.dir_remove(dest + "/" + dir, not(src_svn))
# If a file exists in the src but not in the dest, it needs to be copied.
@ -221,27 +202,17 @@ class SVN:
for file in src_files:
if(os.path.isfile(dest + "/" + file) == False):
# src only
res = self.file_add(src + "/" + file, dest + "/" + file, src_svn)
if(res.status == -1):
return res
self.file_add(src + "/" + file, dest + "/" + file, src_svn)
else:
# in both
res = self.file_copy(src + "/" + file, dest + "/" + file)
if(res.status == -1):
return result
self.file_copy(src + "/" + file, dest + "/" + file)
for file in dest_files:
if(os.path.isfile(src + "/" + file) == False):
# dest only
res = self.file_remove(dest + "/" + file, not(src_svn))
if(res.status == -1):
return res
self.file_remove(dest + "/" + file, not(src_svn))
# FIXME we didn't accumulate the output
return result(1)
"""V Gets a list with files and directories.
"""Gets a list with files and directories.
The function always ignores .svn entries. Items which aren't a directory
are assumed a file.
@ -249,42 +220,35 @@ class SVN:
exclude List with names to ignore.
returns A list with directories and a list with files.
"""
def get_dir_contents(self, dir, exclude ):#= None):
def get_dir_contents(self, dir, exclude):
logging.debug("get dir :\n\tdir = " + dir)
if(exclude != None):
logging.debug("\t exclude = ")
logging.debug(exclude)
logging.debug("get dir contents dir = '%s' exclude = '%s'",
dir, exclude)
items = os.listdir(dir)
dirs = []
files = []
for item in items:
logging.debug("\tTesting item " + item)
# ignore .svn dirs
# Ignore .svn dirs.
if(item == ".svn"):
logging.debug("\t\tIgnore .svn")
continue
# ignore exclude list
# Ignore exclude list.
if(exclude != None and item in exclude):
logging.debug("\t\tIgnore on the exclude list")
continue
# an item is either a directory or not, in the latter case it's
# An item is either a directory or not, in the latter case it's
# assumed to be a file.
if(os.path.isdir(dir + "/" + item)):
logging.debug("\t\tAdded directory")
dirs.append(item)
else:
logging.debug("\t\tAdded file")
files.append(item)
return dirs, files
"""T creates a duplicate of a directory.
"""Creates a duplicate of a directory.
The destination directory shouldn't exist.
src The source directory.
@ -292,129 +256,125 @@ class SVN:
src_svn Either the source or the target is a svn checkout
if True the source is, else the destination.
exclude List with names to ignore.
returns A result object, 0 if nothing has been done, if
something is copied it returns 1 and doesn't check
whether source and target file are different.
returns Nothing.
"""
def dir_add(self, src, dest, src_svn, exclude ):#= None):
def dir_add(self, src, dest, src_svn, exclude):
logging.debug("dir_add :\n\tsvn = " + self.checkout_path + "\n\tsrc = " + src)
logging.debug("dir_add src = '%s' dest = '%s' svn_src = '%s' "
+ "exclude = '%s'", src, dest, src_svn, exclude)
# add parent
os.mkdir(dest)
if(src_svn == False):
res = self.svn_add(dest)
if(res.status == -1):
return res
self.svn_add(dest)
# get sub items
dirs, files = self.get_dir_contents(src, exclude)
# copy files
for file in files:
res = self.file_add(src + "/" + file, dest + "/" + file, src_svn)
if(res.status == -1):
return res
self.file_add(src + "/" + file, dest + "/" + file, src_svn)
# copy dirs
for dir in dirs:
res = self.dir_add(src + "/" + dir, dest + "/" + dir, src_svn, exclude)
if(res.status == -1):
return res
return result(1)
self.dir_add(src + "/" + dir, dest + "/" + dir, src_svn, exclude)
""" FIXME IMPLEMENT
"""
def dir_remove(self, dir):
return result(1)
logging.debug("dir_remove dir = '%s'", dir)
"""T Adds a file.
raise error("dir_remove not implemented")
"""Adds a file.
If src_svn is True it does the same as copy file.
src The source directory.
dest The destination directory.
src_svn Either the source or the target is a svn checkout
if True the source is, else the destination.
returns A result object.
returns Nothing.
"""
def file_add(self, src, dest, src_svn):
logging.debug("file_add src = '%s' dest = '%s' src_svn = '%s'",
src, dest, src_svn)
shutil.copy(src, dest)
if(src_svn):
return result(1)
else:
return self.svn_add(dest)
if(src_svn == False):
self.svn_add(dest)
"""T Copies a file.
"""Copies a file.
src The source directory.
dest The destination directory.
returns A result object.
returns Nothing
"""
def file_copy(self, src, dest):
shutil.copy(src, dest)
return result(1)
""" T Removes a file
logging.debug("file_copy src = '%s' dest = '%s'", src, dest)
shutil.copy(src, dest)
"""Removes a file
file The file to remove.
is_svn Is the file in an svn checkout.
returns A result object.
returns Nothing.
"""
def file_remove(self, file, is_svn):
logging.debug("file_remove file = '%s' is_svn = '%s'", file, is_svn)
if(is_svn):
return self.svn_remove(file)
self.svn_remove(file)
else:
os.remove(file)
return result(1)
"""T adds a file to the repo
"""Add an item to the repo.
note adding an exisiting file is an error now
should return 0 do nothing
The item can either be a file or directory, if the item is already added
this operation is a nop.
NOTE svn info could do the trick if returns out
it under control if returns err not under control
but if scheduled for removal but not commited
svn info still returns info :/
item File or directory to add to svn.
returns Nothing.
"""
def svn_add(self, file):
def svn_add(self, item):
# FIXME we should test whether the file is already in the repo
# execute (repo not required)
out, err = self.execute("svn add " + file)
if(err != ""):
return result(-1, out, err)
return result(1, out)
"""T removes a file from the repo
"""
def svn_remove(self, file):
# FIXME we should test whether the file is in the repo
logging.debug("svn_add item = '%s'", item)
# execute (repo not required)
out, err = self.execute("svn remove --non-interactive " + file)
out, err = self.execute("svn add " + item)
if(err != ""):
return result(-1, out, err)
return result(1, out)
"""\ gets the status from a file in the repo
or a mask of all stausses
raise error("svn_add failed with message:" + err)
"""Removes an item from the repo.
If an item is not in the repo it's silently ignored and the
operation is a nop.
item File or directory to remove from svn.
returns Nothing.
"""
def svn_remove(self, item):
logging.debug("svn_add item = '%s'", item)
# execute (repo not required)
out, err = self.execute("svn remove --non-interactive " + item)
if(err != ""):
raise error("svn_remove failed with message:" + err)
"""Gets the status of an item."""
# note the mask should be send by reference
def svn_status(self, file, mask = None):
# FIXME not used and not tested
result_mask = 0
# if a file is send we only look after that file
@ -448,23 +408,24 @@ class SVN:
return result(1, out, err)
"""\ Cleans up a checkout
"""Cleans up a checkout.
After a commit where a directory is removed the client doesn't remove
this directory. This routine removes all files with the '?' flag.
returns A result object.
"""
def svn_cleanup(self):
# FIXME do something
pass
logging.debug("svn_cleanup")
"""V Executes the command (private)
returns stdout, stderr
raise error("svn_cleanup not implemented")
"""Executes a command.
command The command to execute
returns stdout, stderr
"""
def execute(self, command):
logging.debug("Execute: " + command)
logging.debug("execute command = '%s'", command)
stdin, stdout, stderr = os.popen3(command)
stdin.close()

View File

@ -33,6 +33,7 @@ import wesnoth.libsvn as libsvn
class tempdir:
def __init__(self):
self.path = tempfile.mkdtemp()
logging.debug("created tempdir '%s'", self.path)
# for some reason we need to need a variable to shutil otherwise the
#__del__() will fail. This is caused by import of campaignserver_client
@ -41,6 +42,7 @@ class tempdir:
def __del__(self):
self.dummy.rmtree(self.path)
logging.debug("removed tempdir '%s'", self.path)
if __name__ == "__main__":
@ -50,7 +52,7 @@ if __name__ == "__main__":
campaigns.wesnoth.org:15003.
addon The name of the addon.
path Directory to unpack the campaign in.
returns Nothing, errors are not detected.
returns Nothing.
"""
def extract(server, addon, path):
@ -61,14 +63,13 @@ if __name__ == "__main__":
data = wml.get_campaign(addon)
wml.unpackdir(data, path)
"""Get a list of addons on the server.
server The url of the addon server eg
campaigns.wesnoth.org:15003.
translatable_only If True only returns translatable addons.
returns A dictonary with the addon as key and the translatable
status as value
status as value.
"""
def list(server, translatable_only):
@ -122,7 +123,7 @@ if __name__ == "__main__":
message = "wescamp.py automatic update"
if(os.path.isdir(svn_dir + "/" + addon) == False):
logging.info("Creating directory in svn '%s'",
logging.info("Creating directory in svn '%s'.",
svn_dir + "/" + addon)
svn = libsvn.SVN(svn_dir)
@ -131,26 +132,19 @@ if __name__ == "__main__":
# other campaigns is lost and no more updates are executed.
os.mkdir(svn_dir + "/" + addon)
res = svn.svn_add(svn_dir + "/" + addon)
res = svn.svn_commit("Adding directory for initial wescamp inclusion")
res = svn.svn_commit("wescamp_client: adding directory for initial "
+ "inclusion of addon '" + addon + "'")
# Update the directory
svn = libsvn.SVN(svn_dir + "/" + addon)
res = svn.svn_update()
if(res == -1):
logging.error("svn update of '%s' failed with message: %s",
svn_dir + "/" + addon, res.err)
sys.exit(1)
res = svn.copy_to_svn(temp_dir, ["translations"])
if(res.err != ""):
print "Error :" + res.err
sys.exit(1)
res = svn.svn_commit("wescamp.py automatic update")
if(res.err != ""):
print "Error :" + res.err
sys.exit(1)
svn.svn_update()
if(svn.copy_to_svn(temp_dir, ["translations"])):
svn.svn_commit("wescamp_client: automatic update of addon '"
+ addon + "'")
logging.info("New version of addon '%s' uploaded.", addon)
else:
logging.info("Addon '%s' hasn't been modified, thus not uploaded.",
addon)
"""Update the translations from wescamp to the server.
@ -176,13 +170,10 @@ if __name__ == "__main__":
svn = libsvn.SVN(wescamp + "/" + addon)
res = svn.svn_update()
if(res.status == 0):
logging.info("svn Up to date, nothing to send to server")
if(svn.svn_update() == False):
logging.info("svn up to date, nothing to send to server")
sys.exit(0)
pass
elif(res.status == -1):
sys.exit(1)
# test whether the svn has a translations dir, if not we can stop
@ -197,7 +188,7 @@ if __name__ == "__main__":
if(os.path.isdir(target + "/" + addon + "/translations") == False):
os.mkdir(target + "/" + addon + "/translations")
res = svn.sync_dir(svn.checkout_path + "/" + addon + "/translations" ,
svn.sync_dir(svn.checkout_path + "/" + addon + "/translations" ,
target + "/" + addon + "/translations", True, None)
# upload to the server
@ -208,30 +199,54 @@ if __name__ == "__main__":
optionparser = optparse.OptionParser("%prog [options]")
optionparser.add_option("-u", "--upload", help = "upload a addon to wescamp") # V
optionparser.add_option("-u", "--upload",
help = "Upload a addon to wescamp. Usage: 'addon' WESCAMP-CHECKOUT "
+ "[SERVER [PORT]] [TEMP-DIR] [VERBOSE]")
optionparser.add_option("-d", "--download", help = "download the translations from wescamp") #
optionparser.add_option("-d", "--download",
help = "Download the translations from wescamp and upload to the addon "
+ "server. Usage 'addon' WESCAMP-CHECKOUT PASSWORD [SERVER [PORT]] "
+ "[TEMP-DIR] [VERBOSE]")
optionparser.add_option("-D", "--download-all", action = "store_true", help = "download all translations from wescamp") # \
optionparser.add_option("-D", "--download-all", action = "store_true",
help = "Download all translations from wescamp and upload them to the "
+ "addon server. Usage WESCAMP-CHECKOUT PASSWORD [SERVER [PORT]] " + " [VERBOSE]")
optionparser.add_option("-l", "--list", action = "store_true", help = "list available campaigns" ) # V
optionparser.add_option("-l", "--list", action = "store_true",
help = "List available addons. Usage [SERVER [PORT] [VERBOSE]")
optionparser.add_option("-L", "--list-translatable", action = "store_true", help = "list campaigns available for translation") # V
optionparser.add_option("-L", "--list-translatable", action = "store_true",
help = "List addons available for translation. "
+ "Usage [SERVER [PORT] [VERBOSE]")
optionparser.add_option("-s", "--server", help = "server to connect to") # V
optionparser.add_option("-s", "--server",
help = "Server to connect to [localhost]")
optionparser.add_option("-p", "--port", help = "port on the server to connect to") # V
optionparser.add_option("-p", "--port",
help = "Port on the server to connect to ['']")
optionparser.add_option("-t", "--temp-dir", help = "directory to store the tempory data, if omitted a tempdir is created and destroyed after usage, if specified the data is left in the tempdir") # V
optionparser.add_option("-t", "--temp-dir", help = "Directory to store the "
+ "tempory data, if omitted a tempdir is created and destroyed after "
+ "usage, if specified the data is left in the tempdir. ['']")
optionparser.add_option("-w", "--wescamp-checkout", help = "the directory containing the wescamp checkout root") # V
optionparser.add_option("-w", "--wescamp-checkout",
help = "The directory containing the wescamp checkout root. ['']")
optionparser.add_option("-v", "--verbose", action = "store_true", help = "show more verbose output") # V
optionparser.add_option("-v", "--verbose", action = "store_true",
help = "Show more verbose output. [FALSE]")
optionparser.add_option("-P", "--password", help = "The master password for the campaigne server") # V
optionparser.add_option("-P", "--password",
help = "The master password for the addon server. ['']")
options, args = optionparser.parse_args()
if(options.verbose):
logging.basicConfig(level=logging.DEBUG,
format='[%(levelname)s] %(message)s')
else:
logging.basicConfig(level=logging.INFO,
format='[%(levelname)s] %(message)s')
server = "localhost"
if(options.server != None):
server = options.server
@ -242,6 +257,10 @@ if __name__ == "__main__":
target = None
tmp = tempdir()
if(options.temp_dir != None):
if(options.download_all != None):
logging.error("TEMP-DIR not allowed for DOWNLOAD-ALL.")
sys.exit(2)
target = options.temp_dir
else:
target = tmp.path
@ -250,18 +269,21 @@ if __name__ == "__main__":
if(options.wescamp_checkout):
wescamp = options.wescamp_checkout
if(options.verbose):
logging.basicConfig(level=logging.DEBUG, format='[%(levelname)s] %(message)s')
else:
logging.basicConfig(level=logging.INFO, format='[%(levelname)s] %(message)s')
password = options.password
# List the addons on the server and optional filter on translatable
# addons.
if(options.list or options.list_translatable):
addons = list(server, options.list_translatable)
try:
addons = list(server, options.list_translatable)
except libsvn.error, e:
print "[ERROR svn] " + str(e)
sys.exit(1)
except IOError, e:
print "Unexpected error occured: " + str(e)
sys.exit(e[0])
for k, v in addons.iteritems():
if(v):
print k + " translatable"
@ -275,37 +297,62 @@ if __name__ == "__main__":
logging.error("No wescamp checkout specified")
sys.exit(2)
upload(server, options.upload, target, wescamp)
try:
upload(server, options.upload, target, wescamp)
except libsvn.error, e:
print "[ERROR svn] " + str(e)
sys.exit(1)
except IOError, e:
print "Unexpected error occured: " + str(e)
sys.exit(e[0])
# Download an addon from wescamp.
elif(options.download != None):
if(wescamp == None):
logging.error("No wescamp checkout specified")
logging.error("No wescamp checkout specified.")
sys.exit(2)
if(password == None):
logging.error("No upload password specified")
logging.error("No upload password specified.")
sys.exit(2)
download(server, options.download, target, wescamp, password)
try:
download(server, options.download, target, wescamp, password)
except libsvn.error, e:
print "[ERROR svn] " + str(e)
sys.exit(1)
except IOError, e:
print "Unexpected error occured: " + str(e)
sys.exit(e[0])
# Download all addons from wescamp.
elif(options.download_all != None):
if(wescamp == None):
logging.error("No wescamp checkout specified")
logging.error("No wescamp checkout specified.")
sys.exit(2)
if(password == None):
logging.error("No upload password specified")
logging.error("No upload password specified.")
sys.exit(2)
error = False
addons = list(server, True)
for k, v in addons.iteritems():
download(server, k, target, wescamp, password)
# FIXME clear the tmp dir
try:
# Create a new temp dir for every download.
tmp = tempdir()
download(server, k, tmp.path, wescamp, password)
except libsvn.error, e:
print "[ERROR svn] in addon '" + k + "'" + str(e)
error = True
except IOError, e:
print "Unexpected error occured: " + str(e)
error = True
if(error):
sys.exit(1)
else:
optionparser.print_help()