Package translate :: Package storage :: Module xpi
[hide private]
[frames] | no frames]

Source Code for Module translate.storage.xpi

  1  #!/usr/bin/env python 
  2  # -*- coding: utf-8 -*- 
  3  #  
  4  # Copyright 2004, 2005 Zuza Software Foundation 
  5  #  
  6  # This file is part of translate. 
  7  # 
  8  # translate is free software; you can redistribute it and/or modify 
  9  # it under the terms of the GNU General Public License as published by 
 10  # the Free Software Foundation; either version 2 of the License, or 
 11  # (at your option) any later version. 
 12  #  
 13  # translate is distributed in the hope that it will be useful, 
 14  # but WITHOUT ANY WARRANTY; without even the implied warranty of 
 15  # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the 
 16  # GNU General Public License for more details. 
 17  # 
 18  # You should have received a copy of the GNU General Public License 
 19  # along with translate; if not, write to the Free Software 
 20  # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA 
 21   
 22  """module for accessing mozilla xpi packages""" 
 23   
 24  from __future__ import generators 
 25  import zipfile 
 26  import os.path 
 27  from translate import __version__ 
 28  import StringIO 
 29  import re 
 30   
 31  # we have some enhancements to zipfile in a file called zipfileext 
 32  # hopefully they will be included in a future version of python 
 33  from translate.misc import zipfileext 
 34  ZipFileBase = zipfileext.ZipFileExt 
 35   
 36  from translate.misc import wStringIO 
 37  # this is a fix to the StringIO in Python 2.3.3 
 38  # submitted as patch 951915 on sourceforge 
39 -class FixedStringIO(wStringIO.StringIO):
40 - def truncate(self, size=None):
41 StringIO.StringIO.truncate(self, size) 42 self.len = len(self.buf)
43 44 NamedStringInput = wStringIO.StringIO 45 NamedStringOutput = wStringIO.StringIO 46
47 -def _commonprefix(itemlist):
48 def cp(a, b): 49 l = min(len(a), len(b)) 50 for n in range(l): 51 if a[n] != b[n]: 52 return a[:n] 53 return a[:l]
54 if itemlist: 55 return reduce(cp, itemlist) 56 else: 57 return '' 58
59 -def rememberchanged(self, method):
60 def changed(*args, **kwargs): 61 self.changed = True 62 method(*args, **kwargs)
63 return changed 64
65 -class CatchPotentialOutput(NamedStringInput, object):
66 """catches output if there has been, before closing"""
67 - def __init__(self, contents, onclose):
68 """Set up the output stream, and remember a method to call on closing""" 69 NamedStringInput.__init__(self, contents) 70 self.onclose = onclose 71 self.changed = False 72 s = super(CatchPotentialOutput, self) 73 self.write = rememberchanged(self, s.write) 74 self.writelines = rememberchanged(self, s.writelines) 75 self.truncate = rememberchanged(self, s.truncate)
76
77 - def close(self):
78 """wrap the underlying close method, to pass the value to onclose before it goes""" 79 if self.changed: 80 value = self.getvalue() 81 self.onclose(value) 82 NamedStringInput.close(self)
83
84 - def flush(self):
85 """zip files call flush, not close, on file-like objects""" 86 value = self.getvalue() 87 self.onclose(value) 88 NamedStringInput.flush(self)
89
90 - def slam(self):
91 """use this method to force the closing of the stream if it isn't closed yet""" 92 if not self.closed: 93 self.close()
94
95 -class ZipFileCatcher(ZipFileBase, object):
96 """a ZipFile that calls any methods its instructed to before closing (useful for catching stream output)"""
97 - def __init__(self, *args, **kwargs):
98 """initialize the ZipFileCatcher""" 99 # storing oldclose as attribute, since if close is called from __del__ it has no access to external variables 100 self.oldclose = super(ZipFileCatcher, self).close 101 super(ZipFileCatcher, self).__init__(*args, **kwargs)
102
103 - def addcatcher(self, pendingsave):
104 """remember to call the given method before closing""" 105 if hasattr(self, "pendingsaves"): 106 if not pendingsave in self.pendingsaves: 107 self.pendingsaves.append(pendingsave) 108 else: 109 self.pendingsaves = [pendingsave]
110
111 - def close(self):
112 """close the stream, remembering to call any addcatcher methods first""" 113 if hasattr(self, "pendingsaves"): 114 for pendingsave in self.pendingsaves: 115 pendingsave() 116 # if close is called from __del__, it somehow can't see ZipFileCatcher, so we've cached oldclose... 117 if ZipFileCatcher is None: 118 self.oldclose() 119 else: 120 super(ZipFileCatcher, self).close()
121
122 - def overwritestr(self, zinfo_or_arcname, bytes):
123 """writes the string into the archive, overwriting the file if it exists...""" 124 if isinstance(zinfo_or_arcname, zipfile.ZipInfo): 125 filename = zinfo_or_arcname.filename 126 else: 127 filename = zinfo_or_arcname 128 if filename in self.NameToInfo: 129 self.delete(filename) 130 self.writestr(zinfo_or_arcname, bytes) 131 self.writeendrec()
132
133 -class XpiFile(ZipFileCatcher):
134 - def __init__(self, *args, **kwargs):
135 """sets up the xpi file""" 136 self.includenonloc = kwargs.get("includenonloc", True) 137 if "includenonloc" in kwargs: 138 del kwargs["includenonloc"] 139 if "compression" not in kwargs: 140 kwargs["compression"] = zipfile.ZIP_DEFLATED 141 self.locale = kwargs.pop("locale", None) 142 self.region = kwargs.pop("region", None) 143 super(XpiFile, self).__init__(*args, **kwargs) 144 self.jarfiles = {} 145 self.findlangreg() 146 self.jarprefixes = self.findjarprefixes() 147 self.reverseprefixes = dict([ 148 (prefix,jarfilename) for jarfilename, prefix in self.jarprefixes.iteritems() if prefix]) 149 self.reverseprefixes["package/"] = None
150
151 - def iterjars(self):
152 """iterate through the jar files in the xpi as ZipFile objects""" 153 for filename in self.namelist(): 154 if filename.lower().endswith('.jar'): 155 if filename not in self.jarfiles: 156 jarstream = self.openinputstream(None, filename) 157 jarfile = ZipFileCatcher(jarstream, mode=self.mode) 158 self.jarfiles[filename] = jarfile 159 else: 160 jarfile = self.jarfiles[filename] 161 yield filename, jarfile
162
163 - def islocfile(self, filename):
164 """returns whether the given file is needed for localization (basically .dtd and .properties)""" 165 base, ext = os.path.splitext(filename) 166 return ext in (os.extsep + "dtd", os.extsep + "properties")
167
168 - def findlangreg(self):
169 """finds the common prefix of all the files stored in the jar files""" 170 dirstructure = {} 171 locale = self.locale 172 region = self.region 173 localematch = re.compile("^[a-z]{2,3}(-[a-zA-Z]{2,3}|)$") 174 regionmatch = re.compile("^[a-zA-Z]{2,3}$") 175 # exclude en-mac, en-win, en-unix for seamonkey 176 osmatch = re.compile("^[a-z]{2,3}-(mac|unix|win)$") 177 for jarfilename, jarfile in self.iterjars(): 178 jarname = "".join(jarfilename.split('/')[-1:]).replace(".jar", "", 1) 179 if localematch.match(jarname) and not osmatch.match(jarname): 180 if locale is None: 181 locale = jarname 182 elif locale != jarname: 183 locale = 0 184 elif regionmatch.match(jarname): 185 if region is None: 186 region = jarname 187 elif region != jarname: 188 region = 0 189 for filename in jarfile.namelist(): 190 if filename.endswith('/'): 191 continue 192 if not self.islocfile(filename) and not self.includenonloc: 193 continue 194 parts = filename.split('/')[:-1] 195 treepoint = dirstructure 196 for partnum in range(len(parts)): 197 part = parts[partnum] 198 if part in treepoint: 199 treepoint = treepoint[part] 200 else: 201 treepoint[part] = {} 202 treepoint = treepoint[part] 203 localeentries = {} 204 if 'locale' in dirstructure: 205 for dirname in dirstructure['locale']: 206 localeentries[dirname] = 1 207 if localematch.match(dirname) and not osmatch.match(dirname): 208 if locale is None: 209 locale = dirname 210 elif locale != dirname: 211 print "locale dir mismatch - ", dirname, "but locale is", locale, "setting to 0" 212 locale = 0 213 elif regionmatch.match(dirname): 214 if region is None: 215 region = dirname 216 elif region != dirname: 217 region = 0 218 if locale and locale in localeentries: 219 del localeentries[locale] 220 if region and region in localeentries: 221 del localeentries[region] 222 if locale and not region: 223 if "-" in locale: 224 region = locale.split("-", 1)[1] 225 else: 226 region = "" 227 self.setlangreg(locale, region)
228
229 - def setlangreg(self, locale, region):
230 """set the locale and region of this xpi""" 231 if locale == 0 or locale is None: 232 raise ValueError("unable to determine locale") 233 self.locale = locale 234 self.region = region 235 self.dirmap = {} 236 if self.locale is not None: 237 self.dirmap[('locale', self.locale)] = ('lang-reg',) 238 if self.region: 239 self.dirmap[('locale', self.region)] = ('reg',)
240
241 - def findjarprefixes(self):
242 """checks the uniqueness of the jar files contents""" 243 uniquenames = {} 244 jarprefixes = {} 245 for jarfilename, jarfile in self.iterjars(): 246 jarprefixes[jarfilename] = "" 247 for filename in jarfile.namelist(): 248 if filename.endswith('/'): 249 continue 250 if filename in uniquenames: 251 jarprefixes[jarfilename] = True 252 jarprefixes[uniquenames[filename]] = True 253 else: 254 uniquenames[filename] = jarfilename 255 for jarfilename, hasconflicts in jarprefixes.items(): 256 if hasconflicts: 257 shortjarfilename = os.path.split(jarfilename)[1] 258 shortjarfilename = os.path.splitext(shortjarfilename)[0] 259 jarprefixes[jarfilename] = shortjarfilename+'/' 260 # this is a clever trick that will e.g. remove zu- from zu-win, zu-mac, zu-unix 261 commonjarprefix = _commonprefix([prefix for prefix in jarprefixes.itervalues() if prefix]) 262 if commonjarprefix: 263 for jarfilename, prefix in jarprefixes.items(): 264 if prefix: 265 jarprefixes[jarfilename] = prefix.replace(commonjarprefix, '', 1) 266 return jarprefixes
267
268 - def ziptoospath(self, zippath):
269 """converts a zipfile filepath to an os-style filepath""" 270 return os.path.join(*zippath.split('/'))
271
272 - def ostozippath(self, ospath):
273 """converts an os-style filepath to a zipfile filepath""" 274 return '/'.join(ospath.split(os.sep))
275
276 - def mapfilename(self, filename):
277 """uses a map to simplify the directory structure""" 278 parts = tuple(filename.split('/')) 279 possiblematch = None 280 for prefix, mapto in self.dirmap.iteritems(): 281 if parts[:len(prefix)] == prefix: 282 if possiblematch is None or len(possiblematch[0]) < len(prefix): 283 possiblematch = prefix, mapto 284 if possiblematch is not None: 285 prefix, mapto = possiblematch 286 mapped = mapto + parts[len(prefix):] 287 return '/'.join(mapped) 288 return filename
289
290 - def mapxpifilename(self, filename):
291 """uses a map to rename files that occur straight in the xpi""" 292 if filename.startswith('bin/chrome/') and filename.endswith(".manifest"): 293 return 'bin/chrome/lang-reg.manifest' 294 return filename
295
296 - def reversemapfile(self, filename):
297 """unmaps the filename...""" 298 possiblematch = None 299 parts = tuple(filename.split('/')) 300 for prefix, mapto in self.dirmap.iteritems(): 301 if parts[:len(mapto)] == mapto: 302 if possiblematch is None or len(possiblematch[0]) < len(mapto): 303 possiblematch = (mapto, prefix) 304 if possiblematch is None: 305 return filename 306 mapto, prefix = possiblematch 307 reversemapped = prefix + parts[len(mapto):] 308 return '/'.join(reversemapped)
309
310 - def reversemapxpifilename(self, filename):
311 """uses a map to rename files that occur straight in the xpi""" 312 if filename == 'bin/chrome/lang-reg.manifest': 313 if self.locale: 314 return '/'.join(('bin', 'chrome', self.locale + '.manifest')) 315 else: 316 for otherfilename in self.namelist(): 317 if otherfilename.startswith("bin/chrome/") and otherfilename.endswith(".manifest"): 318 return otherfilename 319 return filename
320
321 - def jartoospath(self, jarfilename, filename):
322 """converts a filename from within a jarfile to an os-style filepath""" 323 if jarfilename: 324 jarprefix = self.jarprefixes[jarfilename] 325 return self.ziptoospath(jarprefix+self.mapfilename(filename)) 326 else: 327 return self.ziptoospath(os.path.join("package", self.mapxpifilename(filename)))
328
329 - def ostojarpath(self, ospath):
330 """converts an extracted os-style filepath to a jarfilename and filename""" 331 zipparts = ospath.split(os.sep) 332 prefix = zipparts[0] + '/' 333 if prefix in self.reverseprefixes: 334 jarfilename = self.reverseprefixes[prefix] 335 filename = self.reversemapfile('/'.join(zipparts[1:])) 336 if jarfilename is None: 337 filename = self.reversemapxpifilename(filename) 338 return jarfilename, filename 339 else: 340 filename = self.ostozippath(ospath) 341 if filename in self.namelist(): 342 return None, filename 343 filename = self.reversemapfile('/'.join(zipparts)) 344 possiblejarfilenames = [jarfilename for jarfilename, prefix in self.jarprefixes.iteritems() if not prefix] 345 for jarfilename in possiblejarfilenames: 346 jarfile = self.jarfiles[jarfilename] 347 if filename in jarfile.namelist(): 348 return jarfilename, filename 349 raise IndexError("ospath not found in xpi file, could not guess location: %r" % ospath)
350
351 - def jarfileexists(self, jarfilename, filename):
352 """checks whether the given file exists inside the xpi""" 353 if jarfilename is None: 354 return filename in self.namelist() 355 else: 356 jarfile = self.jarfiles[jarfilename] 357 return filename in jarfile.namelist()
358
359 - def ospathexists(self, ospath):
360 """checks whether the given file exists inside the xpi""" 361 jarfilename, filename = self.ostojarpath(ospath) 362 if jarfilename is None: 363 return filename in self.namelist() 364 else: 365 jarfile = self.jarfiles[jarfilename] 366 return filename in jarfile.namelist()
367
368 - def openinputstream(self, jarfilename, filename):
369 """opens a file (possibly inside a jarfile as a StringIO""" 370 if jarfilename is None: 371 contents = self.read(filename) 372 def onclose(contents): 373 if contents != self.read(filename): 374 self.overwritestr(filename, contents)
375 inputstream = CatchPotentialOutput(contents, onclose) 376 self.addcatcher(inputstream.slam) 377 else: 378 jarfile = self.jarfiles[jarfilename] 379 contents = jarfile.read(filename) 380 inputstream = NamedStringInput(contents) 381 inputstream.name = self.jartoospath(jarfilename, filename) 382 if hasattr(self.fp, 'name'): 383 inputstream.name = "%s:%s" % (self.fp.name, inputstream.name) 384 return inputstream
385
386 - def openoutputstream(self, jarfilename, filename):
387 """opens a file for writing (possibly inside a jarfile as a StringIO""" 388 if jarfilename is None: 389 def onclose(contents): 390 self.overwritestr(filename, contents)
391 else: 392 if jarfilename in self.jarfiles: 393 jarfile = self.jarfiles[jarfilename] 394 else: 395 jarstream = self.openoutputstream(None, jarfilename) 396 jarfile = ZipFileCatcher(jarstream, "w") 397 self.jarfiles[jarfilename] = jarfile 398 self.addcatcher(jarstream.slam) 399 def onclose(contents): 400 jarfile.overwritestr(filename, contents) 401 outputstream = wStringIO.CatchStringOutput(onclose) 402 outputstream.name = "%s %s" % (jarfilename, filename) 403 if jarfilename is None: 404 self.addcatcher(outputstream.slam) 405 else: 406 jarfile.addcatcher(outputstream.slam) 407 return outputstream 408
409 - def close(self):
410 """Close the file, and for mode "w" and "a" write the ending records.""" 411 for jarfile in self.jarfiles.itervalues(): 412 jarfile.close() 413 super(XpiFile, self).close()
414
415 - def testzip(self):
416 """test the xpi zipfile and all enclosed jar files...""" 417 for jarfile in self.jarfiles.itervalues(): 418 jarfile.testzip() 419 super(XpiFile, self).testzip()
420
421 - def restructurejar(self, origjarfilename, newjarfilename, otherxpi, newlang, newregion):
422 """Create a new .jar file with the same contents as the given name, but rename directories, write to outputstream""" 423 jarfile = self.jarfiles[origjarfilename] 424 origlang = self.locale[:self.locale.find("-")] 425 if newregion: 426 newlocale = "%s-%s" % (newlang, newregion) 427 else: 428 newlocale = newlang 429 for filename in jarfile.namelist(): 430 filenameparts = filename.split("/") 431 for i in range(len(filenameparts)): 432 part = filenameparts[i] 433 if part == origlang: 434 filenameparts[i] = newlang 435 elif part == self.locale: 436 filenameparts[i] = newlocale 437 elif part == self.region: 438 filenameparts[i] = newregion 439 newfilename = '/'.join(filenameparts) 440 fileoutputstream = otherxpi.openoutputstream(newjarfilename, newfilename) 441 fileinputstream = self.openinputstream(origjarfilename, filename) 442 fileoutputstream.write(fileinputstream.read()) 443 fileinputstream.close() 444 fileoutputstream.close()
445
446 - def clone(self, newfilename, newmode=None, newlang=None, newregion=None):
447 """Create a new .xpi file with the same contents as this one...""" 448 other = XpiFile(newfilename, "w", locale=newlang, region=newregion) 449 origlang = self.locale[:self.locale.find("-")] 450 # TODO: check if this language replacement code is still neccessary 451 if newlang is None: 452 newlang = origlang 453 if newregion is None: 454 newregion = self.region 455 if newregion: 456 newlocale = "%s-%s" % (newlang, newregion) 457 else: 458 newlocale = newlang 459 for filename in self.namelist(): 460 filenameparts = filename.split('/') 461 basename = filenameparts[-1] 462 if basename.startswith(self.locale): 463 newbasename = basename.replace(self.locale, newlocale) 464 elif basename.startswith(origlang): 465 newbasename = basename.replace(origlang, newlang) 466 elif basename.startswith(self.region): 467 newbasename = basename.replace(self.region, newregion) 468 else: 469 newbasename = basename 470 if newbasename != basename: 471 filenameparts[-1] = newbasename 472 renamefilename = "/".join(filenameparts) 473 print "cloning", filename, "and renaming to", renamefilename 474 else: 475 print "cloning", filename 476 renamefilename = filename 477 if filename.lower().endswith(".jar"): 478 self.restructurejar(filename, renamefilename, other, newlang, newregion) 479 else: 480 inputstream = self.openinputstream(None, filename) 481 outputstream = other.openoutputstream(None, renamefilename) 482 outputstream.write(inputstream.read()) 483 inputstream.close() 484 outputstream.close() 485 other.close() 486 if newmode is None: 487 newmode = self.mode 488 if newmode == "w": 489 newmode = "a" 490 other = XpiFile(newfilename, newmode) 491 other.setlangreg(newlocale, newregion) 492 return other
493
494 - def iterextractnames(self, includenonjars=False, includedirs=False):
495 """iterates through all the localization files with the common prefix stripped and a jarfile name added if neccessary""" 496 if includenonjars: 497 for filename in self.namelist(): 498 if filename.endswith('/') and not includedirs: 499 continue 500 if not self.islocfile(filename) and not self.includenonloc: 501 continue 502 if not filename.lower().endswith(".jar"): 503 yield self.jartoospath(None, filename) 504 for jarfilename, jarfile in self.iterjars(): 505 for filename in jarfile.namelist(): 506 if filename.endswith('/'): 507 if not includedirs: 508 continue 509 if not self.islocfile(filename) and not self.includenonloc: 510 continue 511 yield self.jartoospath(jarfilename, filename)
512 513 # the following methods are required by translate.convert.ArchiveConvertOptionParser #
514 - def __iter__(self):
515 """iterates through all the files. this is the method use by the converters""" 516 for inputpath in self.iterextractnames(includenonjars=True): 517 yield inputpath
518
519 - def __contains__(self, fullpath):
520 """returns whether the given pathname exists in the archive""" 521 try: 522 jarfilename, filename = self.ostojarpath(fullpath) 523 except IndexError: 524 return False 525 return self.jarfileexists(jarfilename, filename)
526
527 - def openinputfile(self, fullpath):
528 """opens an input file given the full pathname""" 529 jarfilename, filename = self.ostojarpath(fullpath) 530 return self.openinputstream(jarfilename, filename)
531
532 - def openoutputfile(self, fullpath):
533 """opens an output file given the full pathname""" 534 try: 535 jarfilename, filename = self.ostojarpath(fullpath) 536 except IndexError: 537 return None 538 return self.openoutputstream(jarfilename, filename)
539 540 if __name__ == '__main__': 541 import optparse 542 optparser = optparse.OptionParser(version="%prog "+__version__.sver) 543 optparser.usage = "%prog [-l|-x] [options] file.xpi" 544 optparser.add_option("-l", "--list", help="list files", \ 545 action="store_true", dest="listfiles", default=False) 546 optparser.add_option("-p", "--prefix", help="show common prefix", \ 547 action="store_true", dest="showprefix", default=False) 548 optparser.add_option("-x", "--extract", help="extract files", \ 549 action="store_true", dest="extractfiles", default=False) 550 optparser.add_option("-d", "--extractdir", help="extract into EXTRACTDIR", \ 551 default=".", metavar="EXTRACTDIR") 552 (options, args) = optparser.parse_args() 553 if len(args) < 1: 554 optparser.error("need at least one argument") 555 xpifile = XpiFile(args[0]) 556 if options.showprefix: 557 for prefix, mapto in xpifile.dirmap.iteritems(): 558 print "/".join(prefix), "->", "/".join(mapto) 559 if options.listfiles: 560 for name in xpifile.iterextractnames(includenonjars=True, includedirs=True): 561 print name #, xpifile.ostojarpath(name) 562 if options.extractfiles: 563 if options.extractdir and not os.path.isdir(options.extractdir): 564 os.mkdir(options.extractdir) 565 for name in xpifile.iterextractnames(includenonjars=True, includedirs=False): 566 abspath = os.path.join(options.extractdir, name) 567 # check neccessary directories exist - this way we don't create empty directories 568 currentpath = options.extractdir 569 subparts = os.path.dirname(name).split(os.sep) 570 for part in subparts: 571 currentpath = os.path.join(currentpath, part) 572 if not os.path.isdir(currentpath): 573 os.mkdir(currentpath) 574 outputstream = open(abspath, 'w') 575 jarfilename, filename = xpifile.ostojarpath(name) 576 inputstream = xpifile.openinputstream(jarfilename, filename) 577 outputstream.write(inputstream.read()) 578 outputstream.close() 579