Package ewa :: Module clisupport
[hide private]
[frames] | no frames]

Source Code for Module ewa.clisupport

  1  import errno 
  2  import logging 
  3  from optparse import OptionParser 
  4  import os 
  5  from os import path 
  6  import re 
  7  import sys 
  8  import time 
  9   
 10  try: 
 11      import grp 
 12      import pwd 
 13      have_unix = 1 
 14  except ImportError: 
 15      have_unix = 0 
 16   
 17  try: 
 18      from functools import partial 
 19  except ImportError: 
 20   
21 - def partial(func, *args, **kw):
22 23 def inner(*_args, **_kw): 24 d = kw.copy() 25 d.update(_kw) 26 return func(*(args + _args), **d)
27 return inner 28 29 try: 30 from flup.server.scgi_fork import WSGIServer as SCGIServer 31 from flup.server.fcgi_fork import WSGIServer as FCGIServer 32 from flup.server.scgi import WSGIServer as SCGIThreadServer 33 from flup.server.fcgi import WSGIServer as FCGIThreadServer 34 haveflup = True 35 except ImportError: 36 haveflup = False 37 try: 38 from paste import httpserver 39 havepaste = True 40 except ImportError: 41 havepaste = False 42 43 import ewa.mp3 44 import ewa.audio 45 from ewa.config import Config, initConfig 46 from ewa.lighttpd_hack_middleware import LighttpdHackMiddleware 47 from ewa.logutil import (debug, error, exception, info, logger, 48 initLogging, warn) 49 from ewa.wsgiapp import EwaApp 50 from ewa.rules import FileRule 51 from ewa import __version__ 52 53 VERSION_TEXT = """\ 54 %%s %s 55 56 Copyright (C) 2007, 2010 WNYC New York Public Radio. 57 This is free software; see the source for copying conditions. There is NO 58 warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 59 60 Written by Jacob Smullyan and others, not necessarily in that order. 61 """ % __version__ 62 63 64 SERVE_DESCRIPTION = """\ 65 Starts a WSGI application that produces combined MP3 files 66 according to the specified rules. 67 """ 68 69 RULE_DESCRIPTION = """\ 70 Produces a combined MP3 file according to the specified rules. 71 """ 72 73 SPLICE_DESCRIPTION = """\ 74 Splices MP3 files together. 75 """ 76 77
78 -def get_serve_parser():
79 protocols = [] 80 if haveflup: 81 protocols.extend(['fcgi', 'scgi']) 82 if havepaste: 83 protocols.append('http') 84 if not protocols: 85 print >> sys.stderr, ("no protocols available; " 86 "please install paste and/or flup") 87 sys.exit(1) 88 usage = "usage: %prog [options]" 89 parser = OptionParser(usage, description=SERVE_DESCRIPTION) 90 parser.add_option('-c', 91 '--config', 92 dest='configfile', 93 default=None, 94 help="path to ewa config file") 95 parser.add_option('-D', 96 '--nodaemonize', 97 action='store_true', 98 default=False, 99 dest='nodaemonize', 100 help="don't daemonize, regardless of config settings") 101 parser.add_option('--version', 102 action="store_true", 103 dest="version", 104 help="show version and exit") 105 parser.add_option('--lighttpd-hack', 106 action='store_true', 107 default=False, 108 dest='lighttpd_hack', 109 help=("hack for some versions of lighttpd " 110 "to force SCRIPT_NAME to ''")) 111 return parser
112 113
114 -def get_splice_parser():
115 usage = "%prog [options] files" 116 parser = OptionParser(usage, description=SPLICE_DESCRIPTION) 117 parser.add_option('-o', 118 '--output', 119 dest='output', 120 help="output file (default: stdout)", 121 default='-', 122 metavar='OUT') 123 parser.add_option('-t', 124 '--tagfile', 125 dest="tagfile", 126 help="tag file", 127 default=None, 128 metavar="TAGFILE") 129 parser.add_option('-d', 130 '--debug', 131 action='store_true', 132 default=False, 133 help="print debugging information", 134 dest='debugmode') 135 parser.add_option('-s', 136 '--sanitycheck', 137 action='store_true', 138 default=False, 139 help="sanity check the input mp3 files", 140 dest='sanitycheck') 141 parser.add_option('-e', 142 '--engine', 143 default='default', 144 dest='engine', 145 metavar='ENGINE', 146 choices=('default', 'mp3cat', 'sox'), 147 help=("which splicing engine to use (default ewa " 148 "splicer, mp3cat, or sox)")) 149 parser.add_option('--version', 150 action="store_true", 151 dest="version", 152 help="show version and exit") 153 return parser
154 155
156 -def get_ap_parser():
157 usage = "%prog [options] [files]" 158 parser = OptionParser(usage, description=RULE_DESCRIPTION) 159 160 parser.add_option('-c', 161 '--config', 162 dest='configfile', 163 default=None, 164 help="path to ewa config file") 165 parser.add_option('-r', 166 '--recursive', 167 dest='recursive', 168 help="recurse through directories", 169 action='store_true', 170 default=False) 171 parser.add_option('--rulefile', 172 dest='rulefile', 173 help="specify a rulefile", 174 metavar='RULEFILE') 175 parser.add_option('-d', 176 '--debug', 177 action='store_true', 178 default=False, 179 help="print debugging information", 180 dest='debugmode') 181 parser.add_option('-n', 182 '--dry-run', 183 action='store_true', 184 default=False, 185 dest='dryrun', 186 help="don't do anything, just print what would be done") 187 parser.add_option('-e', 188 '--engine', 189 default=None, 190 dest='engine', 191 metavar='ENGINE', 192 choices=('default', 'mp3cat', 'sox'), 193 help=("which splicing engine to use (default ewa " 194 "splicer, mp3cat, or sox)")) 195 parser.add_option('-a', 196 '--absolute', 197 default=False, 198 dest='absolute', 199 action='store_true', 200 help=("interpret file paths relative to the filesystem " 201 "rather than the basedir (default: no)")) 202 203 parser.add_option('-t', 204 '--configtest', 205 default=False, 206 dest='configtest', 207 action='store_true', 208 help="just test the config file for syntax errors") 209 210 parser.add_option('-x', 211 '--max-age', 212 default=0, 213 dest='max_age', 214 type=int, 215 help=('max age before generated files expire and ' 216 'are regenerated. Default is 0, which means no ' 217 'expiration. -1 will force regeneration.')) 218 219 parser.add_option('-D', 220 '--delete', 221 default=False, 222 action='store_true', 223 help=('delete files in combined directory that are ' 224 'not in the the main directory')) 225 226 parser.add_option('-V', '--no-vbr', 227 default=True, 228 action='store_false', 229 dest='tolerate_vbr', 230 help="don't put vbr files in the combined directory") 231 232 parser.add_option('--version', 233 action="store_true", 234 dest="version", 235 help="show version and exit") 236 return parser
237 238
239 -def resolve_engine(enginename):
240 if enginename == 'default': 241 return ewa.mp3._default_splicer 242 elif enginename == 'mp3cat': 243 return ewa.mp3._mp3cat_splicer 244 elif enginename == 'sox': 245 return ewa.mp3._sox_splicer
246 247
248 -def do_splice(args):
249 parser = get_splice_parser() 250 opts, args = parser.parse_args(args) 251 if opts.version: 252 print VERSION_TEXT % path.basename(sys.argv[0]) 253 sys.exit(0) 254 if opts.debugmode: 255 Config.loglevel = logging.DEBUG 256 initLogging(level=Config.loglevel, 257 filename=Config.logfile) 258 259 engine = resolve_engine(opts.engine) 260 if opts.sanitycheck: 261 try: 262 ewa.mp3.mp3_sanity_check(args) 263 except Exception, e: 264 print >> sys.stderr, 'sanity check failed: %s' % str(e) 265 sys.exit(1) 266 267 use_stdout = opts.output == '-' 268 if use_stdout: 269 fp = sys.stdout 270 else: 271 fp = open(options.output, 'wb') 272 for chunk in ewa.mp3.splice(args, opts.tagfile, splicer=engine): 273 fp.write(chunk) 274 if not use_stdout: 275 fp.close()
276 277
278 -def do_audioprovider(args):
279 parser = gs = get_ap_parser() 280 opts, args = parser.parse_args(args) 281 if opts.version: 282 print VERSION_TEXT % path.basename(sys.argv[0]) 283 sys.exit(0) 284 configfile = _find_config(opts.configfile) 285 if not configfile: 286 parser.error('no config file specified or found in the usual places') 287 if not path.exists(configfile): 288 parser.error("config file does not exist: %s" % configfile) 289 initConfig(configfile) 290 291 # override config 292 if opts.debugmode: 293 Config.loglevel = 'debug' 294 if opts.rulefile: 295 Config.rulefile = opts.rulefile 296 if opts.engine: 297 Config.engine = engine 298 engine = resolve_engine(Config.engine) 299 initLogging(level=Config.loglevel) 300 rule = FileRule(Config.rulefile) 301 302 if opts.configtest: 303 print "Config OK" 304 sys.exit(0) 305 306 provider = ewa.audio.FSAudioProvider(Config.basedir, 307 opts.tolerate_vbr, 308 Config.targetdir) 309 mainpath = provider.get_main_path("") 310 if not mainpath.endswith('/'): 311 # currently this won't happen, 312 # but better safe than sorry 313 mainpath += '/' 314 if opts.absolute: 315 abs = [path.abspath(f) for f in args] 316 stripped_main = mainpath[:-1] 317 if not path.commonprefix([stripped_main] + abs) == stripped_main: 318 debug("absolute paths of files: %s", abs) 319 debug("mainpath: %s", mainpath) 320 parser.error("files outside managed directory") 321 idx = len(mainpath) 322 args = [x[idx:] for x in abs] 323 else: 324 args = [re.sub('/*(.*)', r'\1', x) for x in args] 325 326 if opts.delete: 327 if not opts.recursive: 328 parser.error("delete only works in recursive mode") 329 delete_finder = DeleteFinder(args, mainpath) 330 for file, isdir in delete_finder: 331 info('deleting %s', file) 332 if not opts.dryrun: 333 try: 334 if isdir: 335 os.rmdir(file) 336 else: 337 os.unlink(file) 338 except Exception, e: 339 error("couldn't unlink %s: %s", file, e) 340 341 if opts.recursive: 342 # replace args with an iterator that finds mp3 files 343 if opts.max_age == -1: 344 args = RecursiveMp3FileIterator(args, mainpath) 345 else: 346 args = RecursiveChangedMp3FileIterator(args, 347 mainpath, 348 opts.max_age) 349 if not args: 350 parser.error("no files specified") 351 352 if opts.dryrun: 353 for file in args: 354 debug("mp3 file: %s", file) 355 print "playlist for %s:" % file 356 for part in provider.get_playlist(file, rule, False): 357 print "\t%s" % part 358 sys.exit(0) 359 else: 360 _change_user_group() 361 for file in args: 362 try: 363 target = provider.create_combined(file, rule, splicer=engine) 364 except: 365 exception("error creating combined file for %s", file) 366 else: 367 debug('created %s', target) 368 sys.exit(0)
369 370
371 -class DeleteFinder(object):
372 - def __init__(self, files, basedir):
373 self.files = files 374 self.basedir = basedir 375 self.targetdir = Config.targetdir 376 if self.targetdir is None: 377 self.targetdir = path.join(Config.basedir, 'combined')
378
379 - def _yielder(self, root, apath, isdir):
380 targetpath = os.path.join(root, apath) 381 mainpath = path.join(self.basedir, 382 path.relpath(targetpath, self.targetdir)) 383 384 debug('mainpath for %s is %s', apath, mainpath) 385 if not os.path.exists(mainpath): 386 yield targetpath, isdir
387
388 - def __iter__(self):
389 for f in (path.join(self.targetdir, x) for x in self.files): 390 if path.isdir(f): 391 debug('found directory: %s', f) 392 for root, dirs, files in os.walk(f, topdown=False): 393 for thing in files: 394 if thing.endswith('~'): 395 # looks like an ewa temp file. Let it be. 396 continue 397 for res in self._yielder(root, thing, 0): 398 yield res 399 for thing in dirs: 400 for res in self._yielder(root, thing, 1): 401 yield res
402 403
404 -class RecursiveMp3FileIterator(object):
405 - def __init__(self, files, basedir):
406 self.files = files 407 self.basedir = basedir
408
409 - def __iter__(self):
410 length = len(self.basedir) 411 for f in (path.join(self.basedir, x) for x in self.files): 412 if path.isdir(f): 413 debug('found directory: %s', f) 414 for mp3 in self._walk(f): 415 debug('raw value: %s', mp3) 416 yield mp3[length:] 417 else: 418 debug('found non-directory: %s', f) 419 yield f[length:]
420
421 - def _walk(self, f):
422 for root, dirs, files in os.walk(f): 423 for f in files: 424 if f.endswith('.mp3') or f.endswith('MP3'): 425 yield path.join(root, f)
426 427
428 -class RecursiveChangedMp3FileIterator(RecursiveMp3FileIterator):
429
430 - def __init__(self, files, basedir, max_age=0):
431 super(RecursiveChangedMp3FileIterator, self).__init__(files, basedir) 432 self.targetdir = Config.targetdir 433 if self.targetdir is None: 434 self.targetdir = path.join(Config.basedir, 'combined') 435 self.max_age = max_age
436
437 - def _walk(self, f):
438 for root, dirs, files in os.walk(f): 439 for f in files: 440 if f.endswith('.mp3') or f.endswith('MP3'): 441 fullpath = path.join(root, f) 442 debug('fullpath is %s', fullpath) 443 debug('basedir: %s; targetdir: %s', 444 self.basedir, self.targetdir) 445 targetpath = os.path.join(self.targetdir, 446 path.relpath(fullpath, 447 self.basedir)) 448 try: 449 stats = os.stat(targetpath) 450 except OSError, ozzie: 451 if ozzie.errno == errno.ENOENT: 452 # target doesn't exist 453 debug("target doesn't exist: %s", targetpath) 454 yield fullpath 455 else: 456 exception("error statting targetfile") 457 # should we return this, or continue? ???? 458 else: 459 # we know the file exists 460 target_mtime = stats.st_mtime 461 try: 462 original_mtime = path.getmtime(fullpath) 463 except OSError: 464 exception('peculiar error getting mtime of %s', 465 fullpath) 466 continue 467 468 if original_mtime > target_mtime: 469 # if original file has changed, we need to 470 # regenerate regardless 471 yield fullpath 472 elif self.max_age <= 0: 473 # max_age of zero or less means we never 474 # regenerate the files 475 continue 476 else: 477 # regenerate if the file is older than 478 # max_age minutes 479 mtime = stats.st_mtime 480 age = time.time() - mtime 481 if age > self.max_age * 60: 482 yield fullpath
483 484
485 -def _find_config(givenpath):
486 for pth in (p for p in ( 487 givenpath, 488 path.expanduser('~/.ewa.conf'), 489 path.expanduser('~/.ewa/ewa.conf'), 490 '/etc/ewa.conf', 491 '/etc/ewa/ewa.conf', 492 ) if p): 493 if path.exists(pth): 494 return pth
495 496
497 -def do_serve(args):
498 parser = get_serve_parser() 499 opts, args = parser.parse_args(args) 500 if opts.version: 501 print VERSION_TEXT % path.basename(sys.argv[0]) 502 sys.exit(0) 503 configfile = _find_config(opts.configfile) 504 if not configfile: 505 parser.error('no config file specified or found in the usual places') 506 if not path.exists(configfile): 507 parser.error("config file does not exist: %s" % configfile) 508 initConfig(configfile) 509 if opts.nodaemonize: 510 Config.daemonize = False 511 512 if not Config.rulefile: 513 parser.error("a rulefile needs to be specified") 514 rule = FileRule(Config.rulefile) 515 if Config.logfile: 516 initLogging(level=Config.loglevel, 517 filename=Config.logfile, 518 rotate=Config.logrotate) 519 # check for incompatible options 520 if Config.protocol == 'http' and Config.unixsocket: 521 parser.error('unix sockets not supported for http server') 522 if Config.umask and not Config.unixsocket: 523 parser.error('umask only applicable for unix sockets') 524 if Config.unixsocket and (Config.interface or Config.port): 525 parser.error('incompatible mixture of unix socket and tcp options') 526 engine = resolve_engine(Config.engine) 527 528 app = EwaApp(rule=rule, 529 basedir=Config.basedir, 530 targetdir=Config.targetdir, 531 stream=Config.stream, 532 refresh_rate=Config.refresh_rate, 533 use_xsendfile=Config.use_xsendfile, 534 sendfile_header=Config.sendfile_header, 535 content_disposition=Config.content_disposition, 536 splicer=engine) 537 538 if opts.lighttpd_hack: 539 app = LighttpdHackMiddleware(app) 540 541 if Config.protocol == 'http': 542 runner = partial(httpserver.serve, 543 app, 544 Config.interface, 545 Config.port) 546 else: 547 if Config.interface: 548 bindAddress = (Config.interface, Config.port) 549 debug('bindAddress: %s', bindAddress) 550 elif Config.unixsocket: 551 bindAddress = Config.unixsocket 552 if Config.protocol == 'scgi': 553 if Config.use_threads: 554 serverclass = SCGIThreadServer 555 else: 556 serverclass = SCGIServer 557 elif Config.protocol == 'fcgi': 558 if Config.use_threads: 559 serverclass = FCGIThreadServer 560 else: 561 serverclass = FCGIServer 562 if Config.protocol in ('fcgi', 'scgi'): 563 if Config.use_threads: 564 kw = dict(maxSpare=Config.max_spare, 565 minSpare=Config.min_spare, 566 maxThreads=Config.max_threads) 567 else: 568 kw = dict(maxSpare=Config.max_spare, 569 minSpare=Config.min_spare, 570 maxChildren=Config.max_children, 571 maxRequests=Config.max_requests) 572 # clean out Nones 573 kw = dict(i for i in kw.items() if not i[1] is None) 574 else: 575 kw = {} 576 577 runner = serverclass(app, 578 bindAddress=bindAddress, 579 umask=Config.umask, 580 **kw).run 581 582 try: 583 run_server(runner, 584 Config.pidfile, 585 Config.daemonize) 586 except SystemExit: 587 raise 588 except: 589 exception('exception caught') 590 sys.exit(1) 591 else: 592 sys.exit(0)
593 594
595 -def _change_user_group():
596 if not have_unix: 597 # maybe do something else on Windows someday... 598 return 599 600 user = Config.user 601 group = Config.group 602 if user or group: 603 # in case we are seteuid something else, which would 604 # cause setuid or getuid to fail, undo any existing 605 # seteuid. (The only reason to do this is for the case 606 # os.getuid()==0, AFAIK). 607 try: 608 seteuid = os.seteuid 609 except AttributeError: 610 # the OS may not support seteuid, in which 611 # case everything is hotsy-totsy. 612 pass 613 else: 614 seteuid(os.getuid()) 615 if group: 616 gid = grp.getgrnam(group)[2] 617 os.setgid(gid) 618 if user: 619 uid = pwd.getpwnam(user)[2] 620 os.setuid(uid)
621 622
623 -def run_server(func, pidfile=None, daemonize=True):
624 debug("daemonize: %s, pidfile: %s", daemonize, pidfile) 625 if daemonize: 626 try: 627 os.fork 628 except NameError: 629 info("not daemonizing, as the fork() call is not available") 630 daemonize = False 631 if daemonize: 632 if os.fork(): 633 os._exit(0) 634 if os.fork(): 635 os._exit(0) 636 os.setsid() 637 os.chdir('/') 638 os.umask(0) 639 os.open(os.devnull, os.O_RDWR) 640 os.dup2(0, 1) 641 os.dup2(0, 2) 642 if pidfile: 643 pidfp = open(pidfile, 'w') 644 pidfp.write('%s' % os.getpid()) 645 pidfp.close() 646 647 try: 648 _change_user_group() 649 try: 650 func() 651 except KeyboardInterrupt: 652 pass 653 finally: 654 if daemonize and pidfile: 655 try: 656 os.unlink(pidfile) 657 except OSError, e: 658 # if it doesn't exist, that's OK 659 if e.errno != errno.ENOENT: 660 warn("problem unlinking pid file %s", pidfile)
661