Package ewa :: Module clisupport
[hide private]
[frames] | no frames]

Source Code for Module ewa.clisupport

  1  import errno 
  2  import logging 
  3  from optparse import OptionParser 
  4  import os 
  5  from os import path 
  6  import re 
  7  import sys 
  8  import time 
  9   
 10  try: 
 11      import grp 
 12      import pwd 
 13      have_unix = 1 
 14  except ImportError: 
 15      have_unix = 0 
 16   
 17  try: 
 18      from functools import partial 
 19  except ImportError: 
 20   
21 - def partial(func, *args, **kw):
22 23 def inner(*_args, **_kw): 24 d = kw.copy() 25 d.update(_kw) 26 return func(*(args + _args), **d)
27 return inner 28 29 try: 30 from flup.server.scgi_fork import WSGIServer as SCGIServer 31 from flup.server.fcgi_fork import WSGIServer as FCGIServer 32 from flup.server.scgi import WSGIServer as SCGIThreadServer 33 from flup.server.fcgi import WSGIServer as FCGIThreadServer 34 haveflup = True 35 except ImportError: 36 haveflup = False 37 try: 38 from paste import httpserver 39 havepaste = True 40 except ImportError: 41 havepaste = False 42 43 import ewa.mp3 44 import ewa.audio 45 from ewa.config import Config, initConfig 46 from ewa.lighttpd_hack_middleware import LighttpdHackMiddleware 47 from ewa.logutil import (debug, error, exception, info, logger, 48 initLogging, warn) 49 from ewa.wsgiapp import EwaApp 50 from ewa.rules import FileRule 51 from ewa import __version__ 52 53 VERSION_TEXT = """\ 54 %%s %s 55 56 Copyright (C) 2007, 2010 WNYC New York Public Radio. 57 This is free software; see the source for copying conditions. There is NO 58 warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 59 60 Written by Jacob Smullyan and others, not necessarily in that order. 61 """ % __version__ 62 63 64 SERVE_DESCRIPTION = """\ 65 Starts a WSGI application that produces combined MP3 files 66 according to the specified rules. 67 """ 68 69 RULE_DESCRIPTION = """\ 70 Produces a combined MP3 file according to the specified rules. 71 """ 72 73 SPLICE_DESCRIPTION = """\ 74 Splices MP3 files together. 75 """ 76 77
78 -def get_serve_parser():
79 protocols = [] 80 if haveflup: 81 protocols.extend(['fcgi', 'scgi']) 82 if havepaste: 83 protocols.append('http') 84 if not protocols: 85 print >> sys.stderr, ("no protocols available; " 86 "please install paste and/or flup") 87 sys.exit(1) 88 usage = "usage: %prog [options]" 89 parser = OptionParser(usage, description=SERVE_DESCRIPTION) 90 parser.add_option('-c', 91 '--config', 92 dest='configfile', 93 default=None, 94 help="path to ewa config file") 95 parser.add_option('-D', 96 '--nodaemonize', 97 action='store_true', 98 default=False, 99 dest='nodaemonize', 100 help="don't daemonize, regardless of config settings") 101 parser.add_option('--version', 102 action="store_true", 103 dest="version", 104 help="show version and exit") 105 parser.add_option('--lighttpd-hack', 106 action='store_true', 107 default=False, 108 dest='lighttpd_hack', 109 help=("hack for some versions of lighttpd " 110 "to force SCRIPT_NAME to ''")) 111 return parser
112 113
114 -def get_splice_parser():
115 usage = "%prog [options] files" 116 parser = OptionParser(usage, description=SPLICE_DESCRIPTION) 117 parser.add_option('-o', 118 '--output', 119 dest='output', 120 help="output file (default: stdout)", 121 default='-', 122 metavar='OUT') 123 parser.add_option('-t', 124 '--tagfile', 125 dest="tagfile", 126 help="tag file", 127 default=None, 128 metavar="TAGFILE") 129 parser.add_option('-d', 130 '--debug', 131 action='store_true', 132 default=False, 133 help="print debugging information", 134 dest='debugmode') 135 parser.add_option('-s', 136 '--sanitycheck', 137 action='store_true', 138 default=False, 139 help="sanity check the input mp3 files", 140 dest='sanitycheck') 141 parser.add_option('-e', 142 '--engine', 143 default='default', 144 dest='engine', 145 metavar='ENGINE', 146 choices=('default', 'mp3cat', 'sox'), 147 help=("which splicing engine to use (default ewa " 148 "splicer, mp3cat, or sox)")) 149 parser.add_option('--version', 150 action="store_true", 151 dest="version", 152 help="show version and exit") 153 return parser
154 155
156 -def get_ap_parser():
157 usage = "%prog [options] [files]" 158 parser = OptionParser(usage, description=RULE_DESCRIPTION) 159 160 parser.add_option('-c', 161 '--config', 162 dest='configfile', 163 default=None, 164 help="path to ewa config file") 165 parser.add_option('-r', 166 '--recursive', 167 dest='recursive', 168 help="recurse through directories", 169 action='store_true', 170 default=False) 171 parser.add_option('--rulefile', 172 dest='rulefile', 173 help="specify a rulefile", 174 metavar='RULEFILE') 175 parser.add_option('-d', 176 '--debug', 177 action='store_true', 178 default=False, 179 help="print debugging information", 180 dest='debugmode') 181 parser.add_option('-n', 182 '--dry-run', 183 action='store_true', 184 default=False, 185 dest='dryrun', 186 help="don't do anything, just print what would be done") 187 parser.add_option('-e', 188 '--engine', 189 default=None, 190 dest='engine', 191 metavar='ENGINE', 192 choices=('default', 'mp3cat', 'sox'), 193 help=("which splicing engine to use (default ewa " 194 "splicer, mp3cat, or sox)")) 195 parser.add_option('-a', 196 '--absolute', 197 default=False, 198 dest='absolute', 199 action='store_true', 200 help=("interpret file paths relative to the filesystem " 201 "rather than the basedir (default: no)")) 202 203 parser.add_option('-t', 204 '--configtest', 205 default=False, 206 dest='configtest', 207 action='store_true', 208 help="just test the config file for syntax errors") 209 210 parser.add_option('-x', 211 '--max-age', 212 default=0, 213 dest='max_age', 214 type=int, 215 help=('max age before generated files expire and ' 216 'are regenerated. Default is 0, which means no ' 217 'expiration. -1 will force regeneration.')) 218 219 parser.add_option('-D', 220 '--delete', 221 default=False, 222 action='store_true', 223 help=('delete files in combined directory that are ' 224 'not in the the main directory')) 225 226 parser.add_option('-V', '--no-vbr', 227 default=True, 228 action='store_false', 229 dest='tolerate_vbr', 230 help="don't put vbr files in the combined directory") 231 232 parser.add_option('-B', '--no-broken', 233 default=True, 234 action='store_false', 235 dest='tolerate_broken', 236 help="don't put broken files in the combined directory") 237 238 parser.add_option('--version', 239 action="store_true", 240 dest="version", 241 help="show version and exit") 242 parser.add_option('-s', '--sleep', 243 type=float, 244 default=0.0, 245 dest='sleep', 246 help=('number of seconds to sleep between file generations; ' 247 'default: 0.0')) 248 return parser
249 250
251 -def resolve_engine(enginename):
252 if enginename == 'default': 253 return ewa.mp3._default_splicer 254 elif enginename == 'mp3cat': 255 return ewa.mp3._mp3cat_splicer 256 elif enginename == 'sox': 257 return ewa.mp3._sox_splicer
258 259
260 -def do_splice(args):
261 parser = get_splice_parser() 262 opts, args = parser.parse_args(args) 263 if opts.version: 264 print VERSION_TEXT % path.basename(sys.argv[0]) 265 sys.exit(0) 266 if opts.debugmode: 267 Config.loglevel = logging.DEBUG 268 initLogging(level=Config.loglevel, 269 filename=Config.logfile) 270 271 engine = resolve_engine(opts.engine) 272 if opts.sanitycheck: 273 try: 274 ewa.mp3.mp3_sanity_check(args) 275 except Exception, e: 276 print >> sys.stderr, 'sanity check failed: %s' % str(e) 277 sys.exit(1) 278 279 use_stdout = opts.output == '-' 280 if use_stdout: 281 fp = sys.stdout 282 else: 283 fp = open(options.output, 'wb') 284 for chunk in ewa.mp3.splice(args, opts.tagfile, splicer=engine): 285 fp.write(chunk) 286 if not use_stdout: 287 fp.close()
288 289
290 -def do_audioprovider(args):
291 parser = gs = get_ap_parser() 292 opts, args = parser.parse_args(args) 293 if opts.version: 294 print VERSION_TEXT % path.basename(sys.argv[0]) 295 sys.exit(0) 296 configfile = _find_config(opts.configfile) 297 if not configfile: 298 parser.error('no config file specified or found in the usual places') 299 if not path.exists(configfile): 300 parser.error("config file does not exist: %s" % configfile) 301 initConfig(configfile) 302 303 # override config 304 if opts.debugmode: 305 Config.loglevel = 'debug' 306 if opts.rulefile: 307 Config.rulefile = opts.rulefile 308 if opts.engine: 309 Config.engine = engine 310 engine = resolve_engine(Config.engine) 311 initLogging(level=Config.loglevel) 312 rule = FileRule(Config.rulefile) 313 314 if opts.configtest: 315 print "Config OK" 316 sys.exit(0) 317 318 provider = ewa.audio.FSAudioProvider(Config.basedir, 319 opts.tolerate_vbr, 320 opts.tolerate_broken, 321 Config.targetdir) 322 mainpath = provider.get_main_path("") 323 if not mainpath.endswith('/'): 324 # currently this won't happen, 325 # but better safe than sorry 326 mainpath += '/' 327 if opts.absolute: 328 abs = [path.abspath(f) for f in args] 329 stripped_main = mainpath[:-1] 330 if not path.commonprefix([stripped_main] + abs) == stripped_main: 331 debug("absolute paths of files: %s", abs) 332 debug("mainpath: %s", mainpath) 333 parser.error("files outside managed directory") 334 idx = len(mainpath) 335 args = [x[idx:] for x in abs] 336 else: 337 args = [re.sub('/*(.*)', r'\1', x) for x in args] 338 339 if opts.delete: 340 if not opts.recursive: 341 parser.error("delete only works in recursive mode") 342 delete_finder = DeleteFinder(args, mainpath) 343 for file, isdir in delete_finder: 344 info('deleting %s', file) 345 if not opts.dryrun: 346 try: 347 if isdir: 348 os.rmdir(file) 349 else: 350 os.unlink(file) 351 except Exception, e: 352 error("couldn't unlink %s: %s", file, e) 353 354 if opts.recursive: 355 # replace args with an iterator that finds mp3 files 356 if opts.max_age == -1: 357 args = RecursiveMp3FileIterator(args, mainpath) 358 else: 359 args = RecursiveChangedMp3FileIterator(args, 360 mainpath, 361 opts.max_age) 362 if not args: 363 parser.error("no files specified") 364 365 if opts.dryrun: 366 for file in args: 367 debug("mp3 file: %s", file) 368 print "playlist for %s:" % file 369 for part in provider.get_playlist(file, rule, False): 370 print "\t%s" % part 371 sys.exit(0) 372 else: 373 _change_user_group() 374 is_first = True 375 for file in args: 376 if is_first: 377 is_first = False 378 elif opts.sleep: 379 time.sleep(opts.sleep) 380 try: 381 target = provider.create_combined(file, rule, splicer=engine) 382 except: 383 exception("error creating combined file for %s", file) 384 else: 385 debug('created %s', target) 386 sys.exit(0)
387 388
389 -class DeleteFinder(object):
390 - def __init__(self, files, basedir):
391 self.files = files 392 self.basedir = basedir 393 self.targetdir = Config.targetdir 394 if self.targetdir is None: 395 self.targetdir = path.join(Config.basedir, 'combined')
396
397 - def _yielder(self, root, apath, isdir):
398 targetpath = os.path.join(root, apath) 399 mainpath = path.join(self.basedir, 400 path.relpath(targetpath, self.targetdir)) 401 402 debug('mainpath for %s is %s', apath, mainpath) 403 if not os.path.exists(mainpath): 404 yield targetpath, isdir
405
406 - def __iter__(self):
407 for f in (path.join(self.targetdir, x) for x in self.files): 408 if path.isdir(f): 409 debug('found directory: %s', f) 410 for root, dirs, files in os.walk(f, topdown=False): 411 for thing in files: 412 if thing.endswith('~'): 413 # looks like an ewa temp file. Let it be. 414 continue 415 for res in self._yielder(root, thing, 0): 416 yield res 417 for thing in dirs: 418 for res in self._yielder(root, thing, 1): 419 yield res
420 421
422 -class RecursiveMp3FileIterator(object):
423 - def __init__(self, files, basedir):
424 self.files = files 425 self.basedir = basedir
426
427 - def __iter__(self):
428 length = len(self.basedir) 429 for f in (path.join(self.basedir, x) for x in self.files): 430 if path.isdir(f): 431 debug('found directory: %s', f) 432 for mp3 in self._walk(f): 433 debug('raw value: %s', mp3) 434 yield mp3[length:] 435 else: 436 debug('found non-directory: %s', f) 437 yield f[length:]
438
439 - def _walk(self, f):
440 for root, dirs, files in os.walk(f): 441 for f in files: 442 if f.endswith('.mp3') or f.endswith('MP3'): 443 yield path.join(root, f)
444 445
446 -class RecursiveChangedMp3FileIterator(RecursiveMp3FileIterator):
447
448 - def __init__(self, files, basedir, max_age=0):
449 super(RecursiveChangedMp3FileIterator, self).__init__(files, basedir) 450 self.targetdir = Config.targetdir 451 if self.targetdir is None: 452 self.targetdir = path.join(Config.basedir, 'combined') 453 self.max_age = max_age
454
455 - def _walk(self, f):
456 for root, dirs, files in os.walk(f): 457 for f in files: 458 if f.endswith('.mp3') or f.endswith('MP3'): 459 fullpath = path.join(root, f) 460 debug('fullpath is %s', fullpath) 461 debug('basedir: %s; targetdir: %s', 462 self.basedir, self.targetdir) 463 targetpath = os.path.join(self.targetdir, 464 path.relpath(fullpath, 465 self.basedir)) 466 try: 467 stats = os.stat(targetpath) 468 except OSError, ozzie: 469 if ozzie.errno == errno.ENOENT: 470 # target doesn't exist 471 debug("target doesn't exist: %s", targetpath) 472 yield fullpath 473 else: 474 exception("error statting targetfile") 475 # should we return this, or continue? ???? 476 else: 477 # we know the file exists 478 target_mtime = stats.st_mtime 479 try: 480 original_mtime = path.getmtime(fullpath) 481 except OSError: 482 exception('peculiar error getting mtime of %s', 483 fullpath) 484 continue 485 486 if original_mtime > target_mtime: 487 # if original file has changed, we need to 488 # regenerate regardless 489 yield fullpath 490 elif self.max_age <= 0: 491 # max_age of zero or less means we never 492 # regenerate the files 493 continue 494 else: 495 # regenerate if the file is older than 496 # max_age minutes 497 mtime = stats.st_mtime 498 age = time.time() - mtime 499 if age > self.max_age * 60: 500 yield fullpath
501 502
503 -def _find_config(givenpath):
504 for pth in (p for p in ( 505 givenpath, 506 path.expanduser('~/.ewa.conf'), 507 path.expanduser('~/.ewa/ewa.conf'), 508 '/etc/ewa.conf', 509 '/etc/ewa/ewa.conf', 510 ) if p): 511 if path.exists(pth): 512 return pth
513 514
515 -def do_serve(args):
516 parser = get_serve_parser() 517 opts, args = parser.parse_args(args) 518 if opts.version: 519 print VERSION_TEXT % path.basename(sys.argv[0]) 520 sys.exit(0) 521 configfile = _find_config(opts.configfile) 522 if not configfile: 523 parser.error('no config file specified or found in the usual places') 524 if not path.exists(configfile): 525 parser.error("config file does not exist: %s" % configfile) 526 initConfig(configfile) 527 if opts.nodaemonize: 528 Config.daemonize = False 529 530 if not Config.rulefile: 531 parser.error("a rulefile needs to be specified") 532 rule = FileRule(Config.rulefile) 533 if Config.logfile: 534 initLogging(level=Config.loglevel, 535 filename=Config.logfile, 536 rotate=Config.logrotate) 537 # check for incompatible options 538 if Config.protocol == 'http' and Config.unixsocket: 539 parser.error('unix sockets not supported for http server') 540 if Config.umask and not Config.unixsocket: 541 parser.error('umask only applicable for unix sockets') 542 if Config.unixsocket and (Config.interface or Config.port): 543 parser.error('incompatible mixture of unix socket and tcp options') 544 engine = resolve_engine(Config.engine) 545 546 app = EwaApp(rule=rule, 547 basedir=Config.basedir, 548 targetdir=Config.targetdir, 549 stream=Config.stream, 550 refresh_rate=Config.refresh_rate, 551 use_xsendfile=Config.use_xsendfile, 552 sendfile_header=Config.sendfile_header, 553 content_disposition=Config.content_disposition, 554 splicer=engine) 555 556 if opts.lighttpd_hack: 557 app = LighttpdHackMiddleware(app) 558 559 if Config.protocol == 'http': 560 runner = partial(httpserver.serve, 561 app, 562 Config.interface, 563 Config.port) 564 else: 565 if Config.interface: 566 bindAddress = (Config.interface, Config.port) 567 debug('bindAddress: %s', bindAddress) 568 elif Config.unixsocket: 569 bindAddress = Config.unixsocket 570 if Config.protocol == 'scgi': 571 if Config.use_threads: 572 serverclass = SCGIThreadServer 573 else: 574 serverclass = SCGIServer 575 elif Config.protocol == 'fcgi': 576 if Config.use_threads: 577 serverclass = FCGIThreadServer 578 else: 579 serverclass = FCGIServer 580 if Config.protocol in ('fcgi', 'scgi'): 581 if Config.use_threads: 582 kw = dict(maxSpare=Config.max_spare, 583 minSpare=Config.min_spare, 584 maxThreads=Config.max_threads) 585 else: 586 kw = dict(maxSpare=Config.max_spare, 587 minSpare=Config.min_spare, 588 maxChildren=Config.max_children, 589 maxRequests=Config.max_requests) 590 # clean out Nones 591 kw = dict(i for i in kw.items() if not i[1] is None) 592 else: 593 kw = {} 594 595 runner = serverclass(app, 596 bindAddress=bindAddress, 597 umask=Config.umask, 598 **kw).run 599 600 try: 601 run_server(runner, 602 Config.pidfile, 603 Config.daemonize) 604 except SystemExit: 605 raise 606 except: 607 exception('exception caught') 608 sys.exit(1) 609 else: 610 sys.exit(0)
611 612
613 -def _change_user_group():
614 if not have_unix: 615 # maybe do something else on Windows someday... 616 return 617 618 user = Config.user 619 group = Config.group 620 if user or group: 621 # in case we are seteuid something else, which would 622 # cause setuid or getuid to fail, undo any existing 623 # seteuid. (The only reason to do this is for the case 624 # os.getuid()==0, AFAIK). 625 try: 626 seteuid = os.seteuid 627 except AttributeError: 628 # the OS may not support seteuid, in which 629 # case everything is hotsy-totsy. 630 pass 631 else: 632 seteuid(os.getuid()) 633 if group: 634 gid = grp.getgrnam(group)[2] 635 os.setgid(gid) 636 if user: 637 uid = pwd.getpwnam(user)[2] 638 os.setuid(uid)
639 640
641 -def run_server(func, pidfile=None, daemonize=True):
642 debug("daemonize: %s, pidfile: %s", daemonize, pidfile) 643 if daemonize: 644 try: 645 os.fork 646 except NameError: 647 info("not daemonizing, as the fork() call is not available") 648 daemonize = False 649 if daemonize: 650 if os.fork(): 651 os._exit(0) 652 if os.fork(): 653 os._exit(0) 654 os.setsid() 655 os.chdir('/') 656 os.umask(0) 657 os.open(os.devnull, os.O_RDWR) 658 os.dup2(0, 1) 659 os.dup2(0, 2) 660 if pidfile: 661 pidfp = open(pidfile, 'w') 662 pidfp.write('%s' % os.getpid()) 663 pidfp.close() 664 665 try: 666 _change_user_group() 667 try: 668 func() 669 except KeyboardInterrupt: 670 pass 671 finally: 672 if daemonize and pidfile: 673 try: 674 os.unlink(pidfile) 675 except OSError, e: 676 # if it doesn't exist, that's OK 677 if e.errno != errno.ENOENT: 678 warn("problem unlinking pid file %s", pidfile)
679