Package ewa :: Module clisupport
[hide private]
[frames] | no frames]

Source Code for Module ewa.clisupport

  1  import errno 
  2  import logging 
  3  from optparse import OptionParser 
  4  import os 
  5  from os import path 
  6  import re 
  7  import sys 
  8  import time 
  9   
 10  try: 
 11      import grp 
 12      import pwd 
 13      have_unix = 1 
 14  except ImportError: 
 15      have_unix = 0 
 16   
 17  try: 
 18      from functools import partial 
 19  except ImportError: 
 20   
21 - def partial(func, *args, **kw):
22 23 def inner(*_args, **_kw): 24 d = kw.copy() 25 d.update(_kw) 26 return func(*(args + _args), **d)
27 return inner 28 29 try: 30 from flup.server.scgi_fork import WSGIServer as SCGIServer 31 from flup.server.fcgi_fork import WSGIServer as FCGIServer 32 from flup.server.scgi import WSGIServer as SCGIThreadServer 33 from flup.server.fcgi import WSGIServer as FCGIThreadServer 34 haveflup = True 35 except ImportError: 36 haveflup = False 37 try: 38 from paste import httpserver 39 havepaste = True 40 except ImportError: 41 havepaste = False 42 43 import ewa.mp3 44 import ewa.audio 45 from ewa.config import Config, initConfig 46 from ewa.lighttpd_hack_middleware import LighttpdHackMiddleware 47 from ewa.logutil import (debug, error, exception, info, logger, 48 initLogging, warn) 49 from ewa.wsgiapp import EwaApp 50 from ewa.rules import FileRule 51 from ewa import __version__ 52 53 VERSION_TEXT = """\ 54 %%s %s 55 56 Copyright (C) 2007, 2010 WNYC New York Public Radio. 57 This is free software; see the source for copying conditions. There is NO 58 warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 59 60 Written by Jacob Smullyan and others, not necessarily in that order. 61 """ % __version__ 62 63 64 SERVE_DESCRIPTION = """\ 65 Starts a WSGI application that produces combined MP3 files 66 according to the specified rules. 67 """ 68 69 RULE_DESCRIPTION = """\ 70 Produces a combined MP3 file according to the specified rules. 71 """ 72 73 SPLICE_DESCRIPTION = """\ 74 Splices MP3 files together. 75 """ 76 77
78 -def get_serve_parser():
79 protocols = [] 80 if haveflup: 81 protocols.extend(['fcgi', 'scgi']) 82 if havepaste: 83 protocols.append('http') 84 if not protocols: 85 print >> sys.stderr, ("no protocols available; " 86 "please install paste and/or flup") 87 sys.exit(1) 88 usage = "usage: %prog [options]" 89 parser = OptionParser(usage, description=SERVE_DESCRIPTION) 90 parser.add_option('-c', 91 '--config', 92 dest='configfile', 93 default=None, 94 help="path to ewa config file") 95 parser.add_option('-D', 96 '--nodaemonize', 97 action='store_true', 98 default=False, 99 dest='nodaemonize', 100 help="don't daemonize, regardless of config settings") 101 parser.add_option('--version', 102 action="store_true", 103 dest="version", 104 help="show version and exit") 105 parser.add_option('--lighttpd-hack', 106 action='store_true', 107 default=False, 108 dest='lighttpd_hack', 109 help=("hack for some versions of lighttpd " 110 "to force SCRIPT_NAME to ''")) 111 return parser
112 113
114 -def get_splice_parser():
115 usage = "%prog [options] files" 116 parser = OptionParser(usage, description=SPLICE_DESCRIPTION) 117 parser.add_option('-o', 118 '--output', 119 dest='output', 120 help="output file (default: stdout)", 121 default='-', 122 metavar='OUT') 123 parser.add_option('-t', 124 '--tagfile', 125 dest="tagfile", 126 help="tag file", 127 default=None, 128 metavar="TAGFILE") 129 parser.add_option('-d', 130 '--debug', 131 action='store_true', 132 default=False, 133 help="print debugging information", 134 dest='debugmode') 135 parser.add_option('-s', 136 '--sanitycheck', 137 action='store_true', 138 default=False, 139 help="sanity check the input mp3 files", 140 dest='sanitycheck') 141 parser.add_option('-e', 142 '--engine', 143 default='default', 144 dest='engine', 145 metavar='ENGINE', 146 choices=('default', 'mp3cat', 'sox'), 147 help=("which splicing engine to use (default ewa " 148 "splicer, mp3cat, or sox)")) 149 parser.add_option('--version', 150 action="store_true", 151 dest="version", 152 help="show version and exit") 153 return parser
154 155
156 -def get_ap_parser():
157 usage = "%prog [options] [files]" 158 parser = OptionParser(usage, description=RULE_DESCRIPTION) 159 160 parser.add_option('-c', 161 '--config', 162 dest='configfile', 163 default=None, 164 help="path to ewa config file") 165 parser.add_option('-r', 166 '--recursive', 167 dest='recursive', 168 help="recurse through directories", 169 action='store_true', 170 default=False) 171 parser.add_option('--rulefile', 172 dest='rulefile', 173 help="specify a rulefile", 174 metavar='RULEFILE') 175 parser.add_option('-d', 176 '--debug', 177 action='store_true', 178 default=False, 179 help="print debugging information", 180 dest='debugmode') 181 parser.add_option('-n', 182 '--dry-run', 183 action='store_true', 184 default=False, 185 dest='dryrun', 186 help="don't do anything, just print what would be done") 187 parser.add_option('-e', 188 '--engine', 189 default=None, 190 dest='engine', 191 metavar='ENGINE', 192 choices=('default', 'mp3cat', 'sox'), 193 help=("which splicing engine to use (default ewa " 194 "splicer, mp3cat, or sox)")) 195 parser.add_option('-a', 196 '--absolute', 197 default=False, 198 dest='absolute', 199 action='store_true', 200 help=("interpret file paths relative to the filesystem " 201 "rather than the basedir (default: no)")) 202 203 parser.add_option('-t', 204 '--configtest', 205 default=False, 206 dest='configtest', 207 action='store_true', 208 help="just test the config file for syntax errors") 209 210 parser.add_option('-x', 211 '--max-age', 212 default=0, 213 dest='max_age', 214 type=int, 215 help=('max age before generated files expire and ' 216 'are regenerated. Default is 0, which means no ' 217 'expiration. -1 will force regeneration.')) 218 219 parser.add_option('-D', 220 '--delete', 221 default=False, 222 action='store_true', 223 help=('delete files in combined directory that are ' 224 'not in the the main directory')) 225 226 parser.add_option('--version', 227 action="store_true", 228 dest="version", 229 help="show version and exit") 230 return parser
231 232
233 -def resolve_engine(enginename):
234 if enginename == 'default': 235 return ewa.mp3._default_splicer 236 elif enginename == 'mp3cat': 237 return ewa.mp3._mp3cat_splicer 238 elif enginename == 'sox': 239 return ewa.mp3._sox_splicer
240 241
242 -def do_splice(args):
243 parser = get_splice_parser() 244 opts, args = parser.parse_args(args) 245 if opts.version: 246 print VERSION_TEXT % path.basename(sys.argv[0]) 247 sys.exit(0) 248 if opts.debugmode: 249 Config.loglevel = logging.DEBUG 250 initLogging(level=Config.loglevel, 251 filename=Config.logfile) 252 253 engine = resolve_engine(opts.engine) 254 if opts.sanitycheck: 255 try: 256 ewa.mp3.mp3_sanity_check(args) 257 except Exception, e: 258 print >> sys.stderr, 'sanity check failed: %s' % str(e) 259 sys.exit(1) 260 261 use_stdout = opts.output == '-' 262 if use_stdout: 263 fp = sys.stdout 264 else: 265 fp = open(options.output, 'wb') 266 for chunk in ewa.mp3.splice(args, opts.tagfile, splicer=engine): 267 fp.write(chunk) 268 if not use_stdout: 269 fp.close()
270 271
272 -def do_audioprovider(args):
273 parser = gs = get_ap_parser() 274 opts, args = parser.parse_args(args) 275 if opts.version: 276 print VERSION_TEXT % path.basename(sys.argv[0]) 277 sys.exit(0) 278 configfile = _find_config(opts.configfile) 279 if not configfile: 280 parser.error('no config file specified or found in the usual places') 281 if not path.exists(configfile): 282 parser.error("config file does not exist: %s" % configfile) 283 initConfig(configfile) 284 285 # override config 286 if opts.debugmode: 287 Config.loglevel = 'debug' 288 if opts.rulefile: 289 Config.rulefile = opts.rulefile 290 if opts.engine: 291 Config.engine = engine 292 engine = resolve_engine(Config.engine) 293 initLogging(level=Config.loglevel) 294 rule = FileRule(Config.rulefile) 295 296 if opts.configtest: 297 print "Config OK" 298 sys.exit(0) 299 300 provider = ewa.audio.FSAudioProvider(Config.basedir, 301 Config.targetdir) 302 mainpath = provider.get_main_path("") 303 if not mainpath.endswith('/'): 304 # currently this won't happen, 305 # but better safe than sorry 306 mainpath += '/' 307 if opts.absolute: 308 abs = [path.abspath(f) for f in args] 309 stripped_main = mainpath[:-1] 310 if not path.commonprefix([stripped_main] + abs) == stripped_main: 311 debug("absolute paths of files: %s", abs) 312 debug("mainpath: %s", mainpath) 313 parser.error("files outside managed directory") 314 idx = len(mainpath) 315 args = [x[idx:] for x in abs] 316 else: 317 args = [re.sub('/*(.*)', r'\1', x) for x in args] 318 319 if opts.delete: 320 if not opts.recursive: 321 parser.error("delete only works in recursive mode") 322 delete_finder = DeleteFinder(args, mainpath) 323 for file, isdir in delete_finder: 324 info('deleting %s', file) 325 if not opts.dryrun: 326 try: 327 if isdir: 328 os.rmdir(file) 329 else: 330 os.unlink(file) 331 except Exception, e: 332 error("couldn't unlink %s: %s", file, e) 333 334 if opts.recursive: 335 # replace args with an iterator that finds mp3 files 336 if opts.max_age == -1: 337 args = RecursiveMp3FileIterator(args, mainpath) 338 else: 339 args = RecursiveChangedMp3FileIterator(args, 340 mainpath, 341 opts.max_age) 342 if not args: 343 parser.error("no files specified") 344 345 if opts.dryrun: 346 for file in args: 347 debug("mp3 file: %s", file) 348 print "playlist for %s:" % file 349 for part in provider.get_playlist(file, rule, False): 350 print "\t%s" % part 351 sys.exit(0) 352 else: 353 _change_user_group() 354 for file in args: 355 try: 356 target = provider.create_combined(file, rule, splicer=engine) 357 except: 358 exception("error creating combined file for %s", file) 359 else: 360 debug('created %s', target) 361 sys.exit(0)
362 363
364 -class DeleteFinder(object):
365 - def __init__(self, files, basedir):
366 self.files = files 367 self.basedir = basedir 368 self.targetdir = Config.targetdir 369 if self.targetdir is None: 370 self.targetdir = path.join(Config.basedir, 'combined')
371
372 - def _yielder(self, root, apath, isdir):
373 targetpath = os.path.join(root, apath) 374 mainpath = path.join(self.basedir, 375 path.relpath(targetpath, self.targetdir)) 376 377 debug('mainpath for %s is %s', apath, mainpath) 378 if not os.path.exists(mainpath): 379 yield targetpath, isdir
380
381 - def __iter__(self):
382 for f in (path.join(self.targetdir, x) for x in self.files): 383 if path.isdir(f): 384 debug('found directory: %s', f) 385 for root, dirs, files in os.walk(f, topdown=False): 386 for thing in files: 387 if thing.endswith('~'): 388 # looks like an ewa temp file. Let it be. 389 continue 390 for res in self._yielder(root, thing, 0): 391 yield res 392 for thing in dirs: 393 for res in self._yielder(root, thing, 1): 394 yield res
395 396
397 -class RecursiveMp3FileIterator(object):
398 - def __init__(self, files, basedir):
399 self.files = files 400 self.basedir = basedir
401
402 - def __iter__(self):
403 length = len(self.basedir) 404 for f in (path.join(self.basedir, x) for x in self.files): 405 if path.isdir(f): 406 debug('found directory: %s', f) 407 for mp3 in self._walk(f): 408 debug('raw value: %s', mp3) 409 yield mp3[length:] 410 else: 411 debug('found non-directory: %s', f) 412 yield f[length:]
413
414 - def _walk(self, f):
415 for root, dirs, files in os.walk(f): 416 for f in files: 417 if f.endswith('.mp3') or f.endswith('MP3'): 418 yield path.join(root, f)
419 420
421 -class RecursiveChangedMp3FileIterator(RecursiveMp3FileIterator):
422
423 - def __init__(self, files, basedir, max_age=0):
424 super(RecursiveChangedMp3FileIterator, self).__init__(files, basedir) 425 self.targetdir = Config.targetdir 426 if self.targetdir is None: 427 self.targetdir = path.join(Config.basedir, 'combined') 428 self.max_age = max_age
429
430 - def _walk(self, f):
431 for root, dirs, files in os.walk(f): 432 for f in files: 433 if f.endswith('.mp3') or f.endswith('MP3'): 434 fullpath = path.join(root, f) 435 debug('fullpath is %s', fullpath) 436 debug('basedir: %s; targetdir: %s', 437 self.basedir, self.targetdir) 438 targetpath = os.path.join(self.targetdir, 439 path.relpath(fullpath, 440 self.basedir)) 441 try: 442 stats = os.stat(targetpath) 443 except OSError, ozzie: 444 if ozzie.errno == errno.ENOENT: 445 # target doesn't exist 446 debug("target doesn't exist: %s", targetpath) 447 yield fullpath 448 else: 449 exception("error statting targetfile") 450 # should we return this, or continue? ???? 451 else: 452 # we know the file exists 453 target_mtime = stats.st_mtime 454 try: 455 original_mtime = path.getmtime(fullpath) 456 except OSError: 457 exception('peculiar error getting mtime of %s', 458 fullpath) 459 continue 460 461 if original_mtime > target_mtime: 462 # if original file has changed, we need to 463 # regenerate regardless 464 yield fullpath 465 elif self.max_age <= 0: 466 # max_age of zero or less means we never 467 # regenerate the files 468 continue 469 else: 470 # regenerate if the file is older than 471 # max_age minutes 472 mtime = stats.st_mtime 473 age = time.time() - mtime 474 if age > self.max_age * 60: 475 yield fullpath
476 477
478 -def _find_config(givenpath):
479 for pth in (p for p in ( 480 givenpath, 481 path.expanduser('~/.ewa.conf'), 482 path.expanduser('~/.ewa/ewa.conf'), 483 '/etc/ewa.conf', 484 '/etc/ewa/ewa.conf', 485 ) if p): 486 if path.exists(pth): 487 return pth
488 489
490 -def do_serve(args):
491 parser = get_serve_parser() 492 opts, args = parser.parse_args(args) 493 if opts.version: 494 print VERSION_TEXT % path.basename(sys.argv[0]) 495 sys.exit(0) 496 configfile = _find_config(opts.configfile) 497 if not configfile: 498 parser.error('no config file specified or found in the usual places') 499 if not path.exists(configfile): 500 parser.error("config file does not exist: %s" % configfile) 501 initConfig(configfile) 502 if opts.nodaemonize: 503 Config.daemonize = False 504 505 if not Config.rulefile: 506 parser.error("a rulefile needs to be specified") 507 rule = FileRule(Config.rulefile) 508 if Config.logfile: 509 initLogging(level=Config.loglevel, 510 filename=Config.logfile, 511 rotate=Config.logrotate) 512 # check for incompatible options 513 if Config.protocol == 'http' and Config.unixsocket: 514 parser.error('unix sockets not supported for http server') 515 if Config.umask and not Config.unixsocket: 516 parser.error('umask only applicable for unix sockets') 517 if Config.unixsocket and (Config.interface or Config.port): 518 parser.error('incompatible mixture of unix socket and tcp options') 519 engine = resolve_engine(Config.engine) 520 521 app = EwaApp(rule=rule, 522 basedir=Config.basedir, 523 targetdir=Config.targetdir, 524 stream=Config.stream, 525 refresh_rate=Config.refresh_rate, 526 use_xsendfile=Config.use_xsendfile, 527 sendfile_header=Config.sendfile_header, 528 content_disposition=Config.content_disposition, 529 splicer=engine) 530 531 if opts.lighttpd_hack: 532 app = LighttpdHackMiddleware(app) 533 534 if Config.protocol == 'http': 535 runner = partial(httpserver.serve, 536 app, 537 Config.interface, 538 Config.port) 539 else: 540 if Config.interface: 541 bindAddress = (Config.interface, Config.port) 542 debug('bindAddress: %s', bindAddress) 543 elif Config.unixsocket: 544 bindAddress = Config.unixsocket 545 if Config.protocol == 'scgi': 546 if Config.use_threads: 547 serverclass = SCGIThreadServer 548 else: 549 serverclass = SCGIServer 550 elif Config.protocol == 'fcgi': 551 if Config.use_threads: 552 serverclass = FCGIThreadServer 553 else: 554 serverclass = FCGIServer 555 if Config.protocol in ('fcgi', 'scgi'): 556 if Config.use_threads: 557 kw = dict(maxSpare=Config.max_spare, 558 minSpare=Config.min_spare, 559 maxThreads=Config.max_threads) 560 else: 561 kw = dict(maxSpare=Config.max_spare, 562 minSpare=Config.min_spare, 563 maxChildren=Config.max_children, 564 maxRequests=Config.max_requests) 565 # clean out Nones 566 kw = dict(i for i in kw.items() if not i[1] is None) 567 else: 568 kw = {} 569 570 runner = serverclass(app, 571 bindAddress=bindAddress, 572 umask=Config.umask, 573 **kw).run 574 575 try: 576 run_server(runner, 577 Config.pidfile, 578 Config.daemonize) 579 except SystemExit: 580 raise 581 except: 582 exception('exception caught') 583 sys.exit(1) 584 else: 585 sys.exit(0)
586 587
588 -def _change_user_group():
589 if not have_unix: 590 # maybe do something else on Windows someday... 591 return 592 593 user = Config.user 594 group = Config.group 595 if user or group: 596 # in case we are seteuid something else, which would 597 # cause setuid or getuid to fail, undo any existing 598 # seteuid. (The only reason to do this is for the case 599 # os.getuid()==0, AFAIK). 600 try: 601 seteuid = os.seteuid 602 except AttributeError: 603 # the OS may not support seteuid, in which 604 # case everything is hotsy-totsy. 605 pass 606 else: 607 seteuid(os.getuid()) 608 if group: 609 gid = grp.getgrnam(group)[2] 610 os.setgid(gid) 611 if user: 612 uid = pwd.getpwnam(user)[2] 613 os.setuid(uid)
614 615
616 -def run_server(func, pidfile=None, daemonize=True):
617 debug("daemonize: %s, pidfile: %s", daemonize, pidfile) 618 if daemonize: 619 try: 620 os.fork 621 except NameError: 622 info("not daemonizing, as the fork() call is not available") 623 daemonize = False 624 if daemonize: 625 if os.fork(): 626 os._exit(0) 627 if os.fork(): 628 os._exit(0) 629 os.setsid() 630 os.chdir('/') 631 os.umask(0) 632 os.open(os.devnull, os.O_RDWR) 633 os.dup2(0, 1) 634 os.dup2(0, 2) 635 if pidfile: 636 pidfp = open(pidfile, 'w') 637 pidfp.write('%s' % os.getpid()) 638 pidfp.close() 639 640 try: 641 _change_user_group() 642 try: 643 func() 644 except KeyboardInterrupt: 645 pass 646 finally: 647 if daemonize and pidfile: 648 try: 649 os.unlink(pidfile) 650 except OSError, e: 651 # if it doesn't exist, that's OK 652 if e.errno != errno.ENOENT: 653 warn("problem unlinking pid file %s", pidfile)
654