1 import errno
2 import logging
3 from optparse import OptionParser
4 import os
5 from os import path
6 import re
7 import sys
8 import time
9
10 try:
11 import grp
12 import pwd
13 have_unix = 1
14 except ImportError:
15 have_unix = 0
16
17 try:
18 from functools import partial
19 except ImportError:
20
22
23 def inner(*_args, **_kw):
24 d = kw.copy()
25 d.update(_kw)
26 return func(*(args + _args), **d)
27 return inner
28
29 try:
30 from flup.server.scgi_fork import WSGIServer as SCGIServer
31 from flup.server.fcgi_fork import WSGIServer as FCGIServer
32 from flup.server.scgi import WSGIServer as SCGIThreadServer
33 from flup.server.fcgi import WSGIServer as FCGIThreadServer
34 haveflup = True
35 except ImportError:
36 haveflup = False
37 try:
38 from paste import httpserver
39 havepaste = True
40 except ImportError:
41 havepaste = False
42
43 import ewa.mp3
44 import ewa.audio
45 from ewa.config import Config, initConfig
46 from ewa.lighttpd_hack_middleware import LighttpdHackMiddleware
47 from ewa.logutil import (debug, error, exception, info, logger,
48 initLogging, warn)
49 from ewa.wsgiapp import EwaApp
50 from ewa.rules import FileRule
51 from ewa import __version__
52
53 VERSION_TEXT = """\
54 %%s %s
55
56 Copyright (C) 2007, 2010 WNYC New York Public Radio.
57 This is free software; see the source for copying conditions. There is NO
58 warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
59
60 Written by Jacob Smullyan and others, not necessarily in that order.
61 """ % __version__
62
63
64 SERVE_DESCRIPTION = """\
65 Starts a WSGI application that produces combined MP3 files
66 according to the specified rules.
67 """
68
69 RULE_DESCRIPTION = """\
70 Produces a combined MP3 file according to the specified rules.
71 """
72
73 SPLICE_DESCRIPTION = """\
74 Splices MP3 files together.
75 """
76
77
79 protocols = []
80 if haveflup:
81 protocols.extend(['fcgi', 'scgi'])
82 if havepaste:
83 protocols.append('http')
84 if not protocols:
85 print >> sys.stderr, ("no protocols available; "
86 "please install paste and/or flup")
87 sys.exit(1)
88 usage = "usage: %prog [options]"
89 parser = OptionParser(usage, description=SERVE_DESCRIPTION)
90 parser.add_option('-c',
91 '--config',
92 dest='configfile',
93 default=None,
94 help="path to ewa config file")
95 parser.add_option('-D',
96 '--nodaemonize',
97 action='store_true',
98 default=False,
99 dest='nodaemonize',
100 help="don't daemonize, regardless of config settings")
101 parser.add_option('--version',
102 action="store_true",
103 dest="version",
104 help="show version and exit")
105 parser.add_option('--lighttpd-hack',
106 action='store_true',
107 default=False,
108 dest='lighttpd_hack',
109 help=("hack for some versions of lighttpd "
110 "to force SCRIPT_NAME to ''"))
111 return parser
112
113
115 usage = "%prog [options] files"
116 parser = OptionParser(usage, description=SPLICE_DESCRIPTION)
117 parser.add_option('-o',
118 '--output',
119 dest='output',
120 help="output file (default: stdout)",
121 default='-',
122 metavar='OUT')
123 parser.add_option('-t',
124 '--tagfile',
125 dest="tagfile",
126 help="tag file",
127 default=None,
128 metavar="TAGFILE")
129 parser.add_option('-d',
130 '--debug',
131 action='store_true',
132 default=False,
133 help="print debugging information",
134 dest='debugmode')
135 parser.add_option('-s',
136 '--sanitycheck',
137 action='store_true',
138 default=False,
139 help="sanity check the input mp3 files",
140 dest='sanitycheck')
141 parser.add_option('-e',
142 '--engine',
143 default='default',
144 dest='engine',
145 metavar='ENGINE',
146 choices=('default', 'mp3cat', 'sox'),
147 help=("which splicing engine to use (default ewa "
148 "splicer, mp3cat, or sox)"))
149 parser.add_option('--version',
150 action="store_true",
151 dest="version",
152 help="show version and exit")
153 return parser
154
155
157 usage = "%prog [options] [files]"
158 parser = OptionParser(usage, description=RULE_DESCRIPTION)
159
160 parser.add_option('-c',
161 '--config',
162 dest='configfile',
163 default=None,
164 help="path to ewa config file")
165 parser.add_option('-r',
166 '--recursive',
167 dest='recursive',
168 help="recurse through directories",
169 action='store_true',
170 default=False)
171 parser.add_option('--rulefile',
172 dest='rulefile',
173 help="specify a rulefile",
174 metavar='RULEFILE')
175 parser.add_option('-d',
176 '--debug',
177 action='store_true',
178 default=False,
179 help="print debugging information",
180 dest='debugmode')
181 parser.add_option('-n',
182 '--dry-run',
183 action='store_true',
184 default=False,
185 dest='dryrun',
186 help="don't do anything, just print what would be done")
187 parser.add_option('-e',
188 '--engine',
189 default=None,
190 dest='engine',
191 metavar='ENGINE',
192 choices=('default', 'mp3cat', 'sox'),
193 help=("which splicing engine to use (default ewa "
194 "splicer, mp3cat, or sox)"))
195 parser.add_option('-a',
196 '--absolute',
197 default=False,
198 dest='absolute',
199 action='store_true',
200 help=("interpret file paths relative to the filesystem "
201 "rather than the basedir (default: no)"))
202
203 parser.add_option('-t',
204 '--configtest',
205 default=False,
206 dest='configtest',
207 action='store_true',
208 help="just test the config file for syntax errors")
209
210 parser.add_option('-x',
211 '--max-age',
212 default=0,
213 dest='max_age',
214 type=int,
215 help=('max age before generated files expire and '
216 'are regenerated. Default is 0, which means no '
217 'expiration. -1 will force regeneration.'))
218
219 parser.add_option('-D',
220 '--delete',
221 default=False,
222 action='store_true',
223 help=('delete files in combined directory that are '
224 'not in the the main directory'))
225
226 parser.add_option('-V', '--no-vbr',
227 default=True,
228 action='store_false',
229 dest='tolerate_vbr',
230 help="don't put vbr files in the combined directory")
231
232 parser.add_option('-B', '--no-broken',
233 default=True,
234 action='store_false',
235 dest='tolerate_broken',
236 help="don't put broken files in the combined directory")
237
238 parser.add_option('--version',
239 action="store_true",
240 dest="version",
241 help="show version and exit")
242 parser.add_option('-s', '--sleep',
243 type=float,
244 default=0.0,
245 dest='sleep',
246 help=('number of seconds to sleep between file generations; '
247 'default: 0.0'))
248 return parser
249
250
258
259
261 parser = get_splice_parser()
262 opts, args = parser.parse_args(args)
263 if opts.version:
264 print VERSION_TEXT % path.basename(sys.argv[0])
265 sys.exit(0)
266 if opts.debugmode:
267 Config.loglevel = logging.DEBUG
268 initLogging(level=Config.loglevel,
269 filename=Config.logfile)
270
271 engine = resolve_engine(opts.engine)
272 if opts.sanitycheck:
273 try:
274 ewa.mp3.mp3_sanity_check(args)
275 except Exception, e:
276 print >> sys.stderr, 'sanity check failed: %s' % str(e)
277 sys.exit(1)
278
279 use_stdout = opts.output == '-'
280 if use_stdout:
281 fp = sys.stdout
282 else:
283 fp = open(options.output, 'wb')
284 for chunk in ewa.mp3.splice(args, opts.tagfile, splicer=engine):
285 fp.write(chunk)
286 if not use_stdout:
287 fp.close()
288
289
291 parser = gs = get_ap_parser()
292 opts, args = parser.parse_args(args)
293 if opts.version:
294 print VERSION_TEXT % path.basename(sys.argv[0])
295 sys.exit(0)
296 configfile = _find_config(opts.configfile)
297 if not configfile:
298 parser.error('no config file specified or found in the usual places')
299 if not path.exists(configfile):
300 parser.error("config file does not exist: %s" % configfile)
301 initConfig(configfile)
302
303
304 if opts.debugmode:
305 Config.loglevel = 'debug'
306 if opts.rulefile:
307 Config.rulefile = opts.rulefile
308 if opts.engine:
309 Config.engine = engine
310 engine = resolve_engine(Config.engine)
311 initLogging(level=Config.loglevel)
312 rule = FileRule(Config.rulefile)
313
314 if opts.configtest:
315 print "Config OK"
316 sys.exit(0)
317
318 provider = ewa.audio.FSAudioProvider(Config.basedir,
319 opts.tolerate_vbr,
320 opts.tolerate_broken,
321 Config.targetdir)
322 mainpath = provider.get_main_path("")
323 if not mainpath.endswith('/'):
324
325
326 mainpath += '/'
327 if opts.absolute:
328 abs = [path.abspath(f) for f in args]
329 stripped_main = mainpath[:-1]
330 if not path.commonprefix([stripped_main] + abs) == stripped_main:
331 debug("absolute paths of files: %s", abs)
332 debug("mainpath: %s", mainpath)
333 parser.error("files outside managed directory")
334 idx = len(mainpath)
335 args = [x[idx:] for x in abs]
336 else:
337 args = [re.sub('/*(.*)', r'\1', x) for x in args]
338
339 if opts.delete:
340 if not opts.recursive:
341 parser.error("delete only works in recursive mode")
342 delete_finder = DeleteFinder(args, mainpath)
343 for file, isdir in delete_finder:
344 info('deleting %s', file)
345 if not opts.dryrun:
346 try:
347 if isdir:
348 os.rmdir(file)
349 else:
350 os.unlink(file)
351 except Exception, e:
352 error("couldn't unlink %s: %s", file, e)
353
354 if opts.recursive:
355
356 if opts.max_age == -1:
357 args = RecursiveMp3FileIterator(args, mainpath)
358 else:
359 args = RecursiveChangedMp3FileIterator(args,
360 mainpath,
361 opts.max_age)
362 if not args:
363 parser.error("no files specified")
364
365 if opts.dryrun:
366 for file in args:
367 debug("mp3 file: %s", file)
368 print "playlist for %s:" % file
369 for part in provider.get_playlist(file, rule, False):
370 print "\t%s" % part
371 sys.exit(0)
372 else:
373 _change_user_group()
374 is_first = True
375 for file in args:
376 if is_first:
377 is_first = False
378 elif opts.sleep:
379 time.sleep(opts.sleep)
380 try:
381 target = provider.create_combined(file, rule, splicer=engine)
382 except:
383 exception("error creating combined file for %s", file)
384 else:
385 debug('created %s', target)
386 sys.exit(0)
387
388
396
397 - def _yielder(self, root, apath, isdir):
398 targetpath = os.path.join(root, apath)
399 mainpath = path.join(self.basedir,
400 path.relpath(targetpath, self.targetdir))
401
402 debug('mainpath for %s is %s', apath, mainpath)
403 if not os.path.exists(mainpath):
404 yield targetpath, isdir
405
407 for f in (path.join(self.targetdir, x) for x in self.files):
408 if path.isdir(f):
409 debug('found directory: %s', f)
410 for root, dirs, files in os.walk(f, topdown=False):
411 for thing in files:
412 if thing.endswith('~'):
413
414 continue
415 for res in self._yielder(root, thing, 0):
416 yield res
417 for thing in dirs:
418 for res in self._yielder(root, thing, 1):
419 yield res
420
421
426
428 length = len(self.basedir)
429 for f in (path.join(self.basedir, x) for x in self.files):
430 if path.isdir(f):
431 debug('found directory: %s', f)
432 for mp3 in self._walk(f):
433 debug('raw value: %s', mp3)
434 yield mp3[length:]
435 else:
436 debug('found non-directory: %s', f)
437 yield f[length:]
438
440 for root, dirs, files in os.walk(f):
441 for f in files:
442 if f.endswith('.mp3') or f.endswith('MP3'):
443 yield path.join(root, f)
444
445
447
448 - def __init__(self, files, basedir, max_age=0):
454
456 for root, dirs, files in os.walk(f):
457 for f in files:
458 if f.endswith('.mp3') or f.endswith('MP3'):
459 fullpath = path.join(root, f)
460 debug('fullpath is %s', fullpath)
461 debug('basedir: %s; targetdir: %s',
462 self.basedir, self.targetdir)
463 targetpath = os.path.join(self.targetdir,
464 path.relpath(fullpath,
465 self.basedir))
466 try:
467 stats = os.stat(targetpath)
468 except OSError, ozzie:
469 if ozzie.errno == errno.ENOENT:
470
471 debug("target doesn't exist: %s", targetpath)
472 yield fullpath
473 else:
474 exception("error statting targetfile")
475
476 else:
477
478 target_mtime = stats.st_mtime
479 try:
480 original_mtime = path.getmtime(fullpath)
481 except OSError:
482 exception('peculiar error getting mtime of %s',
483 fullpath)
484 continue
485
486 if original_mtime > target_mtime:
487
488
489 yield fullpath
490 elif self.max_age <= 0:
491
492
493 continue
494 else:
495
496
497 mtime = stats.st_mtime
498 age = time.time() - mtime
499 if age > self.max_age * 60:
500 yield fullpath
501
502
504 for pth in (p for p in (
505 givenpath,
506 path.expanduser('~/.ewa.conf'),
507 path.expanduser('~/.ewa/ewa.conf'),
508 '/etc/ewa.conf',
509 '/etc/ewa/ewa.conf',
510 ) if p):
511 if path.exists(pth):
512 return pth
513
514
516 parser = get_serve_parser()
517 opts, args = parser.parse_args(args)
518 if opts.version:
519 print VERSION_TEXT % path.basename(sys.argv[0])
520 sys.exit(0)
521 configfile = _find_config(opts.configfile)
522 if not configfile:
523 parser.error('no config file specified or found in the usual places')
524 if not path.exists(configfile):
525 parser.error("config file does not exist: %s" % configfile)
526 initConfig(configfile)
527 if opts.nodaemonize:
528 Config.daemonize = False
529
530 if not Config.rulefile:
531 parser.error("a rulefile needs to be specified")
532 rule = FileRule(Config.rulefile)
533 if Config.logfile:
534 initLogging(level=Config.loglevel,
535 filename=Config.logfile,
536 rotate=Config.logrotate)
537
538 if Config.protocol == 'http' and Config.unixsocket:
539 parser.error('unix sockets not supported for http server')
540 if Config.umask and not Config.unixsocket:
541 parser.error('umask only applicable for unix sockets')
542 if Config.unixsocket and (Config.interface or Config.port):
543 parser.error('incompatible mixture of unix socket and tcp options')
544 engine = resolve_engine(Config.engine)
545
546 app = EwaApp(rule=rule,
547 basedir=Config.basedir,
548 targetdir=Config.targetdir,
549 stream=Config.stream,
550 refresh_rate=Config.refresh_rate,
551 use_xsendfile=Config.use_xsendfile,
552 sendfile_header=Config.sendfile_header,
553 content_disposition=Config.content_disposition,
554 splicer=engine)
555
556 if opts.lighttpd_hack:
557 app = LighttpdHackMiddleware(app)
558
559 if Config.protocol == 'http':
560 runner = partial(httpserver.serve,
561 app,
562 Config.interface,
563 Config.port)
564 else:
565 if Config.interface:
566 bindAddress = (Config.interface, Config.port)
567 debug('bindAddress: %s', bindAddress)
568 elif Config.unixsocket:
569 bindAddress = Config.unixsocket
570 if Config.protocol == 'scgi':
571 if Config.use_threads:
572 serverclass = SCGIThreadServer
573 else:
574 serverclass = SCGIServer
575 elif Config.protocol == 'fcgi':
576 if Config.use_threads:
577 serverclass = FCGIThreadServer
578 else:
579 serverclass = FCGIServer
580 if Config.protocol in ('fcgi', 'scgi'):
581 if Config.use_threads:
582 kw = dict(maxSpare=Config.max_spare,
583 minSpare=Config.min_spare,
584 maxThreads=Config.max_threads)
585 else:
586 kw = dict(maxSpare=Config.max_spare,
587 minSpare=Config.min_spare,
588 maxChildren=Config.max_children,
589 maxRequests=Config.max_requests)
590
591 kw = dict(i for i in kw.items() if not i[1] is None)
592 else:
593 kw = {}
594
595 runner = serverclass(app,
596 bindAddress=bindAddress,
597 umask=Config.umask,
598 **kw).run
599
600 try:
601 run_server(runner,
602 Config.pidfile,
603 Config.daemonize)
604 except SystemExit:
605 raise
606 except:
607 exception('exception caught')
608 sys.exit(1)
609 else:
610 sys.exit(0)
611
612
614 if not have_unix:
615
616 return
617
618 user = Config.user
619 group = Config.group
620 if user or group:
621
622
623
624
625 try:
626 seteuid = os.seteuid
627 except AttributeError:
628
629
630 pass
631 else:
632 seteuid(os.getuid())
633 if group:
634 gid = grp.getgrnam(group)[2]
635 os.setgid(gid)
636 if user:
637 uid = pwd.getpwnam(user)[2]
638 os.setuid(uid)
639
640
641 -def run_server(func, pidfile=None, daemonize=True):
642 debug("daemonize: %s, pidfile: %s", daemonize, pidfile)
643 if daemonize:
644 try:
645 os.fork
646 except NameError:
647 info("not daemonizing, as the fork() call is not available")
648 daemonize = False
649 if daemonize:
650 if os.fork():
651 os._exit(0)
652 if os.fork():
653 os._exit(0)
654 os.setsid()
655 os.chdir('/')
656 os.umask(0)
657 os.open(os.devnull, os.O_RDWR)
658 os.dup2(0, 1)
659 os.dup2(0, 2)
660 if pidfile:
661 pidfp = open(pidfile, 'w')
662 pidfp.write('%s' % os.getpid())
663 pidfp.close()
664
665 try:
666 _change_user_group()
667 try:
668 func()
669 except KeyboardInterrupt:
670 pass
671 finally:
672 if daemonize and pidfile:
673 try:
674 os.unlink(pidfile)
675 except OSError, e:
676
677 if e.errno != errno.ENOENT:
678 warn("problem unlinking pid file %s", pidfile)
679