1 import errno
2 import logging
3 from optparse import OptionParser
4 import os
5 from os import path
6 import re
7 import sys
8 import time
9
10 try:
11 import grp
12 import pwd
13 have_unix = 1
14 except ImportError:
15 have_unix = 0
16
17 try:
18 from functools import partial
19 except ImportError:
20
22
23 def inner(*_args, **_kw):
24 d = kw.copy()
25 d.update(_kw)
26 return func(*(args + _args), **d)
27 return inner
28
29 try:
30 from flup.server.scgi_fork import WSGIServer as SCGIServer
31 from flup.server.fcgi_fork import WSGIServer as FCGIServer
32 from flup.server.scgi import WSGIServer as SCGIThreadServer
33 from flup.server.fcgi import WSGIServer as FCGIThreadServer
34 haveflup = True
35 except ImportError:
36 haveflup = False
37 try:
38 from paste import httpserver
39 havepaste = True
40 except ImportError:
41 havepaste = False
42
43 import ewa.mp3
44 import ewa.audio
45 from ewa.config import Config, initConfig
46 from ewa.lighttpd_hack_middleware import LighttpdHackMiddleware
47 from ewa.logutil import (debug, error, exception, info, logger,
48 initLogging, warn)
49 from ewa.wsgiapp import EwaApp
50 from ewa.rules import FileRule
51 from ewa import __version__
52
53 VERSION_TEXT = """\
54 %%s %s
55
56 Copyright (C) 2007, 2010 WNYC New York Public Radio.
57 This is free software; see the source for copying conditions. There is NO
58 warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
59
60 Written by Jacob Smullyan and others, not necessarily in that order.
61 """ % __version__
62
63
64 SERVE_DESCRIPTION = """\
65 Starts a WSGI application that produces combined MP3 files
66 according to the specified rules.
67 """
68
69 RULE_DESCRIPTION = """\
70 Produces a combined MP3 file according to the specified rules.
71 """
72
73 SPLICE_DESCRIPTION = """\
74 Splices MP3 files together.
75 """
76
77
79 protocols = []
80 if haveflup:
81 protocols.extend(['fcgi', 'scgi'])
82 if havepaste:
83 protocols.append('http')
84 if not protocols:
85 print >> sys.stderr, ("no protocols available; "
86 "please install paste and/or flup")
87 sys.exit(1)
88 usage = "usage: %prog [options]"
89 parser = OptionParser(usage, description=SERVE_DESCRIPTION)
90 parser.add_option('-c',
91 '--config',
92 dest='configfile',
93 default=None,
94 help="path to ewa config file")
95 parser.add_option('-D',
96 '--nodaemonize',
97 action='store_true',
98 default=False,
99 dest='nodaemonize',
100 help="don't daemonize, regardless of config settings")
101 parser.add_option('--version',
102 action="store_true",
103 dest="version",
104 help="show version and exit")
105 parser.add_option('--lighttpd-hack',
106 action='store_true',
107 default=False,
108 dest='lighttpd_hack',
109 help=("hack for some versions of lighttpd "
110 "to force SCRIPT_NAME to ''"))
111 return parser
112
113
115 usage = "%prog [options] files"
116 parser = OptionParser(usage, description=SPLICE_DESCRIPTION)
117 parser.add_option('-o',
118 '--output',
119 dest='output',
120 help="output file (default: stdout)",
121 default='-',
122 metavar='OUT')
123 parser.add_option('-t',
124 '--tagfile',
125 dest="tagfile",
126 help="tag file",
127 default=None,
128 metavar="TAGFILE")
129 parser.add_option('-d',
130 '--debug',
131 action='store_true',
132 default=False,
133 help="print debugging information",
134 dest='debugmode')
135 parser.add_option('-s',
136 '--sanitycheck',
137 action='store_true',
138 default=False,
139 help="sanity check the input mp3 files",
140 dest='sanitycheck')
141 parser.add_option('-e',
142 '--engine',
143 default='default',
144 dest='engine',
145 metavar='ENGINE',
146 choices=('default', 'mp3cat', 'sox'),
147 help=("which splicing engine to use (default ewa "
148 "splicer, mp3cat, or sox)"))
149 parser.add_option('--version',
150 action="store_true",
151 dest="version",
152 help="show version and exit")
153 return parser
154
155
157 usage = "%prog [options] [files]"
158 parser = OptionParser(usage, description=RULE_DESCRIPTION)
159
160 parser.add_option('-c',
161 '--config',
162 dest='configfile',
163 default=None,
164 help="path to ewa config file")
165 parser.add_option('-r',
166 '--recursive',
167 dest='recursive',
168 help="recurse through directories",
169 action='store_true',
170 default=False)
171 parser.add_option('--rulefile',
172 dest='rulefile',
173 help="specify a rulefile",
174 metavar='RULEFILE')
175 parser.add_option('-d',
176 '--debug',
177 action='store_true',
178 default=False,
179 help="print debugging information",
180 dest='debugmode')
181 parser.add_option('-n',
182 '--dry-run',
183 action='store_true',
184 default=False,
185 dest='dryrun',
186 help="don't do anything, just print what would be done")
187 parser.add_option('-e',
188 '--engine',
189 default=None,
190 dest='engine',
191 metavar='ENGINE',
192 choices=('default', 'mp3cat', 'sox'),
193 help=("which splicing engine to use (default ewa "
194 "splicer, mp3cat, or sox)"))
195 parser.add_option('-a',
196 '--absolute',
197 default=False,
198 dest='absolute',
199 action='store_true',
200 help=("interpret file paths relative to the filesystem "
201 "rather than the basedir (default: no)"))
202
203 parser.add_option('-t',
204 '--configtest',
205 default=False,
206 dest='configtest',
207 action='store_true',
208 help="just test the config file for syntax errors")
209
210 parser.add_option('-x',
211 '--max-age',
212 default=0,
213 dest='max_age',
214 type=int,
215 help=('max age before generated files expire and '
216 'are regenerated. Default is 0, which means no '
217 'expiration. -1 will force regeneration.'))
218
219 parser.add_option('-D',
220 '--delete',
221 default=False,
222 action='store_true',
223 help=('delete files in combined directory that are '
224 'not in the the main directory'))
225
226 parser.add_option('-V', '--no-vbr',
227 default=True,
228 action='store_false',
229 dest='tolerate_vbr',
230 help="don't put vbr files in the combined directory")
231
232 parser.add_option('--version',
233 action="store_true",
234 dest="version",
235 help="show version and exit")
236 return parser
237
238
246
247
249 parser = get_splice_parser()
250 opts, args = parser.parse_args(args)
251 if opts.version:
252 print VERSION_TEXT % path.basename(sys.argv[0])
253 sys.exit(0)
254 if opts.debugmode:
255 Config.loglevel = logging.DEBUG
256 initLogging(level=Config.loglevel,
257 filename=Config.logfile)
258
259 engine = resolve_engine(opts.engine)
260 if opts.sanitycheck:
261 try:
262 ewa.mp3.mp3_sanity_check(args)
263 except Exception, e:
264 print >> sys.stderr, 'sanity check failed: %s' % str(e)
265 sys.exit(1)
266
267 use_stdout = opts.output == '-'
268 if use_stdout:
269 fp = sys.stdout
270 else:
271 fp = open(options.output, 'wb')
272 for chunk in ewa.mp3.splice(args, opts.tagfile, splicer=engine):
273 fp.write(chunk)
274 if not use_stdout:
275 fp.close()
276
277
279 parser = gs = get_ap_parser()
280 opts, args = parser.parse_args(args)
281 if opts.version:
282 print VERSION_TEXT % path.basename(sys.argv[0])
283 sys.exit(0)
284 configfile = _find_config(opts.configfile)
285 if not configfile:
286 parser.error('no config file specified or found in the usual places')
287 if not path.exists(configfile):
288 parser.error("config file does not exist: %s" % configfile)
289 initConfig(configfile)
290
291
292 if opts.debugmode:
293 Config.loglevel = 'debug'
294 if opts.rulefile:
295 Config.rulefile = opts.rulefile
296 if opts.engine:
297 Config.engine = engine
298 engine = resolve_engine(Config.engine)
299 initLogging(level=Config.loglevel)
300 rule = FileRule(Config.rulefile)
301
302 if opts.configtest:
303 print "Config OK"
304 sys.exit(0)
305
306 provider = ewa.audio.FSAudioProvider(Config.basedir,
307 opts.tolerate_vbr,
308 Config.targetdir)
309 mainpath = provider.get_main_path("")
310 if not mainpath.endswith('/'):
311
312
313 mainpath += '/'
314 if opts.absolute:
315 abs = [path.abspath(f) for f in args]
316 stripped_main = mainpath[:-1]
317 if not path.commonprefix([stripped_main] + abs) == stripped_main:
318 debug("absolute paths of files: %s", abs)
319 debug("mainpath: %s", mainpath)
320 parser.error("files outside managed directory")
321 idx = len(mainpath)
322 args = [x[idx:] for x in abs]
323 else:
324 args = [re.sub('/*(.*)', r'\1', x) for x in args]
325
326 if opts.delete:
327 if not opts.recursive:
328 parser.error("delete only works in recursive mode")
329 delete_finder = DeleteFinder(args, mainpath)
330 for file, isdir in delete_finder:
331 info('deleting %s', file)
332 if not opts.dryrun:
333 try:
334 if isdir:
335 os.rmdir(file)
336 else:
337 os.unlink(file)
338 except Exception, e:
339 error("couldn't unlink %s: %s", file, e)
340
341 if opts.recursive:
342
343 if opts.max_age == -1:
344 args = RecursiveMp3FileIterator(args, mainpath)
345 else:
346 args = RecursiveChangedMp3FileIterator(args,
347 mainpath,
348 opts.max_age)
349 if not args:
350 parser.error("no files specified")
351
352 if opts.dryrun:
353 for file in args:
354 debug("mp3 file: %s", file)
355 print "playlist for %s:" % file
356 for part in provider.get_playlist(file, rule, False):
357 print "\t%s" % part
358 sys.exit(0)
359 else:
360 _change_user_group()
361 for file in args:
362 try:
363 target = provider.create_combined(file, rule, splicer=engine)
364 except:
365 exception("error creating combined file for %s", file)
366 else:
367 debug('created %s', target)
368 sys.exit(0)
369
370
378
379 - def _yielder(self, root, apath, isdir):
380 targetpath = os.path.join(root, apath)
381 mainpath = path.join(self.basedir,
382 path.relpath(targetpath, self.targetdir))
383
384 debug('mainpath for %s is %s', apath, mainpath)
385 if not os.path.exists(mainpath):
386 yield targetpath, isdir
387
389 for f in (path.join(self.targetdir, x) for x in self.files):
390 if path.isdir(f):
391 debug('found directory: %s', f)
392 for root, dirs, files in os.walk(f, topdown=False):
393 for thing in files:
394 if thing.endswith('~'):
395
396 continue
397 for res in self._yielder(root, thing, 0):
398 yield res
399 for thing in dirs:
400 for res in self._yielder(root, thing, 1):
401 yield res
402
403
408
410 length = len(self.basedir)
411 for f in (path.join(self.basedir, x) for x in self.files):
412 if path.isdir(f):
413 debug('found directory: %s', f)
414 for mp3 in self._walk(f):
415 debug('raw value: %s', mp3)
416 yield mp3[length:]
417 else:
418 debug('found non-directory: %s', f)
419 yield f[length:]
420
422 for root, dirs, files in os.walk(f):
423 for f in files:
424 if f.endswith('.mp3') or f.endswith('MP3'):
425 yield path.join(root, f)
426
427
429
430 - def __init__(self, files, basedir, max_age=0):
436
438 for root, dirs, files in os.walk(f):
439 for f in files:
440 if f.endswith('.mp3') or f.endswith('MP3'):
441 fullpath = path.join(root, f)
442 debug('fullpath is %s', fullpath)
443 debug('basedir: %s; targetdir: %s',
444 self.basedir, self.targetdir)
445 targetpath = os.path.join(self.targetdir,
446 path.relpath(fullpath,
447 self.basedir))
448 try:
449 stats = os.stat(targetpath)
450 except OSError, ozzie:
451 if ozzie.errno == errno.ENOENT:
452
453 debug("target doesn't exist: %s", targetpath)
454 yield fullpath
455 else:
456 exception("error statting targetfile")
457
458 else:
459
460 target_mtime = stats.st_mtime
461 try:
462 original_mtime = path.getmtime(fullpath)
463 except OSError:
464 exception('peculiar error getting mtime of %s',
465 fullpath)
466 continue
467
468 if original_mtime > target_mtime:
469
470
471 yield fullpath
472 elif self.max_age <= 0:
473
474
475 continue
476 else:
477
478
479 mtime = stats.st_mtime
480 age = time.time() - mtime
481 if age > self.max_age * 60:
482 yield fullpath
483
484
486 for pth in (p for p in (
487 givenpath,
488 path.expanduser('~/.ewa.conf'),
489 path.expanduser('~/.ewa/ewa.conf'),
490 '/etc/ewa.conf',
491 '/etc/ewa/ewa.conf',
492 ) if p):
493 if path.exists(pth):
494 return pth
495
496
498 parser = get_serve_parser()
499 opts, args = parser.parse_args(args)
500 if opts.version:
501 print VERSION_TEXT % path.basename(sys.argv[0])
502 sys.exit(0)
503 configfile = _find_config(opts.configfile)
504 if not configfile:
505 parser.error('no config file specified or found in the usual places')
506 if not path.exists(configfile):
507 parser.error("config file does not exist: %s" % configfile)
508 initConfig(configfile)
509 if opts.nodaemonize:
510 Config.daemonize = False
511
512 if not Config.rulefile:
513 parser.error("a rulefile needs to be specified")
514 rule = FileRule(Config.rulefile)
515 if Config.logfile:
516 initLogging(level=Config.loglevel,
517 filename=Config.logfile,
518 rotate=Config.logrotate)
519
520 if Config.protocol == 'http' and Config.unixsocket:
521 parser.error('unix sockets not supported for http server')
522 if Config.umask and not Config.unixsocket:
523 parser.error('umask only applicable for unix sockets')
524 if Config.unixsocket and (Config.interface or Config.port):
525 parser.error('incompatible mixture of unix socket and tcp options')
526 engine = resolve_engine(Config.engine)
527
528 app = EwaApp(rule=rule,
529 basedir=Config.basedir,
530 targetdir=Config.targetdir,
531 stream=Config.stream,
532 refresh_rate=Config.refresh_rate,
533 use_xsendfile=Config.use_xsendfile,
534 sendfile_header=Config.sendfile_header,
535 content_disposition=Config.content_disposition,
536 splicer=engine)
537
538 if opts.lighttpd_hack:
539 app = LighttpdHackMiddleware(app)
540
541 if Config.protocol == 'http':
542 runner = partial(httpserver.serve,
543 app,
544 Config.interface,
545 Config.port)
546 else:
547 if Config.interface:
548 bindAddress = (Config.interface, Config.port)
549 debug('bindAddress: %s', bindAddress)
550 elif Config.unixsocket:
551 bindAddress = Config.unixsocket
552 if Config.protocol == 'scgi':
553 if Config.use_threads:
554 serverclass = SCGIThreadServer
555 else:
556 serverclass = SCGIServer
557 elif Config.protocol == 'fcgi':
558 if Config.use_threads:
559 serverclass = FCGIThreadServer
560 else:
561 serverclass = FCGIServer
562 if Config.protocol in ('fcgi', 'scgi'):
563 if Config.use_threads:
564 kw = dict(maxSpare=Config.max_spare,
565 minSpare=Config.min_spare,
566 maxThreads=Config.max_threads)
567 else:
568 kw = dict(maxSpare=Config.max_spare,
569 minSpare=Config.min_spare,
570 maxChildren=Config.max_children,
571 maxRequests=Config.max_requests)
572
573 kw = dict(i for i in kw.items() if not i[1] is None)
574 else:
575 kw = {}
576
577 runner = serverclass(app,
578 bindAddress=bindAddress,
579 umask=Config.umask,
580 **kw).run
581
582 try:
583 run_server(runner,
584 Config.pidfile,
585 Config.daemonize)
586 except SystemExit:
587 raise
588 except:
589 exception('exception caught')
590 sys.exit(1)
591 else:
592 sys.exit(0)
593
594
596 if not have_unix:
597
598 return
599
600 user = Config.user
601 group = Config.group
602 if user or group:
603
604
605
606
607 try:
608 seteuid = os.seteuid
609 except AttributeError:
610
611
612 pass
613 else:
614 seteuid(os.getuid())
615 if group:
616 gid = grp.getgrnam(group)[2]
617 os.setgid(gid)
618 if user:
619 uid = pwd.getpwnam(user)[2]
620 os.setuid(uid)
621
622
623 -def run_server(func, pidfile=None, daemonize=True):
624 debug("daemonize: %s, pidfile: %s", daemonize, pidfile)
625 if daemonize:
626 try:
627 os.fork
628 except NameError:
629 info("not daemonizing, as the fork() call is not available")
630 daemonize = False
631 if daemonize:
632 if os.fork():
633 os._exit(0)
634 if os.fork():
635 os._exit(0)
636 os.setsid()
637 os.chdir('/')
638 os.umask(0)
639 os.open(os.devnull, os.O_RDWR)
640 os.dup2(0, 1)
641 os.dup2(0, 2)
642 if pidfile:
643 pidfp = open(pidfile, 'w')
644 pidfp.write('%s' % os.getpid())
645 pidfp.close()
646
647 try:
648 _change_user_group()
649 try:
650 func()
651 except KeyboardInterrupt:
652 pass
653 finally:
654 if daemonize and pidfile:
655 try:
656 os.unlink(pidfile)
657 except OSError, e:
658
659 if e.errno != errno.ENOENT:
660 warn("problem unlinking pid file %s", pidfile)
661