1 import errno
2 import logging
3 from optparse import OptionParser
4 import os
5 from os import path
6 import re
7 import sys
8 import time
9
10 try:
11 import grp
12 import pwd
13 have_unix = 1
14 except ImportError:
15 have_unix = 0
16
17 try:
18 from functools import partial
19 except ImportError:
20
22
23 def inner(*_args, **_kw):
24 d = kw.copy()
25 d.update(_kw)
26 return func(*(args + _args), **d)
27 return inner
28
29 try:
30 from flup.server.scgi_fork import WSGIServer as SCGIServer
31 from flup.server.fcgi_fork import WSGIServer as FCGIServer
32 from flup.server.scgi import WSGIServer as SCGIThreadServer
33 from flup.server.fcgi import WSGIServer as FCGIThreadServer
34 haveflup = True
35 except ImportError:
36 haveflup = False
37 try:
38 from paste import httpserver
39 havepaste = True
40 except ImportError:
41 havepaste = False
42
43 import ewa.mp3
44 import ewa.audio
45 from ewa.config import Config, initConfig
46 from ewa.lighttpd_hack_middleware import LighttpdHackMiddleware
47 from ewa.logutil import (debug, error, exception, info, logger,
48 initLogging, warn)
49 from ewa.wsgiapp import EwaApp
50 from ewa.rules import FileRule
51 from ewa import __version__
52
53 VERSION_TEXT = """\
54 %%s %s
55
56 Copyright (C) 2007, 2010 WNYC New York Public Radio.
57 This is free software; see the source for copying conditions. There is NO
58 warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
59
60 Written by Jacob Smullyan and others, not necessarily in that order.
61 """ % __version__
62
63
64 SERVE_DESCRIPTION = """\
65 Starts a WSGI application that produces combined MP3 files
66 according to the specified rules.
67 """
68
69 RULE_DESCRIPTION = """\
70 Produces a combined MP3 file according to the specified rules.
71 """
72
73 SPLICE_DESCRIPTION = """\
74 Splices MP3 files together.
75 """
76
77
79 protocols = []
80 if haveflup:
81 protocols.extend(['fcgi', 'scgi'])
82 if havepaste:
83 protocols.append('http')
84 if not protocols:
85 print >> sys.stderr, ("no protocols available; "
86 "please install paste and/or flup")
87 sys.exit(1)
88 usage = "usage: %prog [options]"
89 parser = OptionParser(usage, description=SERVE_DESCRIPTION)
90 parser.add_option('-c',
91 '--config',
92 dest='configfile',
93 default=None,
94 help="path to ewa config file")
95 parser.add_option('-D',
96 '--nodaemonize',
97 action='store_true',
98 default=False,
99 dest='nodaemonize',
100 help="don't daemonize, regardless of config settings")
101 parser.add_option('--version',
102 action="store_true",
103 dest="version",
104 help="show version and exit")
105 parser.add_option('--lighttpd-hack',
106 action='store_true',
107 default=False,
108 dest='lighttpd_hack',
109 help=("hack for some versions of lighttpd "
110 "to force SCRIPT_NAME to ''"))
111 return parser
112
113
115 usage = "%prog [options] files"
116 parser = OptionParser(usage, description=SPLICE_DESCRIPTION)
117 parser.add_option('-o',
118 '--output',
119 dest='output',
120 help="output file (default: stdout)",
121 default='-',
122 metavar='OUT')
123 parser.add_option('-t',
124 '--tagfile',
125 dest="tagfile",
126 help="tag file",
127 default=None,
128 metavar="TAGFILE")
129 parser.add_option('-d',
130 '--debug',
131 action='store_true',
132 default=False,
133 help="print debugging information",
134 dest='debugmode')
135 parser.add_option('-s',
136 '--sanitycheck',
137 action='store_true',
138 default=False,
139 help="sanity check the input mp3 files",
140 dest='sanitycheck')
141 parser.add_option('-e',
142 '--engine',
143 default='default',
144 dest='engine',
145 metavar='ENGINE',
146 choices=('default', 'mp3cat', 'sox'),
147 help=("which splicing engine to use (default ewa "
148 "splicer, mp3cat, or sox)"))
149 parser.add_option('--version',
150 action="store_true",
151 dest="version",
152 help="show version and exit")
153 return parser
154
155
157 usage = "%prog [options] [files]"
158 parser = OptionParser(usage, description=RULE_DESCRIPTION)
159
160 parser.add_option('-c',
161 '--config',
162 dest='configfile',
163 default=None,
164 help="path to ewa config file")
165 parser.add_option('-r',
166 '--recursive',
167 dest='recursive',
168 help="recurse through directories",
169 action='store_true',
170 default=False)
171 parser.add_option('--rulefile',
172 dest='rulefile',
173 help="specify a rulefile",
174 metavar='RULEFILE')
175 parser.add_option('-d',
176 '--debug',
177 action='store_true',
178 default=False,
179 help="print debugging information",
180 dest='debugmode')
181 parser.add_option('-n',
182 '--dry-run',
183 action='store_true',
184 default=False,
185 dest='dryrun',
186 help="don't do anything, just print what would be done")
187 parser.add_option('-e',
188 '--engine',
189 default=None,
190 dest='engine',
191 metavar='ENGINE',
192 choices=('default', 'mp3cat', 'sox'),
193 help=("which splicing engine to use (default ewa "
194 "splicer, mp3cat, or sox)"))
195 parser.add_option('-a',
196 '--absolute',
197 default=False,
198 dest='absolute',
199 action='store_true',
200 help=("interpret file paths relative to the filesystem "
201 "rather than the basedir (default: no)"))
202
203 parser.add_option('-t',
204 '--configtest',
205 default=False,
206 dest='configtest',
207 action='store_true',
208 help="just test the config file for syntax errors")
209
210 parser.add_option('-x',
211 '--max-age',
212 default=0,
213 dest='max_age',
214 type=int,
215 help=('max age before generated files expire and '
216 'are regenerated. Default is 0, which means no '
217 'expiration. -1 will force regeneration.'))
218
219 parser.add_option('-D',
220 '--delete',
221 default=False,
222 action='store_true',
223 help=('delete files in combined directory that are '
224 'not in the the main directory'))
225
226 parser.add_option('--version',
227 action="store_true",
228 dest="version",
229 help="show version and exit")
230 return parser
231
232
240
241
243 parser = get_splice_parser()
244 opts, args = parser.parse_args(args)
245 if opts.version:
246 print VERSION_TEXT % path.basename(sys.argv[0])
247 sys.exit(0)
248 if opts.debugmode:
249 Config.loglevel = logging.DEBUG
250 initLogging(level=Config.loglevel,
251 filename=Config.logfile)
252
253 engine = resolve_engine(opts.engine)
254 if opts.sanitycheck:
255 try:
256 ewa.mp3.mp3_sanity_check(args)
257 except Exception, e:
258 print >> sys.stderr, 'sanity check failed: %s' % str(e)
259 sys.exit(1)
260
261 use_stdout = opts.output == '-'
262 if use_stdout:
263 fp = sys.stdout
264 else:
265 fp = open(options.output, 'wb')
266 for chunk in ewa.mp3.splice(args, opts.tagfile, splicer=engine):
267 fp.write(chunk)
268 if not use_stdout:
269 fp.close()
270
271
273 parser = gs = get_ap_parser()
274 opts, args = parser.parse_args(args)
275 if opts.version:
276 print VERSION_TEXT % path.basename(sys.argv[0])
277 sys.exit(0)
278 configfile = _find_config(opts.configfile)
279 if not configfile:
280 parser.error('no config file specified or found in the usual places')
281 if not path.exists(configfile):
282 parser.error("config file does not exist: %s" % configfile)
283 initConfig(configfile)
284
285
286 if opts.debugmode:
287 Config.loglevel = 'debug'
288 if opts.rulefile:
289 Config.rulefile = opts.rulefile
290 if opts.engine:
291 Config.engine = engine
292 engine = resolve_engine(Config.engine)
293 initLogging(level=Config.loglevel)
294 rule = FileRule(Config.rulefile)
295
296 if opts.configtest:
297 print "Config OK"
298 sys.exit(0)
299
300 provider = ewa.audio.FSAudioProvider(Config.basedir,
301 Config.targetdir)
302 mainpath = provider.get_main_path("")
303 if not mainpath.endswith('/'):
304
305
306 mainpath += '/'
307 if opts.absolute:
308 abs = [path.abspath(f) for f in args]
309 stripped_main = mainpath[:-1]
310 if not path.commonprefix([stripped_main] + abs) == stripped_main:
311 debug("absolute paths of files: %s", abs)
312 debug("mainpath: %s", mainpath)
313 parser.error("files outside managed directory")
314 idx = len(mainpath)
315 args = [x[idx:] for x in abs]
316 else:
317 args = [re.sub('/*(.*)', r'\1', x) for x in args]
318
319 if opts.delete:
320 if not opts.recursive:
321 parser.error("delete only works in recursive mode")
322 delete_finder = DeleteFinder(args, mainpath)
323 for file, isdir in delete_finder:
324 info('deleting %s', file)
325 if not opts.dryrun:
326 try:
327 if isdir:
328 os.rmdir(file)
329 else:
330 os.unlink(file)
331 except Exception, e:
332 error("couldn't unlink %s: %s", file, e)
333
334 if opts.recursive:
335
336 if opts.max_age == -1:
337 args = RecursiveMp3FileIterator(args, mainpath)
338 else:
339 args = RecursiveChangedMp3FileIterator(args,
340 mainpath,
341 opts.max_age)
342 if not args:
343 parser.error("no files specified")
344
345 if opts.dryrun:
346 for file in args:
347 debug("mp3 file: %s", file)
348 print "playlist for %s:" % file
349 for part in provider.get_playlist(file, rule, False):
350 print "\t%s" % part
351 sys.exit(0)
352 else:
353 _change_user_group()
354 for file in args:
355 try:
356 target = provider.create_combined(file, rule, splicer=engine)
357 except:
358 exception("error creating combined file for %s", file)
359 else:
360 debug('created %s', target)
361 sys.exit(0)
362
363
371
372 - def _yielder(self, root, apath, isdir):
373 targetpath = os.path.join(root, apath)
374 mainpath = path.join(self.basedir,
375 path.relpath(targetpath, self.targetdir))
376
377 debug('mainpath for %s is %s', apath, mainpath)
378 if not os.path.exists(mainpath):
379 yield targetpath, isdir
380
382 for f in (path.join(self.targetdir, x) for x in self.files):
383 if path.isdir(f):
384 debug('found directory: %s', f)
385 for root, dirs, files in os.walk(f, topdown=False):
386 for thing in files:
387 if thing.endswith('~'):
388
389 continue
390 for res in self._yielder(root, thing, 0):
391 yield res
392 for thing in dirs:
393 for res in self._yielder(root, thing, 1):
394 yield res
395
396
401
403 length = len(self.basedir)
404 for f in (path.join(self.basedir, x) for x in self.files):
405 if path.isdir(f):
406 debug('found directory: %s', f)
407 for mp3 in self._walk(f):
408 debug('raw value: %s', mp3)
409 yield mp3[length:]
410 else:
411 debug('found non-directory: %s', f)
412 yield f[length:]
413
415 for root, dirs, files in os.walk(f):
416 for f in files:
417 if f.endswith('.mp3') or f.endswith('MP3'):
418 yield path.join(root, f)
419
420
422
423 - def __init__(self, files, basedir, max_age=0):
429
431 for root, dirs, files in os.walk(f):
432 for f in files:
433 if f.endswith('.mp3') or f.endswith('MP3'):
434 fullpath = path.join(root, f)
435 debug('fullpath is %s', fullpath)
436 debug('basedir: %s; targetdir: %s',
437 self.basedir, self.targetdir)
438 targetpath = os.path.join(self.targetdir,
439 path.relpath(fullpath,
440 self.basedir))
441 try:
442 stats = os.stat(targetpath)
443 except OSError, ozzie:
444 if ozzie.errno == errno.ENOENT:
445
446 debug("target doesn't exist: %s", targetpath)
447 yield fullpath
448 else:
449 exception("error statting targetfile")
450
451 else:
452
453 target_mtime = stats.st_mtime
454 try:
455 original_mtime = path.getmtime(fullpath)
456 except OSError:
457 exception('peculiar error getting mtime of %s',
458 fullpath)
459 continue
460
461 if original_mtime > target_mtime:
462
463
464 yield fullpath
465 elif self.max_age <= 0:
466
467
468 continue
469 else:
470
471
472 mtime = stats.st_mtime
473 age = time.time() - mtime
474 if age > self.max_age * 60:
475 yield fullpath
476
477
479 for pth in (p for p in (
480 givenpath,
481 path.expanduser('~/.ewa.conf'),
482 path.expanduser('~/.ewa/ewa.conf'),
483 '/etc/ewa.conf',
484 '/etc/ewa/ewa.conf',
485 ) if p):
486 if path.exists(pth):
487 return pth
488
489
491 parser = get_serve_parser()
492 opts, args = parser.parse_args(args)
493 if opts.version:
494 print VERSION_TEXT % path.basename(sys.argv[0])
495 sys.exit(0)
496 configfile = _find_config(opts.configfile)
497 if not configfile:
498 parser.error('no config file specified or found in the usual places')
499 if not path.exists(configfile):
500 parser.error("config file does not exist: %s" % configfile)
501 initConfig(configfile)
502 if opts.nodaemonize:
503 Config.daemonize = False
504
505 if not Config.rulefile:
506 parser.error("a rulefile needs to be specified")
507 rule = FileRule(Config.rulefile)
508 if Config.logfile:
509 initLogging(level=Config.loglevel,
510 filename=Config.logfile,
511 rotate=Config.logrotate)
512
513 if Config.protocol == 'http' and Config.unixsocket:
514 parser.error('unix sockets not supported for http server')
515 if Config.umask and not Config.unixsocket:
516 parser.error('umask only applicable for unix sockets')
517 if Config.unixsocket and (Config.interface or Config.port):
518 parser.error('incompatible mixture of unix socket and tcp options')
519 engine = resolve_engine(Config.engine)
520
521 app = EwaApp(rule=rule,
522 basedir=Config.basedir,
523 targetdir=Config.targetdir,
524 stream=Config.stream,
525 refresh_rate=Config.refresh_rate,
526 use_xsendfile=Config.use_xsendfile,
527 sendfile_header=Config.sendfile_header,
528 content_disposition=Config.content_disposition,
529 splicer=engine)
530
531 if opts.lighttpd_hack:
532 app = LighttpdHackMiddleware(app)
533
534 if Config.protocol == 'http':
535 runner = partial(httpserver.serve,
536 app,
537 Config.interface,
538 Config.port)
539 else:
540 if Config.interface:
541 bindAddress = (Config.interface, Config.port)
542 debug('bindAddress: %s', bindAddress)
543 elif Config.unixsocket:
544 bindAddress = Config.unixsocket
545 if Config.protocol == 'scgi':
546 if Config.use_threads:
547 serverclass = SCGIThreadServer
548 else:
549 serverclass = SCGIServer
550 elif Config.protocol == 'fcgi':
551 if Config.use_threads:
552 serverclass = FCGIThreadServer
553 else:
554 serverclass = FCGIServer
555 if Config.protocol in ('fcgi', 'scgi'):
556 if Config.use_threads:
557 kw = dict(maxSpare=Config.max_spare,
558 minSpare=Config.min_spare,
559 maxThreads=Config.max_threads)
560 else:
561 kw = dict(maxSpare=Config.max_spare,
562 minSpare=Config.min_spare,
563 maxChildren=Config.max_children,
564 maxRequests=Config.max_requests)
565
566 kw = dict(i for i in kw.items() if not i[1] is None)
567 else:
568 kw = {}
569
570 runner = serverclass(app,
571 bindAddress=bindAddress,
572 umask=Config.umask,
573 **kw).run
574
575 try:
576 run_server(runner,
577 Config.pidfile,
578 Config.daemonize)
579 except SystemExit:
580 raise
581 except:
582 exception('exception caught')
583 sys.exit(1)
584 else:
585 sys.exit(0)
586
587
589 if not have_unix:
590
591 return
592
593 user = Config.user
594 group = Config.group
595 if user or group:
596
597
598
599
600 try:
601 seteuid = os.seteuid
602 except AttributeError:
603
604
605 pass
606 else:
607 seteuid(os.getuid())
608 if group:
609 gid = grp.getgrnam(group)[2]
610 os.setgid(gid)
611 if user:
612 uid = pwd.getpwnam(user)[2]
613 os.setuid(uid)
614
615
616 -def run_server(func, pidfile=None, daemonize=True):
617 debug("daemonize: %s, pidfile: %s", daemonize, pidfile)
618 if daemonize:
619 try:
620 os.fork
621 except NameError:
622 info("not daemonizing, as the fork() call is not available")
623 daemonize = False
624 if daemonize:
625 if os.fork():
626 os._exit(0)
627 if os.fork():
628 os._exit(0)
629 os.setsid()
630 os.chdir('/')
631 os.umask(0)
632 os.open(os.devnull, os.O_RDWR)
633 os.dup2(0, 1)
634 os.dup2(0, 2)
635 if pidfile:
636 pidfp = open(pidfile, 'w')
637 pidfp.write('%s' % os.getpid())
638 pidfp.close()
639
640 try:
641 _change_user_group()
642 try:
643 func()
644 except KeyboardInterrupt:
645 pass
646 finally:
647 if daemonize and pidfile:
648 try:
649 os.unlink(pidfile)
650 except OSError, e:
651
652 if e.errno != errno.ENOENT:
653 warn("problem unlinking pid file %s", pidfile)
654