Commit 6fdce115 authored by Skia's avatar Skia

Initial commit: save an old SVN repo

parents
#!/usr/bin/env python
import BaseHTTPServer
import logging
import sys
import threading
import urlparse
import jamendo_downloader
FORM = '''<html><body>
<h1>Jamendo downloader</h1>
<p>%sCurrently <b>%d</b> URLs in queue.</p>
<br>
<form action="/jamendo/" method="POST">
<input type="hidden" name="kind" value="artistname">
Artist name: <input type="text" name="id">
<input type="submit" name="submit" value="Fetch from artist name"
</form>
<br>
<form action="/jamendo/" method="POST">
<input type="hidden" name="kind" value="artist">
Artist ID: <input type="text" name="id">
<input type="submit" name="submit" value="Fetch from artist ID"
</form>
<br>
<form action="/jamendo/" method="POST">
<input type="hidden" name="kind" value="album">
Album ID: <input type="text" name="id">
<input type="submit" name="submit" value="Fetch from album ID"
</form>
<br>
<form action="/jamendo/" method="POST">
<input type="hidden" name="kind" value="track">
Track ID: <input type="text" name="id">
<input type="submit" name="submit" value="Fetch from track ID"
</form>
<br>
</body></html>
'''
def parse_qs(qs):
if not qs:
return {}
else:
return dict(x.split('=', 1) for x in qs.split('&'))
class WebRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler):
def serve_form(self, message=""):
data = FORM % (message, self.server.downloader.queue.qsize())
self.send_response(200)
self.send_header('Content-Type', 'text/html')
self.end_headers()
self.wfile.write(data)
def do_GET(self):
if '?' in self.path:
path, qs = self.path.split('?', 1)
else:
path, qs = self.path, ''
if path != '/jamendo/':
self.send_error(404, 'URL does not exist.')
return
params = parse_qs(qs)
if 'ok' in params:
if params['ok'] == '1':
msg = 'Add successful. '
else:
msg = 'Add failed. '
else:
msg = ''
self.serve_form(msg)
def do_POST(self):
if self.path != '/jamendo/':
self.send_error(404, 'URL does not exist.')
return
if self.headers['Content-Type'] != 'application/x-www-form-urlencoded':
self.send_error(400, 'Bad form data')
return
data = self.rfile.read(int(self.headers['Content-Length']))
print data
params = parse_qs(data)
if self.server.downloader.add_smart(params['kind'], params['id']):
ok = '1'
else:
ok = '0'
self.send_response(303)
self.send_header('Location', '/jamendo/?ok=' + ok)
self.end_headers()
class PermanentDownloader(jamendo_downloader.Downloader):
def start(self):
t = threading.Thread(target=self.run)
t.setDaemon(True)
t.start()
def should_stop(self):
return False
def serve_web(downloader):
addr = ('127.0.0.1', 8000)
httpd = BaseHTTPServer.HTTPServer(addr, WebRequestHandler)
httpd.downloader = downloader
try:
httpd.serve_forever()
except KeyboardInterrupt:
sys.exit(0)
def main():
logging.basicConfig(level=logging.INFO)
downloader = PermanentDownloader(sys.argv[1])
downloader.start()
serve_web(downloader)
if __name__ == '__main__':
main()
#!/usr/bin/env python
# API calls
#
# http://api.jamendo.com/get2/stream/track/m3u/?id=116&streamencoding=ogg2
# http://api.jamendo.com/get2/stream/track/m3u/?album_id=116&streamencoding=ogg2
# http://api.jamendo.com/get2/stream/album/m3u/?artist_id=5&streamencoding=ogg2
import logging
import Queue
import os.path
import sha
import sys
import threading
import time
import urllib
log = logging.getLogger('download_queue').info
URLS = {
'track': 'http://api.jamendo.com/get2/stream/track/m3u/?id=%s&streamencoding=ogg2&n=all',
'album': 'http://api.jamendo.com/get2/stream/track/m3u/?album_id=%s&streamencoding=ogg2&n=all',
'artist': 'http://api.jamendo.com/get2/stream/album/m3u/?artist_id=%s&streamencoding=ogg2&n=all',
}
ARTIST_LOOKUP_URL = 'http://api.jamendo.com/get2/id/artist/plain/?idstr=%s'
class Downloader(object):
def __init__(self, out_dir):
self.queue = Queue.Queue()
self.out_dir = out_dir
self.sleep_time = 0
def add(self, url):
if not url.startswith('http://'):
log('Not adding invalid URL %s' % url)
else:
log('Adding to queue ' + url)
self.queue.put(url)
def add_smart(self, type, identifier):
identifier = urllib.quote_plus(urllib.unquote_plus(identifier))
if type == 'artistname':
try:
id = int(self.get_contents(ARTIST_LOOKUP_URL % identifier))
except ValueError:
log('Unknown artist %s' % identifier)
return False
else:
if id == 0:
log('Uknown artist %s' % identifier)
return False
type = 'artist'
identifier = str(id)
if type not in URLS:
log('Ignoring smart add for unknown type %s' % type)
return False
else:
self.add(URLS[type] % identifier)
return True
def run(self):
while not self.should_stop():
log('%d URLs to fetch' % self.queue.qsize())
time.sleep(self.sleep_time)
url = self.queue.get()
if url.endswith('.ogg'):
self.download_ogg(url)
elif url.endswith('.mp3'):
raise AssertionError('No mp3 should be downloaded')
else:
self.download_m3u(url)
def should_stop(self):
return self.queue.empty()
def get_contents(self, url):
fd = urllib.urlopen(url)
data = fd.read()
fd.close()
return data
def download_ogg(self, url):
log('Downloading OGG ' + url)
data = self.get_contents(url)
filename = os.path.join(self.out_dir,
sha.new(data).hexdigest() + '.ogg')
fd = open(filename, 'w')
fd.write(data)
fd.close()
log('Saved as ' + filename)
self.sleep_time = 5
def download_m3u(self, url):
# Some parameters gets stripped sometimes, check and possibly
# add back
if '&streamencoding' not in url:
url += '&streamencoding=ogg2'
if '&n=' not in url:
url += '&n=all'
log('Downloading M3U ' + url)
data = self.get_contents(url)
for line in data.split('\n'):
if not line.strip() or line.strip().startswith('#'):
continue
self.add(line)
self.sleep_time = 1
def main():
logging.basicConfig(level=logging.INFO)
if len(sys.argv) != 4:
print 'Usage: %s <output dir> (artist|album|track) <entity ID>' % sys.argv[0]
sys.exit(2)
out_dir, type, identifier = sys.argv[1:4]
if not os.path.isdir(out_dir):
print 'Output directory %s does not exist' % out_dir
sys.exit(2)
if type not in URLS:
print 'Unknown entity type %s' % type
sys.exit(2)
d = Downloader(out_dir)
d.add_smart(type, identifier)
d.run()
print 'Retrieval complete.'
if __name__ == '__main__':
main()
import BaseHTTPServer
import logging
import threading
class HealthRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler):
def do_GET(self):
self.send_response(200)
self.send_header('Content-Type', 'text/plain')
self.end_headers()
self.wfile.write('alive')
def log_request(*args, **kwargs):
pass
def log_error(*args, **kwargs):
pass
def log_message(*args, **kwargs):
pass
def serve_health():
addr = ('127.0.0.1', 8001)
httpd = BaseHTTPServer.HTTPServer(addr, HealthRequestHandler)
logging.getLogger('health').info('Serving health info on localhost:8001')
httpd.serve_forever()
def serve_health_thread():
t = threading.Thread(target=serve_health)
t.setDaemon(True)
t.start()
# Music importer.
import logging
import os
import os.path
import sha
import subprocess
import threading
import time
from mutagen import oggvorbis, easyid3, mp3
from django.conf import settings
from django.db import transaction
from streamux.radio import models
IDLE_INTERVAL = 10 # seconds
log = logging.getLogger('importer')
class ProcessingError(RuntimeError):
"""An error occured trying to process an incoming file."""
DEFAULT_TAGS = {
'album': None,
'mbid': '',
'artist_mbid': '',
'album_mbid': ''
}
# All supported extensions, and their associated tag decoders and
# exception base classes.
_SUPPORTED_EXTENSIONS = ('ogg', 'mp3')
def get_tags_mp3(filename):
try:
tag = mp3.MP3(filename)
except (mp3.error, easyid3.error):
raise ProcessingError('Could not parse ID3 tag')
if 'TIT2' not in tag:
raise ProcessingError('No title in ID3 tag')
if 'TPE1' not in tag:
raise ProcessingError('No artist in ID3 tag')
out = DEFAULT_TAGS.copy()
out['title'] = make_unicode(tag['TIT2'].text[0])
out['artist'] = make_unicode(tag['TPE1'].text[0])
if 'TALB' in tag:
out['album'] = make_unicode(tag['TALB'].text[0])
if 'UFID:http://musicbrainz.org' in tag:
out['mbid'] = tag['UFID:http://musicbrainz.org'].data
if 'TXXX:MusicBrainz Album Id' in tag:
out['album_mbid'] = tag['TXXX:MusicBrainz Album Id'].text[0]
if 'TXXX:MusicBrainz Artist Id' in tag:
out['artist_mbid'] = tag['TXXX:MusicBrainz Artist Id'].text[0]
return out
def get_tags_ogg(filename):
try:
tag = oggvorbis.OggVorbis(filename)
except oggvorbis.error:
raise ProcessingError('Could not parse Ogg Vorbis tag')
if 'title' not in tag:
raise ProcessingError('No title in Ogg tag')
if 'artist' not in tag:
raise ProcessingError('No artist in Ogg tag')
out = DEFAULT_TAGS.copy()
out['title'] = make_unicode(tag['title'][0])
out['artist'] = make_unicode(tag['artist'][0])
if 'album' in tag:
out['album'] = make_unicode(tag['album'][0])
if 'musicbrainz_trackid' in tag:
out['mbid'] = tag['musicbrainz_trackid'][0]
if 'musicbrainz_albumid' in tag:
out['album_mbid'] = tag['musicbrainz_albumid'][0]
if 'musicbrainz_artistid' in tag:
out['artist_mbid'] = tag['musicbrainz_artistid'][0]
return out
_TAG_DECODERS = {
'ogg': get_tags_ogg,
'mp3': get_tags_mp3,
}
def replay_gain_ogg(filepath):
output = subprocess.Popen('nice vorbisgain -n -d "%s"' % filepath,
shell=True,
stdout=subprocess.PIPE).stdout.read()
if not output:
raise ProcessingError('vorbisgain failed to run')
log.debug('vorbisgain output: ' + output)
# Given vorbisgain's output, we want the first 'word' of the last
# line.
return output.strip().split('\n')[-1].split()[0]
def replay_gain_mp3(filepath):
log.debug('Running mp3gain on ' + filepath)
output = subprocess.Popen('nice mp3gain -s s -q -o "%s"' % filepath,
shell=True,
stdout=subprocess.PIPE).stdout.read()
if not output:
raise ProcessingError('mp3gain failed to run')
log.debug('mp3gain output: ' + output)
# mp3gain's "database" output places the dB replay gain in the 3rd
# field of the last line. Actually, the last line is "album" gain,
# which is identical to track gain for a single track.
return output.strip().split('\n')[-1].split('\t')[2]
# All supported extensions, and their associated replay gain calculation functions.
_GAIN_CALCULATORS = {
'ogg': replay_gain_ogg,
'mp3': replay_gain_mp3
}
def calculate_replaygain(filepath, extension):
try:
gain = _GAIN_CALCULATORS[extension](filepath)
except IndexError:
raise ProcessingError('Gain calculator did not return gain')
# Check that the gain does look like a floating point number
try:
float(gain)
except ValueError:
raise ProcessingError(
'Invalid gain returned: ' + gain)
return gain
def get_extension(filepath):
if '.' not in filepath:
raise ProcessingError('No extension found')
extension = filepath.rsplit('.', 1)[1]
if extension not in _SUPPORTED_EXTENSIONS:
raise ProcessingError('Unsupported extension .' + extension)
return extension
def make_unicode(input):
if isinstance(input, unicode):
return input
try:
return input.decode('utf-8')
except UnicodeDecodeError:
raise ProcessingError('Unhandled tag encoding')
def content_hash(filepath):
fd = open(filepath)
hash = sha.new(fd.read()).hexdigest()
fd.close()
return hash
FILECHARS_TRANSLATE_TABLE = [chr(x) for x in xrange(256)]
FILECHARS_TRANSLATE_TABLE[:45] = 45*'_'
FILECHARS_TRANSLATE_TABLE[47] = '_'
FILECHARS_TRANSLATE_TABLE[58:65] = 7*'_'
FILECHARS_TRANSLATE_TABLE[91:97] = 6*'_'
FILECHARS_TRANSLATE_TABLE[123:] = 133*'_'
FILECHARS_TRANSLATE_TABLE = ''.join(FILECHARS_TRANSLATE_TABLE)
def install_file(filepath, extension, tags):
# The new name of the file contains sanitized versions of all
# its components, as well as a content hash.
hash = content_hash(filepath)
artist = tags['artist'].encode(
'ascii', 'replace').translate(FILECHARS_TRANSLATE_TABLE)
if tags['album']:
album = tags['album'].encode(
'ascii', 'replace').translate(FILECHARS_TRANSLATE_TABLE)
else:
album = 'NoAlbum'
track = tags['title'].encode(
'ascii', 'replace').translate(FILECHARS_TRANSLATE_TABLE)
new_filename = u'%s_%s_%s_%s.%s' % (hash, artist, album,
track, extension)
new_filedir = os.path.join(settings.MUSIC_DIR, artist)
new_filepath = os.path.join(new_filedir, new_filename)
logging.debug('Installing %s as %s' % (filepath, new_filepath))
if not os.path.exists(new_filedir):
os.makedirs(new_filedir)
os.rename(filepath, new_filepath)
return new_filepath
def register_db_file(filepath, tags, replay_gain):
# First check if we know any of the artist, album or track, by
# name or MBID (if provided)
artist = models.Artist.objects.get_or_create(
name=tags['artist'], mbid=tags['artist_mbid'])[0]
artist.save()
if tags['album']:
album = models.Album.objects.get_or_create(
name=tags['album'], mbid=tags['album_mbid'],
artist=artist)[0]
album.save()
else:
album = None
assert filepath.startswith(settings.MUSIC_DIR)
filepath = filepath[len(settings.MUSIC_DIR):]
track = models.Track.objects.get_or_create(
name=tags['title'], artist=artist, album=album, mbid=tags['mbid'],
replay_gain=replay_gain, file=filepath)[0]
track.save()
def fail(filepath):
hash = content_hash(filepath)
new_filename = '%s_%s' % (hash, os.path.basename(filepath))
new_filepath = os.path.join(settings.MUSIC_FAILED, new_filename)
log.error('Saved failed track as ' + new_filename)
os.rename(filepath, new_filepath)
log.debug('Renamed %s to %s' % (filepath, new_filepath))
def process_file(filepath):
log.info('Processing new file ' + filepath)
extension = get_extension(filepath)
tags = _TAG_DECODERS[extension](filepath)
replay_gain = calculate_replaygain(filepath, extension)
filepath = install_file(filepath, extension, tags)
register_db_file(filepath, tags, replay_gain)
log.info('Successfully registered new track: %s - %s'
% (tags['artist'], tags['title']))
def process_incoming():
processed = 0
for root, _, files in os.walk(settings.MUSIC_INCOMING):
for file in files:
filepath = os.path.join(root, file)
try:
process_file(filepath)
except ProcessingError, e:
log.error('Processing error: %s' % e.args[0])
fail(filepath)
processed += 1
log.debug('Run complete, processed %d new files' % processed)
class MusicImporter(threading.Thread):
def __init__(self):
threading.Thread.__init__(self)
self.setDaemon(True)
def run(self):
log.info('Importer thread started')
log.info('Monitoring ' + settings.MUSIC_INCOMING)
while True:
if not process_incoming():
time.sleep(IDLE_INTERVAL)
#!/usr/bin/env python
import logging
import os
import sys
from django.core import management
from django.utils import daemonize
def run_daemon():
# Add the grandparent of this binary's path to the path, so that
# django finds all of Streamux's bits and pieces.
path = os.path.abspath(os.path.join(os.path.dirname(__file__),
'../..'))
print path
sys.path.insert(0, path)
# Get Django to configure itself, so that we can use the Streamux
# DB models to access the database.
from streamux import settings
management.setup_environ(settings)
from streamux.daemon import importer, health
# Configure logging and possibly background.
if settings.DAEMON_NO_BACKGROUND:
logging.basicConfig(
filename=settings.DAEMON_LOGFILE, level=logging.DEBUG,
format='%(asctime)s %(levelname)-8s %(message)s',
datefmt='%m-%d %H:%M')
console = logging.StreamHandler()
console.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(levelname)s: %(message)s')
console.setFormatter(formatter)
logging.getLogger('').addHandler(console)
logging.info('Not backgrounding, debug output to stderr')
else:
logging.basicConfig(
filename=settings.DAEMON_LOGFILE, level=logging.INFO,
format='%(asctime)s %(levelname)-8s %(message)s',
datefmt='%m-%d %H:%M')
print 'Backgrounding...'
daemonize.become_daemon()
# Write the PID file so that external apps can track us.
pidfile = open(settings.DAEMON_PIDFILE, 'w')
pidfile.write(str(os.getpid()))
pidfile.close()
logging.info('Starting up')
health.serve_health_thread()
importer_service = importer.MusicImporter()
importer_service.run()
# Remove the pidfile if we were daemonized.
if not settings.DAEMON_NO_BACKGROUND:
os.remove(settings.DAEMON_PIDFILE)
logging.info('Shutting down')
def main():
try:
run_daemon()
except KeyboardInterrupt:
print
except SystemExit:
pass
except Exception, e:
logging.critical('Daemon crashed')
import traceback
import StringIO
tb = StringIO.StringIO()
traceback.print_exc(file=tb)
logging.critical(tb.getvalue())
if __name__ == '__main__':
main()
# -*- mode: Python; coding: utf-8 -*-
#
# Fabric deployment script for Streamux.
# ae-streaming is not directly accessible by ssh, so you'll need a
# tunnel from localhost to deploy. If you don't know how to do that
# and the AE-info team won't tell you, you're probably not authorized
# to deploy anyway.
set(fab_hosts = ['localhost'],
fab_port = 2345,
fab_user = 'radio')
def dcommit():
local('git svn dcommit')
def reload():
"Reload the Streamux webapp"
run('touch streamux/streamux.fcgi')
def syncdb():
"Run manage.py syncdb"
run('cd streamux && ./manage.py syncdb')
def update():
"Run svn update"
run('svn update')
def restartdaemon():
"Restart the music processing daemon"
run('cd streamux && ./manage.py restartdaemon')
def genliqconfig():
"Regenerate the liquidsoap configuration"
run('cd streamux && ./manage.py genliqconfig')
def push():
"Push new files to the site, nothing else"
dcommit()
update()
genliqconfig()
def deploy():
"Deploy an updated website only"
push()
syncdb()
reload()
def deploy_daemon():
"Depoy an updated daemon only"
push()
syncdb()
restartdaemon()
def deploy_all():
"Deploy the latest version from Subversion"
push()
syncdb()
reload()
restartdaemon()
def reset_db():
"DESTROY ALL DATA and reset the SQL schema for streamux."
run('cd streamux && ./manage.py reset radio --noinput')
run('cd streamux && ./manage.py reset auth --noinput')
<xsl:stylesheet xmlns:xsl = "http://www.w3.org/1999/XSL/Transform" version = "1.0" >
<xsl:output method="text" indent="no" encoding="UTF-8" />
<xsl:template match = "/icestats" >
<xsl:for-each select="source">
mount=<xsl:value-of select="@mount" />
listeners=<xsl:value-of select="listeners" />