85a0d2e09d008eadf0784ac7884c80de71483afe
[smooth-dl.git] / smooth-dl.py
1 #!/usr/bin/env python
2 #
3 # smooth-dl - download videos served using Smooth Streaming technology
4 #
5 # Copyright (C) 2010  Antonio Ospite <ospite@studenti.unina.it>
6 #
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
11 #
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 # GNU General Public License for more details.
16 #
17 # You should have received a copy of the GNU General Public License
18 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
19 #
20 #
21 # TODO:
22 #  - Handle HTTP errors:
23 #       "Connection reset by peer"
24 #       "Resource not  available"
25 #       "Gateway Time-out"
26 # - Support more Manifest formats:
27 #       WaveFormatEx attribute instead of PrivateCodecdata
28 #       'd' and other attributes in chunk element ('i', 's', 'q')
29 #
30 # basically, write a proper implementation of manifest parsing and chunk
31 # downloading
32
33
34 __description = "Download videos served using Smooth Streaming technology"
35 __version = "0.x"
36 __author_info = "Written by Antonio Ospite http://ao2.it"
37
38 import os
39 import re
40 import sys
41 import xml.etree.ElementTree as etree
42 import urllib2
43 import struct
44 import tempfile
45 from optparse import OptionParser
46 from urlparse import urlparse, urlunparse
47
48
49 def get_chunk_data(data):
50
51     moof_size = struct.unpack(">L", data[0:4])[0]
52     mdat_size = struct.unpack(">L", data[moof_size:moof_size + 4])[0]
53
54     data_start = moof_size + 4 + len('mdat')
55     data_size = mdat_size - 4 - len('mdat')
56
57     #print len(data[data_start:]), \
58     #        len(data[data_start:data_start + data_size]), data_size
59
60     assert(len(data[data_start:]) == data_size)
61
62     return data[data_start:data_start + data_size]
63
64
65 def hexstring_to_bytes(hex_string):
66     res = ""
67     for i in range(0, len(hex_string), 2):
68             res += chr(int(hex_string[i:i + 2], 16))
69
70     return res
71
72
73 def write_wav_header(out_file, fmt, codec_private_data, data_len):
74
75     extradata = hexstring_to_bytes(codec_private_data)
76
77     fmt['cbSize'] = len(extradata)
78     fmt_len = 18 + fmt['cbSize']
79     wave_len = len("WAVEfmt ") + 4 + fmt_len + len('data') + 4
80
81     out_file.write("RIFF")
82     out_file.write(struct.pack('<L', wave_len))
83     out_file.write("WAVEfmt ")
84     out_file.write(struct.pack('<L', fmt_len))
85     out_file.write(struct.pack('<H', fmt['wFormatTag']))
86     out_file.write(struct.pack('<H', fmt['nChannels']))
87     out_file.write(struct.pack('<L', fmt['nSamplesPerSec']))
88     out_file.write(struct.pack('<L', fmt['nAvgBytesPerSec']))
89     out_file.write(struct.pack('<H', fmt['nBlockAlign']))
90     out_file.write(struct.pack('<H', fmt['wBitsPerSample']))
91     out_file.write(struct.pack('<H', fmt['cbSize']))
92     out_file.write(extradata)
93     out_file.write("data")
94     out_file.write(struct.pack('<L', data_len))
95
96 def download_file(src_url, dest_file, mode):
97     try:
98         response = urllib2.urlopen(src_url)
99         data = response.read()
100     except urllib2.HTTPError as e:
101         sys.stderr.write("Error while dowloading URL: %s" % src_url)
102         raise
103
104     if dest_file:
105         f = open(dest_file, mode)
106         f.write(data)
107         f.close()
108
109     return data
110
111 def get_manifest(url, dest_dir=tempfile.gettempdir()):
112     """Returns the manifest and the new URL if this is changed"""
113
114     if os.path.exists(dest_dir) == False:
115         os.mkdir(dest_dir, 0755)
116
117     # Remove the querystring if present
118     manifest_url = urlunparse(urlparse(url)._replace(query=''))
119
120     if not manifest_url.lower().endswith(('/manifest', '.ismc', '.csm')):
121         manifest_url += '/Manifest'
122
123     if manifest_url.startswith('http://'):
124         local_manifest_path = os.path.join(dest_dir, 'Manifest')
125         download_file(manifest_url, local_manifest_path, "w")
126     else:
127         local_manifest_path = url
128
129     manifest = etree.parse(local_manifest_path)
130
131     version = manifest.getroot().attrib['MajorVersion']
132     if version != "2":
133         raise Exception('Only Smooth Streaming version 2 supported')
134
135     try:
136         # if some intermediate client Manifest is used, like in Rai Replay
137         clip = manifest.find("Clip")
138         manifest_url = clip.attrib["Url"]
139     except AttributeError:
140         pass
141
142     manifest_pattern = re.compile("/manifest$", re.IGNORECASE)
143     base_url = manifest_pattern.sub("", manifest_url)
144
145     return (manifest, base_url)
146
147
148 def print_manifest_info(manifest):
149
150     streams = manifest.findall('.//StreamIndex')
151
152     for i, s in enumerate(streams):
153         stream_type = s.attrib["Type"]
154         url = s.attrib["Url"]
155
156         print "Stream: %s Type: %s" % (i, stream_type)
157
158         print "\tQuality Levels:"
159         qualities = s.findall("QualityLevel")
160         for i, q in enumerate(qualities):
161             bitrate = q.attrib["Bitrate"]
162             fourcc = q.attrib["FourCC"]
163
164             if stream_type == "video":
165                 size = "%sx%s" % (q.attrib["MaxWidth"], q.attrib["MaxHeight"])
166                 print "\t%2s: %4s %10s @ %7s bps" % (i, fourcc, size, bitrate)
167             if stream_type == "audio":
168                 channels = q.attrib["Channels"]
169                 sampling_rate = q.attrib["SamplingRate"]
170                 bits_per_sample = q.attrib["BitsPerSample"]
171                 print "\t%2s: %4s %sHz %sbits %sch @ %7s bps" % (i, fourcc,
172                         sampling_rate, bits_per_sample, channels, bitrate)
173
174     print
175
176
177 def get_chunk_quality_string(stream, quality_level):
178     quality = stream.findall("QualityLevel")[quality_level]
179     bitrate = quality.attrib["Bitrate"]
180
181     quality_attributes = quality.findall("CustomAttributes/Attribute")
182     custom_attributes = ""
183     for i in quality_attributes:
184         custom_attributes += "%s=%s," % (i.attrib["Name"], i.attrib["Value"])
185     custom_attributes = custom_attributes.rstrip(',')
186
187     # Assume URLs are in this form:
188     # Url="QualityLevels({bitrate})/Fragments(video={start time})"
189     # or
190     # Url="QualityLevels({bitrate},{CustomAttributes})/Fragments(video={start time})"
191     url = stream.attrib["Url"]
192
193     chunks_quality = url.split('/')[0].replace("{bitrate}", bitrate)
194     chunks_quality = chunks_quality.replace("{CustomAttributes}", custom_attributes)
195
196     return chunks_quality
197
198
199 def get_chunk_name_string(stream, chunk):
200     t = chunk.attrib["t"]
201     url = stream.attrib["Url"]
202     chunk_name = url.split('/')[1].replace("{start time}", t)
203
204     return chunk_name
205
206
207 def download_chunks(base_url, manifest, stream_index, quality_level, dest_dir):
208
209     if os.path.exists(dest_dir) == False:
210         os.mkdir(dest_dir, 0755)
211
212     stream = manifest.findall('.//StreamIndex')[stream_index]
213
214     chunks_quality = get_chunk_quality_string(stream, quality_level)
215
216     chunks_dest_dir = os.path.join(dest_dir, chunks_quality)
217     if os.path.exists(chunks_dest_dir) == False:
218         os.mkdir(chunks_dest_dir, 0755)
219
220     chunks = stream.findall("c")
221     data_size = 0
222     print "\nDownloading Stream %d" % stream_index
223     print "\tChunks %10d/%-10d" % (0, len(chunks)), "\r",
224     sys.stdout.flush()
225     for i, c in enumerate(chunks):
226
227         chunk_name = get_chunk_name_string(stream, c)
228         chunk_file = os.path.join(dest_dir,  chunks_quality, chunk_name)
229
230         if os.path.exists(chunk_file) == False:
231             chunk_url = base_url + '/' + chunks_quality + '/' + chunk_name
232             data = download_file(chunk_url, chunk_file, "wb")
233         else:
234             f = open(chunk_file, "rb")
235             data = f.read()
236             f.close()
237
238         data_size += len(data)
239         print "\tChunks %10d/%-10d" % (i + 1, len(chunks)), "\r",
240         sys.stdout.flush()
241     print "\tDownloaded size:", data_size
242
243
244 def rebuild_stream(manifest, stream_index, quality_level, src_dir,
245         dest_file_name, final_dest_file=None):
246
247     if final_dest_file == None:
248         final_dest_file = dest_file_name
249
250     stream = manifest.findall('.//StreamIndex')[stream_index]
251
252     chunks_quality = get_chunk_quality_string(stream, quality_level)
253
254     chunks_src_dir = os.path.join(src_dir, chunks_quality)
255
256     dest_file = open(dest_file_name, "wb")
257
258     chunks = stream.findall("c")
259     data_size = 0
260     print "\nRebuilding Stream %d" % stream_index
261     print "\tChunks %10d/%-10d" % (0, len(chunks)), "\r",
262     sys.stdout.flush()
263     for i, c in enumerate(chunks):
264
265         chunk_name = get_chunk_name_string(stream, c)
266         chunk_file = os.path.join(chunks_src_dir, chunk_name)
267
268         f = open(chunk_file, "rb")
269         data = get_chunk_data(f.read())
270         f.close()
271         dest_file.write(data)
272         data_size += len(data)
273         print "\tChunks %10d/%-10d" % (i + 1, len(chunks)), "\r",
274         sys.stdout.flush()
275
276     # Add a nice WAV header
277     if stream.attrib['Type'] == "audio":
278         quality = stream.findall("QualityLevel")[quality_level]
279         codec_private_data = quality.attrib['CodecPrivateData']
280
281         fmt = {}
282         fmt['wFormatTag'] = int(quality.attrib['AudioTag'])
283         fmt['nChannels'] = int(quality.attrib['Channels'])
284         fmt['nSamplesPerSec'] = int(quality.attrib['SamplingRate'])
285         fmt['nAvgBytesPerSec'] = int(quality.attrib['Bitrate']) / 8
286         fmt['wBitsPerSample'] = int(quality.attrib['BitsPerSample'])
287         fmt['nBlockAlign'] = int(quality.attrib['PacketSize'])
288         fmt['cbSize'] = 0
289
290         f = open(final_dest_file, "wb")
291         write_wav_header(f, fmt, codec_private_data, data_size)
292         dest_file.close()
293         dest_file = open(dest_file_name, "rb")
294         f.write(dest_file.read())
295         f.close()
296         dest_file.close()
297
298     print
299     print "Stream %d, actual data size: %d\n" % (stream_index, data_size)
300
301
302 def calc_tracks_delay(manifest, stream1_index, stream2_index):
303     streams = manifest.findall('.//StreamIndex')
304
305     s1 = streams[stream1_index]
306     s2 = streams[stream2_index]
307
308     s1_start_chunk = s1.find("c")
309     s2_start_chunk = s2.find("c")
310
311     s1_start_time = int(s1_start_chunk.attrib['t'])
312     s2_start_time = int(s2_start_chunk.attrib['t'])
313
314     s1_timescale = float(s1.attrib['TimeScale'])
315     s2_timescale = float(s2.attrib['TimeScale'])
316
317     # calc difference in seconds
318     delay = s2_start_time / s2_timescale - \
319             s1_start_time / s1_timescale
320
321     return delay
322
323
324 def get_clip_duration(manifest):
325     # TODO: use <Clip ClipBegin="" ClipEnd=""> if Duration is not available
326     duration = manifest.getroot().attrib['Duration']
327
328     return float(duration) / 10000000  # here is the default timescale
329
330
331 def smooth_download(url, manifest, dest_dir=tempfile.gettempdir(),
332         video_stream_index=0, audio_stream_index=1,
333         video_quality_level=0, audio_quality_level=0,
334         chunks_dir=None, download=True,
335         out_video_file='_video.vc1', out_audio_file='_audio.raw'):
336
337         if chunks_dir == None:
338             chunks_dir = dest_dir
339
340         if download:
341             download_chunks(url, manifest, video_stream_index,
342                     video_quality_level, chunks_dir)
343             download_chunks(url, manifest, audio_stream_index,
344                     audio_quality_level, chunks_dir)
345
346         dest_video = os.path.join(dest_dir, out_video_file)
347         dest_audio = os.path.join(dest_dir, out_audio_file)
348
349         rebuild_stream(manifest, video_stream_index, video_quality_level,
350                 chunks_dir, dest_video)
351         rebuild_stream(manifest, audio_stream_index, audio_quality_level,
352                 chunks_dir, dest_audio, dest_audio + '.wav')
353
354         #duration = get_clip_duration(manifest)
355
356         delay = calc_tracks_delay(manifest, video_stream_index,
357                 audio_stream_index)
358
359         # optionally encode audio to vorbis:
360         # ffmpeg -i _audio.raw.wav -acodec libvorbis -aq 60 audio.ogg
361         mux_command = ("ffmpeg -i %s \\\n" +
362                       "  -itsoffset %f -async 1 -i %s \\\n" +
363                       "  -vcodec copy -acodec copy ffout.mkv") % \
364                       (dest_video, delay, dest_audio + '.wav')
365
366         print mux_command
367
368
369 def options_parser():
370     version = "%%prog %s" % __version
371     usage = "usage: %prog [options] <manifest URL or file>"
372     parser = OptionParser(usage=usage, version=version,
373             description=__description, epilog=__author_info)
374     parser.add_option("-i", "--info",
375                       action="store_true", dest="info_only",
376                       default=False, help="print Manifest info and exit")
377     parser.add_option("-m", "--manifest-only",
378                       action="store_true", dest="manifest_only",
379                       default=False, help="download Manifest file and exit")
380     parser.add_option("-n", "--no-download",
381                       action="store_false", dest="download",
382                       default=True, help="disable downloading chunks")
383     parser.add_option("-s", "--sync-delay",
384                       action="store_true", dest="sync_delay",
385                       default=False, help="show the sync delay between the given streams and exit")
386     parser.add_option("-d", "--dest-dir", metavar="<dir>",
387                       dest="dest_dir", default=tempfile.gettempdir(),
388                       help="destination directory")
389     parser.add_option("-c", "--chunks-dir", metavar="<dir>",
390                       dest="chunks_dir", default=None,
391                       help="directory containing chunks, if different from destination dir")
392     parser.add_option("-v", "--video-stream",  metavar="<n>",
393                       type="int", dest="video_stream_index", default=0,
394                       help="index of the video stream")
395     parser.add_option("-a", "--audio-stream", metavar="<n>",
396                       type="int", dest="audio_stream_index", default=1,
397                       help="index of the audio stream")
398     parser.add_option("-q", "--video-quality", metavar="<n>",
399                       type="int", dest="video_quality_level", default=0,
400                       help="index of the video quality level")
401     parser.add_option("-Q", "--audio-quality", metavar="<n>",
402                       type="int", dest="audio_quality_level", default=0,
403                       help="index of the audio quality level")
404
405     return parser
406
407
408 if __name__ == "__main__":
409
410     parser = options_parser()
411     (options, args) = parser.parse_args()
412
413     if len(args) != 1:
414         parser.print_help()
415         parser.exit(1)
416
417     url = args[0]
418     manifest, url = get_manifest(url, options.dest_dir)
419
420     if options.manifest_only:
421         parser.exit(0)
422
423     if options.sync_delay:
424         print calc_tracks_delay(manifest,
425                 options.video_stream_index,
426                 options.audio_stream_index)
427         parser.exit(0)
428
429     if options.info_only:
430         print_manifest_info(manifest)
431         parser.exit(0)
432
433     print_manifest_info(manifest)
434
435     smooth_download(url, manifest, options.dest_dir,
436             options.video_stream_index, options.audio_stream_index,
437             options.video_quality_level, options.audio_quality_level,
438             options.chunks_dir, options.download)