44ef722ead789a7b54872a515caa4864c10c1046
[smooth-dl.git] / smooth-dl.py
1 #!/usr/bin/env python
2 #
3 # smooth-dl - download videos served using Smooth Streaming technology
4 #
5 # Copyright (C) 2010  Antonio Ospite <ospite@studenti.unina.it>
6 #
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
11 #
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 # GNU General Public License for more details.
16 #
17 # You should have received a copy of the GNU General Public License
18 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
19 #
20 #
21 # TODO:
22 #  - Handle HTTP errors:
23 #       "Connection reset by peer"
24 #       "Resource not  available"
25 #       "Gateway Time-out"
26 # - Support more Manifest formats:
27 #       WaveFormatEx attribute instead of PrivateCodecdata
28 #       'd' and other attributes in chunk element ('i', 's', 'q')
29 #
30 # basically, write a proper implementation of manifest parsing and chunk
31 # downloading
32
33 import os
34 import re
35 import sys
36 import xml.etree.ElementTree as etree
37 import urllib2
38 import struct
39 import tempfile
40 from optparse import OptionParser
41 from urlparse import urlparse, urlunparse
42
43 __description__ = "Download videos served using Smooth Streaming technology"
44 __version__ = "0.x"
45 __author_info__ = "Written by Antonio Ospite http://ao2.it"
46
47
48 def get_chunk_data(data):
49
50     moof_size = struct.unpack(">L", data[0:4])[0]
51     mdat_size = struct.unpack(">L", data[moof_size:moof_size + 4])[0]
52
53     data_start = moof_size + 4 + len('mdat')
54     data_size = mdat_size - 4 - len('mdat')
55
56     # print len(data[data_start:]), \
57     #        len(data[data_start:data_start + data_size]), data_size
58
59     assert len(data[data_start:]) == data_size
60
61     return data[data_start:data_start + data_size]
62
63
64 def hexstring_to_bytes(hex_string):
65     res = ""
66     for i in range(0, len(hex_string), 2):
67         res += chr(int(hex_string[i:i + 2], 16))
68
69     return res
70
71
72 def write_wav_header(out_file, fmt, codec_private_data, data_len):
73
74     extradata = hexstring_to_bytes(codec_private_data)
75
76     fmt['cbSize'] = len(extradata)
77     fmt_len = 18 + fmt['cbSize']
78     wave_len = len("WAVEfmt ") + 4 + fmt_len + len('data') + 4
79
80     out_file.write("RIFF")
81     out_file.write(struct.pack('<L', wave_len))
82     out_file.write("WAVEfmt ")
83     out_file.write(struct.pack('<L', fmt_len))
84     out_file.write(struct.pack('<H', fmt['wFormatTag']))
85     out_file.write(struct.pack('<H', fmt['nChannels']))
86     out_file.write(struct.pack('<L', fmt['nSamplesPerSec']))
87     out_file.write(struct.pack('<L', fmt['nAvgBytesPerSec']))
88     out_file.write(struct.pack('<H', fmt['nBlockAlign']))
89     out_file.write(struct.pack('<H', fmt['wBitsPerSample']))
90     out_file.write(struct.pack('<H', fmt['cbSize']))
91     out_file.write(extradata)
92     out_file.write("data")
93     out_file.write(struct.pack('<L', data_len))
94
95
96 def download_file(src_url, dest_file, mode):
97     try:
98         response = urllib2.urlopen(src_url)
99         data = response.read()
100     except urllib2.HTTPError:
101         sys.stderr.write("Error while dowloading URL: %s" % src_url)
102         raise
103
104     if dest_file:
105         f = open(dest_file, mode)
106         f.write(data)
107         f.close()
108
109     return data
110
111
112 def get_manifest(url, dest_dir):
113     """Returns the manifest element and the base content URL"""
114
115     # Remove the querystring if present
116     manifest_url = urlunparse(urlparse(url)._replace(query=''))
117
118     if not manifest_url.lower().endswith(('/manifest', '.ismc', '.csm')):
119         manifest_url += '/Manifest'
120
121     if manifest_url.startswith('http://'):
122         local_manifest_path = os.path.join(dest_dir, 'Manifest')
123         download_file(manifest_url, local_manifest_path, "w")
124     else:
125         local_manifest_path = url
126
127     manifest = etree.parse(local_manifest_path)
128
129     version = manifest.getroot().attrib['MajorVersion']
130     if version != "2":
131         raise Exception('Only Smooth Streaming version 2 supported')
132
133     try:
134         # if some intermediate client Manifest is used, like in Rai Replay
135         clip = manifest.find("Clip")
136         manifest_url = clip.attrib["Url"]
137         manifest = download_file(manifest_url, None, None)
138     except AttributeError:
139         pass
140
141     manifest_pattern = re.compile("/manifest$", re.IGNORECASE)
142     base_url = manifest_pattern.sub("", manifest_url)
143
144     return (manifest, base_url)
145
146
147 def print_manifest_info(manifest):
148
149     streams = manifest.findall('.//StreamIndex')
150
151     for i, s in enumerate(streams):
152         stream_type = s.attrib["Type"]
153
154         print "Stream: %s Type: %s" % (i, stream_type)
155
156         print "\tQuality Levels:"
157         qualities = s.findall("QualityLevel")
158         for i, q in enumerate(qualities):
159             bitrate = q.attrib["Bitrate"]
160             fourcc = q.attrib["FourCC"]
161
162             if stream_type == "video":
163                 size = "%sx%s" % (q.attrib["MaxWidth"], q.attrib["MaxHeight"])
164                 print "\t%2s: %4s %10s @ %7s bps" % (i, fourcc, size, bitrate)
165             if stream_type == "audio":
166                 channels = q.attrib["Channels"]
167                 sampling_rate = q.attrib["SamplingRate"]
168                 bits_per_sample = q.attrib["BitsPerSample"]
169                 print "\t%2s: %4s %sHz %sbits %sch @ %7s bps" % \
170                     (i, fourcc, sampling_rate, bits_per_sample, channels,
171                      bitrate)
172
173     print
174
175
176 def get_chunk_quality_string(stream, quality_level):
177     quality = stream.findall("QualityLevel")[quality_level]
178     bitrate = quality.attrib["Bitrate"]
179
180     quality_attributes = quality.findall("CustomAttributes/Attribute")
181     custom_attributes = ""
182     for i in quality_attributes:
183         custom_attributes += "%s=%s," % (i.attrib["Name"], i.attrib["Value"])
184     custom_attributes = custom_attributes.rstrip(',')
185
186     # Assume URLs are in this form:
187     # Url="QualityLevels({bitrate})/Fragments(video={start time})"
188     # or
189     # Url="QualityLevels({bitrate},{CustomAttributes})/Fragments(video={start time})"
190     url = stream.attrib["Url"]
191
192     chunks_quality = url.split('/')[0].replace("{bitrate}", bitrate)
193     chunks_quality = chunks_quality.replace("{CustomAttributes}",
194                                             custom_attributes)
195
196     return chunks_quality
197
198
199 def get_chunk_name_string(stream, chunk_time):
200     url = stream.attrib["Url"]
201     chunk_name = url.split('/')[1].replace("{start time}", str(chunk_time))
202
203     return chunk_name
204
205
206 def download_chunks(base_url, manifest, stream_index, quality_level, dest_dir):
207     stream = manifest.findall('.//StreamIndex')[stream_index]
208
209     chunks_quality = get_chunk_quality_string(stream, quality_level)
210
211     chunks_dest_dir = os.path.join(dest_dir, chunks_quality)
212     if not os.path.exists(chunks_dest_dir):
213         os.mkdir(chunks_dest_dir, 0755)
214
215     chunks = stream.findall("c")
216     data_size = 0
217     print "\nDownloading Stream %d" % stream_index
218     print "\tChunks %10d/%-10d" % (0, len(chunks)), "\r",
219     sys.stdout.flush()
220
221     stream_duration = 0
222     for i, chunk in enumerate(chunks):
223
224         if "t" in chunk.attrib:
225             chunk_time = chunk.attrib["t"]
226         elif "d" in chunk.attrib:
227             chunk_time = stream_duration
228             stream_duration = chunk_time + int(chunk.attrib["d"])
229
230         chunk_name = get_chunk_name_string(stream, chunk_time)
231         chunk_file = os.path.join(dest_dir, chunks_quality, chunk_name)
232
233         if not os.path.exists(chunk_file):
234             chunk_url = base_url + '/' + chunks_quality + '/' + chunk_name
235             data = download_file(chunk_url, chunk_file, "wb")
236         else:
237             f = open(chunk_file, "rb")
238             data = f.read()
239             f.close()
240
241         data_size += len(data)
242         print "\tChunks %10d/%-10d" % (i + 1, len(chunks)), "\r",
243         sys.stdout.flush()
244     print "\tDownloaded size:", data_size
245
246
247 def rebuild_stream(manifest, stream_index, quality_level, src_dir,
248                    dest_file_name, final_dest_file=None):
249
250     if final_dest_file is None:
251         final_dest_file = dest_file_name
252
253     stream = manifest.findall('.//StreamIndex')[stream_index]
254
255     chunks_quality = get_chunk_quality_string(stream, quality_level)
256
257     chunks_src_dir = os.path.join(src_dir, chunks_quality)
258
259     dest_file = open(dest_file_name, "wb")
260
261     chunks = stream.findall("c")
262     data_size = 0
263     print "\nRebuilding Stream %d" % stream_index
264     print "\tChunks %10d/%-10d" % (0, len(chunks)), "\r",
265     sys.stdout.flush()
266
267     stream_duration = 0
268     for i, chunk in enumerate(chunks):
269
270         if "t" in chunk.attrib:
271             chunk_time = chunk.attrib["t"]
272         elif "d" in chunk.attrib:
273             chunk_time = stream_duration
274             stream_duration = chunk_time + int(chunk.attrib["d"])
275
276         chunk_name = get_chunk_name_string(stream, chunk_time)
277         chunk_file = os.path.join(chunks_src_dir, chunk_name)
278
279         f = open(chunk_file, "rb")
280         data = get_chunk_data(f.read())
281         f.close()
282         dest_file.write(data)
283         data_size += len(data)
284         print "\tChunks %10d/%-10d" % (i + 1, len(chunks)), "\r",
285         sys.stdout.flush()
286
287     # Add a nice WAV header
288     if stream.attrib['Type'] == "audio":
289         quality = stream.findall("QualityLevel")[quality_level]
290         codec_private_data = quality.attrib['CodecPrivateData']
291
292         fmt = {}
293         fmt['wFormatTag'] = int(quality.attrib['AudioTag'])
294         fmt['nChannels'] = int(quality.attrib['Channels'])
295         fmt['nSamplesPerSec'] = int(quality.attrib['SamplingRate'])
296         fmt['nAvgBytesPerSec'] = int(quality.attrib['Bitrate']) / 8
297         fmt['wBitsPerSample'] = int(quality.attrib['BitsPerSample'])
298         fmt['nBlockAlign'] = int(quality.attrib['PacketSize'])
299         fmt['cbSize'] = 0
300
301         f = open(final_dest_file, "wb")
302         write_wav_header(f, fmt, codec_private_data, data_size)
303         dest_file.close()
304         dest_file = open(dest_file_name, "rb")
305         f.write(dest_file.read())
306         f.close()
307         dest_file.close()
308
309     print
310     print "Stream %d, actual data size: %d\n" % (stream_index, data_size)
311
312
313 def calc_tracks_delay(manifest, stream1_index, stream2_index):
314     streams = manifest.findall('.//StreamIndex')
315
316     s1 = streams[stream1_index]
317     s2 = streams[stream2_index]
318
319     if "TimeScale" not in s1 or "TimeScale" not in s2:
320         return 0
321
322     s1_start_chunk = s1.find("c")
323     s2_start_chunk = s2.find("c")
324
325     if "t" not in s1_start_chunk.attrib \
326        or "t" not in s2_start_chunk.attrib:
327         return 0
328
329     s1_start_time = int(s1_start_chunk.attrib['t'])
330     s2_start_time = int(s2_start_chunk.attrib['t'])
331
332     s1_timescale = float(s1.attrib['TimeScale'])
333     s2_timescale = float(s2.attrib['TimeScale'])
334
335     # calc difference in seconds
336     delay = s2_start_time / s2_timescale - \
337         s1_start_time / s1_timescale
338
339     return delay
340
341
342 def get_clip_duration(manifest):
343     # TODO: use <Clip ClipBegin="" ClipEnd=""> if Duration is not available
344     duration = manifest.getroot().attrib['Duration']
345
346     return float(duration) / 10000000  # here is the default timescale
347
348
349 def smooth_download(url, manifest, dest_dir,
350                     video_stream_index=0, audio_stream_index=1,
351                     video_quality_level=0, audio_quality_level=0,
352                     chunks_dir=None, download=True,
353                     out_video_file='_video.vc1', out_audio_file='_audio.raw'):
354
355     if chunks_dir is None:
356         chunks_dir = dest_dir
357
358     if download:
359         download_chunks(url, manifest, video_stream_index,
360                         video_quality_level, chunks_dir)
361         download_chunks(url, manifest, audio_stream_index,
362                         audio_quality_level, chunks_dir)
363
364     dest_video = os.path.join(dest_dir, out_video_file)
365     dest_audio = os.path.join(dest_dir, out_audio_file)
366
367     rebuild_stream(manifest, video_stream_index, video_quality_level,
368                    chunks_dir, dest_video)
369     rebuild_stream(manifest, audio_stream_index, audio_quality_level,
370                    chunks_dir, dest_audio, dest_audio + '.wav')
371
372     # duration = get_clip_duration(manifest)
373
374     delay = calc_tracks_delay(manifest, video_stream_index,
375                               audio_stream_index)
376
377     # optionally encode audio to vorbis:
378     # ffmpeg -i _audio.raw.wav -acodec libvorbis -aq 60 audio.ogg
379     mux_command = ("ffmpeg -i %s \\\n" +
380                    "  -itsoffset %f -async 1 -i %s \\\n" +
381                    "  -vcodec copy -acodec copy ffout.mkv") % \
382         (dest_video, delay, dest_audio + '.wav')
383
384     print mux_command
385
386
387 def options_parser():
388     version = "%%prog %s" % __version__
389     usage = "usage: %prog [options] <manifest URL or file>"
390     parser = OptionParser(usage=usage, version=version,
391                           description=__description__, epilog=__author_info__)
392     parser.add_option("-i", "--info",
393                       action="store_true", dest="info_only",
394                       default=False, help="print Manifest info and exit")
395     parser.add_option("-m", "--manifest-only",
396                       action="store_true", dest="manifest_only",
397                       default=False, help="download Manifest file and exit")
398     parser.add_option("-n", "--no-download",
399                       action="store_false", dest="download",
400                       default=True, help="disable downloading chunks")
401     parser.add_option("-s", "--sync-delay",
402                       action="store_true", dest="sync_delay",
403                       default=False, help="show the sync delay between the given streams and exit")
404     parser.add_option("-d", "--dest-dir", metavar="<dir>",
405                       dest="dest_dir", default=tempfile.gettempdir(),
406                       help="destination directory")
407     parser.add_option("-c", "--chunks-dir", metavar="<dir>",
408                       dest="chunks_dir", default=None,
409                       help="directory containing chunks, if different from destination dir")
410     parser.add_option("-v", "--video-stream", metavar="<n>",
411                       type="int", dest="video_stream_index", default=0,
412                       help="index of the video stream")
413     parser.add_option("-a", "--audio-stream", metavar="<n>",
414                       type="int", dest="audio_stream_index", default=1,
415                       help="index of the audio stream")
416     parser.add_option("-q", "--video-quality", metavar="<n>",
417                       type="int", dest="video_quality_level", default=0,
418                       help="index of the video quality level")
419     parser.add_option("-Q", "--audio-quality", metavar="<n>",
420                       type="int", dest="audio_quality_level", default=0,
421                       help="index of the audio quality level")
422
423     return parser
424
425
426 def main():
427     parser = options_parser()
428     (options, args) = parser.parse_args()
429
430     if len(args) != 1:
431         parser.print_help()
432         parser.exit(1)
433
434     if not os.path.exists(options.dest_dir):
435         os.mkdir(options.dest_dir, 0755)
436
437     url = args[0]
438     manifest, url = get_manifest(url, options.dest_dir)
439
440     if options.manifest_only:
441         parser.exit(0)
442
443     if options.sync_delay:
444         print calc_tracks_delay(manifest,
445                                 options.video_stream_index,
446                                 options.audio_stream_index)
447         parser.exit(0)
448
449     if options.info_only:
450         print_manifest_info(manifest)
451         parser.exit(0)
452
453     print_manifest_info(manifest)
454
455     smooth_download(url, manifest, options.dest_dir,
456                     options.video_stream_index, options.audio_stream_index,
457                     options.video_quality_level, options.audio_quality_level,
458                     options.chunks_dir, options.download)
459
460
461 if __name__ == "__main__":
462     main()