29115b3325c7bcadf8d35127d31e114fa03b1b0c
[smooth-dl.git] / smooth-dl.py
1 #!/usr/bin/env python
2 #
3 # smooth-dl - download videos served using Smooth Streaming technology
4 #
5 # Copyright (C) 2010  Antonio Ospite <ospite@studenti.unina.it>
6 #
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
11 #
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 # GNU General Public License for more details.
16 #
17 # You should have received a copy of the GNU General Public License
18 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
19 #
20 #
21 # TODO:
22 #  - Handle HTTP errors:
23 #       "Connection reset by peer"
24 #       "Resource not  available"
25 #       "Gateway Time-out"
26 # - Support more Manifest formats:
27 #       WaveFormatEx attribute instead of PrivateCodecdata
28 #       'd' and other attributes in chunk element ('i', 's', 'q')
29 #
30 # basically, write a proper implementation of manifest parsing and chunk
31 # downloading
32
33
34 __description = "Download videos served using Smooth Streaming technology"
35 __version = "0.x"
36 __author_info = "Written by Antonio Ospite http://ao2.it"
37
38 import os
39 import sys
40 import xml.etree.ElementTree as etree
41 import urllib2
42 import struct
43 import tempfile
44 from optparse import OptionParser
45
46
47 def get_chunk_data(data):
48
49     moof_size = struct.unpack(">L", data[0:4])[0]
50     mdat_size = struct.unpack(">L", data[moof_size:moof_size + 4])[0]
51
52     data_start = moof_size + 4 + len('mdat')
53     data_size = mdat_size - 4 - len('mdat')
54
55     #print len(data[data_start:]), \
56     #        len(data[data_start:data_start + data_size]), data_size
57
58     assert(len(data[data_start:]) == data_size)
59
60     return data[data_start:data_start + data_size]
61
62
63 def hexstring_to_bytes(hex_string):
64     res = ""
65     for i in range(0, len(hex_string), 2):
66             res += chr(int(hex_string[i:i + 2], 16))
67
68     return res
69
70
71 def write_wav_header(out_file, fmt, codec_private_data, data_len):
72
73     extradata = hexstring_to_bytes(codec_private_data)
74
75     fmt['cbSize'] = len(extradata)
76     fmt_len = 18 + fmt['cbSize']
77     wave_len = len("WAVEfmt ") + 4 + fmt_len + len('data') + 4
78
79     out_file.write("RIFF")
80     out_file.write(struct.pack('<L', wave_len))
81     out_file.write("WAVEfmt ")
82     out_file.write(struct.pack('<L', fmt_len))
83     out_file.write(struct.pack('<H', fmt['wFormatTag']))
84     out_file.write(struct.pack('<H', fmt['nChannels']))
85     out_file.write(struct.pack('<L', fmt['nSamplesPerSec']))
86     out_file.write(struct.pack('<L', fmt['nAvgBytesPerSec']))
87     out_file.write(struct.pack('<H', fmt['nBlockAlign']))
88     out_file.write(struct.pack('<H', fmt['wBitsPerSample']))
89     out_file.write(struct.pack('<H', fmt['cbSize']))
90     out_file.write(extradata)
91     out_file.write("data")
92     out_file.write(struct.pack('<L', data_len))
93
94
95 def get_manifest(base_url, dest_dir=tempfile.gettempdir()):
96     """Returns the manifest and the new URL if this is changed"""
97
98     if os.path.exists(dest_dir) == False:
99         os.mkdir(dest_dir, 0755)
100
101     if base_url.startswith('http://'):
102
103         manifest_url = base_url
104         if not manifest_url.lower().endswith(('/manifest', '.ismc', '.csm')):
105             manifest_url += '/Manifest'
106
107         response = urllib2.urlopen(manifest_url)
108         data = response.read()
109
110         manifest_path = os.path.join(dest_dir, 'Manifest')
111         f = open(manifest_path, "w")
112         f.write(data)
113         f.close()
114     else:
115         manifest_path = base_url
116
117     manifest = etree.parse(manifest_path)
118
119     version = manifest.getroot().attrib['MajorVersion']
120     if version != "2":
121         raise Exception('Only Smooth Streaming version 2 supported')
122
123     try:
124         # if some intermediate client Manifest is used, like in Rai Replay
125         clip = manifest.find("Clip")
126         actual_manifest_url = clip.attrib["Url"]
127         base_url = actual_manifest_url.lower().replace("/manifest", "")
128     except:
129         pass
130
131     return (manifest, base_url)
132
133
134 def print_manifest_info(manifest):
135
136     streams = manifest.findall('.//StreamIndex')
137
138     for i, s in enumerate(streams):
139         stream_type = s.attrib["Type"]
140         url = s.attrib["Url"]
141
142         print "Stream: %s Type: %s" % (i, stream_type)
143
144         print "\tQuality Levels:"
145         qualities = s.findall("QualityLevel")
146         for i, q in enumerate(qualities):
147             bitrate = q.attrib["Bitrate"]
148             fourcc = q.attrib["FourCC"]
149
150             if stream_type == "video":
151                 size = "%sx%s" % (q.attrib["MaxWidth"], q.attrib["MaxHeight"])
152                 print "\t%2s: %4s %10s @ %7s bps" % (i, fourcc, size, bitrate)
153             if stream_type == "audio":
154                 channels = q.attrib["Channels"]
155                 sampling_rate = q.attrib["SamplingRate"]
156                 bits_per_sample = q.attrib["BitsPerSample"]
157                 print "\t%2s: %4s %sHz %sbits %sch @ %7s bps" % (i, fourcc,
158                         sampling_rate, bits_per_sample, channels, bitrate)
159
160     print
161
162
163 def get_chunk_quality_string(stream, quality_level):
164     quality = stream.findall("QualityLevel")[quality_level]
165     bitrate = quality.attrib["Bitrate"]
166
167     quality_attributes = quality.findall("CustomAttributes/Attribute")
168     custom_attributes = ""
169     for i in quality_attributes:
170         custom_attributes += "%s=%s," % (i.attrib["Name"], i.attrib["Value"])
171     custom_attributes = custom_attributes.rstrip(',')
172
173     # Assume URLs are in this form:
174     # Url="QualityLevels({bitrate})/Fragments(video={start time})"
175     # or
176     # Url="QualityLevels({bitrate},{CustomAttributes})/Fragments(video={start time})"
177     url = stream.attrib["Url"]
178
179     chunks_quality = url.split('/')[0].replace("{bitrate}", bitrate)
180     chunks_quality = chunks_quality.replace("{CustomAttributes}", custom_attributes)
181
182     return chunks_quality
183
184
185 def get_chunk_name_string(stream, chunk):
186     t = chunk.attrib["t"]
187     url = stream.attrib["Url"]
188     chunk_name = url.split('/')[1].replace("{start time}", t)
189
190     return chunk_name
191
192
193 def download_chunks(base_url, manifest, stream_index, quality_level, dest_dir):
194
195     if os.path.exists(dest_dir) == False:
196         os.mkdir(dest_dir, 0755)
197
198     stream = manifest.findall('.//StreamIndex')[stream_index]
199
200     chunks_quality = get_chunk_quality_string(stream, quality_level)
201
202     chunks_dest_dir = os.path.join(dest_dir, chunks_quality)
203     if os.path.exists(chunks_dest_dir) == False:
204         os.mkdir(chunks_dest_dir, 0755)
205
206     chunks = stream.findall("c")
207     data_size = 0
208     print "\nDownloading Stream %d" % stream_index
209     print "\tChunks %10d/%-10d" % (0, len(chunks)), "\r",
210     sys.stdout.flush()
211     for i, c in enumerate(chunks):
212
213         chunk_name = get_chunk_name_string(stream, c)
214         chunk_file = os.path.join(dest_dir,  chunks_quality, chunk_name)
215
216         if os.path.exists(chunk_file) == False:
217             chunk_url = base_url + '/' + chunks_quality + '/' + chunk_name
218             try:
219                 response = urllib2.urlopen(chunk_url)
220                 data = response.read()
221
222                 f = open(chunk_file, "wb")
223                 f.write(data)
224                 f.close()
225             except Exception as e:
226                 print e
227
228         else:
229             f = open(chunk_file, "rb")
230             data = f.read()
231             f.close()
232
233         data_size += len(data)
234         print "\tChunks %10d/%-10d" % (i + 1, len(chunks)), "\r",
235         sys.stdout.flush()
236     print "\tDownloaded size:", data_size
237
238
239 def rebuild_stream(manifest, stream_index, quality_level, src_dir,
240         dest_file_name, final_dest_file=None):
241
242     if final_dest_file == None:
243         final_dest_file = dest_file_name
244
245     stream = manifest.findall('.//StreamIndex')[stream_index]
246
247     chunks_quality = get_chunk_quality_string(stream, quality_level)
248
249     chunks_src_dir = os.path.join(src_dir, chunks_quality)
250
251     dest_file = open(dest_file_name, "wb")
252
253     chunks = stream.findall("c")
254     data_size = 0
255     print "\nRebuilding Stream %d" % stream_index
256     print "\tChunks %10d/%-10d" % (0, len(chunks)), "\r",
257     sys.stdout.flush()
258     for i, c in enumerate(chunks):
259
260         chunk_name = get_chunk_name_string(stream, c)
261         chunk_file = os.path.join(chunks_src_dir, chunk_name)
262
263         f = open(chunk_file, "rb")
264         data = get_chunk_data(f.read())
265         f.close()
266         dest_file.write(data)
267         data_size += len(data)
268         print "\tChunks %10d/%-10d" % (i + 1, len(chunks)), "\r",
269         sys.stdout.flush()
270
271     # Add a nice WAV header
272     if stream.attrib['Type'] == "audio":
273         quality = stream.findall("QualityLevel")[quality_level]
274         codec_private_data = quality.attrib['CodecPrivateData']
275
276         fmt = {}
277         fmt['wFormatTag'] = int(quality.attrib['AudioTag'])
278         fmt['nChannels'] = int(quality.attrib['Channels'])
279         fmt['nSamplesPerSec'] = int(quality.attrib['SamplingRate'])
280         fmt['nAvgBytesPerSec'] = int(quality.attrib['Bitrate']) / 8
281         fmt['wBitsPerSample'] = int(quality.attrib['BitsPerSample'])
282         fmt['nBlockAlign'] = int(quality.attrib['PacketSize'])
283         fmt['cbSize'] = 0
284
285         f = open(final_dest_file, "wb")
286         write_wav_header(f, fmt, codec_private_data, data_size)
287         dest_file.close()
288         dest_file = open(dest_file_name, "rb")
289         f.write(dest_file.read())
290         f.close()
291         dest_file.close()
292
293     print
294     print "Stream %d, actual data size: %d\n" % (stream_index, data_size)
295
296
297 def calc_tracks_delay(manifest, stream1_index, stream2_index):
298     streams = manifest.findall('.//StreamIndex')
299
300     s1 = streams[stream1_index]
301     s2 = streams[stream2_index]
302
303     s1_start_chunk = s1.find("c")
304     s2_start_chunk = s2.find("c")
305
306     s1_start_time = int(s1_start_chunk.attrib['t'])
307     s2_start_time = int(s2_start_chunk.attrib['t'])
308
309     s1_timescale = float(s1.attrib['TimeScale'])
310     s2_timescale = float(s2.attrib['TimeScale'])
311
312     # calc difference in seconds
313     delay = s2_start_time / s2_timescale - \
314             s1_start_time / s1_timescale
315
316     return delay
317
318
319 def get_clip_duration(manifest):
320     # TODO: use <Clip ClipBegin="" ClipEnd=""> if Duration is not available
321     duration = manifest.getroot().attrib['Duration']
322
323     return float(duration) / 10000000  # here is the default timescale
324
325
326 def smooth_download(url, manifest, dest_dir=tempfile.gettempdir(),
327         video_stream_index=0, audio_stream_index=1,
328         video_quality_level=0, audio_quality_level=0,
329         chunks_dir=None, download=True,
330         out_video_file='_video.vc1', out_audio_file='_audio.raw'):
331
332         if chunks_dir == None:
333             chunks_dir = dest_dir
334
335         if download:
336             download_chunks(url, manifest, video_stream_index,
337                     video_quality_level, chunks_dir)
338             download_chunks(url, manifest, audio_stream_index,
339                     audio_quality_level, chunks_dir)
340
341         dest_video = os.path.join(dest_dir, out_video_file)
342         dest_audio = os.path.join(dest_dir, out_audio_file)
343
344         rebuild_stream(manifest, video_stream_index, video_quality_level,
345                 chunks_dir, dest_video)
346         rebuild_stream(manifest, audio_stream_index, audio_quality_level,
347                 chunks_dir, dest_audio, dest_audio + '.wav')
348
349         #duration = get_clip_duration(manifest)
350
351         delay = calc_tracks_delay(manifest, video_stream_index,
352                 audio_stream_index)
353
354         # optionally encode audio to vorbis:
355         # ffmpeg -i _audio.raw.wav -acodec libvorbis -aq 60 audio.ogg
356         mux_command = ("ffmpeg -i %s \\\n" +
357                       "  -itsoffset %f -async 1 -i %s \\\n" +
358                       "  -vcodec copy -acodec copy ffout.mkv") % \
359                       (dest_video, delay, dest_audio + '.wav')
360
361         print mux_command
362
363
364 def options_parser():
365     version = "%%prog %s" % __version
366     usage = "usage: %prog [options] <manifest URL or file>"
367     parser = OptionParser(usage=usage, version=version,
368             description=__description, epilog=__author_info)
369     parser.add_option("-i", "--info",
370                       action="store_true", dest="info_only",
371                       default=False, help="print Manifest info and exit")
372     parser.add_option("-m", "--manifest-only",
373                       action="store_true", dest="manifest_only",
374                       default=False, help="download Manifest file and exit")
375     parser.add_option("-n", "--no-download",
376                       action="store_false", dest="download",
377                       default=True, help="disable downloading chunks")
378     parser.add_option("-s", "--sync-delay",
379                       action="store_true", dest="sync_delay",
380                       default=False, help="show the sync delay between the given streams and exit")
381     parser.add_option("-d", "--dest-dir", metavar="<dir>",
382                       dest="dest_dir", default=tempfile.gettempdir(),
383                       help="destination directory")
384     parser.add_option("-c", "--chunks-dir", metavar="<dir>",
385                       dest="chunks_dir", default=None,
386                       help="directory containing chunks, if different from destination dir")
387     parser.add_option("-v", "--video-stream",  metavar="<n>",
388                       type="int", dest="video_stream_index", default=0,
389                       help="index of the video stream")
390     parser.add_option("-a", "--audio-stream", metavar="<n>",
391                       type="int", dest="audio_stream_index", default=1,
392                       help="index of the audio stream")
393     parser.add_option("-q", "--video-quality", metavar="<n>",
394                       type="int", dest="video_quality_level", default=0,
395                       help="index of the video quality level")
396     parser.add_option("-Q", "--audio-quality", metavar="<n>",
397                       type="int", dest="audio_quality_level", default=0,
398                       help="index of the audio quality level")
399
400     return parser
401
402
403 if __name__ == "__main__":
404
405     parser = options_parser()
406     (options, args) = parser.parse_args()
407
408     if len(args) != 1:
409         parser.print_help()
410         parser.exit(1)
411
412     url = args[0]
413     manifest, url = get_manifest(url, options.dest_dir)
414
415     if options.manifest_only:
416         parser.exit(0)
417
418     if options.sync_delay:
419         print calc_tracks_delay(manifest,
420                 options.video_stream_index,
421                 options.audio_stream_index)
422         parser.exit(0)
423
424     if options.info_only:
425         print_manifest_info(manifest)
426         parser.exit(0)
427
428     print_manifest_info(manifest)
429
430     smooth_download(url, manifest, options.dest_dir,
431             options.video_stream_index, options.audio_stream_index,
432             options.video_quality_level, options.audio_quality_level,
433             options.chunks_dir, options.download)