X-Git-Url: https://git.ao2.it/smooth-dl.git/blobdiff_plain/3b8a5827480b918de8ec9a0cac4e615c9d9de6c5..HEAD:/smooth-dl.py

diff --git a/smooth-dl.py b/smooth-dl.py
index 2baa009..d983a28 100755
--- a/smooth-dl.py
+++ b/smooth-dl.py
@@ -1,8 +1,8 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 #
 # smooth-dl - download videos served using Smooth Streaming technology
 #
-# Copyright (C) 2010  Antonio Ospite <ospite@studenti.unina.it>
+# Copyright (C) 2010-2016  Antonio Ospite <ao2@ao2.it>
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -30,18 +30,21 @@
 # basically, write a proper implementation of manifest parsing and chunk
 # downloading
 
-
-__description = "Download videos served using Smooth Streaming technology"
-__version = "0.x"
-__author_info = "Written by Antonio Ospite http://ao2.it"
-
 import os
+import re
 import sys
 import xml.etree.ElementTree as etree
-import urllib2
+import urllib.request
+import urllib.error
+import urllib.parse
 import struct
 import tempfile
 from optparse import OptionParser
+from urllib.parse import urlparse, urlunparse
+
+__description__ = "Download videos served using Smooth Streaming technology"
+__version__ = "0.x"
+__author_info__ = "Written by Antonio Ospite http://ao2.it"
 
 
 def get_chunk_data(data):
@@ -52,20 +55,16 @@ def get_chunk_data(data):
     data_start = moof_size + 4 + len('mdat')
     data_size = mdat_size - 4 - len('mdat')
 
-    #print len(data[data_start:]), \
+    # print len(data[data_start:]), \
     #        len(data[data_start:data_start + data_size]), data_size
 
-    assert(len(data[data_start:]) == data_size)
+    assert len(data[data_start:]) == data_size
 
     return data[data_start:data_start + data_size]
 
 
 def hexstring_to_bytes(hex_string):
-    res = ""
-    for i in range(0, len(hex_string), 2):
-            res += chr(int(hex_string[i:i + 2], 16))
-
-    return res
+    return bytearray.fromhex(hex_string)
 
 
 def write_wav_header(out_file, fmt, codec_private_data, data_len):
@@ -76,9 +75,9 @@ def write_wav_header(out_file, fmt, codec_private_data, data_len):
     fmt_len = 18 + fmt['cbSize']
     wave_len = len("WAVEfmt ") + 4 + fmt_len + len('data') + 4
 
-    out_file.write("RIFF")
+    out_file.write(bytes("RIFF", "ascii"))
     out_file.write(struct.pack('<L', wave_len))
-    out_file.write("WAVEfmt ")
+    out_file.write(bytes("WAVEfmt ", "ascii"))
     out_file.write(struct.pack('<L', fmt_len))
     out_file.write(struct.pack('<H', fmt['wFormatTag']))
     out_file.write(struct.pack('<H', fmt['nChannels']))
@@ -88,46 +87,65 @@ def write_wav_header(out_file, fmt, codec_private_data, data_len):
     out_file.write(struct.pack('<H', fmt['wBitsPerSample']))
     out_file.write(struct.pack('<H', fmt['cbSize']))
     out_file.write(extradata)
-    out_file.write("data")
+    out_file.write(bytes("data", "ascii"))
     out_file.write(struct.pack('<L', data_len))
 
 
-def get_manifest(base_url, dest_dir=tempfile.gettempdir(),
-        manifest_file='Manifest'):
-    """Returns the manifest and the new URL if this is changed"""
+def download_file(src_url, dest_file, mode):
+
+    if os.path.exists(src_url):
+        f = open(src_url, "rb")
+        data = f.read()
+        f.close()
+    else:
+        try:
+            response = urllib.request.urlopen(src_url)
+            data = response.read()
+        except urllib.error.HTTPError:
+            sys.stderr.write("Error while dowloading URL: %s\n" % src_url)
+            raise
+
+    if dest_file:
+        f = open(dest_file, mode)
+        f.write(data)
+        f.close()
 
-    if os.path.exists(dest_dir) == False:
-        os.mkdir(dest_dir, 0755)
+    return data
 
-    if base_url.startswith('http://'):
 
-        manifest_url = base_url
-        if not manifest_url.lower().endswith(('/manifest', '.ismc', '.csm')):
-            manifest_url += '/Manifest'
+def get_manifest(url, dest_dir):
+    """Returns the manifest element and the base content URL"""
 
-        response = urllib2.urlopen(manifest_url)
-        data = response.read()
+    # Remove the querystring if present
+    manifest_url = urlunparse(urlparse(url)._replace(query=''))
 
-        manifest_path = os.path.join(dest_dir, manifest_file)
-        f = open(manifest_path, "w")
-        f.write(data)
-        f.close()
-    else:
-        manifest_path = base_url
+    if not manifest_url.lower().endswith(('/manifest', '.ismc', '.csm')):
+        manifest_url += '/Manifest'
 
-    manifest = etree.parse(manifest_path)
+    local_manifest_path = os.path.join(dest_dir, 'Manifest')
+    download_file(manifest_url, local_manifest_path, "wb")
+
+    manifest = etree.parse(local_manifest_path)
 
     version = manifest.getroot().attrib['MajorVersion']
     if version != "2":
         raise Exception('Only Smooth Streaming version 2 supported')
 
-    try:
-        # if some intermediate client Manifest is used, like in Rai Replay
-        clip = manifest.find("Clip")
-        actual_manifest_url = clip.attrib["Url"]
-        base_url = actual_manifest_url.lower().replace("/manifest", "")
-    except:
-        pass
+    # if some intermediate client Manifest is used, like in Rai Replay
+    # then get the final manifest
+    clip = manifest.find("Clip")
+    if clip is not None and "Url" in clip.attrib:
+        tmp_manifest_url = clip.attrib["Url"]
+        try:
+            tmp_manifest = download_file(tmp_manifest_url, None, None)
+            # set the new values only if the dowload succeded
+            manifest_url = tmp_manifest_url
+            manifest = tmp_manifest
+        except urllib.error.HTTPError:
+            pass
+
+    manifest_pattern = re.compile("/manifest$", re.IGNORECASE)
+    base_url = manifest_pattern.sub("", manifest_url)
 
     return (manifest, base_url)
 
@@ -138,11 +156,10 @@ def print_manifest_info(manifest):
 
     for i, s in enumerate(streams):
         stream_type = s.attrib["Type"]
-        url = s.attrib["Url"]
 
-        print "Stream: %s Type: %s" % (i, stream_type)
+        print("Stream: %s Type: %s" % (i, stream_type))
 
-        print "\tQuality Levels:"
+        print("\tQuality Levels:")
         qualities = s.findall("QualityLevel")
         for i, q in enumerate(qualities):
             bitrate = q.attrib["Bitrate"]
@@ -150,95 +167,119 @@ def print_manifest_info(manifest):
 
             if stream_type == "video":
                 size = "%sx%s" % (q.attrib["MaxWidth"], q.attrib["MaxHeight"])
-                print "\t%2s: %4s %10s @ %7s bps" % (i, fourcc, size, bitrate)
+                print("\t%2s: %4s %10s @ %7s bps" % (i, fourcc, size, bitrate))
             if stream_type == "audio":
                 channels = q.attrib["Channels"]
                 sampling_rate = q.attrib["SamplingRate"]
                 bits_per_sample = q.attrib["BitsPerSample"]
-                print "\t%2s: %4s %sHz %sbits %sch @ %7s bps" % (i, fourcc,
-                        sampling_rate, bits_per_sample, channels, bitrate)
-
-    print
+                print("\t%2s: %4s %sHz %sbits %sch @ %7s bps" %
+                      (i, fourcc, sampling_rate, bits_per_sample, channels,
+                       bitrate))
 
+    print()
 
-def download_chunks(base_url, manifest, stream_index, quality_level, dest_dir):
-
-    if os.path.exists(dest_dir) == False:
-        os.mkdir(dest_dir, 0755)
-
-    stream = manifest.findall('.//StreamIndex')[stream_index]
 
+def get_chunk_quality_string(stream, quality_level):
     quality = stream.findall("QualityLevel")[quality_level]
     bitrate = quality.attrib["Bitrate"]
 
+    quality_attributes = quality.findall("CustomAttributes/Attribute")
+    custom_attributes = ""
+    for i in quality_attributes:
+        custom_attributes += "%s=%s," % (i.attrib["Name"], i.attrib["Value"])
+    custom_attributes = custom_attributes.rstrip(',')
+
     # Assume URLs are in this form:
     # Url="QualityLevels({bitrate})/Fragments(video={start time})"
+    # or
+    # Url="QualityLevels({bitrate},{CustomAttributes})/Fragments(video={start time})"
     url = stream.attrib["Url"]
 
     chunks_quality = url.split('/')[0].replace("{bitrate}", bitrate)
+    chunks_quality = chunks_quality.replace("{CustomAttributes}",
+                                            custom_attributes)
+
+    return chunks_quality
+
+
+def get_chunk_name_string(stream, chunk_time):
+    url = stream.attrib["Url"]
+    chunk_name = url.split('/')[1].replace("{start time}", str(chunk_time))
+
+    return chunk_name
+
+
+def download_chunks(base_url, manifest, stream_index, quality_level, dest_dir):
+    stream = manifest.findall('.//StreamIndex')[stream_index]
+
+    chunks_quality = get_chunk_quality_string(stream, quality_level)
+
     chunks_dest_dir = os.path.join(dest_dir, chunks_quality)
-    if os.path.exists(chunks_dest_dir) == False:
-        os.mkdir(chunks_dest_dir, 0755)
+    if not os.path.exists(chunks_dest_dir):
+        os.mkdir(chunks_dest_dir, 0o755)
 
     chunks = stream.findall("c")
     data_size = 0
-    print "\nDownloading Stream %d" % stream_index
-    print "\tChunks %10d/%-10d" % (0, len(chunks)), "\r",
+    print("\nDownloading Stream %d" % stream_index)
+    print("\tChunks %10d/%-10d" % (0, len(chunks)), "\r", end=' ')
     sys.stdout.flush()
-    for i, c in enumerate(chunks):
-        t = c.attrib["t"]
 
-        chunk_name = url.split('/')[1].replace("{start time}", t)
-        chunk_file = os.path.join(dest_dir,  chunks_quality, chunk_name)
+    stream_duration = 0
+    for i, chunk in enumerate(chunks):
 
-        if os.path.exists(chunk_file) == False:
-            chunk_url = base_url + '/' + chunks_quality + '/' + chunk_name
-            response = urllib2.urlopen(chunk_url)
-            data = response.read()
+        if "t" in chunk.attrib:
+            chunk_time = chunk.attrib["t"]
+        elif "d" in chunk.attrib:
+            chunk_time = stream_duration
+            stream_duration = chunk_time + int(chunk.attrib["d"])
 
-            f = open(chunk_file, "wb")
-            f.write(data)
-            f.close()
+        chunk_name = get_chunk_name_string(stream, chunk_time)
+        chunk_file = os.path.join(dest_dir, chunks_quality, chunk_name)
+
+        if not os.path.exists(chunk_file):
+            chunk_url = base_url + '/' + chunks_quality + '/' + chunk_name
+            data = download_file(chunk_url, chunk_file, "wb")
         else:
             f = open(chunk_file, "rb")
             data = f.read()
             f.close()
 
         data_size += len(data)
-        print "\tChunks %10d/%-10d" % (i + 1, len(chunks)), "\r",
+        print("\tChunks %10d/%-10d" % (i + 1, len(chunks)), "\r", end=' ')
         sys.stdout.flush()
-    print "\tDownloaded size:", data_size
+    print("\tDownloaded size:", data_size)
 
 
 def rebuild_stream(manifest, stream_index, quality_level, src_dir,
-        dest_file_name, final_dest_file=None):
+                   dest_file_name, final_dest_file=None):
 
-    if final_dest_file == None:
+    if final_dest_file is None:
         final_dest_file = dest_file_name
 
     stream = manifest.findall('.//StreamIndex')[stream_index]
 
-    quality = stream.findall("QualityLevel")[quality_level]
-    bitrate = quality.attrib["Bitrate"]
+    chunks_quality = get_chunk_quality_string(stream, quality_level)
 
-    # Assume URLs are in this form:
-    # Url="QualityLevels({bitrate})/Fragments(video={start time})"
-    url = stream.attrib["Url"]
-
-    chunks_quality = url.split('/')[0].replace("{bitrate}", bitrate)
     chunks_src_dir = os.path.join(src_dir, chunks_quality)
 
     dest_file = open(dest_file_name, "wb")
 
     chunks = stream.findall("c")
     data_size = 0
-    print "\nRebuilding Stream %d" % stream_index
-    print "\tChunks %10d/%-10d" % (0, len(chunks)), "\r",
+    print("\nRebuilding Stream %d" % stream_index)
+    print("\tChunks %10d/%-10d" % (0, len(chunks)), "\r", end=' ')
     sys.stdout.flush()
-    for i, c in enumerate(chunks):
-        t = c.attrib["t"]
 
-        chunk_name = url.split('/')[1].replace("{start time}", t)
+    stream_duration = 0
+    for i, chunk in enumerate(chunks):
+
+        if "t" in chunk.attrib:
+            chunk_time = chunk.attrib["t"]
+        elif "d" in chunk.attrib:
+            chunk_time = stream_duration
+            stream_duration = chunk_time + int(chunk.attrib["d"])
+
+        chunk_name = get_chunk_name_string(stream, chunk_time)
         chunk_file = os.path.join(chunks_src_dir, chunk_name)
 
         f = open(chunk_file, "rb")
@@ -246,18 +287,19 @@ def rebuild_stream(manifest, stream_index, quality_level, src_dir,
         f.close()
         dest_file.write(data)
         data_size += len(data)
-        print "\tChunks %10d/%-10d" % (i + 1, len(chunks)), "\r",
+        print("\tChunks %10d/%-10d" % (i + 1, len(chunks)), "\r", end=' ')
         sys.stdout.flush()
 
     # Add a nice WAV header
     if stream.attrib['Type'] == "audio":
+        quality = stream.findall("QualityLevel")[quality_level]
         codec_private_data = quality.attrib['CodecPrivateData']
 
         fmt = {}
         fmt['wFormatTag'] = int(quality.attrib['AudioTag'])
         fmt['nChannels'] = int(quality.attrib['Channels'])
         fmt['nSamplesPerSec'] = int(quality.attrib['SamplingRate'])
-        fmt['nAvgBytesPerSec'] = int(quality.attrib['Bitrate']) / 8
+        fmt['nAvgBytesPerSec'] = int(quality.attrib['Bitrate']) // 8
         fmt['wBitsPerSample'] = int(quality.attrib['BitsPerSample'])
         fmt['nBlockAlign'] = int(quality.attrib['PacketSize'])
         fmt['cbSize'] = 0
@@ -270,8 +312,8 @@ def rebuild_stream(manifest, stream_index, quality_level, src_dir,
         f.close()
         dest_file.close()
 
-    print
-    print "Stream %d, actual data size: %d\n" % (stream_index, data_size)
+    print()
+    print("Stream %d, actual data size: %d\n" % (stream_index, data_size))
 
 
 def calc_tracks_delay(manifest, stream1_index, stream2_index):
@@ -280,9 +322,16 @@ def calc_tracks_delay(manifest, stream1_index, stream2_index):
     s1 = streams[stream1_index]
     s2 = streams[stream2_index]
 
+    if "TimeScale" not in s1 or "TimeScale" not in s2:
+        return 0
+
     s1_start_chunk = s1.find("c")
     s2_start_chunk = s2.find("c")
 
+    if "t" not in s1_start_chunk.attrib \
+       or "t" not in s2_start_chunk.attrib:
+        return 0
+
     s1_start_time = int(s1_start_chunk.attrib['t'])
     s2_start_time = int(s2_start_chunk.attrib['t'])
 
@@ -291,7 +340,7 @@ def calc_tracks_delay(manifest, stream1_index, stream2_index):
 
     # calc difference in seconds
     delay = s2_start_time / s2_timescale - \
-            s1_start_time / s1_timescale
+        s1_start_time / s1_timescale
 
     return delay
 
@@ -303,49 +352,49 @@ def get_clip_duration(manifest):
     return float(duration) / 10000000  # here is the default timescale
 
 
-def smooth_download(url, manifest, dest_dir=tempfile.gettempdir(),
-        video_stream_index=0, audio_stream_index=1,
-        video_quality_level=0, audio_quality_level=0,
-        chunks_dir=None, download=True,
-        out_video_file='_video.vc1', out_audio_file='_audio.raw'):
+def smooth_download(url, manifest, dest_dir,
+                    video_stream_index=0, audio_stream_index=1,
+                    video_quality_level=0, audio_quality_level=0,
+                    chunks_dir=None, download=True,
+                    out_video_file='_video.vc1', out_audio_file='_audio.raw'):
 
-        if chunks_dir == None:
-            chunks_dir = dest_dir
+    if chunks_dir is None:
+        chunks_dir = dest_dir
 
-        if download:
-            download_chunks(url, manifest, video_stream_index,
-                    video_quality_level, chunks_dir)
-            download_chunks(url, manifest, audio_stream_index,
-                    audio_quality_level, chunks_dir)
+    if download:
+        download_chunks(url, manifest, video_stream_index,
+                        video_quality_level, chunks_dir)
+        download_chunks(url, manifest, audio_stream_index,
+                        audio_quality_level, chunks_dir)
 
-        dest_video = os.path.join(dest_dir, out_video_file)
-        dest_audio = os.path.join(dest_dir, out_audio_file)
+    dest_video = os.path.join(dest_dir, out_video_file)
+    dest_audio = os.path.join(dest_dir, out_audio_file)
 
-        rebuild_stream(manifest, video_stream_index, video_quality_level,
-                chunks_dir, dest_video)
-        rebuild_stream(manifest, audio_stream_index, audio_quality_level,
-                chunks_dir, dest_audio, dest_audio + '.wav')
+    rebuild_stream(manifest, video_stream_index, video_quality_level,
+                   chunks_dir, dest_video)
+    rebuild_stream(manifest, audio_stream_index, audio_quality_level,
+                   chunks_dir, dest_audio, dest_audio + '.wav')
 
-        #duration = get_clip_duration(manifest)
+    # duration = get_clip_duration(manifest)
 
-        delay = calc_tracks_delay(manifest, video_stream_index,
-                audio_stream_index)
+    delay = calc_tracks_delay(manifest, video_stream_index,
+                              audio_stream_index)
 
-        # optionally encode audio to vorbis:
-        # ffmpeg -i _audio.raw.wav -acodec libvorbis -aq 60 audio.ogg
-        mux_command = ("ffmpeg -i %s \\\n" +
-                      "  -itsoffset %f -async 1 -i %s \\\n" +
-                      "  -vcodec copy -acodec copy ffout.mkv") % \
-                      (dest_video, delay, dest_audio + '.wav')
+    # optionally encode audio to vorbis:
+    # ffmpeg -i _audio.raw.wav -acodec libvorbis -aq 60 audio.ogg
+    mux_command = ("ffmpeg -i %s \\\n" +
+                   "  -itsoffset %f -async 1 -i %s \\\n" +
+                   "  -vcodec copy -acodec copy ffout.mkv") % \
+        (dest_video, delay, dest_audio + '.wav')
 
-        print mux_command
+    print(mux_command)
 
 
 def options_parser():
-    version = "%%prog %s" % __version
+    version = "%%prog %s" % __version__
     usage = "usage: %prog [options] <manifest URL or file>"
     parser = OptionParser(usage=usage, version=version,
-            description=__description, epilog=__author_info)
+                          description=__description__, epilog=__author_info__)
     parser.add_option("-i", "--info",
                       action="store_true", dest="info_only",
                       default=False, help="print Manifest info and exit")
@@ -364,7 +413,7 @@ def options_parser():
     parser.add_option("-c", "--chunks-dir", metavar="<dir>",
                       dest="chunks_dir", default=None,
                       help="directory containing chunks, if different from destination dir")
-    parser.add_option("-v", "--video-stream",  metavar="<n>",
+    parser.add_option("-v", "--video-stream", metavar="<n>",
                       type="int", dest="video_stream_index", default=0,
                       help="index of the video stream")
     parser.add_option("-a", "--audio-stream", metavar="<n>",
@@ -380,8 +429,7 @@ def options_parser():
     return parser
 
 
-if __name__ == "__main__":
-
+def main():
     parser = options_parser()
     (options, args) = parser.parse_args()
 
@@ -389,6 +437,9 @@ if __name__ == "__main__":
         parser.print_help()
         parser.exit(1)
 
+    if not os.path.exists(options.dest_dir):
+        os.mkdir(options.dest_dir, 0o755)
+
     url = args[0]
     manifest, url = get_manifest(url, options.dest_dir)
 
@@ -396,9 +447,9 @@ if __name__ == "__main__":
         parser.exit(0)
 
     if options.sync_delay:
-        print calc_tracks_delay(manifest,
-                options.video_stream_index,
-                options.audio_stream_index)
+        print(calc_tracks_delay(manifest,
+                                options.video_stream_index,
+                                options.audio_stream_index))
         parser.exit(0)
 
     if options.info_only:
@@ -408,6 +459,10 @@ if __name__ == "__main__":
     print_manifest_info(manifest)
 
     smooth_download(url, manifest, options.dest_dir,
-            options.video_stream_index, options.audio_stream_index,
-            options.video_quality_level, options.audio_quality_level,
-            options.chunks_dir, options.download)
+                    options.video_stream_index, options.audio_stream_index,
+                    options.video_quality_level, options.audio_quality_level,
+                    options.chunks_dir, options.download)
+
+
+if __name__ == "__main__":
+    main()