From 1af7ca5f01989565fefe3bbfb4e5a387f0558602 Mon Sep 17 00:00:00 2001 From: Christopher Rosell Date: Thu, 4 Oct 2012 14:11:17 +0200 Subject: Add support for AkamaiHD's HTTP streaming protocol. --- setup.py | 7 +- src/livestreamer/packages/__init__.py | 0 src/livestreamer/packages/flashmedia/__init__.py | 6 + src/livestreamer/packages/flashmedia/amf0.py | 119 +++ src/livestreamer/packages/flashmedia/box.py | 1083 ++++++++++++++++++++ src/livestreamer/packages/flashmedia/compat.py | 33 + src/livestreamer/packages/flashmedia/error.py | 12 + src/livestreamer/packages/flashmedia/f4v.py | 25 + src/livestreamer/packages/flashmedia/flv.py | 26 + .../packages/flashmedia/ordereddict.py | 260 +++++ src/livestreamer/packages/flashmedia/packet.py | 493 +++++++++ src/livestreamer/packages/flashmedia/tag.py | 391 +++++++ src/livestreamer/packages/flashmedia/util.py | 57 ++ src/livestreamer/stream/__init__.py | 3 +- src/livestreamer/stream/akamaihd.py | 223 ++++ src/livestreamer/utils.py | 36 +- 16 files changed, 2767 insertions(+), 7 deletions(-) create mode 100644 src/livestreamer/packages/__init__.py create mode 100644 src/livestreamer/packages/flashmedia/__init__.py create mode 100644 src/livestreamer/packages/flashmedia/amf0.py create mode 100644 src/livestreamer/packages/flashmedia/box.py create mode 100644 src/livestreamer/packages/flashmedia/compat.py create mode 100644 src/livestreamer/packages/flashmedia/error.py create mode 100644 src/livestreamer/packages/flashmedia/f4v.py create mode 100644 src/livestreamer/packages/flashmedia/flv.py create mode 100644 src/livestreamer/packages/flashmedia/ordereddict.py create mode 100644 src/livestreamer/packages/flashmedia/packet.py create mode 100644 src/livestreamer/packages/flashmedia/tag.py create mode 100644 src/livestreamer/packages/flashmedia/util.py create mode 100644 src/livestreamer/stream/akamaihd.py diff --git a/setup.py b/setup.py index 992b242..8780d8b 100644 --- a/setup.py +++ b/setup.py @@ -5,6 +5,11 @@ from sys import version_info version = "1.3.2" deps = ["pbs", "requests>=0.12.1"] +packages = ["livestreamer", + "livestreamer.stream", + "livestreamer.plugins", + "livestreamer.packages", + "livestreamer.packages.flashmedia"] # require argparse on Python <2.7 and <3.2 if (version_info[0] == 2 and version_info[1] < 7) or \ @@ -18,7 +23,7 @@ setup(name="livestreamer", author="Christopher Rosell", author_email="chrippa@tanuki.se", license="BSD", - packages=["livestreamer", "livestreamer.stream", "livestreamer.plugins"], + packages=packages, package_dir={'': 'src'}, entry_points={ "console_scripts": ['livestreamer=livestreamer.cli:main'] diff --git a/src/livestreamer/packages/__init__.py b/src/livestreamer/packages/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/livestreamer/packages/flashmedia/__init__.py b/src/livestreamer/packages/flashmedia/__init__.py new file mode 100644 index 0000000..0616f9b --- /dev/null +++ b/src/livestreamer/packages/flashmedia/__init__.py @@ -0,0 +1,6 @@ +#!/usr/bin/env python + +from .error import * +from .amf0 import * +from .flv import * +from .f4v import * diff --git a/src/livestreamer/packages/flashmedia/amf0.py b/src/livestreamer/packages/flashmedia/amf0.py new file mode 100644 index 0000000..fa4368e --- /dev/null +++ b/src/livestreamer/packages/flashmedia/amf0.py @@ -0,0 +1,119 @@ +from .compat import * +from .error import * +from .packet import * +from .util import * + +class AMF0Header(Packet): + def __init__(self, name, value, must_understand=False): + self.name = name + self.value = value + self.must_understand = must_understand + + @property + def size(self): + size = 4+1 + size += PacketIO.script_string_size(self.name) + size += PacketIO.script_value_size(self.value) + + return size + + def _serialize(self, packet): + packet.write_script_string(self.name) + packet.write_u8(int(self.must_understand)) + packet.write_u32(self.size) + packet.write_script_value(self.value) + + @classmethod + def _deserialize(cls, io): + name = io.read_script_string() + must_understand = bool(io.read_u8()) + length = io.read_u32() + value = io.read_script_value() + + return cls(name, value, must_understand) + + +class AMF0Message(Packet): + def __init__(self, target_uri, response_uri, value): + self.target_uri = target_uri + self.response_uri = response_uri + self.value = value + + @property + def size(self): + size = 4 + size += PacketIO.script_string_size(self.target_uri) + size += PacketIO.script_string_size(self.response_uri) + size += PacketIO.script_value_size(self.value) + + return size + + def _serialize(self, packet): + packet.write_script_string(self.target_uri) + packet.write_script_string(self.response_uri) + packet.write_u32(self.size) + packet.write_script_value(self.value) + + @classmethod + def _deserialize(cls, io): + target_uri = io.read_script_string() + response_uri = io.read_script_string() + length = io.read_u32() + value = io.read_script_value() + + + return cls(target_uri, response_uri, value) + + +class AMF0Packet(Packet): + def __init__(self, version, headers=[], messages=[]): + self.version = version + self.headers = headers + self.messages = messages + + @property + def size(self): + size = 2+2+2 + + for header in self.headers: + size += header.size + + for message in self.messages: + size += message.size + + return size + + def _serialize(self, packet): + packet.write_u16(self.version) + packet.write_u16(len(self.headers)) + + for header in self.headers: + header.serialize(packet) + + packet.write_u16(len(self.messages)) + for message in self.messages: + message.serialize(packet) + + @classmethod + def _deserialize(cls, io): + version = io.read_u16() + + if version != 0: + raise AMFError("AMF version must be 0") + + headers = [] + header_count = io.read_u16() + + for i in range(header_count): + header = AMF0Header.deserialize(io=io) + headers.append(header) + + messages = [] + message_count = io.read_u16() + for i in range(message_count): + message = AMF0Message.deserialize(io=io) + messages.append(message) + + return cls(version, headers, messages) + +__all__ = ["AMF0Packet", "AMF0Header", "AMF0Message"] diff --git a/src/livestreamer/packages/flashmedia/box.py b/src/livestreamer/packages/flashmedia/box.py new file mode 100644 index 0000000..3a8f19d --- /dev/null +++ b/src/livestreamer/packages/flashmedia/box.py @@ -0,0 +1,1083 @@ +from ctypes import BigEndianStructure, Union, c_uint8, c_uint16, c_uint32 + +from .compat import * +from .error import * +from .packet import * +from .util import * + +class Box(Packet): + def __init__(self, type, total_size, payload, extended_size=False): + self.type = type + self.total_size = total_size + self.payload = payload + self.extended_size = extended_size + + @property + def size(self): + size = 8 + size += self.payload.size + + if size > 0xFFFFFFFF or self.extended_size: + size += 8 + + return size + + @classmethod + def _deserialize(cls, io): + size = io.read_u32() + type_ = io.read_padded(4) + header_size = 8 + extended_size = False + + if size == 1: + size = io.read_u64() + header_size += 8 + extended_size = True + + if type_ in PayloadTypes: + parent_data_left = io.data_left + io.data_left = size - header_size + + payload = PayloadTypes[type_].deserialize(io=io) + + if parent_data_left is not None: + io.data_left = parent_data_left - payload.size + else: + io.data_left = None + else: + if size == 0: + data = io.read() + else: + data = io.read(size - header_size) + + payload = RawPayload(data) + + return cls(type_, size, payload, extended_size) + + def _serialize(self, packet): + size = self.payload.size + + if size > 0xFFFFFFFF or self.extended_size: + packet.write_u32(1) + else: + packet.write_u32(size + 8) + + packet.write_padded(self.type, 4) + + if size > 0xFFFFFFFF or self.extended_size: + packet.write_u64(size + 16) + + if isinstance(self.payload, BoxPayload): + self.payload.serialize(packet) + else: + packet.write(self.payload) + +class BoxPayload(Packet): + @property + def size(self): + return 0 + + @classmethod + def box(cls, *args, **kw): + type_ = None + + for name, kls in PayloadTypes.items(): + if kls == cls: + type_ = name + break + + payload = cls(*args, **kw) + + return Box(type_, 0, payload) + +class BoxContainer(BoxPayload): + def __init__(self, boxes): + self.boxes = boxes + + @property + def size(self): + size = 0 + for box in self.boxes: + size += box.size + + return size + + def _serialize(self, packet): + for box in self.boxes: + box.serialize(packet) + + @classmethod + def _deserialize(cls, io): + boxes = [] + + while io.data_left > 0: + box = Box.deserialize(io=io) + boxes.append(box) + + return cls(boxes) + + +class BoxContainerSingle(BoxPayload): + def __init__(self, box): + self.box = box + + @property + def size(self): + return self.box.size + + def _serialize(self, packet): + self.box.serialize(packet) + + @classmethod + def _deserialize(cls, io): + box = Box.deserialize(io=io) + + return cls(box) + + + + +class RawPayload(BoxPayload): + def __init__(self, data): + self.data = data + + def __repr__(self): + return "".format(self.size) + + @property + def size(self): + return len(self.data) + + @classmethod + def _deserialize(cls, io): + data = io.read() + + return cls(data) + + def _serialize(self, packet): + packet.write(self.data) + +class BoxPayloadFTYP(BoxPayload): + def __init__(self, major_brand="f4v", minor_version=0, + compatible_brands=["isom", "mp42", "m4v"]): + self.major_brand = major_brand + self.minor_version = minor_version + self.compatible_brands = compatible_brands + + @property + def size(self): + return 4+4+(len(self.compatible_brands)*4) + + def _serialize(self, packet): + packet.write_padded(self.major_brand, 4) + packet.write_u32(self.minor_version) + + for brand in self.compatible_brands: + packet.write_padded(brand, 4) + + @classmethod + def _deserialize(cls, io): + major_brand = io.read_padded(4) + minor_version = io.read_u32() + compatible_brands = [] + + while io.data_left > 0: + brand = io.read_padded(4) + compatible_brands.append(brand) + + return cls(major_brand, minor_version, + compatible_brands) + + +class BoxPayloadMVHD(BoxPayload): + def __init__(self, version=0, creation_time=0, modification_time=0, + time_scale=1000, duration=0, rate=1.0, volume=1.0, + matrix=[65536, 0, 0, 0, 65536, 0, 0, 0, 1073741824], + next_track_id=0): + self.version = version + self.creation_time = creation_time + self.modification_time = modification_time + self.time_scale = time_scale + self.duration = duration + self.rate = rate + self.volume = volume + self.matrix = matrix + self.next_track_id = next_track_id + + @property + def size(self): + size = 1+3+4+4+2+2+4+4+(9*4)+(6*4)+4 + + if self.version == 1: + size += 3*8 + else: + size += 3*4 + + return size + + def _serialize(self, packet): + packet.write_u8(self.version) + packet.write_u24(0) # Reserved + + packet.write_u3264(self.version, self.creation_time) + packet.write_u3264(self.version, self.modification_time) + packet.write_u32(self.time_scale) + packet.write_u3264(self.version, self.duration) + + packet.write_s16_16(self.rate) + packet.write_s8_8(self.volume) + + packet.write_u16(0) # Reserved + packet.write_u32(0) # Reserved + packet.write_u32(0) # Reserved + + for m in self.matrix: + packet.write_u32(m) + + for i in range(6): + packet.write_u32(0) # Reserved + + packet.write_u32(self.next_track_id) + + @classmethod + def _deserialize(cls, io): + version = io.read_u8() + io.read_u24() # Reserved + + creation_time = io.read_u3264(version) + modification_time = io.read_u3264(version) + time_scale = io.read_u32() + duration = io.read_u3264(version) + + rate = io.read_s16_16() + volume = io.read_s8_8() + + io.read_u16() # Reserved + io.read_u32() # Reserved + io.read_u32() # Reserved + + matrix = [] + for i in range(9): + matrix.append(io.read_u32()) + + for i in range(6): + io.read_u32() # Reserved + + next_track_id = io.read_u32() + + return cls(version, creation_time, + modification_time, time_scale, duration, + rate, volume, matrix, next_track_id) + + +class SampleFlags(BoxPayload): + class Flags(Union): + class Bits(BigEndianStructure): + _fields_ = [("reserved", c_uint8, 6), + ("sample_depends_on", c_uint8, 2), + ("sample_is_depended_on", c_uint8, 2), + ("sample_has_redundancy", c_uint8, 2), + ("sample_padding_value", c_uint8, 3), + ("sample_is_difference_sample", c_uint8, 1), + ("sample_degradation_priority", c_uint16, 16)] + + _fields_ = [("bit", Bits), ("byte", c_uint32)] + + def __init__(self, sample_depends_on, sample_is_depended_on, + sample_has_redundancy, sample_padding_value, + sample_is_difference_sample, sample_degradation_priority): + + self.flags = self.Flags() + self.flags.bit.reserved = 0 # Reserved + self.flags.bit.sample_depends_on = sample_depends_on + self.flags.bit.sample_is_depended_on = sample_is_depended_on + self.flags.bit.sample_has_redundancy = sample_has_redundancy + self.flags.bit.sample_padding_value = sample_padding_value + self.flags.bit.sample_is_difference_sample = sample_is_difference_sample + self.flags.bit.sample_degradation_priority = sample_degradation_priority + + @property + def size(self): + return 4 + + def _serialize(self, packet): + packet.write_u32(self.flags.byte) + + @classmethod + def _deserialize(cls, io): + flags = cls.Flags() + flags.byte = io.read_u32() + + return cls(flags.bit.sample_depends_on, flags.bit.sample_is_depended_on, + flags.bit.sample_has_redundancy, flags.bit.sample_padding_value, + flags.bit.sample_is_difference_sample, flags.bit.sample_degradation_priority) + + +class BoxPayloadTREX(BoxPayload): + def __init__(self, version, track_id, + default_sample_description_index, + default_sample_duration, default_sample_size, + default_sample_flags): + self.version = version + self.track_id = track_id + self.default_sample_description_index = default_sample_description_index + self.default_sample_duration = default_sample_duration + self.default_sample_size = default_sample_size + self.default_sample_flags = default_sample_flags + + @property + def size(self): + return 1+3+4+4+4+4+self.default_sample_flags.size + + def _serialize(self, packet): + packet.write_u8(self.version) + packet.write_u24(0) # Reserved + packet.write_u32(self.track_id) + packet.write_u32(self.default_sample_description_index) + packet.write_u32(self.default_sample_duration) + packet.write_u32(self.default_sample_size) + self.default_sample_flags.serialize(packet) + + @classmethod + def _deserialize(cls, io): + version = io.read_u8() + flags = io.read_u24() + track_id = io.read_u32() + default_sample_description_index = io.read_u32() + default_sample_duration = io.read_u32() + default_sample_size = io.read_u32() + default_sample_flags = SampleFlags.deserialize(io=io) + + return cls(version, track_id, + default_sample_description_index, + default_sample_duration, default_sample_size, + default_sample_flags) + + +class BoxPayloadTKHD(BoxPayload): + def __init__(self, version=0, flags=1, creation_time=0, modification_time=0, + track_id=1, duration=0, layer=0, alternate_group=0, volume=0.0, + transform_matrix=[65536, 0, 0, 0, 65536, 0, 0, 0, 1073741824], + width=0.0, height=0.0): + self.version = version + self.flags = flags + self.creation_time = creation_time + self.modification_time = modification_time + self.track_id = track_id + self.duration = duration + self.layer = layer + self.alternate_group = alternate_group + self.volume = volume + self.transform_matrix = transform_matrix + self.width = width + self.height = height + + @property + def size(self): + size = 1+3+4+4+4+4+4+(4*2)+2+2+2+2+(9*4)+4+4 + + if self.version == 1: + size += 4*3 + + return size + + def _serialize(self, packet): + packet.write_u8(self.version) + packet.write_u24(self.flags) + + packet.write_u3264(self.version, self.creation_time) + packet.write_u3264(self.version, self.modification_time) + packet.write_u32(self.track_id) + packet.write_u32(0) # Reserved + packet.write_u3264(self.version, self.duration) + + for i in range(2): + packet.write_u32(0) # Reserved + + packet.write_s16(self.layer) + packet.write_s16(self.alternate_group) + packet.write_s8_8(self.volume) + packet.write_u16(0) # Reserved + + for i in range(9): + packet.write_u32(self.transform_matrix[i]) + + packet.write_s16_16(self.width) + packet.write_s16_16(self.height) + + @classmethod + def _deserialize(cls, io): + version = io.read_u8() + flags = io.read_u24() + + creation_time = io.read_u3264(version) + modification_time = io.read_u3264(version) + track_id = io.read_u32() + io.read_u32() # Reserved + duration = io.read_u3264(version) + + for i in range(2): + io.read_u32() # Reserved + + layer = io.read_s16() + alternate_group = io.read_s16() + volume = io.read_s8_8() + io.read_u16() # Reserved + + transform_matrix = [] + for i in range(9): + transform_matrix.append(io.read_s32()) + + width = io.read_s16_16() + height = io.read_s16_16() + + return cls(version, flags, creation_time, modification_time, + track_id, duration, layer, alternate_group, volume, + transform_matrix, width, height) + + +class BoxPayloadMDHD(BoxPayload): + def __init__(self, version=0, creation_time=0, modification_time=0, + time_scale=1000, duration=0, language="eng"): + self.version = version + self.creation_time = creation_time + self.modification_time = modification_time + self.time_scale = time_scale + self.duration = duration + self.language = language + + @property + def size(self): + size = 1+3+4+4+4+4+2+2 + + if self.version == 1: + size += 4*3 + + return size + + def _serialize(self, packet): + packet.write_u8(self.version) + packet.write_u24(0) # Reserved + + packet.write_u3264(self.version, self.creation_time) + packet.write_u3264(self.version, self.modification_time) + packet.write_u32(self.time_scale) + packet.write_u3264(self.version, self.duration) + + packet.write_s16(iso639_to_lang(self.language)) + packet.write_u16(0) # Reserved + + @classmethod + def _deserialize(cls, io): + version = io.read_u8() + io.read_u24() # Reserved + + creation_time = io.read_u3264(version) + modification_time = io.read_u3264(version) + time_scale = io.read_u32() + duration = io.read_u3264(version) + + language = lang_to_iso639(io.read_u16()) + io.read_u16() # Reserved + + return cls(version, creation_time, modification_time, + time_scale, duration, language) + + +class BoxPayloadHDLR(BoxPayload): + def __init__(self, version=0, predefined=0, handler_type="vide", + name=""): + self.version = version + self.predefined = predefined + self.handler_type = handler_type + self.name = name + + @property + def size(self): + size = 1+3+4+4+(3*4) + size += len(self.name) + + return size + + def _serialize(self, packet): + packet.write_u8(self.version) + packet.write_u24(0) # Reserved + packet.write_u32(self.predefined) + packet.write_padded(self.handler_type, 4) + + for i in range(3): + packet.write_u32(0) # Reserved + + packet.write(bytes(self.name, "utf8")) + #packet.write_string(self.name) + + @classmethod + def _deserialize(cls, io): + version = io.read_u8() + flags = io.read_u24() # Reserved + + predefined = io.read_u32() + handler_type = io.read_padded(4) + + for i in range(3): + io.read_u32() # Reserved + + name = io.read_string() + + return cls(version, predefined, handler_type, + name) + + +class BoxPayloadVMHD(BoxPayload): + def __init__(self, version=0, flags=1, graphics_mode=0, op_color=[0, 0, 0]): + self.version = version + self.flags = flags + self.graphics_mode = graphics_mode + self.op_color = op_color + + @property + def size(self): + return 1+3+2+(3*2) + + def _serialize(self, packet): + packet.write_u8(self.version) + packet.write_u24(self.flags) + packet.write_u16(self.graphics_mode) + + for i in range(3): + packet.write_u16(self.op_color[i]) + + @classmethod + def _deserialize(cls, io): + version = io.read_u8() + flags = io.read_u24() + + graphics_mode = io.read_u16() + op_color = [] + for i in range(3): + op_color.append(io.read_u16()) + + return cls(version, flags, graphics_mode, op_color) + + +class BoxPayloadDREF(BoxContainer): + def __init__(self, version=0, boxes=[]): + self.version = version + self.boxes = boxes + + @property + def size(self): + size = 1+3+4 + + for box in self.boxes: + size += box.size + + return size + + def _serialize(self, packet): + packet.write_u8(self.version) + packet.write_u24(0) # Reserved + packet.write_u32(len(self.boxes)) + + for box in self.boxes: + box.serialize(packet) + + @classmethod + def _deserialize(cls, io): + version = io.read_u8() + flags = io.read_u24() + + entry_count = io.read_u32() + boxes = [] + for i in range(entry_count): + box = Box.deserialize(io=io) + boxes.append(box) + + return cls(version, boxes) + +class BoxPayloadURL(BoxPayload): + def __init__(self, version=0, flags=1): + self.version = version + self.flags = flags + + @property + def size(self): + return 4 + + def _serialize(self, packet): + packet.write_u8(self.version) + packet.write_u24(self.flags) + + @classmethod + def _deserialize(cls, io): + version = io.read_u8() + flags = io.read_u24() + + return cls(version, flags) + +class BoxPayloadSTSD(BoxContainer): + def __init__(self, version=0, descriptions=[]): + self.version = version + self.descriptions = descriptions + + @property + def size(self): + size = 4+4 + + for description in self.descriptions: + size += description.size + + return size + + @property + def boxes(self): + return self.descriptions + + def _serialize(self, packet): + packet.write_u8(self.version) + packet.write_u24(0) # Reserved + packet.write_u32(len(self.descriptions)) + + for description in self.descriptions: + description.serialize(packet) + + @classmethod + def _deserialize(cls, io): + version = io.read_u8() + flags = io.read_u24() + count = io.read_u32() + + descriptions = [] + for i in range(count): + box = Box.deserialize(io=io) + descriptions.append(box) + + return cls(version, descriptions) + +class BoxPayloadVisualSample(BoxContainer): + def __init__(self, data_reference_index=0, width=0, height=0, + horiz_resolution=0.0, vert_resolution=0.0, frame_count=0, + compressor_name="", depth=0, boxes=[]): + self.data_reference_index = data_reference_index + self.width = width + self.height = height + self.horiz_resolution = horiz_resolution + self.vert_resolution = vert_resolution + self.frame_count = frame_count + self.compressor_name = compressor_name + slef.depth = depth + self.boxes = boxes + + @property + def size(self): + return 4 + + def _serialize(self, packet): + packet.write_u8(self.version) + packet.write_u24(self.flags) + + @classmethod + def _deserialize(cls, io): + for i in range(4): + io.read_u8() + + + return cls(version, flags) + + + + +class BoxPayloadMDAT(RawPayload): + def __repr__(self): + return "".format(self.size) + +class BoxPayloadSKIP(RawPayload): + def __repr__(self): + return "".format(self.size) + +class BoxPayloadFREE(RawPayload): + def __repr__(self): + return "".format(self.size) + + +class BoxPayloadABST(BoxPayload): + class Flags(Union): + class Bits(BigEndianStructure): + _fields_ = [("profile", c_uint8, 2), + ("live", c_uint8, 1), + ("update", c_uint8, 1), + ("reserved", c_uint8, 4)] + + _fields_ = [("bit", Bits), ("byte", c_uint8)] + + def __init__(self, version, bootstrap_info_version, profile, live, update, + time_scale, current_media_time, smpte_time_code_offset, + movie_identifier, server_entry_table, quality_entry_table, + drm_data, metadata, segment_run_table_entries, + fragment_run_table_entries): + self.version = version + self.bootstrap_info_version = bootstrap_info_version + self.flags = self.Flags() + self.flags.bit.profile = profile + self.flags.bit.live = live + self.flags.bit.update = update + self.flags.bit.reserved = 0 + self.time_scale = time_scale + self.current_media_time = current_media_time + self.smpte_time_code_offset = smpte_time_code_offset + self.movie_identifier = movie_identifier + self.server_entry_table = server_entry_table + self.quality_entry_table = quality_entry_table + self.drm_data = drm_data + self.metadata = metadata + self.segment_run_table_entries = segment_run_table_entries + self.fragment_run_table_entries = fragment_run_table_entries + + profile = flagproperty("flags", "profile") + update = flagproperty("flags", "update", True) + live = flagproperty("flags", "live", True) + + @property + def size(self): + size = 1+3+4+1+4+8+8 + size += len(self.movie_identifier) + 1 + + size += 1 + for server in self.server_entry_table: + size += len(server) + 1 + + size += 1 + for quality_entry in self.quality_entry_table: + size += len(quality_entry) + 1 + + size += len(self.drm_data) + 1 + size += len(self.metadata) + 1 + + size += 1 + for segment_run_table in self.segment_run_table_entries: + size += segment_run_table.size + + size += 1 + for fragment_run_table in self.fragment_run_table_entries: + size += fragment_run_table.size + + return size + + def _serialize(self, packet): + packet.write_u8(self.version) + packet.write_u24(0) # Reserved + packet.write_u32(self.bootstrap_info_version) + packet.write_u8(self.flags.byte) + packet.write_u32(self.time_scale) + packet.write_u64(self.current_media_time) + packet.write_u64(self.smpte_time_code_offset) + packet.write_string(self.movie_identifier) + + packet.write_u8(len(self.server_entry_table)) + for server_entry in self.server_entry_table: + packet.write_string(server_entry) + + packet.write_u8(len(self.quality_entry_table)) + for quality_entry in self.quality_entry_table: + packet.write_string(quality_entry) + + packet.write_string(self.drm_data) + packet.write_string(self.metadata) + + packet.write_u8(len(self.segment_run_table_entries)) + for segment_run_table in self.segment_run_table_entries: + segment_run_table.serialize(packet) + + packet.write_u8(len(self.fragment_run_table_entries)) + for fragment_run_table in self.fragment_run_table_entries: + fragment_run_table.serialize(packet) + + @classmethod + def _deserialize(cls, io): + version = io.read_u8() + io.read_u24() # Reserved + bootstrap_info_version = io.read_u32() + flags = cls.Flags() + flags.byte = io.read_u8() + time_scale = io.read_u32() + current_media_time = io.read_u64() + smpte_time_code_offset = io.read_u64() + movie_identifier = io.read_string() + + server_entry_table = [] + server_entry_count = io.read_u8() + + for i in range(server_entry_count): + server_entry = io.read_string() + server_entry_table.append(server) + + quality_entry_table = [] + quality_entry_count = io.read_u8() + + for i in range(quality_entry_count): + quality_entry = io.read_string() + quality_entry_table.append(quality) + + drm_data = io.read_string() + metadata = io.read_string() + + segment_run_table_entries = [] + segment_run_table_count = io.read_u8() + + for i in range(segment_run_table_count): + segment_run_table = Box.deserialize(io=io) + segment_run_table_entries.append(segment_run_table) + + fragment_run_table_entries = [] + fragment_run_table_count = io.read_u8() + + for i in range(fragment_run_table_count): + fragment_run_table = Box.deserialize(io=io) + fragment_run_table_entries.append(fragment_run_table) + + return cls(version, bootstrap_info_version, flags.bit.profile, + flags.bit.live, flags.bit.update, time_scale, + current_media_time, smpte_time_code_offset, movie_identifier, + server_entry_table, quality_entry_table, drm_data, + metadata, segment_run_table_entries, fragment_run_table_entries) + + +class SegmentRunEntry(BoxPayload): + def __init__(self, first_segment, fragments_per_segment): + self.first_segment = first_segment + self.fragments_per_segment = fragments_per_segment + + @property + def size(self): + return 8 + + def _serialize(self, packet): + packet.write_u32(self.first_segment) + packet.write_u32(self.fragments_per_segment) + + @classmethod + def _deserialize(cls, io): + first_segment = io.read_u32() + fragments_per_segment = io.read_u32() + + return cls(first_segment, fragments_per_segment) + + +class BoxPayloadASRT(BoxPayload): + def __init__(self, version, flags, quality_segment_url_modifiers, + segment_run_entry_table): + self.version = version + self.flags = flags + self.quality_segment_url_modifiers = quality_segment_url_modifiers + self.segment_run_entry_table = segment_run_entry_table + + @property + def size(self): + size = 1+3+1+4 + + for quality in self.quality_segment_url_modifiers: + size += len(quality) + 1 + + for segment_run_entry in self.segment_run_entry_table: + size += segment_run_entry.size + + return size + + def _serialize(self, packet): + packet.write_u8(self.version) + packet.write_u24(self.flags) + packet.write_u8(len(self.quality_segment_url_modifiers)) + + for quality in self.quality_segment_url_modifiers: + packet.write_string(quality) + + packet.write_u32(len(self.segment_run_entry_table)) + for segment_run_entry in self.segment_run_entry_table: + segment_run_entry.serialize(packet) + + @classmethod + def _deserialize(cls, io): + version = io.read_u8() + flags = io.read_u24() + + quality_segment_url_modifiers = [] + quality_entry_count = io.read_u8() + + for i in range(quality_entry_count): + quality = io.read_string() + quality_segment_url_modifiers.append(quality) + + segment_run_entry_count = io.read_u32() + segment_run_entry_table = [] + + for i in range(segment_run_entry_count): + segment_run_entry = SegmentRunEntry.deserialize(io=io) + segment_run_entry_table.append(segment_run_entry) + + return cls(version, flags, quality_segment_url_modifiers, + segment_run_entry_table) + + +class FragmentRunEntry(BoxPayload): + def __init__(self, first_fragment, first_fragment_timestamp, + fragment_duration, discontinuity_indicator): + self.first_fragment = first_fragment + self.first_fragment_timestamp = first_fragment_timestamp + self.fragment_duration = fragment_duration + self.discontinuity_indicator = discontinuity_indicator + + @property + def size(self): + size = 4+8+4 + + if self.fragment_duration == 0: + size += 1 + + return size + + def _serialize(self, packet): + packet.write_u32(self.first_fragment) + packet.write_u64(self.first_fragment_timestamp) + packet.write_u32(self.fragment_duration) + + if self.fragment_duration == 0: + packet.write_u8(self.discontinuity_indicator) + + @classmethod + def _deserialize(cls, io): + first_fragment = io.read_u32() + first_fragment_timestamp = io.read_u64() + fragment_duration = io.read_u32() + + if fragment_duration == 0: + discontinuity_indicator = io.read_u8() + else: + discontinuity_indicator = None + + return cls(first_fragment, first_fragment_timestamp, + fragment_duration, discontinuity_indicator) + + +class BoxPayloadAFRT(BoxPayload): + def __init__(self, version, flags, time_scale, + quality_segment_url_modifiers, + fragment_run_entry_table): + self.version = version + self.flags = flags + self.time_scale = time_scale + self.quality_segment_url_modifiers = quality_segment_url_modifiers + self.fragment_run_entry_table = fragment_run_entry_table + + @property + def size(self): + size = 1+3+4+1+4 + + for quality in self.quality_segment_url_modifiers: + size += len(quality) + 1 + + for fragment_run_entry in self.fragment_run_entry_table: + size += fragment_run_entry.size + + return size + + def _serialize(self, packet): + packet.write_u8(self.version) + packet.write_u24(self.flags) + packet.write_u32(self.time_scale) + packet.write_u8(len(self.quality_segment_url_modifiers)) + + for quality in self.quality_segment_url_modifiers: + packet.write_string(quality) + + packet.write_u32(len(self.fragment_run_entry_table)) + for fragment_run_entry in self.fragment_run_entry_table: + fragment_run_entry.serialize(packet) + + @classmethod + def _deserialize(cls, io): + version = io.read_u8() + flags = io.read_u24() + time_scale = io.read_u32() + + quality_segment_url_modifiers = [] + quality_entry_count = io.read_u8() + + for i in range(quality_entry_count): + quality = io.read_string() + quality_segment_url_modifiers.append(quality) + + fragment_run_entry_count = io.read_u32() + fragment_run_entry_table = [] + + for i in range(fragment_run_entry_count): + fragment_run_entry = FragmentRunEntry.deserialize(io=io) + fragment_run_entry_table.append(fragment_run_entry) + + return cls(version, flags, time_scale, + quality_segment_url_modifiers, + fragment_run_entry_table) + + +class BoxPayloadMVEX(BoxContainer): + pass + +class BoxPayloadMFRA(BoxContainer): + pass + +class BoxPayloadTRAK(BoxContainer): + pass + +class BoxPayloadMDIA(BoxContainer): + pass + +class BoxPayloadMINF(BoxContainer): + pass + +class BoxPayloadSTBL(BoxContainer): + pass + +class BoxPayloadMOOV(BoxContainer): + pass + +class BoxPayloadMOOF(BoxContainer): + pass + +class BoxPayloadMETA(BoxContainer): + pass + + +class BoxPayloadDINF(BoxContainerSingle): + pass + +PayloadTypes = { + "ftyp": BoxPayloadFTYP, + "mvhd": BoxPayloadMVHD, + "trex": BoxPayloadTREX, + "tkhd": BoxPayloadTKHD, + "mdhd": BoxPayloadMDHD, + "hdlr": BoxPayloadHDLR, + "vmhd": BoxPayloadVMHD, + "dref": BoxPayloadDREF, + "url": BoxPayloadURL, + "stsd": BoxPayloadSTSD, + "mdat": BoxPayloadMDAT, + + "abst": BoxPayloadABST, + "asrt": BoxPayloadASRT, + "afrt": BoxPayloadAFRT, + "skip": BoxPayloadSKIP, + "free": BoxPayloadFREE, + + # Containers + "moov": BoxPayloadMOOV, + "moof": BoxPayloadMOOF, + "mvex": BoxPayloadMVEX, + "mdia": BoxPayloadMDIA, + "minf": BoxPayloadMINF, + "meta": BoxPayloadMETA, + "mfra": BoxPayloadMFRA, + "stbl": BoxPayloadSTBL, + "trak": BoxPayloadTRAK, + "dinf": BoxPayloadDINF, +} + diff --git a/src/livestreamer/packages/flashmedia/compat.py b/src/livestreamer/packages/flashmedia/compat.py new file mode 100644 index 0000000..739e286 --- /dev/null +++ b/src/livestreamer/packages/flashmedia/compat.py @@ -0,0 +1,33 @@ +import os +import sys + +is_py2 = (sys.version_info[0] == 2) +is_py3 = (sys.version_info[0] == 3) +is_win32 = os.name == "nt" + +if is_py2: + _str = str + str = unicode + + def bytes(b=None, enc="ascii"): + if b is None: + return "" + elif isinstance(b, list) or isinstance(b, tuple): + return "".join([chr(i) for i in b]) + else: + return _str(b) + + from StringIO import StringIO as BytesIO + +elif is_py3: + bytes = bytes + str = str + from io import BytesIO + + +try: + from collections import OrderedDict +except ImportError: + from .ordereddict import OrderedDict + +__all__ = ["is_py2", "is_py3", "is_win32", "str", "bytes", "BytesIO", "OrderedDict"] diff --git a/src/livestreamer/packages/flashmedia/error.py b/src/livestreamer/packages/flashmedia/error.py new file mode 100644 index 0000000..a55c96a --- /dev/null +++ b/src/livestreamer/packages/flashmedia/error.py @@ -0,0 +1,12 @@ +#!/usr/bin/env python + +class FLVError(Exception): + pass + +class F4VError(Exception): + pass + +class AMFError(Exception): + pass + +__all__ = ["FLVError", "F4VError", "AMFError"] diff --git a/src/livestreamer/packages/flashmedia/f4v.py b/src/livestreamer/packages/flashmedia/f4v.py new file mode 100644 index 0000000..c4e57ef --- /dev/null +++ b/src/livestreamer/packages/flashmedia/f4v.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python + +from .box import Box +from .compat import is_py2 + +class F4V(object): + def __init__(self, fd=None): + self.fd = fd + + def __iter__(self): + return self + + def __next__(self): + try: + box = Box.deserialize(self.fd) + except IOError: + raise StopIteration + + return box + + if is_py2: + next = __next__ + + +__all__ = ["F4V"] diff --git a/src/livestreamer/packages/flashmedia/flv.py b/src/livestreamer/packages/flashmedia/flv.py new file mode 100644 index 0000000..7671fe0 --- /dev/null +++ b/src/livestreamer/packages/flashmedia/flv.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python + +from .tag import Header, Tag +from .compat import is_py2 + +class FLV(object): + def __init__(self, fd=None): + self.fd = fd + self.header = Header.deserialize(self.fd) + + def __iter__(self): + return self + + def __next__(self): + try: + tag = Tag.deserialize(self.fd) + except IOError: + raise StopIteration + + return tag + + if is_py2: + next = __next__ + + +__all__ = ["FLV"] diff --git a/src/livestreamer/packages/flashmedia/ordereddict.py b/src/livestreamer/packages/flashmedia/ordereddict.py new file mode 100644 index 0000000..f497ed7 --- /dev/null +++ b/src/livestreamer/packages/flashmedia/ordereddict.py @@ -0,0 +1,260 @@ +# Source: http://code.activestate.com/recipes/576693/ + +# Backport of OrderedDict() class that runs on Python 2.4, 2.5, 2.6, 2.7 and pypy. +# Passes Python2.7's test suite and incorporates all the latest updates. + +try: + from thread import get_ident as _get_ident +except ImportError: + from dummy_thread import get_ident as _get_ident + +try: + from _abcoll import KeysView, ValuesView, ItemsView +except ImportError: + pass + + +class OrderedDict(dict): + 'Dictionary that remembers insertion order' + # An inherited dict maps keys to values. + # The inherited dict provides __getitem__, __len__, __contains__, and get. + # The remaining methods are order-aware. + # Big-O running times for all methods are the same as for regular dictionaries. + + # The internal self.__map dictionary maps keys to links in a doubly linked list. + # The circular doubly linked list starts and ends with a sentinel element. + # The sentinel element never gets deleted (this simplifies the algorithm). + # Each link is stored as a list of length three: [PREV, NEXT, KEY]. + + def __init__(self, *args, **kwds): + '''Initialize an ordered dictionary. Signature is the same as for + regular dictionaries, but keyword arguments are not recommended + because their insertion order is arbitrary. + + ''' + if len(args) > 1: + raise TypeError('expected at most 1 arguments, got %d' % len(args)) + try: + self.__root + except AttributeError: + self.__root = root = [] # sentinel node + root[:] = [root, root, None] + self.__map = {} + self.__update(*args, **kwds) + + def __setitem__(self, key, value, dict_setitem=dict.__setitem__): + 'od.__setitem__(i, y) <==> od[i]=y' + # Setting a new item creates a new link which goes at the end of the linked + # list, and the inherited dictionary is updated with the new key/value pair. + if key not in self: + root = self.__root + last = root[0] + last[1] = root[0] = self.__map[key] = [last, root, key] + dict_setitem(self, key, value) + + def __delitem__(self, key, dict_delitem=dict.__delitem__): + 'od.__delitem__(y) <==> del od[y]' + # Deleting an existing item uses self.__map to find the link which is + # then removed by updating the links in the predecessor and successor nodes. + dict_delitem(self, key) + link_prev, link_next, key = self.__map.pop(key) + link_prev[1] = link_next + link_next[0] = link_prev + + def __iter__(self): + 'od.__iter__() <==> iter(od)' + root = self.__root + curr = root[1] + while curr is not root: + yield curr[2] + curr = curr[1] + + def __reversed__(self): + 'od.__reversed__() <==> reversed(od)' + root = self.__root + curr = root[0] + while curr is not root: + yield curr[2] + curr = curr[0] + + def clear(self): + 'od.clear() -> None. Remove all items from od.' + try: + for node in self.__map.itervalues(): + del node[:] + root = self.__root + root[:] = [root, root, None] + self.__map.clear() + except AttributeError: + pass + dict.clear(self) + + def popitem(self, last=True): + '''od.popitem() -> (k, v), return and remove a (key, value) pair. + Pairs are returned in LIFO order if last is true or FIFO order if false. + + ''' + if not self: + raise KeyError('dictionary is empty') + root = self.__root + if last: + link = root[0] + link_prev = link[0] + link_prev[1] = root + root[0] = link_prev + else: + link = root[1] + link_next = link[1] + root[1] = link_next + link_next[0] = root + key = link[2] + del self.__map[key] + value = dict.pop(self, key) + return key, value + + # -- the following methods do not depend on the internal structure -- + + def keys(self): + 'od.keys() -> list of keys in od' + return list(self) + + def values(self): + 'od.values() -> list of values in od' + return [self[key] for key in self] + + def items(self): + 'od.items() -> list of (key, value) pairs in od' + return [(key, self[key]) for key in self] + + def iterkeys(self): + 'od.iterkeys() -> an iterator over the keys in od' + return iter(self) + + def itervalues(self): + 'od.itervalues -> an iterator over the values in od' + for k in self: + yield self[k] + + def iteritems(self): + 'od.iteritems -> an iterator over the (key, value) items in od' + for k in self: + yield (k, self[k]) + + def update(*args, **kwds): + '''od.update(E, **F) -> None. Update od from dict/iterable E and F. + + If E is a dict instance, does: for k in E: od[k] = E[k] + If E has a .keys() method, does: for k in E.keys(): od[k] = E[k] + Or if E is an iterable of items, does: for k, v in E: od[k] = v + In either case, this is followed by: for k, v in F.items(): od[k] = v + + ''' + if len(args) > 2: + raise TypeError('update() takes at most 2 positional ' + 'arguments (%d given)' % (len(args),)) + elif not args: + raise TypeError('update() takes at least 1 argument (0 given)') + self = args[0] + # Make progressively weaker assumptions about "other" + other = () + if len(args) == 2: + other = args[1] + if isinstance(other, dict): + for key in other: + self[key] = other[key] + elif hasattr(other, 'keys'): + for key in other.keys(): + self[key] = other[key] + else: + for key, value in other: + self[key] = value + for key, value in kwds.items(): + self[key] = value + + __update = update # let subclasses override update without breaking __init__ + + __marker = object() + + def pop(self, key, default=__marker): + '''od.pop(k[,d]) -> v, remove specified key and return the corresponding value. + If key is not found, d is returned if given, otherwise KeyError is raised. + + ''' + if key in self: + result = self[key] + del self[key] + return result + if default is self.__marker: + raise KeyError(key) + return default + + def setdefault(self, key, default=None): + 'od.setdefault(k[,d]) -> od.get(k,d), also set od[k]=d if k not in od' + if key in self: + return self[key] + self[key] = default + return default + + def __repr__(self, _repr_running={}): + 'od.__repr__() <==> repr(od)' + call_key = id(self), _get_ident() + if call_key in _repr_running: + return '...' + _repr_running[call_key] = 1 + try: + if not self: + return '%s()' % (self.__class__.__name__,) + return '%s(%r)' % (self.__class__.__name__, self.items()) + finally: + del _repr_running[call_key] + + def __reduce__(self): + 'Return state information for pickling' + items = [[k, self[k]] for k in self] + inst_dict = vars(self).copy() + for k in vars(OrderedDict()): + inst_dict.pop(k, None) + if inst_dict: + return (self.__class__, (items,), inst_dict) + return self.__class__, (items,) + + def copy(self): + 'od.copy() -> a shallow copy of od' + return self.__class__(self) + + @classmethod + def fromkeys(cls, iterable, value=None): + '''OD.fromkeys(S[, v]) -> New ordered dictionary with keys from S + and values equal to v (which defaults to None). + + ''' + d = cls() + for key in iterable: + d[key] = value + return d + + def __eq__(self, other): + '''od.__eq__(y) <==> od==y. Comparison to another OD is order-sensitive + while comparison to a regular mapping is order-insensitive. + + ''' + if isinstance(other, OrderedDict): + return len(self)==len(other) and self.items() == other.items() + return dict.__eq__(self, other) + + def __ne__(self, other): + return not self == other + + # -- the following methods are only used in Python 2.7 -- + + def viewkeys(self): + "od.viewkeys() -> a set-like object providing a view on od's keys" + return KeysView(self) + + def viewvalues(self): + "od.viewvalues() -> an object providing a view on od's values" + return ValuesView(self) + + def viewitems(self): + "od.viewitems() -> a set-like object providing a view on od's items" + return ItemsView(self) diff --git a/src/livestreamer/packages/flashmedia/packet.py b/src/livestreamer/packages/flashmedia/packet.py new file mode 100644 index 0000000..c10714f --- /dev/null +++ b/src/livestreamer/packages/flashmedia/packet.py @@ -0,0 +1,493 @@ +#!/usr/bin/env python + +import struct + +from .compat import * +from .util import * + + +class ScriptData: + NUMBER, BOOLEAN, STRING, OBJECT, RESERVED, NULL, \ + UNDEFINED, REFERENCE, ECMAARRAY, OBJECTEND, STRICTARRAY, \ + DATE, LONGSTRING = range(13) + + class Object(OrderedDict): + pass + + class ECMAArray(OrderedDict): + pass + + class ObjectEnd(IOError): + pass + + class Date(object): + def __init__(self, timestamp, offset): + self.timestamp = timestamp + self.offset = offset + +class Packet(object): + @classmethod + def _deserialize(cls): + raise NotImplementedError + + @classmethod + def deserialize(cls, fd=None, io=None): + if not io: + if not fd: + raise IOError("Missing fd parameter") + + io = PacketIO(fd) + + return cls._deserialize(io) + + def _serialize(self): + raise NotImplementedError + + def serialize(self, packet=None): + if not packet: + packet = PacketIO() + + self._serialize(packet) + + return packet.getvalue() + + def __bytes__(self): + return self.serialize() + +# __str__ = __bytes__ + + +class PacketIO(object): + def __init__(self, fd=None): + if fd: + self.fd = fd + else: + self.io = BytesIO() + + self.data_left = None + self._objects = [] + + def getvalue(self): + return self.io.getvalue() + + def read(self, size=None): + if self.data_left is not None: + if size is None: + size = self.data_left + + if size > self.data_left: + raise IOError + + self.data_left -= size + + return self.fd.read(size) + + @property + def written(self): + return len(self.getvalue()) + + def write(self, data): + return self.io.write(data) + + + # Primitives + + def write_u8(self, num): + return self.write(struct.pack("B", int(num))) + + def write_u16(self, num): + return self.write(struct.pack(">H", int(num))) + + def write_s16(self, num): + return self.write(struct.pack(">h", int(num))) + + def write_u24(self, num): + ret = struct.pack(">I", int(num)) + return self.write(ret[1:]) + + def write_s24(self, num): + ret = struct.pack(">i", int(num)) + return self.write(ret[1:]) + + def write_s32(self, num): + return self.write(struct.pack(">i", int(num))) + + def write_u32(self, num): + return self.write(struct.pack(">I", int(num))) + + def write_s32e(self, num): + ret = struct.pack(">i", int(num)) + return self.write(ret[1:] + byte(ret[0])) + + def write_u64(self, num): + return self.write(struct.pack(">Q", int(num))) + + def write_s8_8(self, num): + num = float(num) * float(2**8) + return self.write_s16(int(num)) + + def write_s16_16(self, num): + num = float(num) * float(2**16) + return self.write_s32(int(num)) + + def write_u3264(self, version, num): + if version == 1: + self.write_u64(num) + else: + self.write_u32(num) + + def write_double(self, num): + return self.write(struct.pack(">d", num)) + + def write_string(self, string): + string = bytes(string, "utf8") + return self.write(string + b"\x00") + + def write_padded(self, value, length, padding=b" "): + for i in range(length): + try: + v = value[i] + self.write(bytes(v, "ascii")) + except IndexError: + self.write(padding) + + def read_u8(self): + try: + ret = struct.unpack("B", self.read(1))[0] + except struct.error: + raise IOError + + return ret + + def read_u16(self): + try: + ret = struct.unpack(">H", self.read(2))[0] + except struct.error: + raise IOError + + return ret + + def read_s16(self): + try: + ret = struct.unpack(">h", self.read(2))[0] + except struct.error: + raise IOError + + return ret + + def read_s8_8(self): + return float(self.read_s16()) / float(2**8) + + def read_s16_16(self): + return float(self.read_s32()) / float(2**16) + + def read_s24(self): + try: + high, low = struct.unpack(">Bh", self.read(3)) + except struct.error: + raise IOError + + ret = (high << 16) + low + + return ret + + def read_u24(self): + try: + high, low = struct.unpack(">BH", self.read(3)) + except struct.error: + raise IOError + + ret = (high << 16) + low + + return ret + + def read_s32(self): + try: + ret = struct.unpack(">i", self.read(4))[0] + except struct.error: + raise IOError + + return ret + + def read_u32(self): + try: + ret = struct.unpack(">I", self.read(4))[0] + except struct.error: + raise IOError + + return ret + + def read_s32e(self): + low_high = self.read(4) + + if len(low_high) > 4: + raise IOError + + combined = byte(low_high[3]) + low_high[:3] + + try: + ret = struct.unpack(">i", combined)[0] + except struct.error: + raise IOError + + return ret + + def read_u3264(self, version): + if version == 1: + return self.read_u64() + else: + return self.read_u32() + + def read_u64(self): + try: + ret = struct.unpack(">Q", self.read(8))[0] + except struct.error: + raise IOError + + return ret + + def read_double(self): + try: + ret = struct.unpack(">d", self.read(8))[0] + except struct.error: + raise IOError + + return ret + + def read_string(self): + ret = b"" + + while True: + try: + ch = self.read(1) + except IOError: + break + + if ord(ch) == 0: + break + + ret += ch + + return str(ret, "utf8") + + def read_padded(self, length): + return str(self.read(length), "ascii").rstrip() + + # ScriptData values + + def read_script_value(self): + typ = self.read_u8() + + if typ == ScriptData.NUMBER: + return self.read_script_number() + + elif typ == ScriptData.BOOLEAN: + return self.read_script_boolean() + + elif typ == ScriptData.STRING: + return self.read_script_string() + + elif typ == ScriptData.OBJECT: + return self.read_script_object(ScriptData.Object()) + + elif typ == ScriptData.NULL or typ == ScriptData.UNDEFINED: + return None + + elif typ == ScriptData.REFERENCE: + ref = self.read_u16() + + return self._objects[ref] + + elif typ == ScriptData.ECMAARRAY: + container = ScriptData.ECMAArray() + container.length = self.read_u32() + + return self.read_script_object(container) + + elif typ == ScriptData.OBJECTEND: + raise ScriptData.ObjectEnd + + elif typ == ScriptData.STRICTARRAY: + length = self.read_u32() + rval = [] + + for i in range(length): + val = self.read_script_value() + rval.append(val) + + return rval + + elif typ == ScriptData.DATE: + date = self.read_double() + offset = self.read_s16() + + return ScriptData.Date(date, offset) + + elif typ == ScriptData.LONGSTRING: + return self.read_script_string(True) + + raise IOError("Unhandled script data type: %d" % typ) + + def read_script_number(self): + return self.read_double() + + def read_script_boolean(self): + val = self.read_u8() + + return bool(val) + + def read_script_string(self, full=False): + if full: + length = self.read_u32() + else: + length = self.read_u16() + + val = self.read(length) + + return str(val, "utf8") + + def read_script_object(self, container): + while True: + try: + key = self.read_script_string() + value = self.read_script_value() + except IOError: + break + + if len(key) == 0: + break + + container[key] = value + + self._objects.append(container) + + return container + + def write_script_value(self, val): + if isinstance(val, bool): + self.write_u8(ScriptData.BOOLEAN) + self.write_script_boolean(val) + + elif isinstance(val, int) or isinstance(val, float): + self.write_u8(ScriptData.NUMBER) + self.write_script_number(val) + + elif isstring(val): + if len(val) > 65535: + self.write_u8(ScriptData.LONGSTRING) + else: + self.write_u8(ScriptData.STRING) + + self.write_script_string(val) + + elif isinstance(val, list): + self.write_u8(ScriptData.STRICTARRAY) + self.write_u32(len(val)) + + for value in val: + self.write_script_value(value) + + elif isinstance(val, ScriptData.Object): + self.write_u8(ScriptData.OBJECT) + self.write_script_object(val) + + elif isinstance(val, ScriptData.ECMAArray): + self.write_u8(ScriptData.ECMAARRAY) + self.write_u32(len(val)) + self.write_script_object(val) + + elif isinstance(val, ScriptData.Date): + self.write_u8(ScriptData.DATE) + self.write_double(val.timestamp) + self.write_s16(val.offset) + + else: + raise IOError("Cannot convert {0} to ScriptData value").format(type(val).__name__) + + + def write_script_number(self, val): + self.write_double(float(val)) + + def write_script_boolean(self, val): + self.write_u8(int(val)) + + def write_script_string(self, val): + length = len(val) + + if length > 65535: + self.write_u32(length) + else: + self.write_u16(length) + + self.write(bytes(val, "utf8")) + + def write_script_object(self, val): + for key, value in val.items(): + self.write_script_string(key) + self.write_script_value(value) + + # Empty string + marker ends object + self.write_script_string("") + self.write_u8(ScriptData.OBJECTEND) + + @classmethod + def script_value_size(cls, val): + size = 1 + + if isinstance(val, bool): + size += 1 + elif isinstance(val, int) or isinstance(val, float): + size += 8 + elif isstring(val): + size += cls.script_string_size(val) + + elif isinstance(val, list): + size += 4 + + for value in val: + size += cls.script_value_size(value) + + elif isinstance(val, ScriptData.Object): + size += cls.script_object_size(val) + + elif isinstance(val, ScriptData.ECMAArray): + size += 4 + size += cls.script_object_size(val) + + elif isinstance(val, ScriptData.Date): + size += 10 + + return size + + @classmethod + def script_string_size(cls, val): + size = len(val) + + if size > 65535: + size += 4 + else: + size += 2 + + return size + + @classmethod + def script_object_size(cls, val): + size = 3 + + for key, value in val.items(): + size += cls.script_string_size(key) + size += cls.script_value_size(value) + + return size + + +class TagData(Packet): + @property + def size(self): + if isinstance(self.data, Packet): + return self.data.size + else: + return len(self.data) + + +__all__ = ["Packet", "PacketIO", "TagData", "ScriptData"] diff --git a/src/livestreamer/packages/flashmedia/tag.py b/src/livestreamer/packages/flashmedia/tag.py new file mode 100644 index 0000000..2461628 --- /dev/null +++ b/src/livestreamer/packages/flashmedia/tag.py @@ -0,0 +1,391 @@ +#!/usr/bin/env python + +from ctypes import BigEndianStructure, Union, c_uint8 + +from .compat import * +from .error import * +from .packet import * +from .util import * + +TAG_TYPE_AUDIO = 8 +TAG_TYPE_VIDEO = 9 +TAG_TYPE_SCRIPT = 18 + +class TypeFlags(Union): + class Bits(BigEndianStructure): + _fields_ = [("rsv1", c_uint8, 5), + ("audio", c_uint8, 1), + ("rsv2", c_uint8, 1), + ("video", c_uint8, 1)] + + _fields_ = [("bit", Bits), ("byte", c_uint8)] + + +class TagFlags(Union): + class Bits(BigEndianStructure): + _fields_ = [("rsv", c_uint8, 2), + ("filter", c_uint8, 1), + ("type", c_uint8, 5)] + + _fields_ = [("bit", Bits), ("byte", c_uint8)] + + +class AudioFlags(Union): + class Bits(BigEndianStructure): + _fields_ = [("codec", c_uint8, 4), + ("rate", c_uint8, 2), + ("bits", c_uint8, 1), + ("type", c_uint8, 1)] + + _fields_ = [("bit", Bits), ("byte", c_uint8)] + + +class VideoFlags(Union): + class Bits(BigEndianStructure): + _fields_ = [("type", c_uint8, 4), + ("codec", c_uint8, 4)] + + _fields_ = [("bit", Bits), ("byte", c_uint8)] + + +class Header(Packet): + def __init__(self, version=1, has_audio=False, has_video=False, data_offset=9, tag0_size=0): + self.version = version + self.flags = TypeFlags() + self.flags.bit.audio = int(has_audio) + self.flags.bit.video = int(has_video) + self.data_offset = data_offset + self.tag0_size = tag0_size + + def __repr__(self): + reprformat = "
" + return reprformat.format(version=self.version, offset=self.data_offset, + has_audio=self.has_audio, has_video=self.has_video) + + + has_audio = flagproperty("flags", "audio", True) + has_video = flagproperty("flags", "video", True) + + @classmethod + def _deserialize(cls, io): + head = io.read(3) + + if head != b"FLV": + raise FLVError("Invalid FLV header") + + version = io.read_u8() + flags = TypeFlags() + flags.byte = io.read_u8() + offset = io.read_u32() + tag0_size = io.read_u32() + + return Header(version, bool(flags.bit.audio), bool(flags.bit.video), + offset, tag0_size) + + def _serialize(self, packet): + packet.write(b"FLV") + packet.write_u8(self.version) + packet.write_u8(self.flags.byte) + packet.write_u32(self.data_offset) + packet.write_u32(self.tag0_size) + + +class Tag(Packet): + def __init__(self, typ=TAG_TYPE_SCRIPT, timestamp=0, data=None, streamid=0, filter=False): + self.flags = TagFlags() + self.flags.bit.rsv = 0 + self.flags.bit.type = typ + self.flags.bit.filter = int(filter) + + self.data = data + self.streamid = streamid + self.timestamp = timestamp + + def __repr__(self): + reprformat = "" + return reprformat.format(type=self.type, timestamp=self.timestamp, + streamid=self.streamid, filter=self.filter, + data=repr(self.data)) + + type = flagproperty("flags", "type") + filter = flagproperty("flags", "filter", True) + + @property + def data_size(self): + return self.data.size + + @property + def tag_size(self): + return 11 + self.data_size + + @classmethod + def _deserialize(cls, io): + flags = TagFlags() + flags.byte = io.read_u8() + data_size = io.read_u24() + timestamp = io.read_s32e() + streamid = io.read_u24() + + if flags.bit.filter == 1: + raise FLVError("Encrypted tags are not supported") + + if flags.bit.type in TagDataTypes: + datacls = TagDataTypes[flags.bit.type] + else: + raise FLVError("Unknown tag type!") + + if data_size > 0: + io.data_left = data_size + data = datacls.deserialize(io=io) + io.data_left = None + else: + data = EmptyData() + + tag = Tag(flags.bit.type, timestamp, data, + streamid, bool(flags.bit.filter)) + + tag_size = io.read_u32() + + if tag.tag_size != tag_size: + raise FLVError("Data size mismatch when deserialising tag") + + return tag + + def _serialize(self, packet): + packet.write_u8(self.flags.byte) + packet.write_u24(self.data_size) + packet.write_s32e(self.timestamp) + packet.write_u24(self.streamid) + + self.data.serialize(packet) + + if self.tag_size != packet.written: + raise FLVError("Data size mismatch when serialising tag") + + packet.write_u32(packet.written) + + +class FrameData(TagData): + def __init__(self, type=1, data=b""): + self.type = type + self.data = data + + def __repr__(self): + if not isinstance(self.data, Packet): + data = ("<{0}>").format(type(self.data).__name__) + else: + data = repr(self.data) + + reprformat = "<{cls} type={type} data={data}>" + return reprformat.format(cls=type(self).__name__, type=self.type, data=data) + + @property + def size(self): + return TagData.size.__get__(self) + 1 + + @classmethod + def _deserialize(cls, io): + typ = io.read_u8() + data = io.read() + + return cls(typ, data) + + def _serialize(self, packet): + packet.write_u8(self.type) + packet.write(self.data) + + +class EmptyData(TagData): + def __init__(self): + self.data = b"" + + def __repr__(self): + return "" + + @classmethod + def _deserialize(cls, io): + return EmptyData() + + def _serialize(self, packet): + packet.write(self.data) + + +class AudioData(TagData): + def __init__(self, codec=0, rate=0, bits=0, type=0, data=None): + self.flags = AudioFlags() + self.flags.bit.codec = codec + self.flags.bit.rate = rate + self.flags.bit.bits = bits + self.flags.bit.type = type + self.data = data + + codec = flagproperty("flags", "codec") + rate = flagproperty("flags", "rate") + bits = flagproperty("flags", "bits") + type = flagproperty("flags", "type") + + def __repr__(self): + if not isinstance(self.data, Packet): + data = ("<{0}>").format(type(self.data).__name__) + else: + data = repr(self.data) + + reprformat = "" + return reprformat.format(type=self.type, codec=self.codec, rate=self.rate, + bits=self.bits, data=data) + + @property + def size(self): + return TagData.size.__get__(self) + 1 + + @classmethod + def _deserialize(cls, io): + flags = AudioFlags() + flags.byte = io.read_u8() + + if flags.bit.codec == 10: + data = AACAudioData.deserialize(io) + else: + data = io.read() + + return cls(flags.bit.codec, flags.bit.rate, flags.bit.bits, + flags.bit.type, data) + + def _serialize(self, packet): + packet.write_u8(self.flags.byte) + + if isinstance(self.data, Packet): + self.data.serialize(packet) + else: + packet.write(self.data) + + +class AACAudioData(FrameData): + pass + + +class VideoData(TagData): + def __init__(self, type=0, codec=0, data=b""): + self.flags = VideoFlags() + self.flags.bit.type = type + self.flags.bit.codec = codec + self.data = data + + def __repr__(self): + if not isinstance(self.data, Packet): + data = ("<{0}>").format(type(self.data).__name__) + else: + data = repr(self.data) + + reprformat = "" + return reprformat.format(type=self.type, codec=self.codec, data=data) + + type = flagproperty("flags", "type") + codec = flagproperty("flags", "codec") + + @property + def size(self): + return TagData.size.__get__(self) + 1 + + @classmethod + def _deserialize(cls, io): + flags = VideoFlags() + flags.byte = io.read_u8() + + if flags.bit.type == 5: + data = VideoCommandFrame.deserialize(io) + else: + if flags.bit.codec == 7: + data = AVCVideoData.deserialize(io) + else: + data = io.read() + + return cls(flags.bit.type, flags.bit.codec, data) + + def _serialize(self, packet): + packet.write_u8(self.flags.byte) + + if isinstance(self.data, Packet): + self.data.serialize(packet) + else: + packet.write(self.data) + + +class VideoCommandFrame(FrameData): + pass + + +class AVCVideoData(TagData): + def __init__(self, type=1, composition_time=0, data=b""): + self.type = type + self.composition_time = composition_time + self.data = data + + def __repr__(self): + if not isinstance(self.data, Packet): + data = ("<{0}>").format(type(self.data).__name__) + else: + data = repr(self.data) + + reprformat = "" + return reprformat.format(type=self.type, composition_time=self.composition_time, + data=data) + + @property + def size(self): + return TagData.size.__get__(self) + 4 + + @classmethod + def _deserialize(cls, io): + typ = io.read_u8() + composition_time = io.read_s24() + data = io.read() + + return cls(typ, composition_time, data) + + def _serialize(self, packet): + packet.write_u8(self.type) + packet.write_s24(self.composition_time) + packet.write(self.data) + + +class ScriptData(TagData): + def __init__(self, name=None, value=None): + self.name = name + self.value = value + + def __repr__(self): + reprformat = "" + return reprformat.format(name=self.name, value=self.value) + + @property + def size(self): + size = PacketIO.script_value_size(self.name) + size += PacketIO.script_value_size(self.value) + + return size + + @classmethod + def _deserialize(cls, io): + io._objects = [] + name = io.read_script_value() + value = io.read_script_value() + + return ScriptData(name, value) + + def _serialize(self, packet): + packet.write_script_value(self.name) + packet.write_script_value(self.value) + + +TagDataTypes = { + TAG_TYPE_AUDIO: AudioData, + TAG_TYPE_VIDEO: VideoData, + TAG_TYPE_SCRIPT: ScriptData +} + + + +__all__ = ["Header", "Tag", "FrameData", "AudioData", "AACAudioData", + "VideoData", "VideoCommandFrame", "AVCVideoData", + "ScriptData", "TAG_TYPE_VIDEO", "TAG_TYPE_AUDIO", "TAG_TYPE_SCRIPT"] diff --git a/src/livestreamer/packages/flashmedia/util.py b/src/livestreamer/packages/flashmedia/util.py new file mode 100644 index 0000000..237cf74 --- /dev/null +++ b/src/livestreamer/packages/flashmedia/util.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python + +from .compat import bytes, is_py2 + +def isstring(val): + if is_py2: + return isinstance(val, str) or isinstance(val, unicode) + else: + return isinstance(val, str) + +def byte(ordinal): + if isstring(ordinal): + ordinal = ord(ordinal) + + return bytes((ordinal,)) + +class flagproperty(object): + def __init__(self, flags, attr, boolean=False): + self.flags = flags + self.attr = attr + self.boolean = boolean + + def __get__(self, obj, cls): + flags = getattr(obj, self.flags) + val = getattr(flags.bit, self.attr) + + if self.boolean: + val = bool(val) + + return val + + def __set__(self, obj, val): + flags = getattr(obj, self.flags) + setattr(flags.bit, self.attr, int(val)) + +def lang_to_iso639(lang): + res = [0, 0, 0] + + for i in reversed(range(3)): + res[i] = chr(0x60 + (lang & 0x1f)) + lang = lang >> 5 + + return "".join(res) + + +def iso639_to_lang(iso639): + res = 0 + + for i in range(3): + c = ord(iso639[i]) - 0x60 + res = res << 5 + res = res | c + + return res + +__all__ = ["byte", "isstring", "flagproperty", "lang_to_iso639", "iso639_to_lang"] + diff --git a/src/livestreamer/stream/__init__.py b/src/livestreamer/stream/__init__.py index 24eaa7b..3342661 100644 --- a/src/livestreamer/stream/__init__.py +++ b/src/livestreamer/stream/__init__.py @@ -58,9 +58,10 @@ class StreamProcess(Stream): return stream.process.stdout +from .akamaihd import AkamaiHDStream from .hls import HLSStream from .http import HTTPStream from .rtmpdump import RTMPStream __all__ = ["StreamError", "Stream", "StreamProcess", - "RTMPStream", "HTTPStream"] + "AkamaiHDStream", "HLSStream", "HTTPStream", "RTMPStream"] diff --git a/src/livestreamer/stream/akamaihd.py b/src/livestreamer/stream/akamaihd.py new file mode 100644 index 0000000..a48ccb7 --- /dev/null +++ b/src/livestreamer/stream/akamaihd.py @@ -0,0 +1,223 @@ +#!/usr/bin/env python + +from . import Stream, StreamError +from ..compat import str, bytes, urlparse +from ..utils import RingBuffer, swfdecompress, swfverify, urlget, urlopen + +from ..packages.flashmedia import FLV, FLVError +from ..packages.flashmedia.tag import ScriptData + +import base64 +import hashlib +import hmac +import random +import zlib + +class TokenGenerator(object): + def __init__(self, stream): + self.stream = stream + + def generate(self): + raise NotImplementedError + +class Auth3TokenGenerator(TokenGenerator): + def generate(self): + if not self.stream.swf: + raise StreamError("A SWF URL is required to create session token") + + res = urlget(self.stream.swf, exception=StreamError) + data = swfdecompress(res.content) + + md5 = hashlib.md5() + md5.update(data) + + data = bytes(self.stream.sessionid, "ascii") + md5.digest() + sig = hmac.new(b"foo", data, hashlib.sha1) + b64 = base64.encodestring(sig.digest()) + token = str(b64, "ascii").replace("\n", "") + + return token + +def cache_bust_string(length): + rval = "" + + for i in range(length): + rval += chr(65 + int(round(random.random() * 25))) + + return rval + +class AkamaiHDStream(Stream): + Version = "2.5.8" + FlashVersion = "LNX 11,1,102,63" + + StreamURLFormat = "{host}/{streamname}" + ControlURLFormat = "{host}/control/{streamname}" + ControlData = b":)" + + TokenGenerators = { + "c11e59dea648d56e864fc07a19f717b9": Auth3TokenGenerator + } + + StatusComplete = 3 + StatusError = 4 + + Errors = { + 1: "Stream not found", + 2: "Track not found", + 3: "Seek out of bounds", + 4: "Authentication failed", + 5: "DVR disabled", + 6: "Invalid bitrate test" + } + + def __init__(self, session, url, swf=None, seek=None): + Stream.__init__(self, session) + + parsed = urlparse(url) + + self.logger = self.session.logger.new_module("stream.akamaihd") + self.host = ("{scheme}://{netloc}").format(scheme=parsed.scheme, netloc=parsed.netloc) + self.streamname = parsed.path[1:] + self.swf = swf + self.seek = seek + + def open(self): + self.guid = cache_bust_string(12) + self.islive = None + self.sessionid = None + self.flv = None + + self.buffer = RingBuffer() + self.completed_handshake = False + + url = self.StreamURLFormat.format(host=self.host, streamname=self.streamname) + params = self._create_params(seek=self.seek) + + self.logger.debug("Opening host={host} streamname={streamname}", host=self.host, streamname=self.streamname) + + try: + res = urlget(url, prefetch=False, params=params) + except Exception as err: + raise StreamError(str(err)) + + self.handshake(res.raw) + + return self + + def handshake(self, fd): + try: + self.flv = FLV(fd) + except FLVError as err: + raise StreamError(str(err)) + + self.buffer.write(self.flv.header.serialize()) + self.logger.debug("Attempting to handshake") + + for i, tag in enumerate(self.flv): + if i == 10: + raise StreamError("No OnEdge metadata in FLV after 10 tags, probably not a AkamaiHD stream") + + self.process_tag(tag) + + if self.completed_handshake: + self.logger.debug("Handshake successful") + break + + def process_tag(self, tag): + if isinstance(tag.data, ScriptData) and tag.data.name == "onEdge": + self._on_edge(tag.data.value) + + self.buffer.write(tag.serialize()) + + def send_token(self, token): + headers = { "x-Akamai-Streaming-SessionToken": token } + + self.logger.debug("Sending new session token") + self.send_control("sendingNewToken", headers=headers, + swf=self.swf) + + def send_control(self, cmd, headers={}, **params): + url = self.ControlURLFormat.format(host=self.host, + streamname=self.streamname) + + headers["x-Akamai-Streaming-SessionID"] = self.sessionid + + params = self._create_params(cmd=cmd, **params) + + return urlopen(url, headers=headers, params=params, + data=self.ControlData, exception=StreamError) + + def read(self, size=0): + if not self.flv: + return b"" + + if self.buffer.length == 0: + try: + tag = next(self.flv) + except StopIteration: + return b"" + + self.process_tag(tag) + + return self.buffer.read(size) + + def _create_params(self, **extra): + params = dict(v=self.Version, fp=self.FlashVersion, + r=cache_bust_string(5), g=self.guid) + params.update(extra) + + return params + + def _generate_session_token(self, data64): + swfdata = base64.decodestring(bytes(data64, "ascii")) + md5 = hashlib.md5() + md5.update(swfdata) + hash = md5.hexdigest() + + if hash in self.TokenGenerators: + generator = self.TokenGenerators[hash](self) + + return generator.generate() + else: + raise StreamError(("No token generator available for hash '{0}'").format(hash)) + + def _on_edge(self, data): + def updateattr(attr, key): + if key in data: + setattr(self, attr, data[key]) + + self.logger.debug("onEdge data") + for key, val in data.items(): + if isinstance(val, str): + val = val[:50] + + self.logger.debug(" {key}={val}", + key=key, val=val) + + updateattr("islive", "isLive") + updateattr("sessionid", "session") + updateattr("status", "status") + updateattr("streamname", "streamName") + + if self.status == self.StatusComplete: + self.flv = None + elif self.status == self.StatusError: + errornum = data["errorNumber"] + + if errornum in self.Errors: + msg = self.Errors[errornum] + else: + msg = "Unknown error" + + raise StreamError(msg) + + if not self.completed_handshake: + if "data64" in data: + sessiontoken = self._generate_session_token(data["data64"]) + else: + sessiontoken = None + + self.send_token(sessiontoken) + self.completed_handshake = True + +__all__ = ["AkamaiHDStream"] diff --git a/src/livestreamer/utils.py b/src/livestreamer/utils.py index 8f6e58e..121a365 100644 --- a/src/livestreamer/utils.py +++ b/src/livestreamer/utils.py @@ -90,6 +90,28 @@ class NamedPipe(object): else: os.unlink(self.path) +class RingBuffer(object): + def __init__(self): + self.buffer = b"" + + def read(self, size=0): + if size < 1: + ret = self.buffer[:] + self.buffer = b"" + else: + ret = self.buffer[:size] + self.buffer = self.buffer[size:] + + return ret + + def write(self, data): + self.buffer += data + + @property + def length(self): + return len(self.buffer) + + def urlopen(url, method="get", exception=PluginError, **args): if "data" in args and args["data"] is not None: method = "post" @@ -105,12 +127,15 @@ def urlget(url, prefetch=True, **args): return urlopen(url, method="get", prefetch=prefetch, **args) +def swfdecompress(data): + if data[:3] == b"CWS": + data = b"F" + data[1:8] + zlib.decompress(data[8:]) + + return data + def swfverify(url): res = urlopen(url) - swf = res.content - - if swf[:3] == b"CWS": - swf = b"F" + swf[1:8] + zlib.decompress(swf[8:]) + swf = swfdecompress(res.content) h = hmac.new(SWFKey, swf, hashlib.sha256) @@ -122,4 +147,5 @@ def verifyjson(json, key): return json[key] -__all__ = ["ArgumentParser", "urlopen", "urlget", "swfverify", "verifyjson"] +__all__ = ["ArgumentParser", "NamedPipe", "RingBuffer", + "urlopen", "urlget", "swfdecompress", "swfverify", "verifyjson"] -- cgit v1.2.3