* * should pop to 'tr', not the first 'td'
+ """
+
+ nestingResetTriggers = self.NESTABLE_TAGS.get(name)
+ isNestable = nestingResetTriggers != None
+ isResetNesting = self.RESET_NESTING_TAGS.has_key(name)
+ popTo = None
+ inclusive = True
+ for i in range(len(self.tagStack)-1, 0, -1):
+ p = self.tagStack[i]
+ if (not p or p.name == name) and not isNestable:
+ #Non-nestable tags get popped to the top or to their
+ #last occurance.
+ popTo = name
+ break
+ if (nestingResetTriggers != None
+ and p.name in nestingResetTriggers) \
+ or (nestingResetTriggers == None and isResetNesting
+ and self.RESET_NESTING_TAGS.has_key(p.name)):
+
+ #If we encounter one of the nesting reset triggers
+ #peculiar to this tag, or we encounter another tag
+ #that causes nesting to reset, pop up to but not
+ #including that tag.
+
+ popTo = p.name
+ inclusive = False
+ break
+ p = p.parent
+ if popTo:
+ self._popToTag(popTo, inclusive)
+
+ def unknown_starttag(self, name, attrs, selfClosing=0):
+ #print "Start tag %s" % name
+ if self.quoteStack:
+ #This is not a real tag.
+ #print "<%s> is not real!" % name
+ attrs = ''.join(map(lambda(x, y): ' %s="%s"' % (x, y), attrs))
+ self.handle_data('<%s%s>' % (name, attrs))
+ return
+ self.endData()
+ if not name in self.SELF_CLOSING_TAGS and not selfClosing:
+ self._smartPop(name)
+ tag = Tag(name, attrs, self.currentTag, self.previous)
+ if self.previous:
+ self.previous.next = tag
+ self.previous = tag
+ self.pushTag(tag)
+ if selfClosing or name in self.SELF_CLOSING_TAGS:
+ self.popTag()
+ if name in self.QUOTE_TAGS:
+ #print "Beginning quote (%s)" % name
+ self.quoteStack.append(name)
+ self.literal = 1
+
+ def unknown_endtag(self, name):
+ if self.quoteStack and self.quoteStack[-1] != name:
+ #This is not a real end tag.
+ #print " is not real!" % name
+ self.handle_data('' % name)
+ return
+ self.endData()
+ self._popToTag(name)
+ if self.quoteStack and self.quoteStack[-1] == name:
+ self.quoteStack.pop()
+ self.literal = (len(self.quoteStack) > 0)
+
+ def handle_data(self, data):
+ self.currentData.append(data)
+
+ def handle_pi(self, text):
+ "Propagate processing instructions right through."
+ self.handle_data("" % text)
+
+ def handle_comment(self, text):
+ "Propagate comments right through."
+ self.handle_data("" % text)
+
+ def handle_charref(self, ref):
+ "Propagate char refs right through."
+ self.handle_data('&#%s;' % ref)
+
+ def handle_entityref(self, ref):
+ "Propagate entity refs right through."
+ self.handle_data('&%s;' % ref)
+
+ def handle_decl(self, data):
+ "Propagate DOCTYPEs and the like right through."
+ self.handle_data('' % data)
+
+ def parse_declaration(self, i):
+ """Treat a bogus SGML declaration as raw data. Treat a CDATA
+ declaration as regular data."""
+ j = None
+ if self.rawdata[i:i+9] == '', i)
+ if k == -1:
+ k = len(self.rawdata)
+ self.handle_data(self.rawdata[i+9:k])
+ j = k+3
+ else:
+ try:
+ j = SGMLParser.parse_declaration(self, i)
+ except SGMLParseError:
+ toHandle = self.rawdata[i:]
+ self.handle_data(toHandle)
+ j = i + len(toHandle)
+ return j
+
+class BeautifulSoup(BeautifulStoneSoup):
+
+ """This parser knows the following facts about HTML:
+
+ * Some tags have no closing tag and should be interpreted as being
+ closed as soon as they are encountered.
+
+ * The text inside some tags (ie. 'script') may contain tags which
+ are not really part of the document and which should be parsed
+ as text, not tags. If you want to parse the text as tags, you can
+ always fetch it and parse it explicitly.
+
+ * Tag nesting rules:
+
+ Most tags can't be nested at all. For instance, the occurance of
+ a tag should implicitly close the previous
tag.
+
+
Para1
Para2
+ should be transformed into:
+
Para1
Para2
+
+ Some tags can be nested arbitrarily. For instance, the occurance
+ of a
tag should _not_ implicitly close the previous
+ tag.
+
+ Alice said: Bob said: Blah
+ should NOT be transformed into:
+ Alice said: Bob said: Blah
+
+ Some tags can be nested, but the nesting is reset by the
+ interposition of other tags. For instance, a tag should
+ implicitly close the previous tag within the same ,
+ but not close a tag in another table.
+
+ Blah Blah
+ should be transformed into:
+ Blah Blah
+ but,
+ BlahBlah
+ should NOT be transformed into
+ BlahBlah
+
+ Differing assumptions about tag nesting rules are a major source
+ of problems with the BeautifulSoup class. If BeautifulSoup is not
+ treating as nestable a tag your page author treats as nestable,
+ try ICantBelieveItsBeautifulSoup before writing your own
+ subclass."""
+
+ SELF_CLOSING_TAGS = buildTagMap(None, ['br' , 'hr', 'input', 'img', 'meta',
+ 'spacer', 'link', 'frame', 'base'])
+
+ QUOTE_TAGS = {'script': None}
+
+ #According to the HTML standard, each of these inline tags can
+ #contain another tag of the same type. Furthermore, it's common
+ #to actually use these tags this way.
+ NESTABLE_INLINE_TAGS = ['span', 'font', 'q', 'object', 'bdo', 'sub', 'sup',
+ 'center']
+
+ #According to the HTML standard, these block tags can contain
+ #another tag of the same type. Furthermore, it's common
+ #to actually use these tags this way.
+ NESTABLE_BLOCK_TAGS = ['blockquote', 'div', 'fieldset', 'ins', 'del']
+
+ #Lists can contain other lists, but there are restrictions.
+ NESTABLE_LIST_TAGS = { 'ol' : [],
+ 'ul' : [],
+ 'li' : ['ul', 'ol'],
+ 'dl' : [],
+ 'dd' : ['dl'],
+ 'dt' : ['dl'] }
+
+ #Tables can contain other tables, but there are restrictions.
+ NESTABLE_TABLE_TAGS = {'table' : [],
+ 'tr' : ['table', 'tbody', 'tfoot', 'thead'],
+ 'td' : ['tr'],
+ 'th' : ['tr'],
+ }
+
+ NON_NESTABLE_BLOCK_TAGS = ['address', 'form', 'p', 'pre']
+
+ #If one of these tags is encountered, all tags up to the next tag of
+ #this type are popped.
+ RESET_NESTING_TAGS = buildTagMap(None, NESTABLE_BLOCK_TAGS, 'noscript',
+ NON_NESTABLE_BLOCK_TAGS,
+ NESTABLE_LIST_TAGS,
+ NESTABLE_TABLE_TAGS)
+
+ NESTABLE_TAGS = buildTagMap([], NESTABLE_INLINE_TAGS, NESTABLE_BLOCK_TAGS,
+ NESTABLE_LIST_TAGS, NESTABLE_TABLE_TAGS)
+
+class ICantBelieveItsBeautifulSoup(BeautifulSoup):
+
+ """The BeautifulSoup class is oriented towards skipping over
+ common HTML errors like unclosed tags. However, sometimes it makes
+ errors of its own. For instance, consider this fragment:
+
+ FooBar
+
+ This is perfectly valid (if bizarre) HTML. However, the
+ BeautifulSoup class will implicitly close the first b tag when it
+ encounters the second 'b'. It will think the author wrote
+ "FooBar", and didn't close the first 'b' tag, because
+ there's no real-world reason to bold something that's already
+ bold. When it encounters ' ' it will close two more 'b'
+ tags, for a grand total of three tags closed instead of two. This
+ can throw off the rest of your document structure. The same is
+ true of a number of other tags, listed below.
+
+ It's much more common for someone to forget to close (eg.) a 'b'
+ tag than to actually use nested 'b' tags, and the BeautifulSoup
+ class handles the common case. This class handles the
+ not-co-common case: where you can't believe someone wrote what
+ they did, but it's valid HTML and BeautifulSoup screwed up by
+ assuming it wouldn't be.
+
+ If this doesn't do what you need, try subclassing this class or
+ BeautifulSoup, and providing your own list of NESTABLE_TAGS."""
+
+ I_CANT_BELIEVE_THEYRE_NESTABLE_INLINE_TAGS = \
+ ['em', 'big', 'i', 'small', 'tt', 'abbr', 'acronym', 'strong',
+ 'cite', 'code', 'dfn', 'kbd', 'samp', 'strong', 'var', 'b',
+ 'big']
+
+ I_CANT_BELIEVE_THEYRE_NESTABLE_BLOCK_TAGS = ['noscript']
+
+ NESTABLE_TAGS = buildTagMap([], BeautifulSoup.NESTABLE_TAGS,
+ I_CANT_BELIEVE_THEYRE_NESTABLE_BLOCK_TAGS,
+ I_CANT_BELIEVE_THEYRE_NESTABLE_INLINE_TAGS)
+
+class BeautifulSOAP(BeautifulStoneSoup):
+ """This class will push a tag with only a single string child into
+ the tag's parent as an attribute. The attribute's name is the tag
+ name, and the value is the string child. An example should give
+ the flavor of the change:
+
+ baz baz .
+
+
+Copyright 2002-2006 John J Lee
+Copyright 1997-1999 Gisle Aas (original libwww-perl code)
+Copyright 2002-2003 Johnny Lee (original MSIE Perl code)
+
+This code is free software; you can redistribute it and/or modify it
+under the terms of the BSD or ZPL 2.1 licenses (see the file
+COPYING.txt included with the distribution).
+
+"""
+
+import sys, re, copy, time, urllib, types, logging
+try:
+ import threading
+ _threading = threading; del threading
+except ImportError:
+ import dummy_threading
+ _threading = dummy_threading; del dummy_threading
+
+MISSING_FILENAME_TEXT = ("a filename was not supplied (nor was the CookieJar "
+ "instance initialised with one)")
+DEFAULT_HTTP_PORT = "80"
+
+from _headersutil import split_header_words, parse_ns_headers
+from _util import isstringlike
+import _rfc3986
+
+debug = logging.getLogger("mechanize.cookies").debug
+
+
+def reraise_unmasked_exceptions(unmasked=()):
+ # There are a few catch-all except: statements in this module, for
+ # catching input that's bad in unexpected ways.
+ # This function re-raises some exceptions we don't want to trap.
+ import mechanize, warnings
+ if not mechanize.USE_BARE_EXCEPT:
+ raise
+ unmasked = unmasked + (KeyboardInterrupt, SystemExit, MemoryError)
+ etype = sys.exc_info()[0]
+ if issubclass(etype, unmasked):
+ raise
+ # swallowed an exception
+ import traceback, StringIO
+ f = StringIO.StringIO()
+ traceback.print_exc(None, f)
+ msg = f.getvalue()
+ warnings.warn("mechanize bug!\n%s" % msg, stacklevel=2)
+
+
+IPV4_RE = re.compile(r"\.\d+$")
+def is_HDN(text):
+ """Return True if text is a host domain name."""
+ # XXX
+ # This may well be wrong. Which RFC is HDN defined in, if any (for
+ # the purposes of RFC 2965)?
+ # For the current implementation, what about IPv6? Remember to look
+ # at other uses of IPV4_RE also, if change this.
+ return not (IPV4_RE.search(text) or
+ text == "" or
+ text[0] == "." or text[-1] == ".")
+
+def domain_match(A, B):
+ """Return True if domain A domain-matches domain B, according to RFC 2965.
+
+ A and B may be host domain names or IP addresses.
+
+ RFC 2965, section 1:
+
+ Host names can be specified either as an IP address or a HDN string.
+ Sometimes we compare one host name with another. (Such comparisons SHALL
+ be case-insensitive.) Host A's name domain-matches host B's if
+
+ * their host name strings string-compare equal; or
+
+ * A is a HDN string and has the form NB, where N is a non-empty
+ name string, B has the form .B', and B' is a HDN string. (So,
+ x.y.com domain-matches .Y.com but not Y.com.)
+
+ Note that domain-match is not a commutative operation: a.b.c.com
+ domain-matches .c.com, but not the reverse.
+
+ """
+ # Note that, if A or B are IP addresses, the only relevant part of the
+ # definition of the domain-match algorithm is the direct string-compare.
+ A = A.lower()
+ B = B.lower()
+ if A == B:
+ return True
+ if not is_HDN(A):
+ return False
+ i = A.rfind(B)
+ has_form_nb = not (i == -1 or i == 0)
+ return (
+ has_form_nb and
+ B.startswith(".") and
+ is_HDN(B[1:])
+ )
+
+def liberal_is_HDN(text):
+ """Return True if text is a sort-of-like a host domain name.
+
+ For accepting/blocking domains.
+
+ """
+ return not IPV4_RE.search(text)
+
+def user_domain_match(A, B):
+ """For blocking/accepting domains.
+
+ A and B may be host domain names or IP addresses.
+
+ """
+ A = A.lower()
+ B = B.lower()
+ if not (liberal_is_HDN(A) and liberal_is_HDN(B)):
+ if A == B:
+ # equal IP addresses
+ return True
+ return False
+ initial_dot = B.startswith(".")
+ if initial_dot and A.endswith(B):
+ return True
+ if not initial_dot and A == B:
+ return True
+ return False
+
+cut_port_re = re.compile(r":\d+$")
+def request_host(request):
+ """Return request-host, as defined by RFC 2965.
+
+ Variation from RFC: returned value is lowercased, for convenient
+ comparison.
+
+ """
+ url = request.get_full_url()
+ host = _rfc3986.urlsplit(url)[1]
+ if host is None:
+ host = request.get_header("Host", "")
+ # remove port, if present
+ return cut_port_re.sub("", host, 1)
+
+def request_host_lc(request):
+ return request_host(request).lower()
+
+def eff_request_host(request):
+ """Return a tuple (request-host, effective request-host name)."""
+ erhn = req_host = request_host(request)
+ if req_host.find(".") == -1 and not IPV4_RE.search(req_host):
+ erhn = req_host + ".local"
+ return req_host, erhn
+
+def eff_request_host_lc(request):
+ req_host, erhn = eff_request_host(request)
+ return req_host.lower(), erhn.lower()
+
+def effective_request_host(request):
+ """Return the effective request-host, as defined by RFC 2965."""
+ return eff_request_host(request)[1]
+
+def request_path(request):
+ """Return path component of request-URI, as defined by RFC 2965."""
+ url = request.get_full_url()
+ path = escape_path(_rfc3986.urlsplit(url)[2])
+ if not path.startswith("/"):
+ path = "/" + path
+ return path
+
+def request_port(request):
+ host = request.get_host()
+ i = host.find(':')
+ if i >= 0:
+ port = host[i+1:]
+ try:
+ int(port)
+ except ValueError:
+ debug("nonnumeric port: '%s'", port)
+ return None
+ else:
+ port = DEFAULT_HTTP_PORT
+ return port
+
+def request_is_unverifiable(request):
+ try:
+ return request.is_unverifiable()
+ except AttributeError:
+ if hasattr(request, "unverifiable"):
+ return request.unverifiable
+ else:
+ raise
+
+# Characters in addition to A-Z, a-z, 0-9, '_', '.', and '-' that don't
+# need to be escaped to form a valid HTTP URL (RFCs 2396 and 1738).
+HTTP_PATH_SAFE = "%/;:@&=+$,!~*'()"
+ESCAPED_CHAR_RE = re.compile(r"%([0-9a-fA-F][0-9a-fA-F])")
+def uppercase_escaped_char(match):
+ return "%%%s" % match.group(1).upper()
+def escape_path(path):
+ """Escape any invalid characters in HTTP URL, and uppercase all escapes."""
+ # There's no knowing what character encoding was used to create URLs
+ # containing %-escapes, but since we have to pick one to escape invalid
+ # path characters, we pick UTF-8, as recommended in the HTML 4.0
+ # specification:
+ # http://www.w3.org/TR/REC-html40/appendix/notes.html#h-B.2.1
+ # And here, kind of: draft-fielding-uri-rfc2396bis-03
+ # (And in draft IRI specification: draft-duerst-iri-05)
+ # (And here, for new URI schemes: RFC 2718)
+ if isinstance(path, types.UnicodeType):
+ path = path.encode("utf-8")
+ path = urllib.quote(path, HTTP_PATH_SAFE)
+ path = ESCAPED_CHAR_RE.sub(uppercase_escaped_char, path)
+ return path
+
+def reach(h):
+ """Return reach of host h, as defined by RFC 2965, section 1.
+
+ The reach R of a host name H is defined as follows:
+
+ * If
+
+ - H is the host domain name of a host; and,
+
+ - H has the form A.B; and
+
+ - A has no embedded (that is, interior) dots; and
+
+ - B has at least one embedded dot, or B is the string "local".
+ then the reach of H is .B.
+
+ * Otherwise, the reach of H is H.
+
+ >>> reach("www.acme.com")
+ '.acme.com'
+ >>> reach("acme.com")
+ 'acme.com'
+ >>> reach("acme.local")
+ '.local'
+
+ """
+ i = h.find(".")
+ if i >= 0:
+ #a = h[:i] # this line is only here to show what a is
+ b = h[i+1:]
+ i = b.find(".")
+ if is_HDN(h) and (i >= 0 or b == "local"):
+ return "."+b
+ return h
+
+def is_third_party(request):
+ """
+
+ RFC 2965, section 3.3.6:
+
+ An unverifiable transaction is to a third-party host if its request-
+ host U does not domain-match the reach R of the request-host O in the
+ origin transaction.
+
+ """
+ req_host = request_host_lc(request)
+ # the origin request's request-host was stuffed into request by
+ # _urllib2_support.AbstractHTTPHandler
+ return not domain_match(req_host, reach(request.origin_req_host))
+
+
+try:
+ all
+except NameError:
+ # python 2.4
+ def all(iterable):
+ for x in iterable:
+ if not x:
+ return False
+ return True
+
+
+class Cookie:
+ """HTTP Cookie.
+
+ This class represents both Netscape and RFC 2965 cookies.
+
+ This is deliberately a very simple class. It just holds attributes. It's
+ possible to construct Cookie instances that don't comply with the cookie
+ standards. CookieJar.make_cookies is the factory function for Cookie
+ objects -- it deals with cookie parsing, supplying defaults, and
+ normalising to the representation used in this class. CookiePolicy is
+ responsible for checking them to see whether they should be accepted from
+ and returned to the server.
+
+ version: integer;
+ name: string;
+ value: string (may be None);
+ port: string; None indicates no attribute was supplied (e.g. "Port", rather
+ than eg. "Port=80"); otherwise, a port string (eg. "80") or a port list
+ string (e.g. "80,8080")
+ port_specified: boolean; true if a value was supplied with the Port
+ cookie-attribute
+ domain: string;
+ domain_specified: boolean; true if Domain was explicitly set
+ domain_initial_dot: boolean; true if Domain as set in HTTP header by server
+ started with a dot (yes, this really is necessary!)
+ path: string;
+ path_specified: boolean; true if Path was explicitly set
+ secure: boolean; true if should only be returned over secure connection
+ expires: integer; seconds since epoch (RFC 2965 cookies should calculate
+ this value from the Max-Age attribute)
+ discard: boolean, true if this is a session cookie; (if no expires value,
+ this should be true)
+ comment: string;
+ comment_url: string;
+ rfc2109: boolean; true if cookie arrived in a Set-Cookie: (not
+ Set-Cookie2:) header, but had a version cookie-attribute of 1
+ rest: mapping of other cookie-attributes
+
+ Note that the port may be present in the headers, but unspecified ("Port"
+ rather than"Port=80", for example); if this is the case, port is None.
+
+ """
+
+
+ _attrs = ("version", "name", "value",
+ "port", "port_specified",
+ "domain", "domain_specified", "domain_initial_dot",
+ "path", "path_specified",
+ "secure", "expires", "discard", "comment", "comment_url",
+ "rfc2109", "_rest")
+
+ def __init__(self, version, name, value,
+ port, port_specified,
+ domain, domain_specified, domain_initial_dot,
+ path, path_specified,
+ secure,
+ expires,
+ discard,
+ comment,
+ comment_url,
+ rest,
+ rfc2109=False,
+ ):
+
+ if version is not None: version = int(version)
+ if expires is not None: expires = int(expires)
+ if port is None and port_specified is True:
+ raise ValueError("if port is None, port_specified must be false")
+
+ self.version = version
+ self.name = name
+ self.value = value
+ self.port = port
+ self.port_specified = port_specified
+ # normalise case, as per RFC 2965 section 3.3.3
+ self.domain = domain.lower()
+ self.domain_specified = domain_specified
+ # Sigh. We need to know whether the domain given in the
+ # cookie-attribute had an initial dot, in order to follow RFC 2965
+ # (as clarified in draft errata). Needed for the returned $Domain
+ # value.
+ self.domain_initial_dot = domain_initial_dot
+ self.path = path
+ self.path_specified = path_specified
+ self.secure = secure
+ self.expires = expires
+ self.discard = discard
+ self.comment = comment
+ self.comment_url = comment_url
+ self.rfc2109 = rfc2109
+
+ self._rest = copy.copy(rest)
+
+ def has_nonstandard_attr(self, name):
+ return self._rest.has_key(name)
+ def get_nonstandard_attr(self, name, default=None):
+ return self._rest.get(name, default)
+ def set_nonstandard_attr(self, name, value):
+ self._rest[name] = value
+ def nonstandard_attr_keys(self):
+ return self._rest.keys()
+
+ def is_expired(self, now=None):
+ if now is None: now = time.time()
+ return (self.expires is not None) and (self.expires <= now)
+
+ def __eq__(self, other):
+ return all(getattr(self, a) == getattr(other, a) for a in self._attrs)
+
+ def __ne__(self, other):
+ return not (self == other)
+
+ def __str__(self):
+ if self.port is None: p = ""
+ else: p = ":"+self.port
+ limit = self.domain + p + self.path
+ if self.value is not None:
+ namevalue = "%s=%s" % (self.name, self.value)
+ else:
+ namevalue = self.name
+ return "" % (namevalue, limit)
+
+ def __repr__(self):
+ args = []
+ for name in ["version", "name", "value",
+ "port", "port_specified",
+ "domain", "domain_specified", "domain_initial_dot",
+ "path", "path_specified",
+ "secure", "expires", "discard", "comment", "comment_url",
+ ]:
+ attr = getattr(self, name)
+ args.append("%s=%s" % (name, repr(attr)))
+ args.append("rest=%s" % repr(self._rest))
+ args.append("rfc2109=%s" % repr(self.rfc2109))
+ return "Cookie(%s)" % ", ".join(args)
+
+
+class CookiePolicy:
+ """Defines which cookies get accepted from and returned to server.
+
+ May also modify cookies.
+
+ The subclass DefaultCookiePolicy defines the standard rules for Netscape
+ and RFC 2965 cookies -- override that if you want a customised policy.
+
+ As well as implementing set_ok and return_ok, implementations of this
+ interface must also supply the following attributes, indicating which
+ protocols should be used, and how. These can be read and set at any time,
+ though whether that makes complete sense from the protocol point of view is
+ doubtful.
+
+ Public attributes:
+
+ netscape: implement netscape protocol
+ rfc2965: implement RFC 2965 protocol
+ rfc2109_as_netscape:
+ WARNING: This argument will change or go away if is not accepted into
+ the Python standard library in this form!
+ If true, treat RFC 2109 cookies as though they were Netscape cookies. The
+ default is for this attribute to be None, which means treat 2109 cookies
+ as RFC 2965 cookies unless RFC 2965 handling is switched off (which it is,
+ by default), and as Netscape cookies otherwise.
+ hide_cookie2: don't add Cookie2 header to requests (the presence of
+ this header indicates to the server that we understand RFC 2965
+ cookies)
+
+ """
+ def set_ok(self, cookie, request):
+ """Return true if (and only if) cookie should be accepted from server.
+
+ Currently, pre-expired cookies never get this far -- the CookieJar
+ class deletes such cookies itself.
+
+ cookie: mechanize.Cookie object
+ request: object implementing the interface defined by
+ CookieJar.extract_cookies.__doc__
+
+ """
+ raise NotImplementedError()
+
+ def return_ok(self, cookie, request):
+ """Return true if (and only if) cookie should be returned to server.
+
+ cookie: mechanize.Cookie object
+ request: object implementing the interface defined by
+ CookieJar.add_cookie_header.__doc__
+
+ """
+ raise NotImplementedError()
+
+ def domain_return_ok(self, domain, request):
+ """Return false if cookies should not be returned, given cookie domain.
+
+ This is here as an optimization, to remove the need for checking every
+ cookie with a particular domain (which may involve reading many files).
+ The default implementations of domain_return_ok and path_return_ok
+ (return True) leave all the work to return_ok.
+
+ If domain_return_ok returns true for the cookie domain, path_return_ok
+ is called for the cookie path. Otherwise, path_return_ok and return_ok
+ are never called for that cookie domain. If path_return_ok returns
+ true, return_ok is called with the Cookie object itself for a full
+ check. Otherwise, return_ok is never called for that cookie path.
+
+ Note that domain_return_ok is called for every *cookie* domain, not
+ just for the *request* domain. For example, the function might be
+ called with both ".acme.com" and "www.acme.com" if the request domain
+ is "www.acme.com". The same goes for path_return_ok.
+
+ For argument documentation, see the docstring for return_ok.
+
+ """
+ return True
+
+ def path_return_ok(self, path, request):
+ """Return false if cookies should not be returned, given cookie path.
+
+ See the docstring for domain_return_ok.
+
+ """
+ return True
+
+
+class DefaultCookiePolicy(CookiePolicy):
+ """Implements the standard rules for accepting and returning cookies.
+
+ Both RFC 2965 and Netscape cookies are covered. RFC 2965 handling is
+ switched off by default.
+
+ The easiest way to provide your own policy is to override this class and
+ call its methods in your overriden implementations before adding your own
+ additional checks.
+
+ import mechanize
+ class MyCookiePolicy(mechanize.DefaultCookiePolicy):
+ def set_ok(self, cookie, request):
+ if not mechanize.DefaultCookiePolicy.set_ok(
+ self, cookie, request):
+ return False
+ if i_dont_want_to_store_this_cookie():
+ return False
+ return True
+
+ In addition to the features required to implement the CookiePolicy
+ interface, this class allows you to block and allow domains from setting
+ and receiving cookies. There are also some strictness switches that allow
+ you to tighten up the rather loose Netscape protocol rules a little bit (at
+ the cost of blocking some benign cookies).
+
+ A domain blacklist and whitelist is provided (both off by default). Only
+ domains not in the blacklist and present in the whitelist (if the whitelist
+ is active) participate in cookie setting and returning. Use the
+ blocked_domains constructor argument, and blocked_domains and
+ set_blocked_domains methods (and the corresponding argument and methods for
+ allowed_domains). If you set a whitelist, you can turn it off again by
+ setting it to None.
+
+ Domains in block or allow lists that do not start with a dot must
+ string-compare equal. For example, "acme.com" matches a blacklist entry of
+ "acme.com", but "www.acme.com" does not. Domains that do start with a dot
+ are matched by more specific domains too. For example, both "www.acme.com"
+ and "www.munitions.acme.com" match ".acme.com" (but "acme.com" itself does
+ not). IP addresses are an exception, and must match exactly. For example,
+ if blocked_domains contains "192.168.1.2" and ".168.1.2" 192.168.1.2 is
+ blocked, but 193.168.1.2 is not.
+
+ Additional Public Attributes:
+
+ General strictness switches
+
+ strict_domain: don't allow sites to set two-component domains with
+ country-code top-level domains like .co.uk, .gov.uk, .co.nz. etc.
+ This is far from perfect and isn't guaranteed to work!
+
+ RFC 2965 protocol strictness switches
+
+ strict_rfc2965_unverifiable: follow RFC 2965 rules on unverifiable
+ transactions (usually, an unverifiable transaction is one resulting from
+ a redirect or an image hosted on another site); if this is false, cookies
+ are NEVER blocked on the basis of verifiability
+
+ Netscape protocol strictness switches
+
+ strict_ns_unverifiable: apply RFC 2965 rules on unverifiable transactions
+ even to Netscape cookies
+ strict_ns_domain: flags indicating how strict to be with domain-matching
+ rules for Netscape cookies:
+ DomainStrictNoDots: when setting cookies, host prefix must not contain a
+ dot (e.g. www.foo.bar.com can't set a cookie for .bar.com, because
+ www.foo contains a dot)
+ DomainStrictNonDomain: cookies that did not explicitly specify a Domain
+ cookie-attribute can only be returned to a domain that string-compares
+ equal to the domain that set the cookie (e.g. rockets.acme.com won't
+ be returned cookies from acme.com that had no Domain cookie-attribute)
+ DomainRFC2965Match: when setting cookies, require a full RFC 2965
+ domain-match
+ DomainLiberal and DomainStrict are the most useful combinations of the
+ above flags, for convenience
+ strict_ns_set_initial_dollar: ignore cookies in Set-Cookie: headers that
+ have names starting with '$'
+ strict_ns_set_path: don't allow setting cookies whose path doesn't
+ path-match request URI
+
+ """
+
+ DomainStrictNoDots = 1
+ DomainStrictNonDomain = 2
+ DomainRFC2965Match = 4
+
+ DomainLiberal = 0
+ DomainStrict = DomainStrictNoDots|DomainStrictNonDomain
+
+ def __init__(self,
+ blocked_domains=None, allowed_domains=None,
+ netscape=True, rfc2965=False,
+ # WARNING: this argument will change or go away if is not
+ # accepted into the Python standard library in this form!
+ # default, ie. treat 2109 as netscape iff not rfc2965
+ rfc2109_as_netscape=None,
+ hide_cookie2=False,
+ strict_domain=False,
+ strict_rfc2965_unverifiable=True,
+ strict_ns_unverifiable=False,
+ strict_ns_domain=DomainLiberal,
+ strict_ns_set_initial_dollar=False,
+ strict_ns_set_path=False,
+ ):
+ """
+ Constructor arguments should be used as keyword arguments only.
+
+ blocked_domains: sequence of domain names that we never accept cookies
+ from, nor return cookies to
+ allowed_domains: if not None, this is a sequence of the only domains
+ for which we accept and return cookies
+
+ For other arguments, see CookiePolicy.__doc__ and
+ DefaultCookiePolicy.__doc__..
+
+ """
+ self.netscape = netscape
+ self.rfc2965 = rfc2965
+ self.rfc2109_as_netscape = rfc2109_as_netscape
+ self.hide_cookie2 = hide_cookie2
+ self.strict_domain = strict_domain
+ self.strict_rfc2965_unverifiable = strict_rfc2965_unverifiable
+ self.strict_ns_unverifiable = strict_ns_unverifiable
+ self.strict_ns_domain = strict_ns_domain
+ self.strict_ns_set_initial_dollar = strict_ns_set_initial_dollar
+ self.strict_ns_set_path = strict_ns_set_path
+
+ if blocked_domains is not None:
+ self._blocked_domains = tuple(blocked_domains)
+ else:
+ self._blocked_domains = ()
+
+ if allowed_domains is not None:
+ allowed_domains = tuple(allowed_domains)
+ self._allowed_domains = allowed_domains
+
+ def blocked_domains(self):
+ """Return the sequence of blocked domains (as a tuple)."""
+ return self._blocked_domains
+ def set_blocked_domains(self, blocked_domains):
+ """Set the sequence of blocked domains."""
+ self._blocked_domains = tuple(blocked_domains)
+
+ def is_blocked(self, domain):
+ for blocked_domain in self._blocked_domains:
+ if user_domain_match(domain, blocked_domain):
+ return True
+ return False
+
+ def allowed_domains(self):
+ """Return None, or the sequence of allowed domains (as a tuple)."""
+ return self._allowed_domains
+ def set_allowed_domains(self, allowed_domains):
+ """Set the sequence of allowed domains, or None."""
+ if allowed_domains is not None:
+ allowed_domains = tuple(allowed_domains)
+ self._allowed_domains = allowed_domains
+
+ def is_not_allowed(self, domain):
+ if self._allowed_domains is None:
+ return False
+ for allowed_domain in self._allowed_domains:
+ if user_domain_match(domain, allowed_domain):
+ return False
+ return True
+
+ def set_ok(self, cookie, request):
+ """
+ If you override set_ok, be sure to call this method. If it returns
+ false, so should your subclass (assuming your subclass wants to be more
+ strict about which cookies to accept).
+
+ """
+ debug(" - checking cookie %s", cookie)
+
+ assert cookie.name is not None
+
+ for n in "version", "verifiability", "name", "path", "domain", "port":
+ fn_name = "set_ok_"+n
+ fn = getattr(self, fn_name)
+ if not fn(cookie, request):
+ return False
+
+ return True
+
+ def set_ok_version(self, cookie, request):
+ if cookie.version is None:
+ # Version is always set to 0 by parse_ns_headers if it's a Netscape
+ # cookie, so this must be an invalid RFC 2965 cookie.
+ debug(" Set-Cookie2 without version attribute (%s)", cookie)
+ return False
+ if cookie.version > 0 and not self.rfc2965:
+ debug(" RFC 2965 cookies are switched off")
+ return False
+ elif cookie.version == 0 and not self.netscape:
+ debug(" Netscape cookies are switched off")
+ return False
+ return True
+
+ def set_ok_verifiability(self, cookie, request):
+ if request_is_unverifiable(request) and is_third_party(request):
+ if cookie.version > 0 and self.strict_rfc2965_unverifiable:
+ debug(" third-party RFC 2965 cookie during "
+ "unverifiable transaction")
+ return False
+ elif cookie.version == 0 and self.strict_ns_unverifiable:
+ debug(" third-party Netscape cookie during "
+ "unverifiable transaction")
+ return False
+ return True
+
+ def set_ok_name(self, cookie, request):
+ # Try and stop servers setting V0 cookies designed to hack other
+ # servers that know both V0 and V1 protocols.
+ if (cookie.version == 0 and self.strict_ns_set_initial_dollar and
+ cookie.name.startswith("$")):
+ debug(" illegal name (starts with '$'): '%s'", cookie.name)
+ return False
+ return True
+
+ def set_ok_path(self, cookie, request):
+ if cookie.path_specified:
+ req_path = request_path(request)
+ if ((cookie.version > 0 or
+ (cookie.version == 0 and self.strict_ns_set_path)) and
+ not req_path.startswith(cookie.path)):
+ debug(" path attribute %s is not a prefix of request "
+ "path %s", cookie.path, req_path)
+ return False
+ return True
+
+ def set_ok_countrycode_domain(self, cookie, request):
+ """Return False if explicit cookie domain is not acceptable.
+
+ Called by set_ok_domain, for convenience of overriding by
+ subclasses.
+
+ """
+ if cookie.domain_specified and self.strict_domain:
+ domain = cookie.domain
+ # since domain was specified, we know that:
+ assert domain.startswith(".")
+ if domain.count(".") == 2:
+ # domain like .foo.bar
+ i = domain.rfind(".")
+ tld = domain[i+1:]
+ sld = domain[1:i]
+ if (sld.lower() in [
+ "co", "ac",
+ "com", "edu", "org", "net", "gov", "mil", "int",
+ "aero", "biz", "cat", "coop", "info", "jobs", "mobi",
+ "museum", "name", "pro", "travel",
+ ] and
+ len(tld) == 2):
+ # domain like .co.uk
+ return False
+ return True
+
+ def set_ok_domain(self, cookie, request):
+ if self.is_blocked(cookie.domain):
+ debug(" domain %s is in user block-list", cookie.domain)
+ return False
+ if self.is_not_allowed(cookie.domain):
+ debug(" domain %s is not in user allow-list", cookie.domain)
+ return False
+ if not self.set_ok_countrycode_domain(cookie, request):
+ debug(" country-code second level domain %s", cookie.domain)
+ return False
+ if cookie.domain_specified:
+ req_host, erhn = eff_request_host_lc(request)
+ domain = cookie.domain
+ if domain.startswith("."):
+ undotted_domain = domain[1:]
+ else:
+ undotted_domain = domain
+ embedded_dots = (undotted_domain.find(".") >= 0)
+ if not embedded_dots and domain != ".local":
+ debug(" non-local domain %s contains no embedded dot",
+ domain)
+ return False
+ if cookie.version == 0:
+ if (not erhn.endswith(domain) and
+ (not erhn.startswith(".") and
+ not ("."+erhn).endswith(domain))):
+ debug(" effective request-host %s (even with added "
+ "initial dot) does not end end with %s",
+ erhn, domain)
+ return False
+ if (cookie.version > 0 or
+ (self.strict_ns_domain & self.DomainRFC2965Match)):
+ if not domain_match(erhn, domain):
+ debug(" effective request-host %s does not domain-match "
+ "%s", erhn, domain)
+ return False
+ if (cookie.version > 0 or
+ (self.strict_ns_domain & self.DomainStrictNoDots)):
+ host_prefix = req_host[:-len(domain)]
+ if (host_prefix.find(".") >= 0 and
+ not IPV4_RE.search(req_host)):
+ debug(" host prefix %s for domain %s contains a dot",
+ host_prefix, domain)
+ return False
+ return True
+
+ def set_ok_port(self, cookie, request):
+ if cookie.port_specified:
+ req_port = request_port(request)
+ if req_port is None:
+ req_port = "80"
+ else:
+ req_port = str(req_port)
+ for p in cookie.port.split(","):
+ try:
+ int(p)
+ except ValueError:
+ debug(" bad port %s (not numeric)", p)
+ return False
+ if p == req_port:
+ break
+ else:
+ debug(" request port (%s) not found in %s",
+ req_port, cookie.port)
+ return False
+ return True
+
+ def return_ok(self, cookie, request):
+ """
+ If you override return_ok, be sure to call this method. If it returns
+ false, so should your subclass (assuming your subclass wants to be more
+ strict about which cookies to return).
+
+ """
+ # Path has already been checked by path_return_ok, and domain blocking
+ # done by domain_return_ok.
+ debug(" - checking cookie %s", cookie)
+
+ for n in ("version", "verifiability", "secure", "expires", "port",
+ "domain"):
+ fn_name = "return_ok_"+n
+ fn = getattr(self, fn_name)
+ if not fn(cookie, request):
+ return False
+ return True
+
+ def return_ok_version(self, cookie, request):
+ if cookie.version > 0 and not self.rfc2965:
+ debug(" RFC 2965 cookies are switched off")
+ return False
+ elif cookie.version == 0 and not self.netscape:
+ debug(" Netscape cookies are switched off")
+ return False
+ return True
+
+ def return_ok_verifiability(self, cookie, request):
+ if request_is_unverifiable(request) and is_third_party(request):
+ if cookie.version > 0 and self.strict_rfc2965_unverifiable:
+ debug(" third-party RFC 2965 cookie during unverifiable "
+ "transaction")
+ return False
+ elif cookie.version == 0 and self.strict_ns_unverifiable:
+ debug(" third-party Netscape cookie during unverifiable "
+ "transaction")
+ return False
+ return True
+
+ def return_ok_secure(self, cookie, request):
+ if cookie.secure and request.get_type() != "https":
+ debug(" secure cookie with non-secure request")
+ return False
+ return True
+
+ def return_ok_expires(self, cookie, request):
+ if cookie.is_expired(self._now):
+ debug(" cookie expired")
+ return False
+ return True
+
+ def return_ok_port(self, cookie, request):
+ if cookie.port:
+ req_port = request_port(request)
+ if req_port is None:
+ req_port = "80"
+ for p in cookie.port.split(","):
+ if p == req_port:
+ break
+ else:
+ debug(" request port %s does not match cookie port %s",
+ req_port, cookie.port)
+ return False
+ return True
+
+ def return_ok_domain(self, cookie, request):
+ req_host, erhn = eff_request_host_lc(request)
+ domain = cookie.domain
+
+ # strict check of non-domain cookies: Mozilla does this, MSIE5 doesn't
+ if (cookie.version == 0 and
+ (self.strict_ns_domain & self.DomainStrictNonDomain) and
+ not cookie.domain_specified and domain != erhn):
+ debug(" cookie with unspecified domain does not string-compare "
+ "equal to request domain")
+ return False
+
+ if cookie.version > 0 and not domain_match(erhn, domain):
+ debug(" effective request-host name %s does not domain-match "
+ "RFC 2965 cookie domain %s", erhn, domain)
+ return False
+ if cookie.version == 0 and not ("."+erhn).endswith(domain):
+ debug(" request-host %s does not match Netscape cookie domain "
+ "%s", req_host, domain)
+ return False
+ return True
+
+ def domain_return_ok(self, domain, request):
+ # Liberal check of domain. This is here as an optimization to avoid
+ # having to load lots of MSIE cookie files unless necessary.
+
+ # Munge req_host and erhn to always start with a dot, so as to err on
+ # the side of letting cookies through.
+ dotted_req_host, dotted_erhn = eff_request_host_lc(request)
+ if not dotted_req_host.startswith("."):
+ dotted_req_host = "."+dotted_req_host
+ if not dotted_erhn.startswith("."):
+ dotted_erhn = "."+dotted_erhn
+ if not (dotted_req_host.endswith(domain) or
+ dotted_erhn.endswith(domain)):
+ #debug(" request domain %s does not match cookie domain %s",
+ # req_host, domain)
+ return False
+
+ if self.is_blocked(domain):
+ debug(" domain %s is in user block-list", domain)
+ return False
+ if self.is_not_allowed(domain):
+ debug(" domain %s is not in user allow-list", domain)
+ return False
+
+ return True
+
+ def path_return_ok(self, path, request):
+ debug("- checking cookie path=%s", path)
+ req_path = request_path(request)
+ if not req_path.startswith(path):
+ debug(" %s does not path-match %s", req_path, path)
+ return False
+ return True
+
+
+def vals_sorted_by_key(adict):
+ keys = adict.keys()
+ keys.sort()
+ return map(adict.get, keys)
+
+class MappingIterator:
+ """Iterates over nested mapping, depth-first, in sorted order by key."""
+ def __init__(self, mapping):
+ self._s = [(vals_sorted_by_key(mapping), 0, None)] # LIFO stack
+
+ def __iter__(self): return self
+
+ def next(self):
+ # this is hairy because of lack of generators
+ while 1:
+ try:
+ vals, i, prev_item = self._s.pop()
+ except IndexError:
+ raise StopIteration()
+ if i < len(vals):
+ item = vals[i]
+ i = i + 1
+ self._s.append((vals, i, prev_item))
+ try:
+ item.items
+ except AttributeError:
+ # non-mapping
+ break
+ else:
+ # mapping
+ self._s.append((vals_sorted_by_key(item), 0, item))
+ continue
+ return item
+
+
+# Used as second parameter to dict.get method, to distinguish absent
+# dict key from one with a None value.
+class Absent: pass
+
+class CookieJar:
+ """Collection of HTTP cookies.
+
+ You may not need to know about this class: try mechanize.urlopen().
+
+ The major methods are extract_cookies and add_cookie_header; these are all
+ you are likely to need.
+
+ CookieJar supports the iterator protocol:
+
+ for cookie in cookiejar:
+ # do something with cookie
+
+ Methods:
+
+ add_cookie_header(request)
+ extract_cookies(response, request)
+ get_policy()
+ set_policy(policy)
+ cookies_for_request(request)
+ make_cookies(response, request)
+ set_cookie_if_ok(cookie, request)
+ set_cookie(cookie)
+ clear_session_cookies()
+ clear_expired_cookies()
+ clear(domain=None, path=None, name=None)
+
+ Public attributes
+
+ policy: CookiePolicy object
+
+ """
+
+ non_word_re = re.compile(r"\W")
+ quote_re = re.compile(r"([\"\\])")
+ strict_domain_re = re.compile(r"\.?[^.]*")
+ domain_re = re.compile(r"[^.]*")
+ dots_re = re.compile(r"^\.+")
+
+ def __init__(self, policy=None):
+ """
+ See CookieJar.__doc__ for argument documentation.
+
+ """
+ if policy is None:
+ policy = DefaultCookiePolicy()
+ self._policy = policy
+
+ self._cookies = {}
+
+ # for __getitem__ iteration in pre-2.2 Pythons
+ self._prev_getitem_index = 0
+
+ def get_policy(self):
+ return self._policy
+
+ def set_policy(self, policy):
+ self._policy = policy
+
+ def _cookies_for_domain(self, domain, request):
+ cookies = []
+ if not self._policy.domain_return_ok(domain, request):
+ return []
+ debug("Checking %s for cookies to return", domain)
+ cookies_by_path = self._cookies[domain]
+ for path in cookies_by_path.keys():
+ if not self._policy.path_return_ok(path, request):
+ continue
+ cookies_by_name = cookies_by_path[path]
+ for cookie in cookies_by_name.values():
+ if not self._policy.return_ok(cookie, request):
+ debug(" not returning cookie")
+ continue
+ debug(" it's a match")
+ cookies.append(cookie)
+ return cookies
+
+ def cookies_for_request(self, request):
+ """Return a list of cookies to be returned to server.
+
+ The returned list of cookie instances is sorted in the order they
+ should appear in the Cookie: header for return to the server.
+
+ See add_cookie_header.__doc__ for the interface required of the
+ request argument.
+
+ New in version 0.1.10
+
+ """
+ self._policy._now = self._now = int(time.time())
+ cookies = self._cookies_for_request(request)
+ # add cookies in order of most specific (i.e. longest) path first
+ def decreasing_size(a, b): return cmp(len(b.path), len(a.path))
+ cookies.sort(decreasing_size)
+ return cookies
+
+ def _cookies_for_request(self, request):
+ """Return a list of cookies to be returned to server."""
+ # this method still exists (alongside cookies_for_request) because it
+ # is part of an implied protected interface for subclasses of cookiejar
+ # XXX document that implied interface, or provide another way of
+ # implementing cookiejars than subclassing
+ cookies = []
+ for domain in self._cookies.keys():
+ cookies.extend(self._cookies_for_domain(domain, request))
+ return cookies
+
+ def _cookie_attrs(self, cookies):
+ """Return a list of cookie-attributes to be returned to server.
+
+ The $Version attribute is also added when appropriate (currently only
+ once per request).
+
+ >>> jar = CookieJar()
+ >>> ns_cookie = Cookie(0, "foo", '"bar"', None, False,
+ ... "example.com", False, False,
+ ... "/", False, False, None, True,
+ ... None, None, {})
+ >>> jar._cookie_attrs([ns_cookie])
+ ['foo="bar"']
+ >>> rfc2965_cookie = Cookie(1, "foo", "bar", None, False,
+ ... ".example.com", True, False,
+ ... "/", False, False, None, True,
+ ... None, None, {})
+ >>> jar._cookie_attrs([rfc2965_cookie])
+ ['$Version=1', 'foo=bar', '$Domain="example.com"']
+
+ """
+ version_set = False
+
+ attrs = []
+ for cookie in cookies:
+ # set version of Cookie header
+ # XXX
+ # What should it be if multiple matching Set-Cookie headers have
+ # different versions themselves?
+ # Answer: there is no answer; was supposed to be settled by
+ # RFC 2965 errata, but that may never appear...
+ version = cookie.version
+ if not version_set:
+ version_set = True
+ if version > 0:
+ attrs.append("$Version=%s" % version)
+
+ # quote cookie value if necessary
+ # (not for Netscape protocol, which already has any quotes
+ # intact, due to the poorly-specified Netscape Cookie: syntax)
+ if ((cookie.value is not None) and
+ self.non_word_re.search(cookie.value) and version > 0):
+ value = self.quote_re.sub(r"\\\1", cookie.value)
+ else:
+ value = cookie.value
+
+ # add cookie-attributes to be returned in Cookie header
+ if cookie.value is None:
+ attrs.append(cookie.name)
+ else:
+ attrs.append("%s=%s" % (cookie.name, value))
+ if version > 0:
+ if cookie.path_specified:
+ attrs.append('$Path="%s"' % cookie.path)
+ if cookie.domain.startswith("."):
+ domain = cookie.domain
+ if (not cookie.domain_initial_dot and
+ domain.startswith(".")):
+ domain = domain[1:]
+ attrs.append('$Domain="%s"' % domain)
+ if cookie.port is not None:
+ p = "$Port"
+ if cookie.port_specified:
+ p = p + ('="%s"' % cookie.port)
+ attrs.append(p)
+
+ return attrs
+
+ def add_cookie_header(self, request):
+ """Add correct Cookie: header to request (mechanize.Request object).
+
+ The Cookie2 header is also added unless policy.hide_cookie2 is true.
+
+ The request object (usually a mechanize.Request instance) must support
+ the methods get_full_url, get_host, is_unverifiable, get_type,
+ has_header, get_header, header_items and add_unredirected_header, as
+ documented by urllib2.
+ """
+ debug("add_cookie_header")
+ cookies = self.cookies_for_request(request)
+
+ attrs = self._cookie_attrs(cookies)
+ if attrs:
+ if not request.has_header("Cookie"):
+ request.add_unredirected_header("Cookie", "; ".join(attrs))
+
+ # if necessary, advertise that we know RFC 2965
+ if self._policy.rfc2965 and not self._policy.hide_cookie2:
+ for cookie in cookies:
+ if cookie.version != 1 and not request.has_header("Cookie2"):
+ request.add_unredirected_header("Cookie2", '$Version="1"')
+ break
+
+ self.clear_expired_cookies()
+
+ def _normalized_cookie_tuples(self, attrs_set):
+ """Return list of tuples containing normalised cookie information.
+
+ attrs_set is the list of lists of key,value pairs extracted from
+ the Set-Cookie or Set-Cookie2 headers.
+
+ Tuples are name, value, standard, rest, where name and value are the
+ cookie name and value, standard is a dictionary containing the standard
+ cookie-attributes (discard, secure, version, expires or max-age,
+ domain, path and port) and rest is a dictionary containing the rest of
+ the cookie-attributes.
+
+ """
+ cookie_tuples = []
+
+ boolean_attrs = "discard", "secure"
+ value_attrs = ("version",
+ "expires", "max-age",
+ "domain", "path", "port",
+ "comment", "commenturl")
+
+ for cookie_attrs in attrs_set:
+ name, value = cookie_attrs[0]
+
+ # Build dictionary of standard cookie-attributes (standard) and
+ # dictionary of other cookie-attributes (rest).
+
+ # Note: expiry time is normalised to seconds since epoch. V0
+ # cookies should have the Expires cookie-attribute, and V1 cookies
+ # should have Max-Age, but since V1 includes RFC 2109 cookies (and
+ # since V0 cookies may be a mish-mash of Netscape and RFC 2109), we
+ # accept either (but prefer Max-Age).
+ max_age_set = False
+
+ bad_cookie = False
+
+ standard = {}
+ rest = {}
+ for k, v in cookie_attrs[1:]:
+ lc = k.lower()
+ # don't lose case distinction for unknown fields
+ if lc in value_attrs or lc in boolean_attrs:
+ k = lc
+ if k in boolean_attrs and v is None:
+ # boolean cookie-attribute is present, but has no value
+ # (like "discard", rather than "port=80")
+ v = True
+ if standard.has_key(k):
+ # only first value is significant
+ continue
+ if k == "domain":
+ if v is None:
+ debug(" missing value for domain attribute")
+ bad_cookie = True
+ break
+ # RFC 2965 section 3.3.3
+ v = v.lower()
+ if k == "expires":
+ if max_age_set:
+ # Prefer max-age to expires (like Mozilla)
+ continue
+ if v is None:
+ debug(" missing or invalid value for expires "
+ "attribute: treating as session cookie")
+ continue
+ if k == "max-age":
+ max_age_set = True
+ if v is None:
+ debug(" missing value for max-age attribute")
+ bad_cookie = True
+ break
+ try:
+ v = int(v)
+ except ValueError:
+ debug(" missing or invalid (non-numeric) value for "
+ "max-age attribute")
+ bad_cookie = True
+ break
+ # convert RFC 2965 Max-Age to seconds since epoch
+ # XXX Strictly you're supposed to follow RFC 2616
+ # age-calculation rules. Remember that zero Max-Age is a
+ # is a request to discard (old and new) cookie, though.
+ k = "expires"
+ v = self._now + v
+ if (k in value_attrs) or (k in boolean_attrs):
+ if (v is None and
+ k not in ["port", "comment", "commenturl"]):
+ debug(" missing value for %s attribute" % k)
+ bad_cookie = True
+ break
+ standard[k] = v
+ else:
+ rest[k] = v
+
+ if bad_cookie:
+ continue
+
+ cookie_tuples.append((name, value, standard, rest))
+
+ return cookie_tuples
+
+ def _cookie_from_cookie_tuple(self, tup, request):
+ # standard is dict of standard cookie-attributes, rest is dict of the
+ # rest of them
+ name, value, standard, rest = tup
+
+ domain = standard.get("domain", Absent)
+ path = standard.get("path", Absent)
+ port = standard.get("port", Absent)
+ expires = standard.get("expires", Absent)
+
+ # set the easy defaults
+ version = standard.get("version", None)
+ if version is not None:
+ try:
+ version = int(version)
+ except ValueError:
+ return None # invalid version, ignore cookie
+ secure = standard.get("secure", False)
+ # (discard is also set if expires is Absent)
+ discard = standard.get("discard", False)
+ comment = standard.get("comment", None)
+ comment_url = standard.get("commenturl", None)
+
+ # set default path
+ if path is not Absent and path != "":
+ path_specified = True
+ path = escape_path(path)
+ else:
+ path_specified = False
+ path = request_path(request)
+ i = path.rfind("/")
+ if i != -1:
+ if version == 0:
+ # Netscape spec parts company from reality here
+ path = path[:i]
+ else:
+ path = path[:i+1]
+ if len(path) == 0: path = "/"
+
+ # set default domain
+ domain_specified = domain is not Absent
+ # but first we have to remember whether it starts with a dot
+ domain_initial_dot = False
+ if domain_specified:
+ domain_initial_dot = bool(domain.startswith("."))
+ if domain is Absent:
+ req_host, erhn = eff_request_host_lc(request)
+ domain = erhn
+ elif not domain.startswith("."):
+ domain = "."+domain
+
+ # set default port
+ port_specified = False
+ if port is not Absent:
+ if port is None:
+ # Port attr present, but has no value: default to request port.
+ # Cookie should then only be sent back on that port.
+ port = request_port(request)
+ else:
+ port_specified = True
+ port = re.sub(r"\s+", "", port)
+ else:
+ # No port attr present. Cookie can be sent back on any port.
+ port = None
+
+ # set default expires and discard
+ if expires is Absent:
+ expires = None
+ discard = True
+
+ return Cookie(version,
+ name, value,
+ port, port_specified,
+ domain, domain_specified, domain_initial_dot,
+ path, path_specified,
+ secure,
+ expires,
+ discard,
+ comment,
+ comment_url,
+ rest)
+
+ def _cookies_from_attrs_set(self, attrs_set, request):
+ cookie_tuples = self._normalized_cookie_tuples(attrs_set)
+
+ cookies = []
+ for tup in cookie_tuples:
+ cookie = self._cookie_from_cookie_tuple(tup, request)
+ if cookie: cookies.append(cookie)
+ return cookies
+
+ def _process_rfc2109_cookies(self, cookies):
+ if self._policy.rfc2109_as_netscape is None:
+ rfc2109_as_netscape = not self._policy.rfc2965
+ else:
+ rfc2109_as_netscape = self._policy.rfc2109_as_netscape
+ for cookie in cookies:
+ if cookie.version == 1:
+ cookie.rfc2109 = True
+ if rfc2109_as_netscape:
+ # treat 2109 cookies as Netscape cookies rather than
+ # as RFC2965 cookies
+ cookie.version = 0
+
+ def _make_cookies(self, response, request):
+ # get cookie-attributes for RFC 2965 and Netscape protocols
+ headers = response.info()
+ rfc2965_hdrs = headers.getheaders("Set-Cookie2")
+ ns_hdrs = headers.getheaders("Set-Cookie")
+
+ rfc2965 = self._policy.rfc2965
+ netscape = self._policy.netscape
+
+ if ((not rfc2965_hdrs and not ns_hdrs) or
+ (not ns_hdrs and not rfc2965) or
+ (not rfc2965_hdrs and not netscape) or
+ (not netscape and not rfc2965)):
+ return [] # no relevant cookie headers: quick exit
+
+ try:
+ cookies = self._cookies_from_attrs_set(
+ split_header_words(rfc2965_hdrs), request)
+ except:
+ reraise_unmasked_exceptions()
+ cookies = []
+
+ if ns_hdrs and netscape:
+ try:
+ # RFC 2109 and Netscape cookies
+ ns_cookies = self._cookies_from_attrs_set(
+ parse_ns_headers(ns_hdrs), request)
+ except:
+ reraise_unmasked_exceptions()
+ ns_cookies = []
+ self._process_rfc2109_cookies(ns_cookies)
+
+ # Look for Netscape cookies (from Set-Cookie headers) that match
+ # corresponding RFC 2965 cookies (from Set-Cookie2 headers).
+ # For each match, keep the RFC 2965 cookie and ignore the Netscape
+ # cookie (RFC 2965 section 9.1). Actually, RFC 2109 cookies are
+ # bundled in with the Netscape cookies for this purpose, which is
+ # reasonable behaviour.
+ if rfc2965:
+ lookup = {}
+ for cookie in cookies:
+ lookup[(cookie.domain, cookie.path, cookie.name)] = None
+
+ def no_matching_rfc2965(ns_cookie, lookup=lookup):
+ key = ns_cookie.domain, ns_cookie.path, ns_cookie.name
+ return not lookup.has_key(key)
+ ns_cookies = filter(no_matching_rfc2965, ns_cookies)
+
+ if ns_cookies:
+ cookies.extend(ns_cookies)
+
+ return cookies
+
+ def make_cookies(self, response, request):
+ """Return sequence of Cookie objects extracted from response object.
+
+ See extract_cookies.__doc__ for the interface required of the
+ response and request arguments.
+
+ """
+ self._policy._now = self._now = int(time.time())
+ return [cookie for cookie in self._make_cookies(response, request)
+ if cookie.expires is None or not cookie.expires <= self._now]
+
+ def set_cookie_if_ok(self, cookie, request):
+ """Set a cookie if policy says it's OK to do so.
+
+ cookie: mechanize.Cookie instance
+ request: see extract_cookies.__doc__ for the required interface
+
+ """
+ self._policy._now = self._now = int(time.time())
+
+ if self._policy.set_ok(cookie, request):
+ self.set_cookie(cookie)
+
+ def set_cookie(self, cookie):
+ """Set a cookie, without checking whether or not it should be set.
+
+ cookie: mechanize.Cookie instance
+ """
+ c = self._cookies
+ if not c.has_key(cookie.domain): c[cookie.domain] = {}
+ c2 = c[cookie.domain]
+ if not c2.has_key(cookie.path): c2[cookie.path] = {}
+ c3 = c2[cookie.path]
+ c3[cookie.name] = cookie
+
+ def extract_cookies(self, response, request):
+ """Extract cookies from response, where allowable given the request.
+
+ Look for allowable Set-Cookie: and Set-Cookie2: headers in the response
+ object passed as argument. Any of these headers that are found are
+ used to update the state of the object (subject to the policy.set_ok
+ method's approval).
+
+ The response object (usually be the result of a call to
+ mechanize.urlopen, or similar) should support an info method, which
+ returns a mimetools.Message object (in fact, the 'mimetools.Message
+ object' may be any object that provides a getheaders method).
+
+ The request object (usually a mechanize.Request instance) must support
+ the methods get_full_url, get_type, get_host, and is_unverifiable, as
+ documented by mechanize, and the port attribute (the port number). The
+ request is used to set default values for cookie-attributes as well as
+ for checking that the cookie is OK to be set.
+
+ """
+ debug("extract_cookies: %s", response.info())
+ self._policy._now = self._now = int(time.time())
+
+ for cookie in self._make_cookies(response, request):
+ if cookie.expires is not None and cookie.expires <= self._now:
+ # Expiry date in past is request to delete cookie. This can't be
+ # in DefaultCookiePolicy, because can't delete cookies there.
+ try:
+ self.clear(cookie.domain, cookie.path, cookie.name)
+ except KeyError:
+ pass
+ debug("Expiring cookie, domain='%s', path='%s', name='%s'",
+ cookie.domain, cookie.path, cookie.name)
+ elif self._policy.set_ok(cookie, request):
+ debug(" setting cookie: %s", cookie)
+ self.set_cookie(cookie)
+
+ def clear(self, domain=None, path=None, name=None):
+ """Clear some cookies.
+
+ Invoking this method without arguments will clear all cookies. If
+ given a single argument, only cookies belonging to that domain will be
+ removed. If given two arguments, cookies belonging to the specified
+ path within that domain are removed. If given three arguments, then
+ the cookie with the specified name, path and domain is removed.
+
+ Raises KeyError if no matching cookie exists.
+
+ """
+ if name is not None:
+ if (domain is None) or (path is None):
+ raise ValueError(
+ "domain and path must be given to remove a cookie by name")
+ del self._cookies[domain][path][name]
+ elif path is not None:
+ if domain is None:
+ raise ValueError(
+ "domain must be given to remove cookies by path")
+ del self._cookies[domain][path]
+ elif domain is not None:
+ del self._cookies[domain]
+ else:
+ self._cookies = {}
+
+ def clear_session_cookies(self):
+ """Discard all session cookies.
+
+ Discards all cookies held by object which had either no Max-Age or
+ Expires cookie-attribute or an explicit Discard cookie-attribute, or
+ which otherwise have ended up with a true discard attribute. For
+ interactive browsers, the end of a session usually corresponds to
+ closing the browser window.
+
+ Note that the save method won't save session cookies anyway, unless you
+ ask otherwise by passing a true ignore_discard argument.
+
+ """
+ for cookie in self:
+ if cookie.discard:
+ self.clear(cookie.domain, cookie.path, cookie.name)
+
+ def clear_expired_cookies(self):
+ """Discard all expired cookies.
+
+ You probably don't need to call this method: expired cookies are never
+ sent back to the server (provided you're using DefaultCookiePolicy),
+ this method is called by CookieJar itself every so often, and the save
+ method won't save expired cookies anyway (unless you ask otherwise by
+ passing a true ignore_expires argument).
+
+ """
+ now = time.time()
+ for cookie in self:
+ if cookie.is_expired(now):
+ self.clear(cookie.domain, cookie.path, cookie.name)
+
+ def __getitem__(self, i):
+ if i == 0:
+ self._getitem_iterator = self.__iter__()
+ elif self._prev_getitem_index != i-1: raise IndexError(
+ "CookieJar.__getitem__ only supports sequential iteration")
+ self._prev_getitem_index = i
+ try:
+ return self._getitem_iterator.next()
+ except StopIteration:
+ raise IndexError()
+
+ def __iter__(self):
+ return MappingIterator(self._cookies)
+
+ def __len__(self):
+ """Return number of contained cookies."""
+ i = 0
+ for cookie in self: i = i + 1
+ return i
+
+ def __repr__(self):
+ r = []
+ for cookie in self: r.append(repr(cookie))
+ return "<%s[%s]>" % (self.__class__, ", ".join(r))
+
+ def __str__(self):
+ r = []
+ for cookie in self: r.append(str(cookie))
+ return "<%s[%s]>" % (self.__class__, ", ".join(r))
+
+
+class LoadError(Exception): pass
+
+class FileCookieJar(CookieJar):
+ """CookieJar that can be loaded from and saved to a file.
+
+ Additional methods
+
+ save(filename=None, ignore_discard=False, ignore_expires=False)
+ load(filename=None, ignore_discard=False, ignore_expires=False)
+ revert(filename=None, ignore_discard=False, ignore_expires=False)
+
+ Additional public attributes
+
+ filename: filename for loading and saving cookies
+
+ Additional public readable attributes
+
+ delayload: request that cookies are lazily loaded from disk; this is only
+ a hint since this only affects performance, not behaviour (unless the
+ cookies on disk are changing); a CookieJar object may ignore it (in fact,
+ only MSIECookieJar lazily loads cookies at the moment)
+
+ """
+
+ def __init__(self, filename=None, delayload=False, policy=None):
+ """
+ See FileCookieJar.__doc__ for argument documentation.
+
+ Cookies are NOT loaded from the named file until either the load or
+ revert method is called.
+
+ """
+ CookieJar.__init__(self, policy)
+ if filename is not None and not isstringlike(filename):
+ raise ValueError("filename must be string-like")
+ self.filename = filename
+ self.delayload = bool(delayload)
+
+ def save(self, filename=None, ignore_discard=False, ignore_expires=False):
+ """Save cookies to a file.
+
+ filename: name of file in which to save cookies
+ ignore_discard: save even cookies set to be discarded
+ ignore_expires: save even cookies that have expired
+
+ The file is overwritten if it already exists, thus wiping all its
+ cookies. Saved cookies can be restored later using the load or revert
+ methods. If filename is not specified, self.filename is used; if
+ self.filename is None, ValueError is raised.
+
+ """
+ raise NotImplementedError()
+
+ def load(self, filename=None, ignore_discard=False, ignore_expires=False):
+ """Load cookies from a file.
+
+ Old cookies are kept unless overwritten by newly loaded ones.
+
+ Arguments are as for .save().
+
+ If filename is not specified, self.filename is used; if self.filename
+ is None, ValueError is raised. The named file must be in the format
+ understood by the class, or LoadError will be raised. This format will
+ be identical to that written by the save method, unless the load format
+ is not sufficiently well understood (as is the case for MSIECookieJar).
+
+ """
+ if filename is None:
+ if self.filename is not None: filename = self.filename
+ else: raise ValueError(MISSING_FILENAME_TEXT)
+
+ f = open(filename)
+ try:
+ self._really_load(f, filename, ignore_discard, ignore_expires)
+ finally:
+ f.close()
+
+ def revert(self, filename=None,
+ ignore_discard=False, ignore_expires=False):
+ """Clear all cookies and reload cookies from a saved file.
+
+ Raises LoadError (or IOError) if reversion is not successful; the
+ object's state will not be altered if this happens.
+
+ """
+ if filename is None:
+ if self.filename is not None: filename = self.filename
+ else: raise ValueError(MISSING_FILENAME_TEXT)
+
+ old_state = copy.deepcopy(self._cookies)
+ self._cookies = {}
+ try:
+ self.load(filename, ignore_discard, ignore_expires)
+ except (LoadError, IOError):
+ self._cookies = old_state
+ raise
diff --git a/couchpotato/core/media/movie/providers/trailer/mechanize/_debug.py b/couchpotato/core/media/movie/providers/trailer/mechanize/_debug.py
new file mode 100644
index 0000000000..8243969990
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/mechanize/_debug.py
@@ -0,0 +1,28 @@
+import logging
+
+from _response import response_seek_wrapper
+from _urllib2_fork import BaseHandler
+
+
+class HTTPResponseDebugProcessor(BaseHandler):
+ handler_order = 900 # before redirections, after everything else
+
+ def http_response(self, request, response):
+ if not hasattr(response, "seek"):
+ response = response_seek_wrapper(response)
+ info = logging.getLogger("mechanize.http_responses").info
+ try:
+ info(response.read())
+ finally:
+ response.seek(0)
+ info("*****************************************************")
+ return response
+
+ https_response = http_response
+
+class HTTPRedirectDebugProcessor(BaseHandler):
+ def http_request(self, request):
+ if hasattr(request, "redirect_dict"):
+ info = logging.getLogger("mechanize.http_redirects").info
+ info("redirecting to %s", request.get_full_url())
+ return request
diff --git a/couchpotato/core/media/movie/providers/trailer/mechanize/_firefox3cookiejar.py b/couchpotato/core/media/movie/providers/trailer/mechanize/_firefox3cookiejar.py
new file mode 100644
index 0000000000..a64d70f35d
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/mechanize/_firefox3cookiejar.py
@@ -0,0 +1,248 @@
+"""Firefox 3 "cookies.sqlite" cookie persistence.
+
+Copyright 2008 John J Lee
+
+This code is free software; you can redistribute it and/or modify it
+under the terms of the BSD or ZPL 2.1 licenses (see the file
+COPYING.txt included with the distribution).
+
+"""
+
+import logging
+import time
+
+from _clientcookie import CookieJar, Cookie, MappingIterator
+from _util import isstringlike, experimental
+debug = logging.getLogger("mechanize.cookies").debug
+
+
+class Firefox3CookieJar(CookieJar):
+
+ """Firefox 3 cookie jar.
+
+ The cookies are stored in Firefox 3's "cookies.sqlite" format.
+
+ Constructor arguments:
+
+ filename: filename of cookies.sqlite (typically found at the top level
+ of a firefox profile directory)
+ autoconnect: as a convenience, connect to the SQLite cookies database at
+ Firefox3CookieJar construction time (default True)
+ policy: an object satisfying the mechanize.CookiePolicy interface
+
+ Note that this is NOT a FileCookieJar, and there are no .load(),
+ .save() or .restore() methods. The database is in sync with the
+ cookiejar object's state after each public method call.
+
+ Following Firefox's own behaviour, session cookies are never saved to
+ the database.
+
+ The file is created, and an sqlite database written to it, if it does
+ not already exist. The moz_cookies database table is created if it does
+ not already exist.
+ """
+
+ # XXX
+ # handle DatabaseError exceptions
+ # add a FileCookieJar (explicit .save() / .revert() / .load() methods)
+
+ def __init__(self, filename, autoconnect=True, policy=None):
+ experimental("Firefox3CookieJar is experimental code")
+ CookieJar.__init__(self, policy)
+ if filename is not None and not isstringlike(filename):
+ raise ValueError("filename must be string-like")
+ self.filename = filename
+ self._conn = None
+ if autoconnect:
+ self.connect()
+
+ def connect(self):
+ import sqlite3 # not available in Python 2.4 stdlib
+ self._conn = sqlite3.connect(self.filename)
+ self._conn.isolation_level = "DEFERRED"
+ self._create_table_if_necessary()
+
+ def close(self):
+ self._conn.close()
+
+ def _transaction(self, func):
+ try:
+ cur = self._conn.cursor()
+ try:
+ result = func(cur)
+ finally:
+ cur.close()
+ except:
+ self._conn.rollback()
+ raise
+ else:
+ self._conn.commit()
+ return result
+
+ def _execute(self, query, params=()):
+ return self._transaction(lambda cur: cur.execute(query, params))
+
+ def _query(self, query, params=()):
+ # XXX should we bother with a transaction?
+ cur = self._conn.cursor()
+ try:
+ cur.execute(query, params)
+ return cur.fetchall()
+ finally:
+ cur.close()
+
+ def _create_table_if_necessary(self):
+ self._execute("""\
+CREATE TABLE IF NOT EXISTS moz_cookies (id INTEGER PRIMARY KEY, name TEXT,
+ value TEXT, host TEXT, path TEXT,expiry INTEGER,
+ lastAccessed INTEGER, isSecure INTEGER, isHttpOnly INTEGER)""")
+
+ def _cookie_from_row(self, row):
+ (pk, name, value, domain, path, expires,
+ last_accessed, secure, http_only) = row
+
+ version = 0
+ domain = domain.encode("ascii", "ignore")
+ path = path.encode("ascii", "ignore")
+ name = name.encode("ascii", "ignore")
+ value = value.encode("ascii", "ignore")
+ secure = bool(secure)
+
+ # last_accessed isn't a cookie attribute, so isn't added to rest
+ rest = {}
+ if http_only:
+ rest["HttpOnly"] = None
+
+ if name == "":
+ name = value
+ value = None
+
+ initial_dot = domain.startswith(".")
+ domain_specified = initial_dot
+
+ discard = False
+ if expires == "":
+ expires = None
+ discard = True
+
+ return Cookie(version, name, value,
+ None, False,
+ domain, domain_specified, initial_dot,
+ path, False,
+ secure,
+ expires,
+ discard,
+ None,
+ None,
+ rest)
+
+ def clear(self, domain=None, path=None, name=None):
+ CookieJar.clear(self, domain, path, name)
+ where_parts = []
+ sql_params = []
+ if domain is not None:
+ where_parts.append("host = ?")
+ sql_params.append(domain)
+ if path is not None:
+ where_parts.append("path = ?")
+ sql_params.append(path)
+ if name is not None:
+ where_parts.append("name = ?")
+ sql_params.append(name)
+ where = " AND ".join(where_parts)
+ if where:
+ where = " WHERE " + where
+ def clear(cur):
+ cur.execute("DELETE FROM moz_cookies%s" % where,
+ tuple(sql_params))
+ self._transaction(clear)
+
+ def _row_from_cookie(self, cookie, cur):
+ expires = cookie.expires
+ if cookie.discard:
+ expires = ""
+
+ domain = unicode(cookie.domain)
+ path = unicode(cookie.path)
+ name = unicode(cookie.name)
+ value = unicode(cookie.value)
+ secure = bool(int(cookie.secure))
+
+ if value is None:
+ value = name
+ name = ""
+
+ last_accessed = int(time.time())
+ http_only = cookie.has_nonstandard_attr("HttpOnly")
+
+ query = cur.execute("""SELECT MAX(id) + 1 from moz_cookies""")
+ pk = query.fetchone()[0]
+ if pk is None:
+ pk = 1
+
+ return (pk, name, value, domain, path, expires,
+ last_accessed, secure, http_only)
+
+ def set_cookie(self, cookie):
+ if cookie.discard:
+ CookieJar.set_cookie(self, cookie)
+ return
+
+ def set_cookie(cur):
+ # XXX
+ # is this RFC 2965-correct?
+ # could this do an UPDATE instead?
+ row = self._row_from_cookie(cookie, cur)
+ name, unused, domain, path = row[1:5]
+ cur.execute("""\
+DELETE FROM moz_cookies WHERE host = ? AND path = ? AND name = ?""",
+ (domain, path, name))
+ cur.execute("""\
+INSERT INTO moz_cookies VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+""", row)
+ self._transaction(set_cookie)
+
+ def __iter__(self):
+ # session (non-persistent) cookies
+ for cookie in MappingIterator(self._cookies):
+ yield cookie
+ # persistent cookies
+ for row in self._query("""\
+SELECT * FROM moz_cookies ORDER BY name, path, host"""):
+ yield self._cookie_from_row(row)
+
+ def _cookies_for_request(self, request):
+ session_cookies = CookieJar._cookies_for_request(self, request)
+ def get_cookies(cur):
+ query = cur.execute("SELECT host from moz_cookies")
+ domains = [row[0] for row in query.fetchall()]
+ cookies = []
+ for domain in domains:
+ cookies += self._persistent_cookies_for_domain(domain,
+ request, cur)
+ return cookies
+ persistent_coookies = self._transaction(get_cookies)
+ return session_cookies + persistent_coookies
+
+ def _persistent_cookies_for_domain(self, domain, request, cur):
+ cookies = []
+ if not self._policy.domain_return_ok(domain, request):
+ return []
+ debug("Checking %s for cookies to return", domain)
+ query = cur.execute("""\
+SELECT * from moz_cookies WHERE host = ? ORDER BY path""",
+ (domain,))
+ cookies = [self._cookie_from_row(row) for row in query.fetchall()]
+ last_path = None
+ r = []
+ for cookie in cookies:
+ if (cookie.path != last_path and
+ not self._policy.path_return_ok(cookie.path, request)):
+ last_path = cookie.path
+ continue
+ if not self._policy.return_ok(cookie, request):
+ debug(" not returning cookie")
+ continue
+ debug(" it's a match")
+ r.append(cookie)
+ return r
diff --git a/couchpotato/core/media/movie/providers/trailer/mechanize/_form.py b/couchpotato/core/media/movie/providers/trailer/mechanize/_form.py
new file mode 100644
index 0000000000..d45bdfc395
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/mechanize/_form.py
@@ -0,0 +1,3280 @@
+"""HTML form handling for web clients.
+
+HTML form handling for web clients: useful for parsing HTML forms, filling them
+in and returning the completed forms to the server. This code developed from a
+port of Gisle Aas' Perl module HTML::Form, from the libwww-perl library, but
+the interface is not the same.
+
+The most useful docstring is the one for HTMLForm.
+
+RFC 1866: HTML 2.0
+RFC 1867: Form-based File Upload in HTML
+RFC 2388: Returning Values from Forms: multipart/form-data
+HTML 3.2 Specification, W3C Recommendation 14 January 1997 (for ISINDEX)
+HTML 4.01 Specification, W3C Recommendation 24 December 1999
+
+
+Copyright 2002-2007 John J. Lee
+Copyright 2005 Gary Poster
+Copyright 2005 Zope Corporation
+Copyright 1998-2000 Gisle Aas.
+
+This code is free software; you can redistribute it and/or modify it
+under the terms of the BSD or ZPL 2.1 licenses (see the file
+COPYING.txt included with the distribution).
+
+"""
+
+# TODO:
+# Clean up post the merge into mechanize
+# * Remove code that was duplicated in ClientForm and mechanize
+# * Remove weird import stuff
+# * Remove pre-Python 2.4 compatibility cruft
+# * Clean up tests
+# * Later release: Remove the ClientForm 0.1 backwards-compatibility switch
+# Remove parser testing hack
+# Clean action URI
+# Switch to unicode throughout
+# See Wichert Akkerman's 2004-01-22 message to c.l.py.
+# Apply recommendations from google code project CURLIES
+# Apply recommendations from HTML 5 spec
+# Add charset parameter to Content-type headers? How to find value??
+# Functional tests to add:
+# Single and multiple file upload
+# File upload with missing name (check standards)
+# mailto: submission & enctype text/plain??
+
+# Replace by_label etc. with moniker / selector concept. Allows, e.g., a
+# choice between selection by value / id / label / element contents. Or
+# choice between matching labels exactly or by substring. etc.
+
+
+__all__ = ['AmbiguityError', 'CheckboxControl', 'Control',
+ 'ControlNotFoundError', 'FileControl', 'FormParser', 'HTMLForm',
+ 'HiddenControl', 'IgnoreControl', 'ImageControl', 'IsindexControl',
+ 'Item', 'ItemCountError', 'ItemNotFoundError', 'Label',
+ 'ListControl', 'LocateError', 'Missing', 'ParseError', 'ParseFile',
+ 'ParseFileEx', 'ParseResponse', 'ParseResponseEx','PasswordControl',
+ 'RadioControl', 'ScalarControl', 'SelectControl',
+ 'SubmitButtonControl', 'SubmitControl', 'TextControl',
+ 'TextareaControl', 'XHTMLCompatibleFormParser']
+
+import HTMLParser
+from cStringIO import StringIO
+import inspect
+import logging
+import random
+import re
+import sys
+import urllib
+import urlparse
+import warnings
+
+import _beautifulsoup
+import _request
+
+# from Python itself, for backwards compatibility of raised exceptions
+import sgmllib
+# bundled copy of sgmllib
+import _sgmllib_copy
+
+
+VERSION = "0.2.11"
+
+CHUNK = 1024 # size of chunks fed to parser, in bytes
+
+DEFAULT_ENCODING = "latin-1"
+
+_logger = logging.getLogger("mechanize.forms")
+OPTIMIZATION_HACK = True
+
+def debug(msg, *args, **kwds):
+ if OPTIMIZATION_HACK:
+ return
+
+ caller_name = inspect.stack()[1][3]
+ extended_msg = '%%s %s' % msg
+ extended_args = (caller_name,)+args
+ _logger.debug(extended_msg, *extended_args, **kwds)
+
+def _show_debug_messages():
+ global OPTIMIZATION_HACK
+ OPTIMIZATION_HACK = False
+ _logger.setLevel(logging.DEBUG)
+ handler = logging.StreamHandler(sys.stdout)
+ handler.setLevel(logging.DEBUG)
+ _logger.addHandler(handler)
+
+
+def deprecation(message, stack_offset=0):
+ warnings.warn(message, DeprecationWarning, stacklevel=3+stack_offset)
+
+
+class Missing: pass
+
+_compress_re = re.compile(r"\s+")
+def compress_text(text): return _compress_re.sub(" ", text.strip())
+
+def normalize_line_endings(text):
+ return re.sub(r"(?:(?
+ w = MimeWriter(f)
+ ...call w.addheader(key, value) 0 or more times...
+
+ followed by either:
+
+ f = w.startbody(content_type)
+ ...call f.write(data) for body data...
+
+ or:
+
+ w.startmultipartbody(subtype)
+ for each part:
+ subwriter = w.nextpart()
+ ...use the subwriter's methods to create the subpart...
+ w.lastpart()
+
+ The subwriter is another MimeWriter instance, and should be
+ treated in the same way as the toplevel MimeWriter. This way,
+ writing recursive body parts is easy.
+
+ Warning: don't forget to call lastpart()!
+
+ XXX There should be more state so calls made in the wrong order
+ are detected.
+
+ Some special cases:
+
+ - startbody() just returns the file passed to the constructor;
+ but don't use this knowledge, as it may be changed.
+
+ - startmultipartbody() actually returns a file as well;
+ this can be used to write the initial 'if you can read this your
+ mailer is not MIME-aware' message.
+
+ - If you call flushheaders(), the headers accumulated so far are
+ written out (and forgotten); this is useful if you don't need a
+ body part at all, e.g. for a subpart of type message/rfc822
+ that's (mis)used to store some header-like information.
+
+ - Passing a keyword argument 'prefix=' to addheader(),
+ start*body() affects where the header is inserted; 0 means
+ append at the end, 1 means insert at the start; default is
+ append for addheader(), but insert for start*body(), which use
+ it to determine where the Content-type header goes.
+
+ """
+
+ def __init__(self, fp, http_hdrs=None):
+ self._http_hdrs = http_hdrs
+ self._fp = fp
+ self._headers = []
+ self._boundary = []
+ self._first_part = True
+
+ def addheader(self, key, value, prefix=0,
+ add_to_http_hdrs=0):
+ """
+ prefix is ignored if add_to_http_hdrs is true.
+ """
+ lines = value.split("\r\n")
+ while lines and not lines[-1]: del lines[-1]
+ while lines and not lines[0]: del lines[0]
+ if add_to_http_hdrs:
+ value = "".join(lines)
+ # 2.2 urllib2 doesn't normalize header case
+ self._http_hdrs.append((key.capitalize(), value))
+ else:
+ for i in range(1, len(lines)):
+ lines[i] = " " + lines[i].strip()
+ value = "\r\n".join(lines) + "\r\n"
+ line = key.title() + ": " + value
+ if prefix:
+ self._headers.insert(0, line)
+ else:
+ self._headers.append(line)
+
+ def flushheaders(self):
+ self._fp.writelines(self._headers)
+ self._headers = []
+
+ def startbody(self, ctype=None, plist=[], prefix=1,
+ add_to_http_hdrs=0, content_type=1):
+ """
+ prefix is ignored if add_to_http_hdrs is true.
+ """
+ if content_type and ctype:
+ for name, value in plist:
+ ctype = ctype + ';\r\n %s=%s' % (name, value)
+ self.addheader("Content-Type", ctype, prefix=prefix,
+ add_to_http_hdrs=add_to_http_hdrs)
+ self.flushheaders()
+ if not add_to_http_hdrs: self._fp.write("\r\n")
+ self._first_part = True
+ return self._fp
+
+ def startmultipartbody(self, subtype, boundary=None, plist=[], prefix=1,
+ add_to_http_hdrs=0, content_type=1):
+ boundary = boundary or choose_boundary()
+ self._boundary.append(boundary)
+ return self.startbody("multipart/" + subtype,
+ [("boundary", boundary)] + plist,
+ prefix=prefix,
+ add_to_http_hdrs=add_to_http_hdrs,
+ content_type=content_type)
+
+ def nextpart(self):
+ boundary = self._boundary[-1]
+ if self._first_part:
+ self._first_part = False
+ else:
+ self._fp.write("\r\n")
+ self._fp.write("--" + boundary + "\r\n")
+ return self.__class__(self._fp)
+
+ def lastpart(self):
+ if self._first_part:
+ self.nextpart()
+ boundary = self._boundary.pop()
+ self._fp.write("\r\n--" + boundary + "--\r\n")
+
+
+class LocateError(ValueError): pass
+class AmbiguityError(LocateError): pass
+class ControlNotFoundError(LocateError): pass
+class ItemNotFoundError(LocateError): pass
+
+class ItemCountError(ValueError): pass
+
+# for backwards compatibility, ParseError derives from exceptions that were
+# raised by versions of ClientForm <= 0.2.5
+# TODO: move to _html
+class ParseError(sgmllib.SGMLParseError,
+ HTMLParser.HTMLParseError):
+
+ def __init__(self, *args, **kwds):
+ Exception.__init__(self, *args, **kwds)
+
+ def __str__(self):
+ return Exception.__str__(self)
+
+
+class _AbstractFormParser:
+ """forms attribute contains HTMLForm instances on completion."""
+ # thanks to Moshe Zadka for an example of sgmllib/htmllib usage
+ def __init__(self, entitydefs=None, encoding=DEFAULT_ENCODING):
+ if entitydefs is None:
+ entitydefs = get_entitydefs()
+ self._entitydefs = entitydefs
+ self._encoding = encoding
+
+ self.base = None
+ self.forms = []
+ self.labels = []
+ self._current_label = None
+ self._current_form = None
+ self._select = None
+ self._optgroup = None
+ self._option = None
+ self._textarea = None
+
+ # forms[0] will contain all controls that are outside of any form
+ # self._global_form is an alias for self.forms[0]
+ self._global_form = None
+ self.start_form([])
+ self.end_form()
+ self._current_form = self._global_form = self.forms[0]
+
+ def do_base(self, attrs):
+ debug("%s", attrs)
+ for key, value in attrs:
+ if key == "href":
+ self.base = self.unescape_attr_if_required(value)
+
+ def end_body(self):
+ debug("")
+ if self._current_label is not None:
+ self.end_label()
+ if self._current_form is not self._global_form:
+ self.end_form()
+
+ def start_form(self, attrs):
+ debug("%s", attrs)
+ if self._current_form is not self._global_form:
+ raise ParseError("nested FORMs")
+ name = None
+ action = None
+ enctype = "application/x-www-form-urlencoded"
+ method = "GET"
+ d = {}
+ for key, value in attrs:
+ if key == "name":
+ name = self.unescape_attr_if_required(value)
+ elif key == "action":
+ action = self.unescape_attr_if_required(value)
+ elif key == "method":
+ method = self.unescape_attr_if_required(value.upper())
+ elif key == "enctype":
+ enctype = self.unescape_attr_if_required(value.lower())
+ d[key] = self.unescape_attr_if_required(value)
+ controls = []
+ self._current_form = (name, action, method, enctype), d, controls
+
+ def end_form(self):
+ debug("")
+ if self._current_label is not None:
+ self.end_label()
+ if self._current_form is self._global_form:
+ raise ParseError("end of FORM before start")
+ self.forms.append(self._current_form)
+ self._current_form = self._global_form
+
+ def start_select(self, attrs):
+ debug("%s", attrs)
+ if self._select is not None:
+ raise ParseError("nested SELECTs")
+ if self._textarea is not None:
+ raise ParseError("SELECT inside TEXTAREA")
+ d = {}
+ for key, val in attrs:
+ d[key] = self.unescape_attr_if_required(val)
+
+ self._select = d
+ self._add_label(d)
+
+ self._append_select_control({"__select": d})
+
+ def end_select(self):
+ debug("")
+ if self._select is None:
+ raise ParseError("end of SELECT before start")
+
+ if self._option is not None:
+ self._end_option()
+
+ self._select = None
+
+ def start_optgroup(self, attrs):
+ debug("%s", attrs)
+ if self._select is None:
+ raise ParseError("OPTGROUP outside of SELECT")
+ d = {}
+ for key, val in attrs:
+ d[key] = self.unescape_attr_if_required(val)
+
+ self._optgroup = d
+
+ def end_optgroup(self):
+ debug("")
+ if self._optgroup is None:
+ raise ParseError("end of OPTGROUP before start")
+ self._optgroup = None
+
+ def _start_option(self, attrs):
+ debug("%s", attrs)
+ if self._select is None:
+ raise ParseError("OPTION outside of SELECT")
+ if self._option is not None:
+ self._end_option()
+
+ d = {}
+ for key, val in attrs:
+ d[key] = self.unescape_attr_if_required(val)
+
+ self._option = {}
+ self._option.update(d)
+ if (self._optgroup and self._optgroup.has_key("disabled") and
+ not self._option.has_key("disabled")):
+ self._option["disabled"] = None
+
+ def _end_option(self):
+ debug("")
+ if self._option is None:
+ raise ParseError("end of OPTION before start")
+
+ contents = self._option.get("contents", "").strip()
+ self._option["contents"] = contents
+ if not self._option.has_key("value"):
+ self._option["value"] = contents
+ if not self._option.has_key("label"):
+ self._option["label"] = contents
+ # stuff dict of SELECT HTML attrs into a special private key
+ # (gets deleted again later)
+ self._option["__select"] = self._select
+ self._append_select_control(self._option)
+ self._option = None
+
+ def _append_select_control(self, attrs):
+ debug("%s", attrs)
+ controls = self._current_form[2]
+ name = self._select.get("name")
+ controls.append(("select", name, attrs))
+
+ def start_textarea(self, attrs):
+ debug("%s", attrs)
+ if self._textarea is not None:
+ raise ParseError("nested TEXTAREAs")
+ if self._select is not None:
+ raise ParseError("TEXTAREA inside SELECT")
+ d = {}
+ for key, val in attrs:
+ d[key] = self.unescape_attr_if_required(val)
+ self._add_label(d)
+
+ self._textarea = d
+
+ def end_textarea(self):
+ debug("")
+ if self._textarea is None:
+ raise ParseError("end of TEXTAREA before start")
+ controls = self._current_form[2]
+ name = self._textarea.get("name")
+ controls.append(("textarea", name, self._textarea))
+ self._textarea = None
+
+ def start_label(self, attrs):
+ debug("%s", attrs)
+ if self._current_label:
+ self.end_label()
+ d = {}
+ for key, val in attrs:
+ d[key] = self.unescape_attr_if_required(val)
+ taken = bool(d.get("for")) # empty id is invalid
+ d["__text"] = ""
+ d["__taken"] = taken
+ if taken:
+ self.labels.append(d)
+ self._current_label = d
+
+ def end_label(self):
+ debug("")
+ label = self._current_label
+ if label is None:
+ # something is ugly in the HTML, but we're ignoring it
+ return
+ self._current_label = None
+ # if it is staying around, it is True in all cases
+ del label["__taken"]
+
+ def _add_label(self, d):
+ #debug("%s", d)
+ if self._current_label is not None:
+ if not self._current_label["__taken"]:
+ self._current_label["__taken"] = True
+ d["__label"] = self._current_label
+
+ def handle_data(self, data):
+ debug("%s", data)
+
+ if self._option is not None:
+ # self._option is a dictionary of the OPTION element's HTML
+ # attributes, but it has two special keys, one of which is the
+ # special "contents" key contains text between OPTION tags (the
+ # other is the "__select" key: see the end_option method)
+ map = self._option
+ key = "contents"
+ elif self._textarea is not None:
+ map = self._textarea
+ key = "value"
+ data = normalize_line_endings(data)
+ # not if within option or textarea
+ elif self._current_label is not None:
+ map = self._current_label
+ key = "__text"
+ else:
+ return
+
+ if data and not map.has_key(key):
+ # according to
+ # http://www.w3.org/TR/html4/appendix/notes.html#h-B.3.1 line break
+ # immediately after start tags or immediately before end tags must
+ # be ignored, but real browsers only ignore a line break after a
+ # start tag, so we'll do that.
+ if data[0:2] == "\r\n":
+ data = data[2:]
+ elif data[0:1] in ["\n", "\r"]:
+ data = data[1:]
+ map[key] = data
+ else:
+ map[key] = map[key] + data
+
+ def do_button(self, attrs):
+ debug("%s", attrs)
+ d = {}
+ d["type"] = "submit" # default
+ for key, val in attrs:
+ d[key] = self.unescape_attr_if_required(val)
+ controls = self._current_form[2]
+
+ type = d["type"]
+ name = d.get("name")
+ # we don't want to lose information, so use a type string that
+ # doesn't clash with INPUT TYPE={SUBMIT,RESET,BUTTON}
+ # e.g. type for BUTTON/RESET is "resetbutton"
+ # (type for INPUT/RESET is "reset")
+ type = type+"button"
+ self._add_label(d)
+ controls.append((type, name, d))
+
+ def do_input(self, attrs):
+ debug("%s", attrs)
+ d = {}
+ d["type"] = "text" # default
+ for key, val in attrs:
+ d[key] = self.unescape_attr_if_required(val)
+ controls = self._current_form[2]
+
+ type = d["type"]
+ name = d.get("name")
+ self._add_label(d)
+ controls.append((type, name, d))
+
+ def do_isindex(self, attrs):
+ debug("%s", attrs)
+ d = {}
+ for key, val in attrs:
+ d[key] = self.unescape_attr_if_required(val)
+ controls = self._current_form[2]
+
+ self._add_label(d)
+ # isindex doesn't have type or name HTML attributes
+ controls.append(("isindex", None, d))
+
+ def handle_entityref(self, name):
+ #debug("%s", name)
+ self.handle_data(unescape(
+ '&%s;' % name, self._entitydefs, self._encoding))
+
+ def handle_charref(self, name):
+ #debug("%s", name)
+ self.handle_data(unescape_charref(name, self._encoding))
+
+ def unescape_attr(self, name):
+ #debug("%s", name)
+ return unescape(name, self._entitydefs, self._encoding)
+
+ def unescape_attrs(self, attrs):
+ #debug("%s", attrs)
+ escaped_attrs = {}
+ for key, val in attrs.items():
+ try:
+ val.items
+ except AttributeError:
+ escaped_attrs[key] = self.unescape_attr(val)
+ else:
+ # e.g. "__select" -- yuck!
+ escaped_attrs[key] = self.unescape_attrs(val)
+ return escaped_attrs
+
+ def unknown_entityref(self, ref): self.handle_data("&%s;" % ref)
+ def unknown_charref(self, ref): self.handle_data("&#%s;" % ref)
+
+
+class XHTMLCompatibleFormParser(_AbstractFormParser, HTMLParser.HTMLParser):
+ """Good for XHTML, bad for tolerance of incorrect HTML."""
+ # thanks to Michael Howitz for this!
+ def __init__(self, entitydefs=None, encoding=DEFAULT_ENCODING):
+ HTMLParser.HTMLParser.__init__(self)
+ _AbstractFormParser.__init__(self, entitydefs, encoding)
+
+ def feed(self, data):
+ try:
+ HTMLParser.HTMLParser.feed(self, data)
+ except HTMLParser.HTMLParseError, exc:
+ raise ParseError(exc)
+
+ def start_option(self, attrs):
+ _AbstractFormParser._start_option(self, attrs)
+
+ def end_option(self):
+ _AbstractFormParser._end_option(self)
+
+ def handle_starttag(self, tag, attrs):
+ try:
+ method = getattr(self, "start_" + tag)
+ except AttributeError:
+ try:
+ method = getattr(self, "do_" + tag)
+ except AttributeError:
+ pass # unknown tag
+ else:
+ method(attrs)
+ else:
+ method(attrs)
+
+ def handle_endtag(self, tag):
+ try:
+ method = getattr(self, "end_" + tag)
+ except AttributeError:
+ pass # unknown tag
+ else:
+ method()
+
+ def unescape(self, name):
+ # Use the entitydefs passed into constructor, not
+ # HTMLParser.HTMLParser's entitydefs.
+ return self.unescape_attr(name)
+
+ def unescape_attr_if_required(self, name):
+ return name # HTMLParser.HTMLParser already did it
+ def unescape_attrs_if_required(self, attrs):
+ return attrs # ditto
+
+ def close(self):
+ HTMLParser.HTMLParser.close(self)
+ self.end_body()
+
+
+class _AbstractSgmllibParser(_AbstractFormParser):
+
+ def do_option(self, attrs):
+ _AbstractFormParser._start_option(self, attrs)
+
+ # we override this attr to decode hex charrefs
+ entity_or_charref = re.compile(
+ '&(?:([a-zA-Z][-.a-zA-Z0-9]*)|#(x?[0-9a-fA-F]+))(;?)')
+ def convert_entityref(self, name):
+ return unescape("&%s;" % name, self._entitydefs, self._encoding)
+ def convert_charref(self, name):
+ return unescape_charref("%s" % name, self._encoding)
+ def unescape_attr_if_required(self, name):
+ return name # sgmllib already did it
+ def unescape_attrs_if_required(self, attrs):
+ return attrs # ditto
+
+
+class FormParser(_AbstractSgmllibParser, _sgmllib_copy.SGMLParser):
+ """Good for tolerance of incorrect HTML, bad for XHTML."""
+ def __init__(self, entitydefs=None, encoding=DEFAULT_ENCODING):
+ _sgmllib_copy.SGMLParser.__init__(self)
+ _AbstractFormParser.__init__(self, entitydefs, encoding)
+
+ def feed(self, data):
+ try:
+ _sgmllib_copy.SGMLParser.feed(self, data)
+ except _sgmllib_copy.SGMLParseError, exc:
+ raise ParseError(exc)
+
+ def close(self):
+ _sgmllib_copy.SGMLParser.close(self)
+ self.end_body()
+
+
+class _AbstractBSFormParser(_AbstractSgmllibParser):
+
+ bs_base_class = None
+
+ def __init__(self, entitydefs=None, encoding=DEFAULT_ENCODING):
+ _AbstractFormParser.__init__(self, entitydefs, encoding)
+ self.bs_base_class.__init__(self)
+
+ def handle_data(self, data):
+ _AbstractFormParser.handle_data(self, data)
+ self.bs_base_class.handle_data(self, data)
+
+ def feed(self, data):
+ try:
+ self.bs_base_class.feed(self, data)
+ except _sgmllib_copy.SGMLParseError, exc:
+ raise ParseError(exc)
+
+ def close(self):
+ self.bs_base_class.close(self)
+ self.end_body()
+
+
+class RobustFormParser(_AbstractBSFormParser, _beautifulsoup.BeautifulSoup):
+
+ """Tries to be highly tolerant of incorrect HTML."""
+
+ bs_base_class = _beautifulsoup.BeautifulSoup
+
+
+class NestingRobustFormParser(_AbstractBSFormParser,
+ _beautifulsoup.ICantBelieveItsBeautifulSoup):
+
+ """Tries to be highly tolerant of incorrect HTML.
+
+ Different from RobustFormParser in that it more often guesses nesting
+ above missing end tags (see BeautifulSoup docs).
+ """
+
+ bs_base_class = _beautifulsoup.ICantBelieveItsBeautifulSoup
+
+
+#FormParser = XHTMLCompatibleFormParser # testing hack
+#FormParser = RobustFormParser # testing hack
+
+
+def ParseResponseEx(response,
+ select_default=False,
+ form_parser_class=FormParser,
+ request_class=_request.Request,
+ entitydefs=None,
+ encoding=DEFAULT_ENCODING,
+
+ # private
+ _urljoin=urlparse.urljoin,
+ _urlparse=urlparse.urlparse,
+ _urlunparse=urlparse.urlunparse,
+ ):
+ """Identical to ParseResponse, except that:
+
+ 1. The returned list contains an extra item. The first form in the list
+ contains all controls not contained in any FORM element.
+
+ 2. The arguments ignore_errors and backwards_compat have been removed.
+
+ 3. Backwards-compatibility mode (backwards_compat=True) is not available.
+ """
+ return _ParseFileEx(response, response.geturl(),
+ select_default,
+ False,
+ form_parser_class,
+ request_class,
+ entitydefs,
+ False,
+ encoding,
+ _urljoin=_urljoin,
+ _urlparse=_urlparse,
+ _urlunparse=_urlunparse,
+ )
+
+def ParseFileEx(file, base_uri,
+ select_default=False,
+ form_parser_class=FormParser,
+ request_class=_request.Request,
+ entitydefs=None,
+ encoding=DEFAULT_ENCODING,
+
+ # private
+ _urljoin=urlparse.urljoin,
+ _urlparse=urlparse.urlparse,
+ _urlunparse=urlparse.urlunparse,
+ ):
+ """Identical to ParseFile, except that:
+
+ 1. The returned list contains an extra item. The first form in the list
+ contains all controls not contained in any FORM element.
+
+ 2. The arguments ignore_errors and backwards_compat have been removed.
+
+ 3. Backwards-compatibility mode (backwards_compat=True) is not available.
+ """
+ return _ParseFileEx(file, base_uri,
+ select_default,
+ False,
+ form_parser_class,
+ request_class,
+ entitydefs,
+ False,
+ encoding,
+ _urljoin=_urljoin,
+ _urlparse=_urlparse,
+ _urlunparse=_urlunparse,
+ )
+
+def ParseString(text, base_uri, *args, **kwds):
+ fh = StringIO(text)
+ return ParseFileEx(fh, base_uri, *args, **kwds)
+
+def ParseResponse(response, *args, **kwds):
+ """Parse HTTP response and return a list of HTMLForm instances.
+
+ The return value of mechanize.urlopen can be conveniently passed to this
+ function as the response parameter.
+
+ mechanize.ParseError is raised on parse errors.
+
+ response: file-like object (supporting read() method) with a method
+ geturl(), returning the URI of the HTTP response
+ select_default: for multiple-selection SELECT controls and RADIO controls,
+ pick the first item as the default if none are selected in the HTML
+ form_parser_class: class to instantiate and use to pass
+ request_class: class to return from .click() method (default is
+ mechanize.Request)
+ entitydefs: mapping like {"&": "&", ...} containing HTML entity
+ definitions (a sensible default is used)
+ encoding: character encoding used for encoding numeric character references
+ when matching link text. mechanize does not attempt to find the encoding
+ in a META HTTP-EQUIV attribute in the document itself (mechanize, for
+ example, does do that and will pass the correct value to mechanize using
+ this parameter).
+
+ backwards_compat: boolean that determines whether the returned HTMLForm
+ objects are backwards-compatible with old code. If backwards_compat is
+ true:
+
+ - ClientForm 0.1 code will continue to work as before.
+
+ - Label searches that do not specify a nr (number or count) will always
+ get the first match, even if other controls match. If
+ backwards_compat is False, label searches that have ambiguous results
+ will raise an AmbiguityError.
+
+ - Item label matching is done by strict string comparison rather than
+ substring matching.
+
+ - De-selecting individual list items is allowed even if the Item is
+ disabled.
+
+ The backwards_compat argument will be removed in a future release.
+
+ Pass a true value for select_default if you want the behaviour specified by
+ RFC 1866 (the HTML 2.0 standard), which is to select the first item in a
+ RADIO or multiple-selection SELECT control if none were selected in the
+ HTML. Most browsers (including Microsoft Internet Explorer (IE) and
+ Netscape Navigator) instead leave all items unselected in these cases. The
+ W3C HTML 4.0 standard leaves this behaviour undefined in the case of
+ multiple-selection SELECT controls, but insists that at least one RADIO
+ button should be checked at all times, in contradiction to browser
+ behaviour.
+
+ There is a choice of parsers. mechanize.XHTMLCompatibleFormParser (uses
+ HTMLParser.HTMLParser) works best for XHTML, mechanize.FormParser (uses
+ bundled copy of sgmllib.SGMLParser) (the default) works better for ordinary
+ grubby HTML. Note that HTMLParser is only available in Python 2.2 and
+ later. You can pass your own class in here as a hack to work around bad
+ HTML, but at your own risk: there is no well-defined interface.
+
+ """
+ return _ParseFileEx(response, response.geturl(), *args, **kwds)[1:]
+
+def ParseFile(file, base_uri, *args, **kwds):
+ """Parse HTML and return a list of HTMLForm instances.
+
+ mechanize.ParseError is raised on parse errors.
+
+ file: file-like object (supporting read() method) containing HTML with zero
+ or more forms to be parsed
+ base_uri: the URI of the document (note that the base URI used to submit
+ the form will be that given in the BASE element if present, not that of
+ the document)
+
+ For the other arguments and further details, see ParseResponse.__doc__.
+
+ """
+ return _ParseFileEx(file, base_uri, *args, **kwds)[1:]
+
+def _ParseFileEx(file, base_uri,
+ select_default=False,
+ ignore_errors=False,
+ form_parser_class=FormParser,
+ request_class=_request.Request,
+ entitydefs=None,
+ backwards_compat=True,
+ encoding=DEFAULT_ENCODING,
+ _urljoin=urlparse.urljoin,
+ _urlparse=urlparse.urlparse,
+ _urlunparse=urlparse.urlunparse,
+ ):
+ if backwards_compat:
+ deprecation("operating in backwards-compatibility mode", 1)
+ fp = form_parser_class(entitydefs, encoding)
+ while 1:
+ data = file.read(CHUNK)
+ try:
+ fp.feed(data)
+ except ParseError, e:
+ e.base_uri = base_uri
+ raise
+ if len(data) != CHUNK: break
+ fp.close()
+ if fp.base is not None:
+ # HTML BASE element takes precedence over document URI
+ base_uri = fp.base
+ labels = [] # Label(label) for label in fp.labels]
+ id_to_labels = {}
+ for l in fp.labels:
+ label = Label(l)
+ labels.append(label)
+ for_id = l["for"]
+ coll = id_to_labels.get(for_id)
+ if coll is None:
+ id_to_labels[for_id] = [label]
+ else:
+ coll.append(label)
+ forms = []
+ for (name, action, method, enctype), attrs, controls in fp.forms:
+ if action is None:
+ action = base_uri
+ else:
+ action = _urljoin(base_uri, action)
+ # would be nice to make HTMLForm class (form builder) pluggable
+ form = HTMLForm(
+ action, method, enctype, name, attrs, request_class,
+ forms, labels, id_to_labels, backwards_compat)
+ form._urlparse = _urlparse
+ form._urlunparse = _urlunparse
+ for ii in range(len(controls)):
+ type, name, attrs = controls[ii]
+ # index=ii*10 allows ImageControl to return multiple ordered pairs
+ form.new_control(
+ type, name, attrs, select_default=select_default, index=ii*10)
+ forms.append(form)
+ for form in forms:
+ form.fixup()
+ return forms
+
+
+class Label:
+ def __init__(self, attrs):
+ self.id = attrs.get("for")
+ self._text = attrs.get("__text").strip()
+ self._ctext = compress_text(self._text)
+ self.attrs = attrs
+ self._backwards_compat = False # maintained by HTMLForm
+
+ def __getattr__(self, name):
+ if name == "text":
+ if self._backwards_compat:
+ return self._text
+ else:
+ return self._ctext
+ return getattr(Label, name)
+
+ def __setattr__(self, name, value):
+ if name == "text":
+ # don't see any need for this, so make it read-only
+ raise AttributeError("text attribute is read-only")
+ self.__dict__[name] = value
+
+ def __str__(self):
+ return "" % (self.id, self.text)
+
+
+def _get_label(attrs):
+ text = attrs.get("__label")
+ if text is not None:
+ return Label(text)
+ else:
+ return None
+
+class Control:
+ """An HTML form control.
+
+ An HTMLForm contains a sequence of Controls. The Controls in an HTMLForm
+ are accessed using the HTMLForm.find_control method or the
+ HTMLForm.controls attribute.
+
+ Control instances are usually constructed using the ParseFile /
+ ParseResponse functions. If you use those functions, you can ignore the
+ rest of this paragraph. A Control is only properly initialised after the
+ fixup method has been called. In fact, this is only strictly necessary for
+ ListControl instances. This is necessary because ListControls are built up
+ from ListControls each containing only a single item, and their initial
+ value(s) can only be known after the sequence is complete.
+
+ The types and values that are acceptable for assignment to the value
+ attribute are defined by subclasses.
+
+ If the disabled attribute is true, this represents the state typically
+ represented by browsers by 'greying out' a control. If the disabled
+ attribute is true, the Control will raise AttributeError if an attempt is
+ made to change its value. In addition, the control will not be considered
+ 'successful' as defined by the W3C HTML 4 standard -- ie. it will
+ contribute no data to the return value of the HTMLForm.click* methods. To
+ enable a control, set the disabled attribute to a false value.
+
+ If the readonly attribute is true, the Control will raise AttributeError if
+ an attempt is made to change its value. To make a control writable, set
+ the readonly attribute to a false value.
+
+ All controls have the disabled and readonly attributes, not only those that
+ may have the HTML attributes of the same names.
+
+ On assignment to the value attribute, the following exceptions are raised:
+ TypeError, AttributeError (if the value attribute should not be assigned
+ to, because the control is disabled, for example) and ValueError.
+
+ If the name or value attributes are None, or the value is an empty list, or
+ if the control is disabled, the control is not successful.
+
+ Public attributes:
+
+ type: string describing type of control (see the keys of the
+ HTMLForm.type2class dictionary for the allowable values) (readonly)
+ name: name of control (readonly)
+ value: current value of control (subclasses may allow a single value, a
+ sequence of values, or either)
+ disabled: disabled state
+ readonly: readonly state
+ id: value of id HTML attribute
+
+ """
+ def __init__(self, type, name, attrs, index=None):
+ """
+ type: string describing type of control (see the keys of the
+ HTMLForm.type2class dictionary for the allowable values)
+ name: control name
+ attrs: HTML attributes of control's HTML element
+
+ """
+ raise NotImplementedError()
+
+ def add_to_form(self, form):
+ self._form = form
+ form.controls.append(self)
+
+ def fixup(self):
+ pass
+
+ def is_of_kind(self, kind):
+ raise NotImplementedError()
+
+ def clear(self):
+ raise NotImplementedError()
+
+ def __getattr__(self, name): raise NotImplementedError()
+ def __setattr__(self, name, value): raise NotImplementedError()
+
+ def pairs(self):
+ """Return list of (key, value) pairs suitable for passing to urlencode.
+ """
+ return [(k, v) for (i, k, v) in self._totally_ordered_pairs()]
+
+ def _totally_ordered_pairs(self):
+ """Return list of (key, value, index) tuples.
+
+ Like pairs, but allows preserving correct ordering even where several
+ controls are involved.
+
+ """
+ raise NotImplementedError()
+
+ def _write_mime_data(self, mw, name, value):
+ """Write data for a subitem of this control to a MimeWriter."""
+ # called by HTMLForm
+ mw2 = mw.nextpart()
+ mw2.addheader("Content-Disposition",
+ 'form-data; name="%s"' % name, 1)
+ f = mw2.startbody(prefix=0)
+ f.write(value)
+
+ def __str__(self):
+ raise NotImplementedError()
+
+ def get_labels(self):
+ """Return all labels (Label instances) for this control.
+
+ If the control was surrounded by a tag, that will be the first
+ label; all other labels, connected by 'for' and 'id', are in the order
+ that appear in the HTML.
+
+ """
+ res = []
+ if self._label:
+ res.append(self._label)
+ if self.id:
+ res.extend(self._form._id_to_labels.get(self.id, ()))
+ return res
+
+
+#---------------------------------------------------
+class ScalarControl(Control):
+ """Control whose value is not restricted to one of a prescribed set.
+
+ Some ScalarControls don't accept any value attribute. Otherwise, takes a
+ single value, which must be string-like.
+
+ Additional read-only public attribute:
+
+ attrs: dictionary mapping the names of original HTML attributes of the
+ control to their values
+
+ """
+ def __init__(self, type, name, attrs, index=None):
+ self._index = index
+ self._label = _get_label(attrs)
+ self.__dict__["type"] = type.lower()
+ self.__dict__["name"] = name
+ self._value = attrs.get("value")
+ self.disabled = attrs.has_key("disabled")
+ self.readonly = attrs.has_key("readonly")
+ self.id = attrs.get("id")
+
+ self.attrs = attrs.copy()
+
+ self._clicked = False
+
+ self._urlparse = urlparse.urlparse
+ self._urlunparse = urlparse.urlunparse
+
+ def __getattr__(self, name):
+ if name == "value":
+ return self.__dict__["_value"]
+ else:
+ raise AttributeError("%s instance has no attribute '%s'" %
+ (self.__class__.__name__, name))
+
+ def __setattr__(self, name, value):
+ if name == "value":
+ if not isstringlike(value):
+ raise TypeError("must assign a string")
+ elif self.readonly:
+ raise AttributeError("control '%s' is readonly" % self.name)
+ elif self.disabled:
+ raise AttributeError("control '%s' is disabled" % self.name)
+ self.__dict__["_value"] = value
+ elif name in ("name", "type"):
+ raise AttributeError("%s attribute is readonly" % name)
+ else:
+ self.__dict__[name] = value
+
+ def _totally_ordered_pairs(self):
+ name = self.name
+ value = self.value
+ if name is None or value is None or self.disabled:
+ return []
+ return [(self._index, name, value)]
+
+ def clear(self):
+ if self.readonly:
+ raise AttributeError("control '%s' is readonly" % self.name)
+ self.__dict__["_value"] = None
+
+ def __str__(self):
+ name = self.name
+ value = self.value
+ if name is None: name = ""
+ if value is None: value = ""
+
+ infos = []
+ if self.disabled: infos.append("disabled")
+ if self.readonly: infos.append("readonly")
+ info = ", ".join(infos)
+ if info: info = " (%s)" % info
+
+ return "<%s(%s=%s)%s>" % (self.__class__.__name__, name, value, info)
+
+
+#---------------------------------------------------
+class TextControl(ScalarControl):
+ """Textual input control.
+
+ Covers:
+
+ INPUT/TEXT
+ INPUT/PASSWORD
+ INPUT/HIDDEN
+ TEXTAREA
+
+ """
+ def __init__(self, type, name, attrs, index=None):
+ ScalarControl.__init__(self, type, name, attrs, index)
+ if self.type == "hidden": self.readonly = True
+ if self._value is None:
+ self._value = ""
+
+ def is_of_kind(self, kind): return kind == "text"
+
+#---------------------------------------------------
+class FileControl(ScalarControl):
+ """File upload with INPUT TYPE=FILE.
+
+ The value attribute of a FileControl is always None. Use add_file instead.
+
+ Additional public method: add_file
+
+ """
+
+ def __init__(self, type, name, attrs, index=None):
+ ScalarControl.__init__(self, type, name, attrs, index)
+ self._value = None
+ self._upload_data = []
+
+ def is_of_kind(self, kind): return kind == "file"
+
+ def clear(self):
+ if self.readonly:
+ raise AttributeError("control '%s' is readonly" % self.name)
+ self._upload_data = []
+
+ def __setattr__(self, name, value):
+ if name in ("value", "name", "type"):
+ raise AttributeError("%s attribute is readonly" % name)
+ else:
+ self.__dict__[name] = value
+
+ def add_file(self, file_object, content_type=None, filename=None):
+ if not hasattr(file_object, "read"):
+ raise TypeError("file-like object must have read method")
+ if content_type is not None and not isstringlike(content_type):
+ raise TypeError("content type must be None or string-like")
+ if filename is not None and not isstringlike(filename):
+ raise TypeError("filename must be None or string-like")
+ if content_type is None:
+ content_type = "application/octet-stream"
+ self._upload_data.append((file_object, content_type, filename))
+
+ def _totally_ordered_pairs(self):
+ # XXX should it be successful even if unnamed?
+ if self.name is None or self.disabled:
+ return []
+ return [(self._index, self.name, "")]
+
+ # If enctype is application/x-www-form-urlencoded and there's a FILE
+ # control present, what should be sent? Strictly, it should be 'name=data'
+ # (see HTML 4.01 spec., section 17.13.2), but code sends "name=" ATM. What
+ # about multiple file upload?
+ def _write_mime_data(self, mw, _name, _value):
+ # called by HTMLForm
+ # assert _name == self.name and _value == ''
+ if len(self._upload_data) < 2:
+ if len(self._upload_data) == 0:
+ file_object = StringIO()
+ content_type = "application/octet-stream"
+ filename = ""
+ else:
+ file_object, content_type, filename = self._upload_data[0]
+ if filename is None:
+ filename = ""
+ mw2 = mw.nextpart()
+ fn_part = '; filename="%s"' % filename
+ disp = 'form-data; name="%s"%s' % (self.name, fn_part)
+ mw2.addheader("Content-Disposition", disp, prefix=1)
+ fh = mw2.startbody(content_type, prefix=0)
+ fh.write(file_object.read())
+ else:
+ # multiple files
+ mw2 = mw.nextpart()
+ disp = 'form-data; name="%s"' % self.name
+ mw2.addheader("Content-Disposition", disp, prefix=1)
+ fh = mw2.startmultipartbody("mixed", prefix=0)
+ for file_object, content_type, filename in self._upload_data:
+ mw3 = mw2.nextpart()
+ if filename is None:
+ filename = ""
+ fn_part = '; filename="%s"' % filename
+ disp = "file%s" % fn_part
+ mw3.addheader("Content-Disposition", disp, prefix=1)
+ fh2 = mw3.startbody(content_type, prefix=0)
+ fh2.write(file_object.read())
+ mw2.lastpart()
+
+ def __str__(self):
+ name = self.name
+ if name is None: name = ""
+
+ if not self._upload_data:
+ value = ""
+ else:
+ value = []
+ for file, ctype, filename in self._upload_data:
+ if filename is None:
+ value.append("")
+ else:
+ value.append(filename)
+ value = ", ".join(value)
+
+ info = []
+ if self.disabled: info.append("disabled")
+ if self.readonly: info.append("readonly")
+ info = ", ".join(info)
+ if info: info = " (%s)" % info
+
+ return "<%s(%s=%s)%s>" % (self.__class__.__name__, name, value, info)
+
+
+#---------------------------------------------------
+class IsindexControl(ScalarControl):
+ """ISINDEX control.
+
+ ISINDEX is the odd-one-out of HTML form controls. In fact, it isn't really
+ part of regular HTML forms at all, and predates it. You're only allowed
+ one ISINDEX per HTML document. ISINDEX and regular form submission are
+ mutually exclusive -- either submit a form, or the ISINDEX.
+
+ Having said this, since ISINDEX controls may appear in forms (which is
+ probably bad HTML), ParseFile / ParseResponse will include them in the
+ HTMLForm instances it returns. You can set the ISINDEX's value, as with
+ any other control (but note that ISINDEX controls have no name, so you'll
+ need to use the type argument of set_value!). When you submit the form,
+ the ISINDEX will not be successful (ie., no data will get returned to the
+ server as a result of its presence), unless you click on the ISINDEX
+ control, in which case the ISINDEX gets submitted instead of the form:
+
+ form.set_value("my isindex value", type="isindex")
+ mechanize.urlopen(form.click(type="isindex"))
+
+ ISINDEX elements outside of FORMs are ignored. If you want to submit one
+ by hand, do it like so:
+
+ url = urlparse.urljoin(page_uri, "?"+urllib.quote_plus("my isindex value"))
+ result = mechanize.urlopen(url)
+
+ """
+ def __init__(self, type, name, attrs, index=None):
+ ScalarControl.__init__(self, type, name, attrs, index)
+ if self._value is None:
+ self._value = ""
+
+ def is_of_kind(self, kind): return kind in ["text", "clickable"]
+
+ def _totally_ordered_pairs(self):
+ return []
+
+ def _click(self, form, coord, return_type, request_class=_request.Request):
+ # Relative URL for ISINDEX submission: instead of "foo=bar+baz",
+ # want "bar+baz".
+ # This doesn't seem to be specified in HTML 4.01 spec. (ISINDEX is
+ # deprecated in 4.01, but it should still say how to submit it).
+ # Submission of ISINDEX is explained in the HTML 3.2 spec, though.
+ parts = self._urlparse(form.action)
+ rest, (query, frag) = parts[:-2], parts[-2:]
+ parts = rest + (urllib.quote_plus(self.value), None)
+ url = self._urlunparse(parts)
+ req_data = url, None, []
+
+ if return_type == "pairs":
+ return []
+ elif return_type == "request_data":
+ return req_data
+ else:
+ return request_class(url)
+
+ def __str__(self):
+ value = self.value
+ if value is None: value = ""
+
+ infos = []
+ if self.disabled: infos.append("disabled")
+ if self.readonly: infos.append("readonly")
+ info = ", ".join(infos)
+ if info: info = " (%s)" % info
+
+ return "<%s(%s)%s>" % (self.__class__.__name__, value, info)
+
+
+#---------------------------------------------------
+class IgnoreControl(ScalarControl):
+ """Control that we're not interested in.
+
+ Covers:
+
+ INPUT/RESET
+ BUTTON/RESET
+ INPUT/BUTTON
+ BUTTON/BUTTON
+
+ These controls are always unsuccessful, in the terminology of HTML 4 (ie.
+ they never require any information to be returned to the server).
+
+ BUTTON/BUTTON is used to generate events for script embedded in HTML.
+
+ The value attribute of IgnoreControl is always None.
+
+ """
+ def __init__(self, type, name, attrs, index=None):
+ ScalarControl.__init__(self, type, name, attrs, index)
+ self._value = None
+
+ def is_of_kind(self, kind): return False
+
+ def __setattr__(self, name, value):
+ if name == "value":
+ raise AttributeError(
+ "control '%s' is ignored, hence read-only" % self.name)
+ elif name in ("name", "type"):
+ raise AttributeError("%s attribute is readonly" % name)
+ else:
+ self.__dict__[name] = value
+
+
+#---------------------------------------------------
+# ListControls
+
+# helpers and subsidiary classes
+
+class Item:
+ def __init__(self, control, attrs, index=None):
+ label = _get_label(attrs)
+ self.__dict__.update({
+ "name": attrs["value"],
+ "_labels": label and [label] or [],
+ "attrs": attrs,
+ "_control": control,
+ "disabled": attrs.has_key("disabled"),
+ "_selected": False,
+ "id": attrs.get("id"),
+ "_index": index,
+ })
+ control.items.append(self)
+
+ def get_labels(self):
+ """Return all labels (Label instances) for this item.
+
+ For items that represent radio buttons or checkboxes, if the item was
+ surrounded by a tag, that will be the first label; all other
+ labels, connected by 'for' and 'id', are in the order that appear in
+ the HTML.
+
+ For items that represent select options, if the option had a label
+ attribute, that will be the first label. If the option has contents
+ (text within the option tags) and it is not the same as the label
+ attribute (if any), that will be a label. There is nothing in the
+ spec to my knowledge that makes an option with an id unable to be the
+ target of a label's for attribute, so those are included, if any, for
+ the sake of consistency and completeness.
+
+ """
+ res = []
+ res.extend(self._labels)
+ if self.id:
+ res.extend(self._control._form._id_to_labels.get(self.id, ()))
+ return res
+
+ def __getattr__(self, name):
+ if name=="selected":
+ return self._selected
+ raise AttributeError(name)
+
+ def __setattr__(self, name, value):
+ if name == "selected":
+ self._control._set_selected_state(self, value)
+ elif name == "disabled":
+ self.__dict__["disabled"] = bool(value)
+ else:
+ raise AttributeError(name)
+
+ def __str__(self):
+ res = self.name
+ if self.selected:
+ res = "*" + res
+ if self.disabled:
+ res = "(%s)" % res
+ return res
+
+ def __repr__(self):
+ # XXX appending the attrs without distinguishing them from name and id
+ # is silly
+ attrs = [("name", self.name), ("id", self.id)]+self.attrs.items()
+ return "<%s %s>" % (
+ self.__class__.__name__,
+ " ".join(["%s=%r" % (k, v) for k, v in attrs])
+ )
+
+def disambiguate(items, nr, **kwds):
+ msgs = []
+ for key, value in kwds.items():
+ msgs.append("%s=%r" % (key, value))
+ msg = " ".join(msgs)
+ if not items:
+ raise ItemNotFoundError(msg)
+ if nr is None:
+ if len(items) > 1:
+ raise AmbiguityError(msg)
+ nr = 0
+ if len(items) <= nr:
+ raise ItemNotFoundError(msg)
+ return items[nr]
+
+class ListControl(Control):
+ """Control representing a sequence of items.
+
+ The value attribute of a ListControl represents the successful list items
+ in the control. The successful list items are those that are selected and
+ not disabled.
+
+ ListControl implements both list controls that take a length-1 value
+ (single-selection) and those that take length >1 values
+ (multiple-selection).
+
+ ListControls accept sequence values only. Some controls only accept
+ sequences of length 0 or 1 (RADIO, and single-selection SELECT).
+ In those cases, ItemCountError is raised if len(sequence) > 1. CHECKBOXes
+ and multiple-selection SELECTs (those having the "multiple" HTML attribute)
+ accept sequences of any length.
+
+ Note the following mistake:
+
+ control.value = some_value
+ assert control.value == some_value # not necessarily true
+
+ The reason for this is that the value attribute always gives the list items
+ in the order they were listed in the HTML.
+
+ ListControl items can also be referred to by their labels instead of names.
+ Use the label argument to .get(), and the .set_value_by_label(),
+ .get_value_by_label() methods.
+
+ Note that, rather confusingly, though SELECT controls are represented in
+ HTML by SELECT elements (which contain OPTION elements, representing
+ individual list items), CHECKBOXes and RADIOs are not represented by *any*
+ element. Instead, those controls are represented by a collection of INPUT
+ elements. For example, this is a SELECT control, named "control1":
+
+
+ foo
+ bar
+
+
+ and this is a CHECKBOX control, named "control2":
+
+ ")
+ self._set_selected_state(self._get(name, by_label, nr), selected)
+
+ def _set_selected_state(self, item, action):
+ # action:
+ # bool False: off
+ # bool True: on
+ if self.disabled:
+ raise AttributeError("control '%s' is disabled" % self.name)
+ if self.readonly:
+ raise AttributeError("control '%s' is readonly" % self.name)
+ action == bool(action)
+ compat = self._form.backwards_compat
+ if not compat and item.disabled:
+ raise AttributeError("item is disabled")
+ else:
+ if compat and item.disabled and action:
+ raise AttributeError("item is disabled")
+ if self.multiple:
+ item.__dict__["_selected"] = action
+ else:
+ if not action:
+ item.__dict__["_selected"] = False
+ else:
+ for o in self.items:
+ o.__dict__["_selected"] = False
+ item.__dict__["_selected"] = True
+
+ def toggle_single(self, by_label=None):
+ """Deprecated: toggle the selection of the single item in this control.
+
+ Raises ItemCountError if the control does not contain only one item.
+
+ by_label argument is ignored, and included only for backwards
+ compatibility.
+
+ """
+ deprecation(
+ "control.items[0].selected = not control.items[0].selected")
+ if len(self.items) != 1:
+ raise ItemCountError(
+ "'%s' is not a single-item control" % self.name)
+ item = self.items[0]
+ self._set_selected_state(item, not item.selected)
+
+ def set_single(self, selected, by_label=None):
+ """Deprecated: set the selection of the single item in this control.
+
+ Raises ItemCountError if the control does not contain only one item.
+
+ by_label argument is ignored, and included only for backwards
+ compatibility.
+
+ """
+ deprecation(
+ "control.items[0].selected = ")
+ if len(self.items) != 1:
+ raise ItemCountError(
+ "'%s' is not a single-item control" % self.name)
+ self._set_selected_state(self.items[0], selected)
+
+ def get_item_disabled(self, name, by_label=False, nr=None):
+ """Get disabled state of named list item in a ListControl."""
+ deprecation(
+ "control.get(...).disabled")
+ return self._get(name, by_label, nr).disabled
+
+ def set_item_disabled(self, disabled, name, by_label=False, nr=None):
+ """Set disabled state of named list item in a ListControl.
+
+ disabled: boolean disabled state
+
+ """
+ deprecation(
+ "control.get(...).disabled = ")
+ self._get(name, by_label, nr).disabled = disabled
+
+ def set_all_items_disabled(self, disabled):
+ """Set disabled state of all list items in a ListControl.
+
+ disabled: boolean disabled state
+
+ """
+ for o in self.items:
+ o.disabled = disabled
+
+ def get_item_attrs(self, name, by_label=False, nr=None):
+ """Return dictionary of HTML attributes for a single ListControl item.
+
+ The HTML element types that describe list items are: OPTION for SELECT
+ controls, INPUT for the rest. These elements have HTML attributes that
+ you may occasionally want to know about -- for example, the "alt" HTML
+ attribute gives a text string describing the item (graphical browsers
+ usually display this as a tooltip).
+
+ The returned dictionary maps HTML attribute names to values. The names
+ and values are taken from the original HTML.
+
+ """
+ deprecation(
+ "control.get(...).attrs")
+ return self._get(name, by_label, nr).attrs
+
+ def close_control(self):
+ self._closed = True
+
+ def add_to_form(self, form):
+ assert self._form is None or form == self._form, (
+ "can't add control to more than one form")
+ self._form = form
+ if self.name is None:
+ # always count nameless elements as separate controls
+ Control.add_to_form(self, form)
+ else:
+ for ii in range(len(form.controls)-1, -1, -1):
+ control = form.controls[ii]
+ if control.name == self.name and control.type == self.type:
+ if control._closed:
+ Control.add_to_form(self, form)
+ else:
+ control.merge_control(self)
+ break
+ else:
+ Control.add_to_form(self, form)
+
+ def merge_control(self, control):
+ assert bool(control.multiple) == bool(self.multiple)
+ # usually, isinstance(control, self.__class__)
+ self.items.extend(control.items)
+
+ def fixup(self):
+ """
+ ListControls are built up from component list items (which are also
+ ListControls) during parsing. This method should be called after all
+ items have been added. See ListControl.__doc__ for the reason this is
+ required.
+
+ """
+ # Need to set default selection where no item was indicated as being
+ # selected by the HTML:
+
+ # CHECKBOX:
+ # Nothing should be selected.
+ # SELECT/single, SELECT/multiple and RADIO:
+ # RFC 1866 (HTML 2.0): says first item should be selected.
+ # W3C HTML 4.01 Specification: says that client behaviour is
+ # undefined in this case. For RADIO, exactly one must be selected,
+ # though which one is undefined.
+ # Both Netscape and Microsoft Internet Explorer (IE) choose first
+ # item for SELECT/single. However, both IE5 and Mozilla (both 1.0
+ # and Firebird 0.6) leave all items unselected for RADIO and
+ # SELECT/multiple.
+
+ # Since both Netscape and IE all choose the first item for
+ # SELECT/single, we do the same. OTOH, both Netscape and IE
+ # leave SELECT/multiple with nothing selected, in violation of RFC 1866
+ # (but not in violation of the W3C HTML 4 standard); the same is true
+ # of RADIO (which *is* in violation of the HTML 4 standard). We follow
+ # RFC 1866 if the _select_default attribute is set, and Netscape and IE
+ # otherwise. RFC 1866 and HTML 4 are always violated insofar as you
+ # can deselect all items in a RadioControl.
+
+ for o in self.items:
+ # set items' controls to self, now that we've merged
+ o.__dict__["_control"] = self
+
+ def __getattr__(self, name):
+ if name == "value":
+ compat = self._form.backwards_compat
+ if self.name is None:
+ return []
+ return [o.name for o in self.items if o.selected and
+ (not o.disabled or compat)]
+ else:
+ raise AttributeError("%s instance has no attribute '%s'" %
+ (self.__class__.__name__, name))
+
+ def __setattr__(self, name, value):
+ if name == "value":
+ if self.disabled:
+ raise AttributeError("control '%s' is disabled" % self.name)
+ if self.readonly:
+ raise AttributeError("control '%s' is readonly" % self.name)
+ self._set_value(value)
+ elif name in ("name", "type", "multiple"):
+ raise AttributeError("%s attribute is readonly" % name)
+ else:
+ self.__dict__[name] = value
+
+ def _set_value(self, value):
+ if value is None or isstringlike(value):
+ raise TypeError("ListControl, must set a sequence")
+ if not value:
+ compat = self._form.backwards_compat
+ for o in self.items:
+ if not o.disabled or compat:
+ o.selected = False
+ elif self.multiple:
+ self._multiple_set_value(value)
+ elif len(value) > 1:
+ raise ItemCountError(
+ "single selection list, must set sequence of "
+ "length 0 or 1")
+ else:
+ self._single_set_value(value)
+
+ def _get_items(self, name, target=1):
+ all_items = self.get_items(name)
+ items = [o for o in all_items if not o.disabled]
+ if len(items) < target:
+ if len(all_items) < target:
+ raise ItemNotFoundError(
+ "insufficient items with name %r" % name)
+ else:
+ raise AttributeError(
+ "insufficient non-disabled items with name %s" % name)
+ on = []
+ off = []
+ for o in items:
+ if o.selected:
+ on.append(o)
+ else:
+ off.append(o)
+ return on, off
+
+ def _single_set_value(self, value):
+ assert len(value) == 1
+ on, off = self._get_items(value[0])
+ assert len(on) <= 1
+ if not on:
+ off[0].selected = True
+
+ def _multiple_set_value(self, value):
+ compat = self._form.backwards_compat
+ turn_on = [] # transactional-ish
+ turn_off = [item for item in self.items if
+ item.selected and (not item.disabled or compat)]
+ names = {}
+ for nn in value:
+ if nn in names.keys():
+ names[nn] += 1
+ else:
+ names[nn] = 1
+ for name, count in names.items():
+ on, off = self._get_items(name, count)
+ for i in range(count):
+ if on:
+ item = on[0]
+ del on[0]
+ del turn_off[turn_off.index(item)]
+ else:
+ item = off[0]
+ del off[0]
+ turn_on.append(item)
+ for item in turn_off:
+ item.selected = False
+ for item in turn_on:
+ item.selected = True
+
+ def set_value_by_label(self, value):
+ """Set the value of control by item labels.
+
+ value is expected to be an iterable of strings that are substrings of
+ the item labels that should be selected. Before substring matching is
+ performed, the original label text is whitespace-compressed
+ (consecutive whitespace characters are converted to a single space
+ character) and leading and trailing whitespace is stripped. Ambiguous
+ labels are accepted without complaint if the form's backwards_compat is
+ True; otherwise, it will not complain as long as all ambiguous labels
+ share the same item name (e.g. OPTION value).
+
+ """
+ if isstringlike(value):
+ raise TypeError(value)
+ if not self.multiple and len(value) > 1:
+ raise ItemCountError(
+ "single selection list, must set sequence of "
+ "length 0 or 1")
+ items = []
+ for nn in value:
+ found = self.get_items(label=nn)
+ if len(found) > 1:
+ if not self._form.backwards_compat:
+ # ambiguous labels are fine as long as item names (e.g.
+ # OPTION values) are same
+ opt_name = found[0].name
+ if [o for o in found[1:] if o.name != opt_name]:
+ raise AmbiguityError(nn)
+ else:
+ # OK, we'll guess :-( Assume first available item.
+ found = found[:1]
+ for o in found:
+ # For the multiple-item case, we could try to be smarter,
+ # saving them up and trying to resolve, but that's too much.
+ if self._form.backwards_compat or o not in items:
+ items.append(o)
+ break
+ else: # all of them are used
+ raise ItemNotFoundError(nn)
+ # now we have all the items that should be on
+ # let's just turn everything off and then back on.
+ self.value = []
+ for o in items:
+ o.selected = True
+
+ def get_value_by_label(self):
+ """Return the value of the control as given by normalized labels."""
+ res = []
+ compat = self._form.backwards_compat
+ for o in self.items:
+ if (not o.disabled or compat) and o.selected:
+ for l in o.get_labels():
+ if l.text:
+ res.append(l.text)
+ break
+ else:
+ res.append(None)
+ return res
+
+ def possible_items(self, by_label=False):
+ """Deprecated: return the names or labels of all possible items.
+
+ Includes disabled items, which may be misleading for some use cases.
+
+ """
+ deprecation(
+ "[item.name for item in self.items]")
+ if by_label:
+ res = []
+ for o in self.items:
+ for l in o.get_labels():
+ if l.text:
+ res.append(l.text)
+ break
+ else:
+ res.append(None)
+ return res
+ return [o.name for o in self.items]
+
+ def _totally_ordered_pairs(self):
+ if self.disabled or self.name is None:
+ return []
+ else:
+ return [(o._index, self.name, o.name) for o in self.items
+ if o.selected and not o.disabled]
+
+ def __str__(self):
+ name = self.name
+ if name is None: name = ""
+
+ display = [str(o) for o in self.items]
+
+ infos = []
+ if self.disabled: infos.append("disabled")
+ if self.readonly: infos.append("readonly")
+ info = ", ".join(infos)
+ if info: info = " (%s)" % info
+
+ return "<%s(%s=[%s])%s>" % (self.__class__.__name__,
+ name, ", ".join(display), info)
+
+
+class RadioControl(ListControl):
+ """
+ Covers:
+
+ INPUT/RADIO
+
+ """
+ def __init__(self, type, name, attrs, select_default=False, index=None):
+ attrs.setdefault("value", "on")
+ ListControl.__init__(self, type, name, attrs, select_default,
+ called_as_base_class=True, index=index)
+ self.__dict__["multiple"] = False
+ o = Item(self, attrs, index)
+ o.__dict__["_selected"] = attrs.has_key("checked")
+
+ def fixup(self):
+ ListControl.fixup(self)
+ found = [o for o in self.items if o.selected and not o.disabled]
+ if not found:
+ if self._select_default:
+ for o in self.items:
+ if not o.disabled:
+ o.selected = True
+ break
+ else:
+ # Ensure only one item selected. Choose the last one,
+ # following IE and Firefox.
+ for o in found[:-1]:
+ o.selected = False
+
+ def get_labels(self):
+ return []
+
+class CheckboxControl(ListControl):
+ """
+ Covers:
+
+ INPUT/CHECKBOX
+
+ """
+ def __init__(self, type, name, attrs, select_default=False, index=None):
+ attrs.setdefault("value", "on")
+ ListControl.__init__(self, type, name, attrs, select_default,
+ called_as_base_class=True, index=index)
+ self.__dict__["multiple"] = True
+ o = Item(self, attrs, index)
+ o.__dict__["_selected"] = attrs.has_key("checked")
+
+ def get_labels(self):
+ return []
+
+
+class SelectControl(ListControl):
+ """
+ Covers:
+
+ SELECT (and OPTION)
+
+
+ OPTION 'values', in HTML parlance, are Item 'names' in mechanize parlance.
+
+ SELECT control values and labels are subject to some messy defaulting
+ rules. For example, if the HTML representation of the control is:
+
+
+ current year
+ 2001
+ 2000
+
+
+ The items, in order, have labels "2002", "2001" and "2000", whereas their
+ names (the OPTION values) are "0", "1" and "2000" respectively. Note that
+ the value of the last OPTION in this example defaults to its contents, as
+ specified by RFC 1866, as do the labels of the second and third OPTIONs.
+
+ The OPTION labels are sometimes more meaningful than the OPTION values,
+ which can make for more maintainable code.
+
+ Additional read-only public attribute: attrs
+
+ The attrs attribute is a dictionary of the original HTML attributes of the
+ SELECT element. Other ListControls do not have this attribute, because in
+ other cases the control as a whole does not correspond to any single HTML
+ element. control.get(...).attrs may be used as usual to get at the HTML
+ attributes of the HTML elements corresponding to individual list items (for
+ SELECT controls, these are OPTION elements).
+
+ Another special case is that the Item.attrs dictionaries have a special key
+ "contents" which does not correspond to any real HTML attribute, but rather
+ contains the contents of the OPTION element:
+
+ this bit
+
+ """
+ # HTML attributes here are treated slightly differently from other list
+ # controls:
+ # -The SELECT HTML attributes dictionary is stuffed into the OPTION
+ # HTML attributes dictionary under the "__select" key.
+ # -The content of each OPTION element is stored under the special
+ # "contents" key of the dictionary.
+ # After all this, the dictionary is passed to the SelectControl constructor
+ # as the attrs argument, as usual. However:
+ # -The first SelectControl constructed when building up a SELECT control
+ # has a constructor attrs argument containing only the __select key -- so
+ # this SelectControl represents an empty SELECT control.
+ # -Subsequent SelectControls have both OPTION HTML-attribute in attrs and
+ # the __select dictionary containing the SELECT HTML-attributes.
+
+ def __init__(self, type, name, attrs, select_default=False, index=None):
+ # fish out the SELECT HTML attributes from the OPTION HTML attributes
+ # dictionary
+ self.attrs = attrs["__select"].copy()
+ self.__dict__["_label"] = _get_label(self.attrs)
+ self.__dict__["id"] = self.attrs.get("id")
+ self.__dict__["multiple"] = self.attrs.has_key("multiple")
+ # the majority of the contents, label, and value dance already happened
+ contents = attrs.get("contents")
+ attrs = attrs.copy()
+ del attrs["__select"]
+
+ ListControl.__init__(self, type, name, self.attrs, select_default,
+ called_as_base_class=True, index=index)
+ self.disabled = self.attrs.has_key("disabled")
+ self.readonly = self.attrs.has_key("readonly")
+ if attrs.has_key("value"):
+ # otherwise it is a marker 'select started' token
+ o = Item(self, attrs, index)
+ o.__dict__["_selected"] = attrs.has_key("selected")
+ # add 'label' label and contents label, if different. If both are
+ # provided, the 'label' label is used for display in HTML
+ # 4.0-compliant browsers (and any lower spec? not sure) while the
+ # contents are used for display in older or less-compliant
+ # browsers. We make label objects for both, if the values are
+ # different.
+ label = attrs.get("label")
+ if label:
+ o._labels.append(Label({"__text": label}))
+ if contents and contents != label:
+ o._labels.append(Label({"__text": contents}))
+ elif contents:
+ o._labels.append(Label({"__text": contents}))
+
+ def fixup(self):
+ ListControl.fixup(self)
+ # Firefox doesn't exclude disabled items from those considered here
+ # (i.e. from 'found', for both branches of the if below). Note that
+ # IE6 doesn't support the disabled attribute on OPTIONs at all.
+ found = [o for o in self.items if o.selected]
+ if not found:
+ if not self.multiple or self._select_default:
+ for o in self.items:
+ if not o.disabled:
+ was_disabled = self.disabled
+ self.disabled = False
+ try:
+ o.selected = True
+ finally:
+ o.disabled = was_disabled
+ break
+ elif not self.multiple:
+ # Ensure only one item selected. Choose the last one,
+ # following IE and Firefox.
+ for o in found[:-1]:
+ o.selected = False
+
+
+#---------------------------------------------------
+class SubmitControl(ScalarControl):
+ """
+ Covers:
+
+ INPUT/SUBMIT
+ BUTTON/SUBMIT
+
+ """
+ def __init__(self, type, name, attrs, index=None):
+ ScalarControl.__init__(self, type, name, attrs, index)
+ # IE5 defaults SUBMIT value to "Submit Query"; Firebird 0.6 leaves it
+ # blank, Konqueror 3.1 defaults to "Submit". HTML spec. doesn't seem
+ # to define this.
+ if self.value is None: self.value = ""
+ self.readonly = True
+
+ def get_labels(self):
+ res = []
+ if self.value:
+ res.append(Label({"__text": self.value}))
+ res.extend(ScalarControl.get_labels(self))
+ return res
+
+ def is_of_kind(self, kind): return kind == "clickable"
+
+ def _click(self, form, coord, return_type, request_class=_request.Request):
+ self._clicked = coord
+ r = form._switch_click(return_type, request_class)
+ self._clicked = False
+ return r
+
+ def _totally_ordered_pairs(self):
+ if not self._clicked:
+ return []
+ return ScalarControl._totally_ordered_pairs(self)
+
+
+#---------------------------------------------------
+class ImageControl(SubmitControl):
+ """
+ Covers:
+
+ INPUT/IMAGE
+
+ Coordinates are specified using one of the HTMLForm.click* methods.
+
+ """
+ def __init__(self, type, name, attrs, index=None):
+ SubmitControl.__init__(self, type, name, attrs, index)
+ self.readonly = False
+
+ def _totally_ordered_pairs(self):
+ clicked = self._clicked
+ if self.disabled or not clicked:
+ return []
+ name = self.name
+ if name is None: return []
+ pairs = [
+ (self._index, "%s.x" % name, str(clicked[0])),
+ (self._index+1, "%s.y" % name, str(clicked[1])),
+ ]
+ value = self._value
+ if value:
+ pairs.append((self._index+2, name, value))
+ return pairs
+
+ get_labels = ScalarControl.get_labels
+
+# aliases, just to make str(control) and str(form) clearer
+class PasswordControl(TextControl): pass
+class HiddenControl(TextControl): pass
+class TextareaControl(TextControl): pass
+class SubmitButtonControl(SubmitControl): pass
+
+
+def is_listcontrol(control): return control.is_of_kind("list")
+
+
+class HTMLForm:
+ """Represents a single HTML element.
+
+ A form consists of a sequence of controls that usually have names, and
+ which can take on various values. The values of the various types of
+ controls represent variously: text, zero-or-one-of-many or many-of-many
+ choices, and files to be uploaded. Some controls can be clicked on to
+ submit the form, and clickable controls' values sometimes include the
+ coordinates of the click.
+
+ Forms can be filled in with data to be returned to the server, and then
+ submitted, using the click method to generate a request object suitable for
+ passing to mechanize.urlopen (or the click_request_data or click_pairs
+ methods for integration with third-party code).
+
+ import mechanize
+ forms = mechanize.ParseFile(html, base_uri)
+ form = forms[0]
+
+ form["query"] = "Python"
+ form.find_control("nr_results").get("lots").selected = True
+
+ response = mechanize.urlopen(form.click())
+
+ Usually, HTMLForm instances are not created directly. Instead, the
+ ParseFile or ParseResponse factory functions are used. If you do construct
+ HTMLForm objects yourself, however, note that an HTMLForm instance is only
+ properly initialised after the fixup method has been called (ParseFile and
+ ParseResponse do this for you). See ListControl.__doc__ for the reason
+ this is required.
+
+ Indexing a form (form["control_name"]) returns the named Control's value
+ attribute. Assignment to a form index (form["control_name"] = something)
+ is equivalent to assignment to the named Control's value attribute. If you
+ need to be more specific than just supplying the control's name, use the
+ set_value and get_value methods.
+
+ ListControl values are lists of item names (specifically, the names of the
+ items that are selected and not disabled, and hence are "successful" -- ie.
+ cause data to be returned to the server). The list item's name is the
+ value of the corresponding HTML element's"value" attribute.
+
+ Example:
+
+
+ 1
+ cheddar
+
+
+ defines a SELECT control with name "more_cheeses" which has two items,
+ named "1" and "2" (because the OPTION element's value HTML attribute
+ defaults to the element contents -- see SelectControl.__doc__ for more on
+ these defaulting rules).
+
+ To select, deselect or otherwise manipulate individual list items, use the
+ HTMLForm.find_control() and ListControl.get() methods. To set the whole
+ value, do as for any other control: use indexing or the set_/get_value
+ methods.
+
+ Example:
+
+ # select *only* the item named "cheddar"
+ form["cheeses"] = ["cheddar"]
+ # select "cheddar", leave other items unaffected
+ form.find_control("cheeses").get("cheddar").selected = True
+
+ Some controls (RADIO and SELECT without the multiple attribute) can only
+ have zero or one items selected at a time. Some controls (CHECKBOX and
+ SELECT with the multiple attribute) can have multiple items selected at a
+ time. To set the whole value of a ListControl, assign a sequence to a form
+ index:
+
+ form["cheeses"] = ["cheddar", "leicester"]
+
+ If the ListControl is not multiple-selection, the assigned list must be of
+ length one.
+
+ To check if a control has an item, if an item is selected, or if an item is
+ successful (selected and not disabled), respectively:
+
+ "cheddar" in [item.name for item in form.find_control("cheeses").items]
+ "cheddar" in [item.name for item in form.find_control("cheeses").items and
+ item.selected]
+ "cheddar" in form["cheeses"] # (or "cheddar" in form.get_value("cheeses"))
+
+ Note that some list items may be disabled (see below).
+
+ Note the following mistake:
+
+ form[control_name] = control_value
+ assert form[control_name] == control_value # not necessarily true
+
+ The reason for this is that form[control_name] always gives the list items
+ in the order they were listed in the HTML.
+
+ List items (hence list values, too) can be referred to in terms of list
+ item labels rather than list item names using the appropriate label
+ arguments. Note that each item may have several labels.
+
+ The question of default values of OPTION contents, labels and values is
+ somewhat complicated: see SelectControl.__doc__ and
+ ListControl.get_item_attrs.__doc__ if you think you need to know.
+
+ Controls can be disabled or readonly. In either case, the control's value
+ cannot be changed until you clear those flags (see example below).
+ Disabled is the state typically represented by browsers by 'greying out' a
+ control. Disabled controls are not 'successful' -- they don't cause data
+ to get returned to the server. Readonly controls usually appear in
+ browsers as read-only text boxes. Readonly controls are successful. List
+ items can also be disabled. Attempts to select or deselect disabled items
+ fail with AttributeError.
+
+ If a lot of controls are readonly, it can be useful to do this:
+
+ form.set_all_readonly(False)
+
+ To clear a control's value attribute, so that it is not successful (until a
+ value is subsequently set):
+
+ form.clear("cheeses")
+
+ More examples:
+
+ control = form.find_control("cheeses")
+ control.disabled = False
+ control.readonly = False
+ control.get("gruyere").disabled = True
+ control.items[0].selected = True
+
+ See the various Control classes for further documentation. Many methods
+ take name, type, kind, id, label and nr arguments to specify the control to
+ be operated on: see HTMLForm.find_control.__doc__.
+
+ ControlNotFoundError (subclass of ValueError) is raised if the specified
+ control can't be found. This includes occasions where a non-ListControl
+ is found, but the method (set, for example) requires a ListControl.
+ ItemNotFoundError (subclass of ValueError) is raised if a list item can't
+ be found. ItemCountError (subclass of ValueError) is raised if an attempt
+ is made to select more than one item and the control doesn't allow that, or
+ set/get_single are called and the control contains more than one item.
+ AttributeError is raised if a control or item is readonly or disabled and
+ an attempt is made to alter its value.
+
+ Security note: Remember that any passwords you store in HTMLForm instances
+ will be saved to disk in the clear if you pickle them (directly or
+ indirectly). The simplest solution to this is to avoid pickling HTMLForm
+ objects. You could also pickle before filling in any password, or just set
+ the password to "" before pickling.
+
+
+ Public attributes:
+
+ action: full (absolute URI) form action
+ method: "GET" or "POST"
+ enctype: form transfer encoding MIME type
+ name: name of form (None if no name was specified)
+ attrs: dictionary mapping original HTML form attributes to their values
+
+ controls: list of Control instances; do not alter this list
+ (instead, call form.new_control to make a Control and add it to the
+ form, or control.add_to_form if you already have a Control instance)
+
+
+
+ Methods for form filling:
+ -------------------------
+
+ Most of the these methods have very similar arguments. See
+ HTMLForm.find_control.__doc__ for details of the name, type, kind, label
+ and nr arguments.
+
+ def find_control(self,
+ name=None, type=None, kind=None, id=None, predicate=None,
+ nr=None, label=None)
+
+ get_value(name=None, type=None, kind=None, id=None, nr=None,
+ by_label=False, # by_label is deprecated
+ label=None)
+ set_value(value,
+ name=None, type=None, kind=None, id=None, nr=None,
+ by_label=False, # by_label is deprecated
+ label=None)
+
+ clear_all()
+ clear(name=None, type=None, kind=None, id=None, nr=None, label=None)
+
+ set_all_readonly(readonly)
+
+
+ Method applying only to FileControls:
+
+ add_file(file_object,
+ content_type="application/octet-stream", filename=None,
+ name=None, id=None, nr=None, label=None)
+
+
+ Methods applying only to clickable controls:
+
+ click(name=None, type=None, id=None, nr=0, coord=(1,1), label=None)
+ click_request_data(name=None, type=None, id=None, nr=0, coord=(1,1),
+ label=None)
+ click_pairs(name=None, type=None, id=None, nr=0, coord=(1,1), label=None)
+
+ """
+
+ type2class = {
+ "text": TextControl,
+ "password": PasswordControl,
+ "hidden": HiddenControl,
+ "textarea": TextareaControl,
+
+ "isindex": IsindexControl,
+
+ "file": FileControl,
+
+ "button": IgnoreControl,
+ "buttonbutton": IgnoreControl,
+ "reset": IgnoreControl,
+ "resetbutton": IgnoreControl,
+
+ "submit": SubmitControl,
+ "submitbutton": SubmitButtonControl,
+ "image": ImageControl,
+
+ "radio": RadioControl,
+ "checkbox": CheckboxControl,
+ "select": SelectControl,
+ }
+
+#---------------------------------------------------
+# Initialisation. Use ParseResponse / ParseFile instead.
+
+ def __init__(self, action, method="GET",
+ enctype="application/x-www-form-urlencoded",
+ name=None, attrs=None,
+ request_class=_request.Request,
+ forms=None, labels=None, id_to_labels=None,
+ backwards_compat=True):
+ """
+ In the usual case, use ParseResponse (or ParseFile) to create new
+ HTMLForm objects.
+
+ action: full (absolute URI) form action
+ method: "GET" or "POST"
+ enctype: form transfer encoding MIME type
+ name: name of form
+ attrs: dictionary mapping original HTML form attributes to their values
+
+ """
+ self.action = action
+ self.method = method
+ self.enctype = enctype
+ self.name = name
+ if attrs is not None:
+ self.attrs = attrs.copy()
+ else:
+ self.attrs = {}
+ self.controls = []
+ self._request_class = request_class
+
+ # these attributes are used by zope.testbrowser
+ self._forms = forms # this is a semi-public API!
+ self._labels = labels # this is a semi-public API!
+ self._id_to_labels = id_to_labels # this is a semi-public API!
+
+ self.backwards_compat = backwards_compat # note __setattr__
+
+ self._urlunparse = urlparse.urlunparse
+ self._urlparse = urlparse.urlparse
+
+ def __getattr__(self, name):
+ if name == "backwards_compat":
+ return self._backwards_compat
+ return getattr(HTMLForm, name)
+
+ def __setattr__(self, name, value):
+ # yuck
+ if name == "backwards_compat":
+ name = "_backwards_compat"
+ value = bool(value)
+ for cc in self.controls:
+ try:
+ items = cc.items
+ except AttributeError:
+ continue
+ else:
+ for ii in items:
+ for ll in ii.get_labels():
+ ll._backwards_compat = value
+ self.__dict__[name] = value
+
+ def new_control(self, type, name, attrs,
+ ignore_unknown=False, select_default=False, index=None):
+ """Adds a new control to the form.
+
+ This is usually called by ParseFile and ParseResponse. Don't call it
+ youself unless you're building your own Control instances.
+
+ Note that controls representing lists of items are built up from
+ controls holding only a single list item. See ListControl.__doc__ for
+ further information.
+
+ type: type of control (see Control.__doc__ for a list)
+ attrs: HTML attributes of control
+ ignore_unknown: if true, use a dummy Control instance for controls of
+ unknown type; otherwise, use a TextControl
+ select_default: for RADIO and multiple-selection SELECT controls, pick
+ the first item as the default if no 'selected' HTML attribute is
+ present (this defaulting happens when the HTMLForm.fixup method is
+ called)
+ index: index of corresponding element in HTML (see
+ MoreFormTests.test_interspersed_controls for motivation)
+
+ """
+ type = type.lower()
+ klass = self.type2class.get(type)
+ if klass is None:
+ if ignore_unknown:
+ klass = IgnoreControl
+ else:
+ klass = TextControl
+
+ a = attrs.copy()
+ if issubclass(klass, ListControl):
+ control = klass(type, name, a, select_default, index)
+ else:
+ control = klass(type, name, a, index)
+
+ if type == "select" and len(attrs) == 1:
+ for ii in range(len(self.controls)-1, -1, -1):
+ ctl = self.controls[ii]
+ if ctl.type == "select":
+ ctl.close_control()
+ break
+
+ control.add_to_form(self)
+ control._urlparse = self._urlparse
+ control._urlunparse = self._urlunparse
+
+ def fixup(self):
+ """Normalise form after all controls have been added.
+
+ This is usually called by ParseFile and ParseResponse. Don't call it
+ youself unless you're building your own Control instances.
+
+ This method should only be called once, after all controls have been
+ added to the form.
+
+ """
+ for control in self.controls:
+ control.fixup()
+ self.backwards_compat = self._backwards_compat
+
+#---------------------------------------------------
+ def __str__(self):
+ header = "%s%s %s %s" % (
+ (self.name and self.name+" " or ""),
+ self.method, self.action, self.enctype)
+ rep = [header]
+ for control in self.controls:
+ rep.append(" %s" % str(control))
+ return "<%s>" % "\n".join(rep)
+
+#---------------------------------------------------
+# Form-filling methods.
+
+ def __getitem__(self, name):
+ return self.find_control(name).value
+ def __contains__(self, name):
+ return bool(self.find_control(name))
+ def __setitem__(self, name, value):
+ control = self.find_control(name)
+ try:
+ control.value = value
+ except AttributeError, e:
+ raise ValueError(str(e))
+
+ def get_value(self,
+ name=None, type=None, kind=None, id=None, nr=None,
+ by_label=False, # by_label is deprecated
+ label=None):
+ """Return value of control.
+
+ If only name and value arguments are supplied, equivalent to
+
+ form[name]
+
+ """
+ if by_label:
+ deprecation("form.get_value_by_label(...)")
+ c = self.find_control(name, type, kind, id, label=label, nr=nr)
+ if by_label:
+ try:
+ meth = c.get_value_by_label
+ except AttributeError:
+ raise NotImplementedError(
+ "control '%s' does not yet support by_label" % c.name)
+ else:
+ return meth()
+ else:
+ return c.value
+ def set_value(self, value,
+ name=None, type=None, kind=None, id=None, nr=None,
+ by_label=False, # by_label is deprecated
+ label=None):
+ """Set value of control.
+
+ If only name and value arguments are supplied, equivalent to
+
+ form[name] = value
+
+ """
+ if by_label:
+ deprecation("form.get_value_by_label(...)")
+ c = self.find_control(name, type, kind, id, label=label, nr=nr)
+ if by_label:
+ try:
+ meth = c.set_value_by_label
+ except AttributeError:
+ raise NotImplementedError(
+ "control '%s' does not yet support by_label" % c.name)
+ else:
+ meth(value)
+ else:
+ c.value = value
+ def get_value_by_label(
+ self, name=None, type=None, kind=None, id=None, label=None, nr=None):
+ """
+
+ All arguments should be passed by name.
+
+ """
+ c = self.find_control(name, type, kind, id, label=label, nr=nr)
+ return c.get_value_by_label()
+
+ def set_value_by_label(
+ self, value,
+ name=None, type=None, kind=None, id=None, label=None, nr=None):
+ """
+
+ All arguments should be passed by name.
+
+ """
+ c = self.find_control(name, type, kind, id, label=label, nr=nr)
+ c.set_value_by_label(value)
+
+ def set_all_readonly(self, readonly):
+ for control in self.controls:
+ control.readonly = bool(readonly)
+
+ def clear_all(self):
+ """Clear the value attributes of all controls in the form.
+
+ See HTMLForm.clear.__doc__.
+
+ """
+ for control in self.controls:
+ control.clear()
+
+ def clear(self,
+ name=None, type=None, kind=None, id=None, nr=None, label=None):
+ """Clear the value attribute of a control.
+
+ As a result, the affected control will not be successful until a value
+ is subsequently set. AttributeError is raised on readonly controls.
+
+ """
+ c = self.find_control(name, type, kind, id, label=label, nr=nr)
+ c.clear()
+
+
+#---------------------------------------------------
+# Form-filling methods applying only to ListControls.
+
+ def possible_items(self, # deprecated
+ name=None, type=None, kind=None, id=None,
+ nr=None, by_label=False, label=None):
+ """Return a list of all values that the specified control can take."""
+ c = self._find_list_control(name, type, kind, id, label, nr)
+ return c.possible_items(by_label)
+
+ def set(self, selected, item_name, # deprecated
+ name=None, type=None, kind=None, id=None, nr=None,
+ by_label=False, label=None):
+ """Select / deselect named list item.
+
+ selected: boolean selected state
+
+ """
+ self._find_list_control(name, type, kind, id, label, nr).set(
+ selected, item_name, by_label)
+ def toggle(self, item_name, # deprecated
+ name=None, type=None, kind=None, id=None, nr=None,
+ by_label=False, label=None):
+ """Toggle selected state of named list item."""
+ self._find_list_control(name, type, kind, id, label, nr).toggle(
+ item_name, by_label)
+
+ def set_single(self, selected, # deprecated
+ name=None, type=None, kind=None, id=None,
+ nr=None, by_label=None, label=None):
+ """Select / deselect list item in a control having only one item.
+
+ If the control has multiple list items, ItemCountError is raised.
+
+ This is just a convenience method, so you don't need to know the item's
+ name -- the item name in these single-item controls is usually
+ something meaningless like "1" or "on".
+
+ For example, if a checkbox has a single item named "on", the following
+ two calls are equivalent:
+
+ control.toggle("on")
+ control.toggle_single()
+
+ """ # by_label ignored and deprecated
+ self._find_list_control(
+ name, type, kind, id, label, nr).set_single(selected)
+ def toggle_single(self, name=None, type=None, kind=None, id=None,
+ nr=None, by_label=None, label=None): # deprecated
+ """Toggle selected state of list item in control having only one item.
+
+ The rest is as for HTMLForm.set_single.__doc__.
+
+ """ # by_label ignored and deprecated
+ self._find_list_control(name, type, kind, id, label, nr).toggle_single()
+
+#---------------------------------------------------
+# Form-filling method applying only to FileControls.
+
+ def add_file(self, file_object, content_type=None, filename=None,
+ name=None, id=None, nr=None, label=None):
+ """Add a file to be uploaded.
+
+ file_object: file-like object (with read method) from which to read
+ data to upload
+ content_type: MIME content type of data to upload
+ filename: filename to pass to server
+
+ If filename is None, no filename is sent to the server.
+
+ If content_type is None, the content type is guessed based on the
+ filename and the data from read from the file object.
+
+ XXX
+ At the moment, guessed content type is always application/octet-stream.
+ Use sndhdr, imghdr modules. Should also try to guess HTML, XML, and
+ plain text.
+
+ Note the following useful HTML attributes of file upload controls (see
+ HTML 4.01 spec, section 17):
+
+ accept: comma-separated list of content types that the server will
+ handle correctly; you can use this to filter out non-conforming files
+ size: XXX IIRC, this is indicative of whether form wants multiple or
+ single files
+ maxlength: XXX hint of max content length in bytes?
+
+ """
+ self.find_control(name, "file", id=id, label=label, nr=nr).add_file(
+ file_object, content_type, filename)
+
+#---------------------------------------------------
+# Form submission methods, applying only to clickable controls.
+
+ def click(self, name=None, type=None, id=None, nr=0, coord=(1,1),
+ request_class=_request.Request,
+ label=None):
+ """Return request that would result from clicking on a control.
+
+ The request object is a mechanize.Request instance, which you can pass
+ to mechanize.urlopen.
+
+ Only some control types (INPUT/SUBMIT & BUTTON/SUBMIT buttons and
+ IMAGEs) can be clicked.
+
+ Will click on the first clickable control, subject to the name, type
+ and nr arguments (as for find_control). If no name, type, id or number
+ is specified and there are no clickable controls, a request will be
+ returned for the form in its current, un-clicked, state.
+
+ IndexError is raised if any of name, type, id or nr is specified but no
+ matching control is found. ValueError is raised if the HTMLForm has an
+ enctype attribute that is not recognised.
+
+ You can optionally specify a coordinate to click at, which only makes a
+ difference if you clicked on an image.
+
+ """
+ return self._click(name, type, id, label, nr, coord, "request",
+ self._request_class)
+
+ def click_request_data(self,
+ name=None, type=None, id=None,
+ nr=0, coord=(1,1),
+ request_class=_request.Request,
+ label=None):
+ """As for click method, but return a tuple (url, data, headers).
+
+ You can use this data to send a request to the server. This is useful
+ if you're using httplib or urllib rather than mechanize. Otherwise,
+ use the click method.
+
+ # Untested. Have to subclass to add headers, I think -- so use
+ # mechanize instead!
+ import urllib
+ url, data, hdrs = form.click_request_data()
+ r = urllib.urlopen(url, data)
+
+ # Untested. I don't know of any reason to use httplib -- you can get
+ # just as much control with mechanize.
+ import httplib, urlparse
+ url, data, hdrs = form.click_request_data()
+ tup = urlparse(url)
+ host, path = tup[1], urlparse.urlunparse((None, None)+tup[2:])
+ conn = httplib.HTTPConnection(host)
+ if data:
+ httplib.request("POST", path, data, hdrs)
+ else:
+ httplib.request("GET", path, headers=hdrs)
+ r = conn.getresponse()
+
+ """
+ return self._click(name, type, id, label, nr, coord, "request_data",
+ self._request_class)
+
+ def click_pairs(self, name=None, type=None, id=None,
+ nr=0, coord=(1,1),
+ label=None):
+ """As for click_request_data, but returns a list of (key, value) pairs.
+
+ You can use this list as an argument to urllib.urlencode. This is
+ usually only useful if you're using httplib or urllib rather than
+ mechanize. It may also be useful if you want to manually tweak the
+ keys and/or values, but this should not be necessary. Otherwise, use
+ the click method.
+
+ Note that this method is only useful for forms of MIME type
+ x-www-form-urlencoded. In particular, it does not return the
+ information required for file upload. If you need file upload and are
+ not using mechanize, use click_request_data.
+ """
+ return self._click(name, type, id, label, nr, coord, "pairs",
+ self._request_class)
+
+#---------------------------------------------------
+
+ def find_control(self,
+ name=None, type=None, kind=None, id=None,
+ predicate=None, nr=None,
+ label=None):
+ """Locate and return some specific control within the form.
+
+ At least one of the name, type, kind, predicate and nr arguments must
+ be supplied. If no matching control is found, ControlNotFoundError is
+ raised.
+
+ If name is specified, then the control must have the indicated name.
+
+ If type is specified then the control must have the specified type (in
+ addition to the types possible for
+ separators = "(" | ")" | "<" | ">" | "@"
+ | "," | ";" | ":" | "\" | <">
+ | "/" | "[" | "]" | "?" | "="
+ | "{" | "}" | SP | HT
+
+ quoted-string = ( <"> *(qdtext | quoted-pair ) <"> )
+ qdtext = >
+ quoted-pair = "\" CHAR
+
+ parameter = attribute "=" value
+ attribute = token
+ value = token | quoted-string
+
+ Each header is represented by a list of key/value pairs. The value for a
+ simple token (not part of a parameter) is None. Syntactically incorrect
+ headers will not necessarily be parsed as you would want.
+
+ This is easier to describe with some examples:
+
+ >>> split_header_words(['foo="bar"; port="80,81"; discard, bar=baz'])
+ [[('foo', 'bar'), ('port', '80,81'), ('discard', None)], [('bar', 'baz')]]
+ >>> split_header_words(['text/html; charset="iso-8859-1"'])
+ [[('text/html', None), ('charset', 'iso-8859-1')]]
+ >>> split_header_words([r'Basic realm="\"foo\bar\""'])
+ [[('Basic', None), ('realm', '"foobar"')]]
+
+ """
+ assert type(header_values) not in STRING_TYPES
+ result = []
+ for text in header_values:
+ orig_text = text
+ pairs = []
+ while text:
+ m = token_re.search(text)
+ if m:
+ text = unmatched(m)
+ name = m.group(1)
+ m = quoted_value_re.search(text)
+ if m: # quoted value
+ text = unmatched(m)
+ value = m.group(1)
+ value = escape_re.sub(r"\1", value)
+ else:
+ m = value_re.search(text)
+ if m: # unquoted value
+ text = unmatched(m)
+ value = m.group(1)
+ value = value.rstrip()
+ else:
+ # no value, a lone token
+ value = None
+ pairs.append((name, value))
+ elif text.lstrip().startswith(","):
+ # concatenated headers, as per RFC 2616 section 4.2
+ text = text.lstrip()[1:]
+ if pairs: result.append(pairs)
+ pairs = []
+ else:
+ # skip junk
+ non_junk, nr_junk_chars = re.subn("^[=\s;]*", "", text)
+ assert nr_junk_chars > 0, (
+ "split_header_words bug: '%s', '%s', %s" %
+ (orig_text, text, pairs))
+ text = non_junk
+ if pairs: result.append(pairs)
+ return result
+
+join_escape_re = re.compile(r"([\"\\])")
+def join_header_words(lists):
+ """Do the inverse of the conversion done by split_header_words.
+
+ Takes a list of lists of (key, value) pairs and produces a single header
+ value. Attribute values are quoted if needed.
+
+ >>> join_header_words([[("text/plain", None), ("charset", "iso-8859/1")]])
+ 'text/plain; charset="iso-8859/1"'
+ >>> join_header_words([[("text/plain", None)], [("charset", "iso-8859/1")]])
+ 'text/plain, charset="iso-8859/1"'
+
+ """
+ headers = []
+ for pairs in lists:
+ attr = []
+ for k, v in pairs:
+ if v is not None:
+ if not re.search(r"^\w+$", v):
+ v = join_escape_re.sub(r"\\\1", v) # escape " and \
+ v = '"%s"' % v
+ if k is None: # Netscape cookies may have no name
+ k = v
+ else:
+ k = "%s=%s" % (k, v)
+ attr.append(k)
+ if attr: headers.append("; ".join(attr))
+ return ", ".join(headers)
+
+def strip_quotes(text):
+ if text.startswith('"'):
+ text = text[1:]
+ if text.endswith('"'):
+ text = text[:-1]
+ return text
+
+def parse_ns_headers(ns_headers):
+ """Ad-hoc parser for Netscape protocol cookie-attributes.
+
+ The old Netscape cookie format for Set-Cookie can for instance contain
+ an unquoted "," in the expires field, so we have to use this ad-hoc
+ parser instead of split_header_words.
+
+ XXX This may not make the best possible effort to parse all the crap
+ that Netscape Cookie headers contain. Ronald Tschalar's HTTPClient
+ parser is probably better, so could do worse than following that if
+ this ever gives any trouble.
+
+ Currently, this is also used for parsing RFC 2109 cookies.
+
+ """
+ known_attrs = ("expires", "domain", "path", "secure",
+ # RFC 2109 attrs (may turn up in Netscape cookies, too)
+ "version", "port", "max-age")
+
+ result = []
+ for ns_header in ns_headers:
+ pairs = []
+ version_set = False
+ params = re.split(r";\s*", ns_header)
+ for ii in range(len(params)):
+ param = params[ii]
+ param = param.rstrip()
+ if param == "": continue
+ if "=" not in param:
+ k, v = param, None
+ else:
+ k, v = re.split(r"\s*=\s*", param, 1)
+ k = k.lstrip()
+ if ii != 0:
+ lc = k.lower()
+ if lc in known_attrs:
+ k = lc
+ if k == "version":
+ # This is an RFC 2109 cookie.
+ v = strip_quotes(v)
+ version_set = True
+ if k == "expires":
+ # convert expires date to seconds since epoch
+ v = http2time(strip_quotes(v)) # None if invalid
+ pairs.append((k, v))
+
+ if pairs:
+ if not version_set:
+ pairs.append(("version", "0"))
+ result.append(pairs)
+
+ return result
+
+
+def _test():
+ import doctest, _headersutil
+ return doctest.testmod(_headersutil)
+
+if __name__ == "__main__":
+ _test()
diff --git a/couchpotato/core/media/movie/providers/trailer/mechanize/_html.py b/couchpotato/core/media/movie/providers/trailer/mechanize/_html.py
new file mode 100644
index 0000000000..1a4e2c0281
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/mechanize/_html.py
@@ -0,0 +1,629 @@
+"""HTML handling.
+
+Copyright 2003-2006 John J. Lee
+
+This code is free software; you can redistribute it and/or modify it under
+the terms of the BSD or ZPL 2.1 licenses (see the file COPYING.txt
+included with the distribution).
+
+"""
+
+import codecs
+import copy
+import htmlentitydefs
+import re
+
+import _sgmllib_copy as sgmllib
+
+import _beautifulsoup
+import _form
+from _headersutil import split_header_words, is_html as _is_html
+import _request
+import _rfc3986
+
+DEFAULT_ENCODING = "latin-1"
+
+COMPRESS_RE = re.compile(r"\s+")
+
+
+class CachingGeneratorFunction(object):
+ """Caching wrapper around a no-arguments iterable."""
+
+ def __init__(self, iterable):
+ self._cache = []
+ # wrap iterable to make it non-restartable (otherwise, repeated
+ # __call__ would give incorrect results)
+ self._iterator = iter(iterable)
+
+ def __call__(self):
+ cache = self._cache
+ for item in cache:
+ yield item
+ for item in self._iterator:
+ cache.append(item)
+ yield item
+
+
+class EncodingFinder:
+ def __init__(self, default_encoding):
+ self._default_encoding = default_encoding
+ def encoding(self, response):
+ # HTTPEquivProcessor may be in use, so both HTTP and HTTP-EQUIV
+ # headers may be in the response. HTTP-EQUIV headers come last,
+ # so try in order from first to last.
+ for ct in response.info().getheaders("content-type"):
+ for k, v in split_header_words([ct])[0]:
+ if k == "charset":
+ encoding = v
+ try:
+ codecs.lookup(v)
+ except LookupError:
+ continue
+ else:
+ return encoding
+ return self._default_encoding
+
+
+class ResponseTypeFinder:
+ def __init__(self, allow_xhtml):
+ self._allow_xhtml = allow_xhtml
+ def is_html(self, response, encoding):
+ ct_hdrs = response.info().getheaders("content-type")
+ url = response.geturl()
+ # XXX encoding
+ return _is_html(ct_hdrs, url, self._allow_xhtml)
+
+
+class Args(object):
+
+ # idea for this argument-processing trick is from Peter Otten
+
+ def __init__(self, args_map):
+ self.__dict__["dictionary"] = dict(args_map)
+
+ def __getattr__(self, key):
+ try:
+ return self.dictionary[key]
+ except KeyError:
+ return getattr(self.__class__, key)
+
+ def __setattr__(self, key, value):
+ if key == "dictionary":
+ raise AttributeError()
+ self.dictionary[key] = value
+
+
+def form_parser_args(
+ select_default=False,
+ form_parser_class=None,
+ request_class=None,
+ backwards_compat=False,
+ ):
+ return Args(locals())
+
+
+class Link:
+ def __init__(self, base_url, url, text, tag, attrs):
+ assert None not in [url, tag, attrs]
+ self.base_url = base_url
+ self.absolute_url = _rfc3986.urljoin(base_url, url)
+ self.url, self.text, self.tag, self.attrs = url, text, tag, attrs
+ def __cmp__(self, other):
+ try:
+ for name in "url", "text", "tag", "attrs":
+ if getattr(self, name) != getattr(other, name):
+ return -1
+ except AttributeError:
+ return -1
+ return 0
+ def __repr__(self):
+ return "Link(base_url=%r, url=%r, text=%r, tag=%r, attrs=%r)" % (
+ self.base_url, self.url, self.text, self.tag, self.attrs)
+
+
+class LinksFactory:
+
+ def __init__(self,
+ link_parser_class=None,
+ link_class=Link,
+ urltags=None,
+ ):
+ import _pullparser
+ if link_parser_class is None:
+ link_parser_class = _pullparser.TolerantPullParser
+ self.link_parser_class = link_parser_class
+ self.link_class = link_class
+ if urltags is None:
+ urltags = {
+ "a": "href",
+ "area": "href",
+ "frame": "src",
+ "iframe": "src",
+ }
+ self.urltags = urltags
+ self._response = None
+ self._encoding = None
+
+ def set_response(self, response, base_url, encoding):
+ self._response = response
+ self._encoding = encoding
+ self._base_url = base_url
+
+ def links(self):
+ """Return an iterator that provides links of the document."""
+ response = self._response
+ encoding = self._encoding
+ base_url = self._base_url
+ p = self.link_parser_class(response, encoding=encoding)
+
+ try:
+ for token in p.tags(*(self.urltags.keys()+["base"])):
+ if token.type == "endtag":
+ continue
+ if token.data == "base":
+ base_href = dict(token.attrs).get("href")
+ if base_href is not None:
+ base_url = base_href
+ continue
+ attrs = dict(token.attrs)
+ tag = token.data
+ text = None
+ # XXX use attr_encoding for ref'd doc if that doc does not
+ # provide one by other means
+ #attr_encoding = attrs.get("charset")
+ url = attrs.get(self.urltags[tag]) # XXX is "" a valid URL?
+ if not url:
+ # Probably an link or Andy
+
+This code is free software; you can redistribute it and/or modify it
+under the terms of the BSD or ZPL 2.1 licenses (see the file
+COPYING.txt included with the distribution).
+
+"""
+
+import HTMLParser
+from cStringIO import StringIO
+import htmlentitydefs
+import logging
+import robotparser
+import socket
+import time
+
+import _sgmllib_copy as sgmllib
+from _urllib2_fork import HTTPError, BaseHandler
+
+from _headersutil import is_html
+from _html import unescape, unescape_charref
+from _request import Request
+from _response import response_seek_wrapper
+import _rfc3986
+import _sockettimeout
+
+debug = logging.getLogger("mechanize").debug
+debug_robots = logging.getLogger("mechanize.robots").debug
+
+# monkeypatch urllib2.HTTPError to show URL
+## import urllib2
+## def urllib2_str(self):
+## return 'HTTP Error %s: %s (%s)' % (
+## self.code, self.msg, self.geturl())
+## urllib2.HTTPError.__str__ = urllib2_str
+
+
+CHUNK = 1024 # size of chunks fed to HTML HEAD parser, in bytes
+DEFAULT_ENCODING = 'latin-1'
+
+# XXX would self.reset() work, instead of raising this exception?
+class EndOfHeadError(Exception): pass
+class AbstractHeadParser:
+ # only these elements are allowed in or before HEAD of document
+ head_elems = ("html", "head",
+ "title", "base",
+ "script", "style", "meta", "link", "object")
+ _entitydefs = htmlentitydefs.name2codepoint
+ _encoding = DEFAULT_ENCODING
+
+ def __init__(self):
+ self.http_equiv = []
+
+ def start_meta(self, attrs):
+ http_equiv = content = None
+ for key, value in attrs:
+ if key == "http-equiv":
+ http_equiv = self.unescape_attr_if_required(value)
+ elif key == "content":
+ content = self.unescape_attr_if_required(value)
+ if http_equiv is not None and content is not None:
+ self.http_equiv.append((http_equiv, content))
+
+ def end_head(self):
+ raise EndOfHeadError()
+
+ def handle_entityref(self, name):
+ #debug("%s", name)
+ self.handle_data(unescape(
+ '&%s;' % name, self._entitydefs, self._encoding))
+
+ def handle_charref(self, name):
+ #debug("%s", name)
+ self.handle_data(unescape_charref(name, self._encoding))
+
+ def unescape_attr(self, name):
+ #debug("%s", name)
+ return unescape(name, self._entitydefs, self._encoding)
+
+ def unescape_attrs(self, attrs):
+ #debug("%s", attrs)
+ escaped_attrs = {}
+ for key, val in attrs.items():
+ escaped_attrs[key] = self.unescape_attr(val)
+ return escaped_attrs
+
+ def unknown_entityref(self, ref):
+ self.handle_data("&%s;" % ref)
+
+ def unknown_charref(self, ref):
+ self.handle_data("&#%s;" % ref)
+
+
+class XHTMLCompatibleHeadParser(AbstractHeadParser,
+ HTMLParser.HTMLParser):
+ def __init__(self):
+ HTMLParser.HTMLParser.__init__(self)
+ AbstractHeadParser.__init__(self)
+
+ def handle_starttag(self, tag, attrs):
+ if tag not in self.head_elems:
+ raise EndOfHeadError()
+ try:
+ method = getattr(self, 'start_' + tag)
+ except AttributeError:
+ try:
+ method = getattr(self, 'do_' + tag)
+ except AttributeError:
+ pass # unknown tag
+ else:
+ method(attrs)
+ else:
+ method(attrs)
+
+ def handle_endtag(self, tag):
+ if tag not in self.head_elems:
+ raise EndOfHeadError()
+ try:
+ method = getattr(self, 'end_' + tag)
+ except AttributeError:
+ pass # unknown tag
+ else:
+ method()
+
+ def unescape(self, name):
+ # Use the entitydefs passed into constructor, not
+ # HTMLParser.HTMLParser's entitydefs.
+ return self.unescape_attr(name)
+
+ def unescape_attr_if_required(self, name):
+ return name # HTMLParser.HTMLParser already did it
+
+class HeadParser(AbstractHeadParser, sgmllib.SGMLParser):
+
+ def _not_called(self):
+ assert False
+
+ def __init__(self):
+ sgmllib.SGMLParser.__init__(self)
+ AbstractHeadParser.__init__(self)
+
+ def handle_starttag(self, tag, method, attrs):
+ if tag not in self.head_elems:
+ raise EndOfHeadError()
+ if tag == "meta":
+ method(attrs)
+
+ def unknown_starttag(self, tag, attrs):
+ self.handle_starttag(tag, self._not_called, attrs)
+
+ def handle_endtag(self, tag, method):
+ if tag in self.head_elems:
+ method()
+ else:
+ raise EndOfHeadError()
+
+ def unescape_attr_if_required(self, name):
+ return self.unescape_attr(name)
+
+def parse_head(fileobj, parser):
+ """Return a list of key, value pairs."""
+ while 1:
+ data = fileobj.read(CHUNK)
+ try:
+ parser.feed(data)
+ except EndOfHeadError:
+ break
+ if len(data) != CHUNK:
+ # this should only happen if there is no HTML body, or if
+ # CHUNK is big
+ break
+ return parser.http_equiv
+
+class HTTPEquivProcessor(BaseHandler):
+ """Append META HTTP-EQUIV headers to regular HTTP headers."""
+
+ handler_order = 300 # before handlers that look at HTTP headers
+
+ def __init__(self, head_parser_class=HeadParser,
+ i_want_broken_xhtml_support=False,
+ ):
+ self.head_parser_class = head_parser_class
+ self._allow_xhtml = i_want_broken_xhtml_support
+
+ def http_response(self, request, response):
+ if not hasattr(response, "seek"):
+ response = response_seek_wrapper(response)
+ http_message = response.info()
+ url = response.geturl()
+ ct_hdrs = http_message.getheaders("content-type")
+ if is_html(ct_hdrs, url, self._allow_xhtml):
+ try:
+ try:
+ html_headers = parse_head(response,
+ self.head_parser_class())
+ finally:
+ response.seek(0)
+ except (HTMLParser.HTMLParseError,
+ sgmllib.SGMLParseError):
+ pass
+ else:
+ for hdr, val in html_headers:
+ # add a header
+ http_message.dict[hdr.lower()] = val
+ text = hdr + ": " + val
+ for line in text.split("\n"):
+ http_message.headers.append(line + "\n")
+ return response
+
+ https_response = http_response
+
+
+class MechanizeRobotFileParser(robotparser.RobotFileParser):
+
+ def __init__(self, url='', opener=None):
+ robotparser.RobotFileParser.__init__(self, url)
+ self._opener = opener
+ self._timeout = _sockettimeout._GLOBAL_DEFAULT_TIMEOUT
+
+ def set_opener(self, opener=None):
+ import _opener
+ if opener is None:
+ opener = _opener.OpenerDirector()
+ self._opener = opener
+
+ def set_timeout(self, timeout):
+ self._timeout = timeout
+
+ def read(self):
+ """Reads the robots.txt URL and feeds it to the parser."""
+ if self._opener is None:
+ self.set_opener()
+ req = Request(self.url, unverifiable=True, visit=False,
+ timeout=self._timeout)
+ try:
+ f = self._opener.open(req)
+ except HTTPError, f:
+ pass
+ except (IOError, socket.error, OSError), exc:
+ debug_robots("ignoring error opening %r: %s" %
+ (self.url, exc))
+ return
+ lines = []
+ line = f.readline()
+ while line:
+ lines.append(line.strip())
+ line = f.readline()
+ status = f.code
+ if status == 401 or status == 403:
+ self.disallow_all = True
+ debug_robots("disallow all")
+ elif status >= 400:
+ self.allow_all = True
+ debug_robots("allow all")
+ elif status == 200 and lines:
+ debug_robots("parse lines")
+ self.parse(lines)
+
+class RobotExclusionError(HTTPError):
+ def __init__(self, request, *args):
+ apply(HTTPError.__init__, (self,)+args)
+ self.request = request
+
+class HTTPRobotRulesProcessor(BaseHandler):
+ # before redirections, after everything else
+ handler_order = 800
+
+ try:
+ from httplib import HTTPMessage
+ except:
+ from mimetools import Message
+ http_response_class = Message
+ else:
+ http_response_class = HTTPMessage
+
+ def __init__(self, rfp_class=MechanizeRobotFileParser):
+ self.rfp_class = rfp_class
+ self.rfp = None
+ self._host = None
+
+ def http_request(self, request):
+ scheme = request.get_type()
+ if scheme not in ["http", "https"]:
+ # robots exclusion only applies to HTTP
+ return request
+
+ if request.get_selector() == "/robots.txt":
+ # /robots.txt is always OK to fetch
+ return request
+
+ host = request.get_host()
+
+ # robots.txt requests don't need to be allowed by robots.txt :-)
+ origin_req = getattr(request, "_origin_req", None)
+ if (origin_req is not None and
+ origin_req.get_selector() == "/robots.txt" and
+ origin_req.get_host() == host
+ ):
+ return request
+
+ if host != self._host:
+ self.rfp = self.rfp_class()
+ try:
+ self.rfp.set_opener(self.parent)
+ except AttributeError:
+ debug("%r instance does not support set_opener" %
+ self.rfp.__class__)
+ self.rfp.set_url(scheme+"://"+host+"/robots.txt")
+ self.rfp.set_timeout(request.timeout)
+ self.rfp.read()
+ self._host = host
+
+ ua = request.get_header("User-agent", "")
+ if self.rfp.can_fetch(ua, request.get_full_url()):
+ return request
+ else:
+ # XXX This should really have raised URLError. Too late now...
+ msg = "request disallowed by robots.txt"
+ raise RobotExclusionError(
+ request,
+ request.get_full_url(),
+ 403, msg,
+ self.http_response_class(StringIO()), StringIO(msg))
+
+ https_request = http_request
+
+class HTTPRefererProcessor(BaseHandler):
+ """Add Referer header to requests.
+
+ This only makes sense if you use each RefererProcessor for a single
+ chain of requests only (so, for example, if you use a single
+ HTTPRefererProcessor to fetch a series of URLs extracted from a single
+ page, this will break).
+
+ There's a proper implementation of this in mechanize.Browser.
+
+ """
+ def __init__(self):
+ self.referer = None
+
+ def http_request(self, request):
+ if ((self.referer is not None) and
+ not request.has_header("Referer")):
+ request.add_unredirected_header("Referer", self.referer)
+ return request
+
+ def http_response(self, request, response):
+ self.referer = response.geturl()
+ return response
+
+ https_request = http_request
+ https_response = http_response
+
+
+def clean_refresh_url(url):
+ # e.g. Firefox 1.5 does (something like) this
+ if ((url.startswith('"') and url.endswith('"')) or
+ (url.startswith("'") and url.endswith("'"))):
+ url = url[1:-1]
+ return _rfc3986.clean_url(url, "latin-1") # XXX encoding
+
+def parse_refresh_header(refresh):
+ """
+ >>> parse_refresh_header("1; url=http://example.com/")
+ (1.0, 'http://example.com/')
+ >>> parse_refresh_header("1; url='http://example.com/'")
+ (1.0, 'http://example.com/')
+ >>> parse_refresh_header("1")
+ (1.0, None)
+ >>> parse_refresh_header("blah") # doctest: +IGNORE_EXCEPTION_DETAIL
+ Traceback (most recent call last):
+ ValueError: invalid literal for float(): blah
+
+ """
+
+ ii = refresh.find(";")
+ if ii != -1:
+ pause, newurl_spec = float(refresh[:ii]), refresh[ii+1:]
+ jj = newurl_spec.find("=")
+ key = None
+ if jj != -1:
+ key, newurl = newurl_spec[:jj], newurl_spec[jj+1:]
+ newurl = clean_refresh_url(newurl)
+ if key is None or key.strip().lower() != "url":
+ raise ValueError()
+ else:
+ pause, newurl = float(refresh), None
+ return pause, newurl
+
+class HTTPRefreshProcessor(BaseHandler):
+ """Perform HTTP Refresh redirections.
+
+ Note that if a non-200 HTTP code has occurred (for example, a 30x
+ redirect), this processor will do nothing.
+
+ By default, only zero-time Refresh headers are redirected. Use the
+ max_time attribute / constructor argument to allow Refresh with longer
+ pauses. Use the honor_time attribute / constructor argument to control
+ whether the requested pause is honoured (with a time.sleep()) or
+ skipped in favour of immediate redirection.
+
+ Public attributes:
+
+ max_time: see above
+ honor_time: see above
+
+ """
+ handler_order = 1000
+
+ def __init__(self, max_time=0, honor_time=True):
+ self.max_time = max_time
+ self.honor_time = honor_time
+ self._sleep = time.sleep
+
+ def http_response(self, request, response):
+ code, msg, hdrs = response.code, response.msg, response.info()
+
+ if code == 200 and hdrs.has_key("refresh"):
+ refresh = hdrs.getheaders("refresh")[0]
+ try:
+ pause, newurl = parse_refresh_header(refresh)
+ except ValueError:
+ debug("bad Refresh header: %r" % refresh)
+ return response
+
+ if newurl is None:
+ newurl = response.geturl()
+ if (self.max_time is None) or (pause <= self.max_time):
+ if pause > 1E-3 and self.honor_time:
+ self._sleep(pause)
+ hdrs["location"] = newurl
+ # hardcoded http is NOT a bug
+ response = self.parent.error(
+ "http", request, response,
+ "refresh", msg, hdrs)
+ else:
+ debug("Refresh header ignored: %r" % refresh)
+
+ return response
+
+ https_response = http_response
diff --git a/couchpotato/core/media/movie/providers/trailer/mechanize/_lwpcookiejar.py b/couchpotato/core/media/movie/providers/trailer/mechanize/_lwpcookiejar.py
new file mode 100644
index 0000000000..f8d49cf2d4
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/mechanize/_lwpcookiejar.py
@@ -0,0 +1,185 @@
+"""Load / save to libwww-perl (LWP) format files.
+
+Actually, the format is slightly extended from that used by LWP's
+(libwww-perl's) HTTP::Cookies, to avoid losing some RFC 2965 information
+not recorded by LWP.
+
+It uses the version string "2.0", though really there isn't an LWP Cookies
+2.0 format. This indicates that there is extra information in here
+(domain_dot and port_spec) while still being compatible with libwww-perl,
+I hope.
+
+Copyright 2002-2006 John J Lee
+Copyright 1997-1999 Gisle Aas (original libwww-perl code)
+
+This code is free software; you can redistribute it and/or modify it
+under the terms of the BSD or ZPL 2.1 licenses (see the file
+COPYING.txt included with the distribution).
+
+"""
+
+import time, re, logging
+
+from _clientcookie import reraise_unmasked_exceptions, FileCookieJar, Cookie, \
+ MISSING_FILENAME_TEXT, LoadError
+from _headersutil import join_header_words, split_header_words
+from _util import iso2time, time2isoz
+
+debug = logging.getLogger("mechanize").debug
+
+
+def lwp_cookie_str(cookie):
+ """Return string representation of Cookie in an the LWP cookie file format.
+
+ Actually, the format is extended a bit -- see module docstring.
+
+ """
+ h = [(cookie.name, cookie.value),
+ ("path", cookie.path),
+ ("domain", cookie.domain)]
+ if cookie.port is not None: h.append(("port", cookie.port))
+ if cookie.path_specified: h.append(("path_spec", None))
+ if cookie.port_specified: h.append(("port_spec", None))
+ if cookie.domain_initial_dot: h.append(("domain_dot", None))
+ if cookie.secure: h.append(("secure", None))
+ if cookie.expires: h.append(("expires",
+ time2isoz(float(cookie.expires))))
+ if cookie.discard: h.append(("discard", None))
+ if cookie.comment: h.append(("comment", cookie.comment))
+ if cookie.comment_url: h.append(("commenturl", cookie.comment_url))
+ if cookie.rfc2109: h.append(("rfc2109", None))
+
+ keys = cookie.nonstandard_attr_keys()
+ keys.sort()
+ for k in keys:
+ h.append((k, str(cookie.get_nonstandard_attr(k))))
+
+ h.append(("version", str(cookie.version)))
+
+ return join_header_words([h])
+
+class LWPCookieJar(FileCookieJar):
+ """
+ The LWPCookieJar saves a sequence of"Set-Cookie3" lines.
+ "Set-Cookie3" is the format used by the libwww-perl libary, not known
+ to be compatible with any browser, but which is easy to read and
+ doesn't lose information about RFC 2965 cookies.
+
+ Additional methods
+
+ as_lwp_str(ignore_discard=True, ignore_expired=True)
+
+ """
+
+ magic_re = r"^\#LWP-Cookies-(\d+\.\d+)"
+
+ def as_lwp_str(self, ignore_discard=True, ignore_expires=True):
+ """Return cookies as a string of "\n"-separated "Set-Cookie3" headers.
+
+ ignore_discard and ignore_expires: see docstring for FileCookieJar.save
+
+ """
+ now = time.time()
+ r = []
+ for cookie in self:
+ if not ignore_discard and cookie.discard:
+ debug(" Not saving %s: marked for discard", cookie.name)
+ continue
+ if not ignore_expires and cookie.is_expired(now):
+ debug(" Not saving %s: expired", cookie.name)
+ continue
+ r.append("Set-Cookie3: %s" % lwp_cookie_str(cookie))
+ return "\n".join(r+[""])
+
+ def save(self, filename=None, ignore_discard=False, ignore_expires=False):
+ if filename is None:
+ if self.filename is not None: filename = self.filename
+ else: raise ValueError(MISSING_FILENAME_TEXT)
+
+ f = open(filename, "w")
+ try:
+ debug("Saving LWP cookies file")
+ # There really isn't an LWP Cookies 2.0 format, but this indicates
+ # that there is extra information in here (domain_dot and
+ # port_spec) while still being compatible with libwww-perl, I hope.
+ f.write("#LWP-Cookies-2.0\n")
+ f.write(self.as_lwp_str(ignore_discard, ignore_expires))
+ finally:
+ f.close()
+
+ def _really_load(self, f, filename, ignore_discard, ignore_expires):
+ magic = f.readline()
+ if not re.search(self.magic_re, magic):
+ msg = "%s does not seem to contain cookies" % filename
+ raise LoadError(msg)
+
+ now = time.time()
+
+ header = "Set-Cookie3:"
+ boolean_attrs = ("port_spec", "path_spec", "domain_dot",
+ "secure", "discard", "rfc2109")
+ value_attrs = ("version",
+ "port", "path", "domain",
+ "expires",
+ "comment", "commenturl")
+
+ try:
+ while 1:
+ line = f.readline()
+ if line == "": break
+ if not line.startswith(header):
+ continue
+ line = line[len(header):].strip()
+
+ for data in split_header_words([line]):
+ name, value = data[0]
+ standard = {}
+ rest = {}
+ for k in boolean_attrs:
+ standard[k] = False
+ for k, v in data[1:]:
+ if k is not None:
+ lc = k.lower()
+ else:
+ lc = None
+ # don't lose case distinction for unknown fields
+ if (lc in value_attrs) or (lc in boolean_attrs):
+ k = lc
+ if k in boolean_attrs:
+ if v is None: v = True
+ standard[k] = v
+ elif k in value_attrs:
+ standard[k] = v
+ else:
+ rest[k] = v
+
+ h = standard.get
+ expires = h("expires")
+ discard = h("discard")
+ if expires is not None:
+ expires = iso2time(expires)
+ if expires is None:
+ discard = True
+ domain = h("domain")
+ domain_specified = domain.startswith(".")
+ c = Cookie(h("version"), name, value,
+ h("port"), h("port_spec"),
+ domain, domain_specified, h("domain_dot"),
+ h("path"), h("path_spec"),
+ h("secure"),
+ expires,
+ discard,
+ h("comment"),
+ h("commenturl"),
+ rest,
+ h("rfc2109"),
+ )
+ if not ignore_discard and c.discard:
+ continue
+ if not ignore_expires and c.is_expired(now):
+ continue
+ self.set_cookie(c)
+ except:
+ reraise_unmasked_exceptions((IOError,))
+ raise LoadError("invalid Set-Cookie3 format file %s" % filename)
+
diff --git a/couchpotato/core/media/movie/providers/trailer/mechanize/_markupbase.py b/couchpotato/core/media/movie/providers/trailer/mechanize/_markupbase.py
new file mode 100644
index 0000000000..ae9c2a875f
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/mechanize/_markupbase.py
@@ -0,0 +1,393 @@
+# Taken from Python 2.6.4 for use by _sgmllib.py
+"""Shared support for scanning document type declarations in HTML and XHTML.
+
+This module is used as a foundation for the HTMLParser and sgmllib
+modules (indirectly, for htmllib as well). It has no documented
+public API and should not be used directly.
+
+"""
+
+import re
+
+_declname_match = re.compile(r'[a-zA-Z][-_.a-zA-Z0-9]*\s*').match
+_declstringlit_match = re.compile(r'(\'[^\']*\'|"[^"]*")\s*').match
+_commentclose = re.compile(r'--\s*>')
+_markedsectionclose = re.compile(r']\s*]\s*>')
+
+# An analysis of the MS-Word extensions is available at
+# http://www.planetpublish.com/xmlarena/xap/Thursday/WordtoXML.pdf
+
+_msmarkedsectionclose = re.compile(r']\s*>')
+
+del re
+
+
+class ParserBase:
+ """Parser base class which provides some common support methods used
+ by the SGML/HTML and XHTML parsers."""
+
+ def __init__(self):
+ if self.__class__ is ParserBase:
+ raise RuntimeError(
+ "markupbase.ParserBase must be subclassed")
+
+ def error(self, message):
+ raise NotImplementedError(
+ "subclasses of ParserBase must override error()")
+
+ def reset(self):
+ self.lineno = 1
+ self.offset = 0
+
+ def getpos(self):
+ """Return current line number and offset."""
+ return self.lineno, self.offset
+
+ # Internal -- update line number and offset. This should be
+ # called for each piece of data exactly once, in order -- in other
+ # words the concatenation of all the input strings to this
+ # function should be exactly the entire input.
+ def updatepos(self, i, j):
+ if i >= j:
+ return j
+ rawdata = self.rawdata
+ nlines = rawdata.count("\n", i, j)
+ if nlines:
+ self.lineno = self.lineno + nlines
+ pos = rawdata.rindex("\n", i, j) # Should not fail
+ self.offset = j-(pos+1)
+ else:
+ self.offset = self.offset + j-i
+ return j
+
+ _decl_otherchars = ''
+
+ # Internal -- parse declaration (for use by subclasses).
+ def parse_declaration(self, i):
+ # This is some sort of declaration; in "HTML as
+ # deployed," this should only be the document type
+ # declaration ("").
+ # ISO 8879:1986, however, has more complex
+ # declaration syntax for elements in , including:
+ # --comment--
+ # [marked section]
+ # name in the following list: ENTITY, DOCTYPE, ELEMENT,
+ # ATTLIST, NOTATION, SHORTREF, USEMAP,
+ # LINKTYPE, LINK, IDLINK, USELINK, SYSTEM
+ rawdata = self.rawdata
+ j = i + 2
+ assert rawdata[i:j] == "":
+ # the empty comment
+ return j + 1
+ if rawdata[j:j+1] in ("-", ""):
+ # Start of comment followed by buffer boundary,
+ # or just a buffer boundary.
+ return -1
+ # A simple, practical version could look like: ((name|stringlit) S*) + '>'
+ n = len(rawdata)
+ if rawdata[j:j+2] == '--': #comment
+ # Locate --.*-- as the body of the comment
+ return self.parse_comment(i)
+ elif rawdata[j] == '[': #marked section
+ # Locate [statusWord [...arbitrary SGML...]] as the body of the marked section
+ # Where statusWord is one of TEMP, CDATA, IGNORE, INCLUDE, RCDATA
+ # Note that this is extended by Microsoft Office "Save as Web" function
+ # to include [if...] and [endif].
+ return self.parse_marked_section(i)
+ else: #all other declaration elements
+ decltype, j = self._scan_name(j, i)
+ if j < 0:
+ return j
+ if decltype == "doctype":
+ self._decl_otherchars = ''
+ while j < n:
+ c = rawdata[j]
+ if c == ">":
+ # end of declaration syntax
+ data = rawdata[i+2:j]
+ if decltype == "doctype":
+ self.handle_decl(data)
+ else:
+ self.unknown_decl(data)
+ return j + 1
+ if c in "\"'":
+ m = _declstringlit_match(rawdata, j)
+ if not m:
+ return -1 # incomplete
+ j = m.end()
+ elif c in "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ":
+ name, j = self._scan_name(j, i)
+ elif c in self._decl_otherchars:
+ j = j + 1
+ elif c == "[":
+ # this could be handled in a separate doctype parser
+ if decltype == "doctype":
+ j = self._parse_doctype_subset(j + 1, i)
+ elif decltype in ("attlist", "linktype", "link", "element"):
+ # must tolerate []'d groups in a content model in an element declaration
+ # also in data attribute specifications of attlist declaration
+ # also link type declaration subsets in linktype declarations
+ # also link attribute specification lists in link declarations
+ self.error("unsupported '[' char in %s declaration" % decltype)
+ else:
+ self.error("unexpected '[' char in declaration")
+ else:
+ self.error(
+ "unexpected %r char in declaration" % rawdata[j])
+ if j < 0:
+ return j
+ return -1 # incomplete
+
+ # Internal -- parse a marked section
+ # Override this to handle MS-word extension syntax content
+ def parse_marked_section(self, i, report=1):
+ rawdata= self.rawdata
+ assert rawdata[i:i+3] == ' ending
+ match= _markedsectionclose.search(rawdata, i+3)
+ elif sectName in ("if", "else", "endif"):
+ # look for MS Office ]> ending
+ match= _msmarkedsectionclose.search(rawdata, i+3)
+ else:
+ self.error('unknown status keyword %r in marked section' % rawdata[i+3:j])
+ if not match:
+ return -1
+ if report:
+ j = match.start(0)
+ self.unknown_decl(rawdata[i+3: j])
+ return match.end(0)
+
+ # Internal -- parse comment, return length or -1 if not terminated
+ def parse_comment(self, i, report=1):
+ rawdata = self.rawdata
+ if rawdata[i:i+4] != '
+ >>> print Token("decl", "decl")
+
+ >>> print Token("pi", "pi")
+
+ """
+ if self.attrs is not None:
+ attrs = "".join([" %s=%s" % (k, saxutils.quoteattr(v)) for
+ k, v in self.attrs])
+ else:
+ attrs = ""
+ if self.type == "starttag":
+ return "<%s%s>" % (self.data, attrs)
+ elif self.type == "startendtag":
+ return "<%s%s />" % (self.data, attrs)
+ elif self.type == "endtag":
+ return "" % self.data
+ elif self.type == "charref":
+ return "&#%s;" % self.data
+ elif self.type == "entityref":
+ return "&%s;" % self.data
+ elif self.type == "data":
+ return self.data
+ elif self.type == "comment":
+ return "" % self.data
+ elif self.type == "decl":
+ return "" % self.data
+ elif self.type == "pi":
+ return "" % self.data
+ assert False
+
+
+def iter_until_exception(fn, exception, *args, **kwds):
+ while 1:
+ try:
+ yield fn(*args, **kwds)
+ except exception:
+ raise StopIteration
+
+
+class _AbstractParser:
+ chunk = 1024
+ compress_re = re.compile(r"\s+")
+ def __init__(self, fh, textify={"img": "alt", "applet": "alt"},
+ encoding="ascii", entitydefs=None):
+ """
+ fh: file-like object (only a .read() method is required) from which to
+ read HTML to be parsed
+ textify: mapping used by .get_text() and .get_compressed_text() methods
+ to represent opening tags as text
+ encoding: encoding used to encode numeric character references by
+ .get_text() and .get_compressed_text() ("ascii" by default)
+
+ entitydefs: mapping like {"amp": "&", ...} containing HTML entity
+ definitions (a sensible default is used). This is used to unescape
+ entities in .get_text() (and .get_compressed_text()) and attribute
+ values. If the encoding can not represent the character, the entity
+ reference is left unescaped. Note that entity references (both
+ numeric - e.g. { or ઼ - and non-numeric - e.g. &) are
+ unescaped in attribute values and the return value of .get_text(), but
+ not in data outside of tags. Instead, entity references outside of
+ tags are represented as tokens. This is a bit odd, it's true :-/
+
+ If the element name of an opening tag matches a key in the textify
+ mapping then that tag is converted to text. The corresponding value is
+ used to specify which tag attribute to obtain the text from. textify
+ maps from element names to either:
+
+ - an HTML attribute name, in which case the HTML attribute value is
+ used as its text value along with the element name in square
+ brackets (e.g. "alt text goes here[IMG]", or, if the alt attribute
+ were missing, just "[IMG]")
+ - a callable object (e.g. a function) which takes a Token and returns
+ the string to be used as its text value
+
+ If textify has no key for an element name, nothing is substituted for
+ the opening tag.
+
+ Public attributes:
+
+ encoding and textify: see above
+
+ """
+ self._fh = fh
+ self._tokenstack = [] # FIFO
+ self.textify = textify
+ self.encoding = encoding
+ if entitydefs is None:
+ entitydefs = htmlentitydefs.name2codepoint
+ self._entitydefs = entitydefs
+
+ def __iter__(self): return self
+
+ def tags(self, *names):
+ return iter_until_exception(self.get_tag, NoMoreTokensError, *names)
+
+ def tokens(self, *tokentypes):
+ return iter_until_exception(self.get_token, NoMoreTokensError,
+ *tokentypes)
+
+ def next(self):
+ try:
+ return self.get_token()
+ except NoMoreTokensError:
+ raise StopIteration()
+
+ def get_token(self, *tokentypes):
+ """Pop the next Token object from the stack of parsed tokens.
+
+ If arguments are given, they are taken to be token types in which the
+ caller is interested: tokens representing other elements will be
+ skipped. Element names must be given in lower case.
+
+ Raises NoMoreTokensError.
+
+ """
+ while 1:
+ while self._tokenstack:
+ token = self._tokenstack.pop(0)
+ if tokentypes:
+ if token.type in tokentypes:
+ return token
+ else:
+ return token
+ data = self._fh.read(self.chunk)
+ if not data:
+ raise NoMoreTokensError()
+ self.feed(data)
+
+ def unget_token(self, token):
+ """Push a Token back onto the stack."""
+ self._tokenstack.insert(0, token)
+
+ def get_tag(self, *names):
+ """Return the next Token that represents an opening or closing tag.
+
+ If arguments are given, they are taken to be element names in which the
+ caller is interested: tags representing other elements will be skipped.
+ Element names must be given in lower case.
+
+ Raises NoMoreTokensError.
+
+ """
+ while 1:
+ tok = self.get_token()
+ if tok.type not in ["starttag", "endtag", "startendtag"]:
+ continue
+ if names:
+ if tok.data in names:
+ return tok
+ else:
+ return tok
+
+ def get_text(self, endat=None):
+ """Get some text.
+
+ endat: stop reading text at this tag (the tag is included in the
+ returned text); endtag is a tuple (type, name) where type is
+ "starttag", "endtag" or "startendtag", and name is the element name of
+ the tag (element names must be given in lower case)
+
+ If endat is not given, .get_text() will stop at the next opening or
+ closing tag, or when there are no more tokens (no exception is raised).
+ Note that .get_text() includes the text representation (if any) of the
+ opening tag, but pushes the opening tag back onto the stack. As a
+ result, if you want to call .get_text() again, you need to call
+ .get_tag() first (unless you want an empty string returned when you
+ next call .get_text()).
+
+ Entity references are translated using the value of the entitydefs
+ constructor argument (a mapping from names to characters like that
+ provided by the standard module htmlentitydefs). Named entity
+ references that are not in this mapping are left unchanged.
+
+ The textify attribute is used to translate opening tags into text: see
+ the class docstring.
+
+ """
+ text = []
+ tok = None
+ while 1:
+ try:
+ tok = self.get_token()
+ except NoMoreTokensError:
+ # unget last token (not the one we just failed to get)
+ if tok: self.unget_token(tok)
+ break
+ if tok.type == "data":
+ text.append(tok.data)
+ elif tok.type == "entityref":
+ t = unescape("&%s;"%tok.data, self._entitydefs, self.encoding)
+ text.append(t)
+ elif tok.type == "charref":
+ t = unescape_charref(tok.data, self.encoding)
+ text.append(t)
+ elif tok.type in ["starttag", "endtag", "startendtag"]:
+ tag_name = tok.data
+ if tok.type in ["starttag", "startendtag"]:
+ alt = self.textify.get(tag_name)
+ if alt is not None:
+ if callable(alt):
+ text.append(alt(tok))
+ elif tok.attrs is not None:
+ for k, v in tok.attrs:
+ if k == alt:
+ text.append(v)
+ text.append("[%s]" % tag_name.upper())
+ if endat is None or endat == (tok.type, tag_name):
+ self.unget_token(tok)
+ break
+ return "".join(text)
+
+ def get_compressed_text(self, *args, **kwds):
+ """
+ As .get_text(), but collapses each group of contiguous whitespace to a
+ single space character, and removes all initial and trailing
+ whitespace.
+
+ """
+ text = self.get_text(*args, **kwds)
+ text = text.strip()
+ return self.compress_re.sub(" ", text)
+
+ def handle_startendtag(self, tag, attrs):
+ self._tokenstack.append(Token("startendtag", tag, attrs))
+ def handle_starttag(self, tag, attrs):
+ self._tokenstack.append(Token("starttag", tag, attrs))
+ def handle_endtag(self, tag):
+ self._tokenstack.append(Token("endtag", tag))
+ def handle_charref(self, name):
+ self._tokenstack.append(Token("charref", name))
+ def handle_entityref(self, name):
+ self._tokenstack.append(Token("entityref", name))
+ def handle_data(self, data):
+ self._tokenstack.append(Token("data", data))
+ def handle_comment(self, data):
+ self._tokenstack.append(Token("comment", data))
+ def handle_decl(self, decl):
+ self._tokenstack.append(Token("decl", decl))
+ def unknown_decl(self, data):
+ # XXX should this call self.error instead?
+ #self.error("unknown declaration: " + `data`)
+ self._tokenstack.append(Token("decl", data))
+ def handle_pi(self, data):
+ self._tokenstack.append(Token("pi", data))
+
+ def unescape_attr(self, name):
+ return unescape(name, self._entitydefs, self.encoding)
+ def unescape_attrs(self, attrs):
+ escaped_attrs = []
+ for key, val in attrs:
+ escaped_attrs.append((key, self.unescape_attr(val)))
+ return escaped_attrs
+
+class PullParser(_AbstractParser, HTMLParser.HTMLParser):
+ def __init__(self, *args, **kwds):
+ HTMLParser.HTMLParser.__init__(self)
+ _AbstractParser.__init__(self, *args, **kwds)
+ def unescape(self, name):
+ # Use the entitydefs passed into constructor, not
+ # HTMLParser.HTMLParser's entitydefs.
+ return self.unescape_attr(name)
+
+class TolerantPullParser(_AbstractParser, sgmllib.SGMLParser):
+ def __init__(self, *args, **kwds):
+ sgmllib.SGMLParser.__init__(self)
+ _AbstractParser.__init__(self, *args, **kwds)
+ def unknown_starttag(self, tag, attrs):
+ attrs = self.unescape_attrs(attrs)
+ self._tokenstack.append(Token("starttag", tag, attrs))
+ def unknown_endtag(self, tag):
+ self._tokenstack.append(Token("endtag", tag))
+
+
+def _test():
+ import doctest, _pullparser
+ return doctest.testmod(_pullparser)
+
+if __name__ == "__main__":
+ _test()
diff --git a/couchpotato/core/media/movie/providers/trailer/mechanize/_request.py b/couchpotato/core/media/movie/providers/trailer/mechanize/_request.py
new file mode 100644
index 0000000000..ad8acb6cc4
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/mechanize/_request.py
@@ -0,0 +1,40 @@
+"""Integration with Python standard library module urllib2: Request class.
+
+Copyright 2004-2006 John J Lee
+
+This code is free software; you can redistribute it and/or modify it
+under the terms of the BSD or ZPL 2.1 licenses (see the file
+COPYING.txt included with the distribution).
+
+"""
+
+import logging
+
+import _rfc3986
+import _sockettimeout
+import _urllib2_fork
+
+warn = logging.getLogger("mechanize").warning
+
+
+class Request(_urllib2_fork.Request):
+ def __init__(self, url, data=None, headers={},
+ origin_req_host=None, unverifiable=False, visit=None,
+ timeout=_sockettimeout._GLOBAL_DEFAULT_TIMEOUT):
+ # In mechanize 0.2, the interpretation of a unicode url argument will
+ # change: A unicode url argument will be interpreted as an IRI, and a
+ # bytestring as a URI. For now, we accept unicode or bytestring. We
+ # don't insist that the value is always a URI (specifically, must only
+ # contain characters which are legal), because that might break working
+ # code (who knows what bytes some servers want to see, especially with
+ # browser plugins for internationalised URIs).
+ if not _rfc3986.is_clean_uri(url):
+ warn("url argument is not a URI "
+ "(contains illegal characters) %r" % url)
+ _urllib2_fork.Request.__init__(self, url, data, headers)
+ self.selector = None
+ self.visit = visit
+ self.timeout = timeout
+
+ def __str__(self):
+ return "" % self.get_full_url()
diff --git a/couchpotato/core/media/movie/providers/trailer/mechanize/_response.py b/couchpotato/core/media/movie/providers/trailer/mechanize/_response.py
new file mode 100644
index 0000000000..d5ca5f2e4e
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/mechanize/_response.py
@@ -0,0 +1,525 @@
+"""Response classes.
+
+The seek_wrapper code is not used if you're using UserAgent with
+.set_seekable_responses(False), or if you're using the urllib2-level interface
+HTTPEquivProcessor. Class closeable_response is instantiated by some handlers
+(AbstractHTTPHandler), but the closeable_response interface is only depended
+upon by Browser-level code. Function upgrade_response is only used if you're
+using Browser.
+
+
+Copyright 2006 John J. Lee
+
+This code is free software; you can redistribute it and/or modify it
+under the terms of the BSD or ZPL 2.1 licenses (see the file COPYING.txt
+included with the distribution).
+
+"""
+
+import copy, mimetools, urllib2
+from cStringIO import StringIO
+
+
+def len_of_seekable(file_):
+ # this function exists because evaluation of len(file_.getvalue()) on every
+ # .read() from seek_wrapper would be O(N**2) in number of .read()s
+ pos = file_.tell()
+ file_.seek(0, 2) # to end
+ try:
+ return file_.tell()
+ finally:
+ file_.seek(pos)
+
+
+# XXX Andrew Dalke kindly sent me a similar class in response to my request on
+# comp.lang.python, which I then proceeded to lose. I wrote this class
+# instead, but I think he's released his code publicly since, could pinch the
+# tests from it, at least...
+
+# For testing seek_wrapper invariant (note that
+# test_urllib2.HandlerTest.test_seekable is expected to fail when this
+# invariant checking is turned on). The invariant checking is done by module
+# ipdc, which is available here:
+# http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/436834
+## from ipdbc import ContractBase
+## class seek_wrapper(ContractBase):
+class seek_wrapper:
+ """Adds a seek method to a file object.
+
+ This is only designed for seeking on readonly file-like objects.
+
+ Wrapped file-like object must have a read method. The readline method is
+ only supported if that method is present on the wrapped object. The
+ readlines method is always supported. xreadlines and iteration are
+ supported only for Python 2.2 and above.
+
+ Public attributes:
+
+ wrapped: the wrapped file object
+ is_closed: true iff .close() has been called
+
+ WARNING: All other attributes of the wrapped object (ie. those that are not
+ one of wrapped, read, readline, readlines, xreadlines, __iter__ and next)
+ are passed through unaltered, which may or may not make sense for your
+ particular file object.
+
+ """
+ # General strategy is to check that cache is full enough, then delegate to
+ # the cache (self.__cache, which is a cStringIO.StringIO instance). A seek
+ # position (self.__pos) is maintained independently of the cache, in order
+ # that a single cache may be shared between multiple seek_wrapper objects.
+ # Copying using module copy shares the cache in this way.
+
+ def __init__(self, wrapped):
+ self.wrapped = wrapped
+ self.__read_complete_state = [False]
+ self.__is_closed_state = [False]
+ self.__have_readline = hasattr(self.wrapped, "readline")
+ self.__cache = StringIO()
+ self.__pos = 0 # seek position
+
+ def invariant(self):
+ # The end of the cache is always at the same place as the end of the
+ # wrapped file (though the .tell() method is not required to be present
+ # on wrapped file).
+ return self.wrapped.tell() == len(self.__cache.getvalue())
+
+ def close(self):
+ self.wrapped.close()
+ self.is_closed = True
+
+ def __getattr__(self, name):
+ if name == "is_closed":
+ return self.__is_closed_state[0]
+ elif name == "read_complete":
+ return self.__read_complete_state[0]
+
+ wrapped = self.__dict__.get("wrapped")
+ if wrapped:
+ return getattr(wrapped, name)
+
+ return getattr(self.__class__, name)
+
+ def __setattr__(self, name, value):
+ if name == "is_closed":
+ self.__is_closed_state[0] = bool(value)
+ elif name == "read_complete":
+ if not self.is_closed:
+ self.__read_complete_state[0] = bool(value)
+ else:
+ self.__dict__[name] = value
+
+ def seek(self, offset, whence=0):
+ assert whence in [0,1,2]
+
+ # how much data, if any, do we need to read?
+ if whence == 2: # 2: relative to end of *wrapped* file
+ if offset < 0: raise ValueError("negative seek offset")
+ # since we don't know yet where the end of that file is, we must
+ # read everything
+ to_read = None
+ else:
+ if whence == 0: # 0: absolute
+ if offset < 0: raise ValueError("negative seek offset")
+ dest = offset
+ else: # 1: relative to current position
+ pos = self.__pos
+ if pos < offset:
+ raise ValueError("seek to before start of file")
+ dest = pos + offset
+ end = len_of_seekable(self.__cache)
+ to_read = dest - end
+ if to_read < 0:
+ to_read = 0
+
+ if to_read != 0:
+ self.__cache.seek(0, 2)
+ if to_read is None:
+ assert whence == 2
+ self.__cache.write(self.wrapped.read())
+ self.read_complete = True
+ self.__pos = self.__cache.tell() - offset
+ else:
+ data = self.wrapped.read(to_read)
+ if not data:
+ self.read_complete = True
+ else:
+ self.__cache.write(data)
+ # Don't raise an exception even if we've seek()ed past the end
+ # of .wrapped, since fseek() doesn't complain in that case.
+ # Also like fseek(), pretend we have seek()ed past the end,
+ # i.e. not:
+ #self.__pos = self.__cache.tell()
+ # but rather:
+ self.__pos = dest
+ else:
+ self.__pos = dest
+
+ def tell(self):
+ return self.__pos
+
+ def __copy__(self):
+ cpy = self.__class__(self.wrapped)
+ cpy.__cache = self.__cache
+ cpy.__read_complete_state = self.__read_complete_state
+ cpy.__is_closed_state = self.__is_closed_state
+ return cpy
+
+ def get_data(self):
+ pos = self.__pos
+ try:
+ self.seek(0)
+ return self.read(-1)
+ finally:
+ self.__pos = pos
+
+ def read(self, size=-1):
+ pos = self.__pos
+ end = len_of_seekable(self.__cache)
+ available = end - pos
+
+ # enough data already cached?
+ if size <= available and size != -1:
+ self.__cache.seek(pos)
+ self.__pos = pos+size
+ return self.__cache.read(size)
+
+ # no, so read sufficient data from wrapped file and cache it
+ self.__cache.seek(0, 2)
+ if size == -1:
+ self.__cache.write(self.wrapped.read())
+ self.read_complete = True
+ else:
+ to_read = size - available
+ assert to_read > 0
+ data = self.wrapped.read(to_read)
+ if not data:
+ self.read_complete = True
+ else:
+ self.__cache.write(data)
+ self.__cache.seek(pos)
+
+ data = self.__cache.read(size)
+ self.__pos = self.__cache.tell()
+ assert self.__pos == pos + len(data)
+ return data
+
+ def readline(self, size=-1):
+ if not self.__have_readline:
+ raise NotImplementedError("no readline method on wrapped object")
+
+ # line we're about to read might not be complete in the cache, so
+ # read another line first
+ pos = self.__pos
+ self.__cache.seek(0, 2)
+ data = self.wrapped.readline()
+ if not data:
+ self.read_complete = True
+ else:
+ self.__cache.write(data)
+ self.__cache.seek(pos)
+
+ data = self.__cache.readline()
+ if size != -1:
+ r = data[:size]
+ self.__pos = pos+size
+ else:
+ r = data
+ self.__pos = pos+len(data)
+ return r
+
+ def readlines(self, sizehint=-1):
+ pos = self.__pos
+ self.__cache.seek(0, 2)
+ self.__cache.write(self.wrapped.read())
+ self.read_complete = True
+ self.__cache.seek(pos)
+ data = self.__cache.readlines(sizehint)
+ self.__pos = self.__cache.tell()
+ return data
+
+ def __iter__(self): return self
+ def next(self):
+ line = self.readline()
+ if line == "": raise StopIteration
+ return line
+
+ xreadlines = __iter__
+
+ def __repr__(self):
+ return ("<%s at %s whose wrapped object = %r>" %
+ (self.__class__.__name__, hex(abs(id(self))), self.wrapped))
+
+
+class response_seek_wrapper(seek_wrapper):
+
+ """
+ Supports copying response objects and setting response body data.
+
+ """
+
+ def __init__(self, wrapped):
+ seek_wrapper.__init__(self, wrapped)
+ self._headers = self.wrapped.info()
+
+ def __copy__(self):
+ cpy = seek_wrapper.__copy__(self)
+ # copy headers from delegate
+ cpy._headers = copy.copy(self.info())
+ return cpy
+
+ # Note that .info() and .geturl() (the only two urllib2 response methods
+ # that are not implemented by seek_wrapper) must be here explicitly rather
+ # than by seek_wrapper's __getattr__ delegation) so that the nasty
+ # dynamically-created HTTPError classes in get_seek_wrapper_class() get the
+ # wrapped object's implementation, and not HTTPError's.
+
+ def info(self):
+ return self._headers
+
+ def geturl(self):
+ return self.wrapped.geturl()
+
+ def set_data(self, data):
+ self.seek(0)
+ self.read()
+ self.close()
+ cache = self._seek_wrapper__cache = StringIO()
+ cache.write(data)
+ self.seek(0)
+
+
+class eoffile:
+ # file-like object that always claims to be at end-of-file...
+ def read(self, size=-1): return ""
+ def readline(self, size=-1): return ""
+ def __iter__(self): return self
+ def next(self): return ""
+ def close(self): pass
+
+class eofresponse(eoffile):
+ def __init__(self, url, headers, code, msg):
+ self._url = url
+ self._headers = headers
+ self.code = code
+ self.msg = msg
+ def geturl(self): return self._url
+ def info(self): return self._headers
+
+
+class closeable_response:
+ """Avoids unnecessarily clobbering urllib.addinfourl methods on .close().
+
+ Only supports responses returned by mechanize.HTTPHandler.
+
+ After .close(), the following methods are supported:
+
+ .read()
+ .readline()
+ .info()
+ .geturl()
+ .__iter__()
+ .next()
+ .close()
+
+ and the following attributes are supported:
+
+ .code
+ .msg
+
+ Also supports pickling (but the stdlib currently does something to prevent
+ it: http://python.org/sf/1144636).
+
+ """
+ # presence of this attr indicates is useable after .close()
+ closeable_response = None
+
+ def __init__(self, fp, headers, url, code, msg):
+ self._set_fp(fp)
+ self._headers = headers
+ self._url = url
+ self.code = code
+ self.msg = msg
+
+ def _set_fp(self, fp):
+ self.fp = fp
+ self.read = self.fp.read
+ self.readline = self.fp.readline
+ if hasattr(self.fp, "readlines"): self.readlines = self.fp.readlines
+ if hasattr(self.fp, "fileno"):
+ self.fileno = self.fp.fileno
+ else:
+ self.fileno = lambda: None
+ self.__iter__ = self.fp.__iter__
+ self.next = self.fp.next
+
+ def __repr__(self):
+ return '<%s at %s whose fp = %r>' % (
+ self.__class__.__name__, hex(abs(id(self))), self.fp)
+
+ def info(self):
+ return self._headers
+
+ def geturl(self):
+ return self._url
+
+ def close(self):
+ wrapped = self.fp
+ wrapped.close()
+ new_wrapped = eofresponse(
+ self._url, self._headers, self.code, self.msg)
+ self._set_fp(new_wrapped)
+
+ def __getstate__(self):
+ # There are three obvious options here:
+ # 1. truncate
+ # 2. read to end
+ # 3. close socket, pickle state including read position, then open
+ # again on unpickle and use Range header
+ # XXXX um, 4. refuse to pickle unless .close()d. This is better,
+ # actually ("errors should never pass silently"). Pickling doesn't
+ # work anyway ATM, because of http://python.org/sf/1144636 so fix
+ # this later
+
+ # 2 breaks pickle protocol, because one expects the original object
+ # to be left unscathed by pickling. 3 is too complicated and
+ # surprising (and too much work ;-) to happen in a sane __getstate__.
+ # So we do 1.
+
+ state = self.__dict__.copy()
+ new_wrapped = eofresponse(
+ self._url, self._headers, self.code, self.msg)
+ state["wrapped"] = new_wrapped
+ return state
+
+def test_response(data='test data', headers=[],
+ url="http://example.com/", code=200, msg="OK"):
+ return make_response(data, headers, url, code, msg)
+
+def test_html_response(data='test data', headers=[],
+ url="http://example.com/", code=200, msg="OK"):
+ headers += [("Content-type", "text/html")]
+ return make_response(data, headers, url, code, msg)
+
+def make_response(data, headers, url, code, msg):
+ """Convenient factory for objects implementing response interface.
+
+ data: string containing response body data
+ headers: sequence of (name, value) pairs
+ url: URL of response
+ code: integer response code (e.g. 200)
+ msg: string response code message (e.g. "OK")
+
+ """
+ mime_headers = make_headers(headers)
+ r = closeable_response(StringIO(data), mime_headers, url, code, msg)
+ return response_seek_wrapper(r)
+
+
+def make_headers(headers):
+ """
+ headers: sequence of (name, value) pairs
+ """
+ hdr_text = []
+ for name_value in headers:
+ hdr_text.append("%s: %s" % name_value)
+ return mimetools.Message(StringIO("\n".join(hdr_text)))
+
+
+# Rest of this module is especially horrible, but needed, at least until fork
+# urllib2. Even then, may want to preseve urllib2 compatibility.
+
+def get_seek_wrapper_class(response):
+ # in order to wrap response objects that are also exceptions, we must
+ # dynamically subclass the exception :-(((
+ if (isinstance(response, urllib2.HTTPError) and
+ not hasattr(response, "seek")):
+ if response.__class__.__module__ == "__builtin__":
+ exc_class_name = response.__class__.__name__
+ else:
+ exc_class_name = "%s.%s" % (
+ response.__class__.__module__, response.__class__.__name__)
+
+ class httperror_seek_wrapper(response_seek_wrapper, response.__class__):
+ # this only derives from HTTPError in order to be a subclass --
+ # the HTTPError behaviour comes from delegation
+
+ _exc_class_name = exc_class_name
+
+ def __init__(self, wrapped):
+ response_seek_wrapper.__init__(self, wrapped)
+ # be compatible with undocumented HTTPError attributes :-(
+ self.hdrs = wrapped.info()
+ self.filename = wrapped.geturl()
+
+ def __repr__(self):
+ return (
+ "<%s (%s instance) at %s "
+ "whose wrapped object = %r>" % (
+ self.__class__.__name__, self._exc_class_name,
+ hex(abs(id(self))), self.wrapped)
+ )
+ wrapper_class = httperror_seek_wrapper
+ else:
+ wrapper_class = response_seek_wrapper
+ return wrapper_class
+
+def seek_wrapped_response(response):
+ """Return a copy of response that supports seekable response interface.
+
+ Accepts responses from both mechanize and urllib2 handlers.
+
+ Copes with both ordinary response instances and HTTPError instances (which
+ can't be simply wrapped due to the requirement of preserving the exception
+ base class).
+ """
+ if not hasattr(response, "seek"):
+ wrapper_class = get_seek_wrapper_class(response)
+ response = wrapper_class(response)
+ assert hasattr(response, "get_data")
+ return response
+
+def upgrade_response(response):
+ """Return a copy of response that supports Browser response interface.
+
+ Browser response interface is that of "seekable responses"
+ (response_seek_wrapper), plus the requirement that responses must be
+ useable after .close() (closeable_response).
+
+ Accepts responses from both mechanize and urllib2 handlers.
+
+ Copes with both ordinary response instances and HTTPError instances (which
+ can't be simply wrapped due to the requirement of preserving the exception
+ base class).
+ """
+ wrapper_class = get_seek_wrapper_class(response)
+ if hasattr(response, "closeable_response"):
+ if not hasattr(response, "seek"):
+ response = wrapper_class(response)
+ assert hasattr(response, "get_data")
+ return copy.copy(response)
+
+ # a urllib2 handler constructed the response, i.e. the response is an
+ # urllib.addinfourl or a urllib2.HTTPError, instead of a
+ # _Util.closeable_response as returned by e.g. mechanize.HTTPHandler
+ try:
+ code = response.code
+ except AttributeError:
+ code = None
+ try:
+ msg = response.msg
+ except AttributeError:
+ msg = None
+
+ # may have already-.read() data from .seek() cache
+ data = None
+ get_data = getattr(response, "get_data", None)
+ if get_data:
+ data = get_data()
+
+ response = closeable_response(
+ response.fp, response.info(), response.geturl(), code, msg)
+ response = wrapper_class(response)
+ if data:
+ response.set_data(data)
+ return response
diff --git a/couchpotato/core/media/movie/providers/trailer/mechanize/_rfc3986.py b/couchpotato/core/media/movie/providers/trailer/mechanize/_rfc3986.py
new file mode 100644
index 0000000000..91fcd197f4
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/mechanize/_rfc3986.py
@@ -0,0 +1,245 @@
+"""RFC 3986 URI parsing and relative reference resolution / absolutization.
+
+(aka splitting and joining)
+
+Copyright 2006 John J. Lee
+
+This code is free software; you can redistribute it and/or modify it under
+the terms of the BSD or ZPL 2.1 licenses (see the file COPYING.txt
+included with the distribution).
+
+"""
+
+# XXX Wow, this is ugly. Overly-direct translation of the RFC ATM.
+
+import re, urllib
+
+## def chr_range(a, b):
+## return "".join(map(chr, range(ord(a), ord(b)+1)))
+
+## UNRESERVED_URI_CHARS = ("ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+## "abcdefghijklmnopqrstuvwxyz"
+## "0123456789"
+## "-_.~")
+## RESERVED_URI_CHARS = "!*'();:@&=+$,/?#[]"
+## URI_CHARS = RESERVED_URI_CHARS+UNRESERVED_URI_CHARS+'%'
+# this re matches any character that's not in URI_CHARS
+BAD_URI_CHARS_RE = re.compile("[^A-Za-z0-9\-_.~!*'();:@&=+$,/?%#[\]]")
+
+
+def clean_url(url, encoding):
+ # percent-encode illegal URI characters
+ # Trying to come up with test cases for this gave me a headache, revisit
+ # when do switch to unicode.
+ # Somebody else's comments (lost the attribution):
+## - IE will return you the url in the encoding you send it
+## - Mozilla/Firefox will send you latin-1 if there's no non latin-1
+## characters in your link. It will send you utf-8 however if there are...
+ if type(url) == type(""):
+ url = url.decode(encoding, "replace")
+ url = url.strip()
+ # for second param to urllib.quote(), we want URI_CHARS, minus the
+ # 'always_safe' characters that urllib.quote() never percent-encodes
+ return urllib.quote(url.encode(encoding), "!*'();:@&=+$,/?%#[]~")
+
+def is_clean_uri(uri):
+ """
+ >>> is_clean_uri("ABC!")
+ True
+ >>> is_clean_uri(u"ABC!")
+ True
+ >>> is_clean_uri("ABC|")
+ False
+ >>> is_clean_uri(u"ABC|")
+ False
+ >>> is_clean_uri("http://example.com/0")
+ True
+ >>> is_clean_uri(u"http://example.com/0")
+ True
+ """
+ # note module re treats bytestrings as through they were decoded as latin-1
+ # so this function accepts both unicode and bytestrings
+ return not bool(BAD_URI_CHARS_RE.search(uri))
+
+
+SPLIT_MATCH = re.compile(
+ r"^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?").match
+def urlsplit(absolute_uri):
+ """Return scheme, authority, path, query, fragment."""
+ match = SPLIT_MATCH(absolute_uri)
+ if match:
+ g = match.groups()
+ return g[1], g[3], g[4], g[6], g[8]
+
+def urlunsplit(parts):
+ scheme, authority, path, query, fragment = parts
+ r = []
+ append = r.append
+ if scheme is not None:
+ append(scheme)
+ append(":")
+ if authority is not None:
+ append("//")
+ append(authority)
+ append(path)
+ if query is not None:
+ append("?")
+ append(query)
+ if fragment is not None:
+ append("#")
+ append(fragment)
+ return "".join(r)
+
+def urljoin(base_uri, uri_reference):
+ """Join a base URI with a URI reference and return the resulting URI.
+
+ See RFC 3986.
+ """
+ return urlunsplit(urljoin_parts(urlsplit(base_uri),
+ urlsplit(uri_reference)))
+
+# oops, this doesn't do the same thing as the literal translation
+# from the RFC below
+## import posixpath
+## def urljoin_parts(base_parts, reference_parts):
+## scheme, authority, path, query, fragment = base_parts
+## rscheme, rauthority, rpath, rquery, rfragment = reference_parts
+
+## # compute target URI path
+## if rpath == "":
+## tpath = path
+## else:
+## tpath = rpath
+## if not tpath.startswith("/"):
+## tpath = merge(authority, path, tpath)
+## tpath = posixpath.normpath(tpath)
+
+## if rscheme is not None:
+## return (rscheme, rauthority, tpath, rquery, rfragment)
+## elif rauthority is not None:
+## return (scheme, rauthority, tpath, rquery, rfragment)
+## elif rpath == "":
+## if rquery is not None:
+## tquery = rquery
+## else:
+## tquery = query
+## return (scheme, authority, tpath, tquery, rfragment)
+## else:
+## return (scheme, authority, tpath, rquery, rfragment)
+
+def urljoin_parts(base_parts, reference_parts):
+ scheme, authority, path, query, fragment = base_parts
+ rscheme, rauthority, rpath, rquery, rfragment = reference_parts
+
+ if rscheme == scheme:
+ rscheme = None
+
+ if rscheme is not None:
+ tscheme, tauthority, tpath, tquery = (
+ rscheme, rauthority, remove_dot_segments(rpath), rquery)
+ else:
+ if rauthority is not None:
+ tauthority, tpath, tquery = (
+ rauthority, remove_dot_segments(rpath), rquery)
+ else:
+ if rpath == "":
+ tpath = path
+ if rquery is not None:
+ tquery = rquery
+ else:
+ tquery = query
+ else:
+ if rpath.startswith("/"):
+ tpath = remove_dot_segments(rpath)
+ else:
+ tpath = merge(authority, path, rpath)
+ tpath = remove_dot_segments(tpath)
+ tquery = rquery
+ tauthority = authority
+ tscheme = scheme
+ tfragment = rfragment
+ return (tscheme, tauthority, tpath, tquery, tfragment)
+
+# um, something *vaguely* like this is what I want, but I have to generate
+# lots of test cases first, if only to understand what it is that
+# remove_dot_segments really does...
+## def remove_dot_segments(path):
+## if path == '':
+## return ''
+## comps = path.split('/')
+## new_comps = []
+## for comp in comps:
+## if comp in ['.', '']:
+## if not new_comps or new_comps[-1]:
+## new_comps.append('')
+## continue
+## if comp != '..':
+## new_comps.append(comp)
+## elif new_comps:
+## new_comps.pop()
+## return '/'.join(new_comps)
+
+
+def remove_dot_segments(path):
+ r = []
+ while path:
+ # A
+ if path.startswith("../"):
+ path = path[3:]
+ continue
+ if path.startswith("./"):
+ path = path[2:]
+ continue
+ # B
+ if path.startswith("/./"):
+ path = path[2:]
+ continue
+ if path == "/.":
+ path = "/"
+ continue
+ # C
+ if path.startswith("/../"):
+ path = path[3:]
+ if r:
+ r.pop()
+ continue
+ if path == "/..":
+ path = "/"
+ if r:
+ r.pop()
+ continue
+ # D
+ if path == ".":
+ path = path[1:]
+ continue
+ if path == "..":
+ path = path[2:]
+ continue
+ # E
+ start = 0
+ if path.startswith("/"):
+ start = 1
+ ii = path.find("/", start)
+ if ii < 0:
+ ii = None
+ r.append(path[:ii])
+ if ii is None:
+ break
+ path = path[ii:]
+ return "".join(r)
+
+def merge(base_authority, base_path, ref_path):
+ # XXXX Oddly, the sample Perl implementation of this by Roy Fielding
+ # doesn't even take base_authority as a parameter, despite the wording in
+ # the RFC suggesting otherwise. Perhaps I'm missing some obvious identity.
+ #if base_authority is not None and base_path == "":
+ if base_path == "":
+ return "/" + ref_path
+ ii = base_path.rfind("/")
+ if ii >= 0:
+ return base_path[:ii+1] + ref_path
+ return ref_path
+
+if __name__ == "__main__":
+ import doctest
+ doctest.testmod()
diff --git a/couchpotato/core/media/movie/providers/trailer/mechanize/_sgmllib_copy.py b/couchpotato/core/media/movie/providers/trailer/mechanize/_sgmllib_copy.py
new file mode 100644
index 0000000000..a545d25eb9
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/mechanize/_sgmllib_copy.py
@@ -0,0 +1,559 @@
+# Taken from Python 2.6.4 and regexp module constants modified
+"""A parser for SGML, using the derived class as a static DTD."""
+
+# XXX This only supports those SGML features used by HTML.
+
+# XXX There should be a way to distinguish between PCDATA (parsed
+# character data -- the normal case), RCDATA (replaceable character
+# data -- only char and entity references and end tags are special)
+# and CDATA (character data -- only end tags are special). RCDATA is
+# not supported at all.
+
+
+# from warnings import warnpy3k
+# warnpy3k("the sgmllib module has been removed in Python 3.0",
+# stacklevel=2)
+# del warnpy3k
+
+import markupbase
+import re
+
+__all__ = ["SGMLParser", "SGMLParseError"]
+
+# Regular expressions used for parsing
+
+interesting = re.compile('[&<]')
+incomplete = re.compile('&([a-zA-Z][a-zA-Z0-9]*|#[0-9]*)?|'
+ '<([a-zA-Z][^<>]*|'
+ '/([a-zA-Z][^<>]*)?|'
+ '![^<>]*)?')
+
+entityref = re.compile('&([a-zA-Z][-.a-zA-Z0-9]*)[^a-zA-Z0-9]')
+# hack to fix http://bugs.python.org/issue803422
+# charref = re.compile('&#([0-9]+)[^0-9]')
+charref = re.compile("&#(x?[0-9a-fA-F]+)[^0-9a-fA-F]")
+
+starttagopen = re.compile('<[>a-zA-Z]')
+shorttagopen = re.compile('<[a-zA-Z][-.a-zA-Z0-9]*/')
+shorttag = re.compile('<([a-zA-Z][-.a-zA-Z0-9]*)/([^/]*)/')
+piclose = re.compile('>')
+endbracket = re.compile('[<>]')
+# hack moved from _beautifulsoup.py (bundled BeautifulSoup version 2)
+#This code makes Beautiful Soup able to parse XML with namespaces
+# tagfind = re.compile('[a-zA-Z][-_.a-zA-Z0-9]*')
+tagfind = re.compile('[a-zA-Z][-_.:a-zA-Z0-9]*')
+attrfind = re.compile(
+ r'\s*([a-zA-Z_][-:.a-zA-Z_0-9]*)(\s*=\s*'
+ r'(\'[^\']*\'|"[^"]*"|[][\-a-zA-Z0-9./,:;+*%?!&$\(\)_#=~\'"@]*))?')
+
+
+class SGMLParseError(RuntimeError):
+ """Exception raised for all parse errors."""
+ pass
+
+
+# SGML parser base class -- find tags and call handler functions.
+# Usage: p = SGMLParser(); p.feed(data); ...; p.close().
+# The dtd is defined by deriving a class which defines methods
+# with special names to handle tags: start_foo and end_foo to handle
+# and , respectively, or do_foo to handle by itself.
+# (Tags are converted to lower case for this purpose.) The data
+# between tags is passed to the parser by calling self.handle_data()
+# with some data as argument (the data may be split up in arbitrary
+# chunks). Entity references are passed by calling
+# self.handle_entityref() with the entity reference as argument.
+
+class SGMLParser(markupbase.ParserBase):
+ # Definition of entities -- derived classes may override
+ entity_or_charref = re.compile('&(?:'
+ '([a-zA-Z][-.a-zA-Z0-9]*)|#([0-9]+)'
+ ')(;?)')
+
+ def __init__(self, verbose=0):
+ """Initialize and reset this instance."""
+ self.verbose = verbose
+ self.reset()
+
+ def reset(self):
+ """Reset this instance. Loses all unprocessed data."""
+ self.__starttag_text = None
+ self.rawdata = ''
+ self.stack = []
+ self.lasttag = '???'
+ self.nomoretags = 0
+ self.literal = 0
+ markupbase.ParserBase.reset(self)
+
+ def setnomoretags(self):
+ """Enter literal mode (CDATA) till EOF.
+
+ Intended for derived classes only.
+ """
+ self.nomoretags = self.literal = 1
+
+ def setliteral(self, *args):
+ """Enter literal mode (CDATA).
+
+ Intended for derived classes only.
+ """
+ self.literal = 1
+
+ def feed(self, data):
+ """Feed some data to the parser.
+
+ Call this as often as you want, with as little or as much text
+ as you want (may include '\n'). (This just saves the text,
+ all the processing is done by goahead().)
+ """
+
+ self.rawdata = self.rawdata + data
+ self.goahead(0)
+
+ def close(self):
+ """Handle the remaining data."""
+ self.goahead(1)
+
+ def error(self, message):
+ raise SGMLParseError(message)
+
+ # Internal -- handle data as far as reasonable. May leave state
+ # and data to be processed by a subsequent call. If 'end' is
+ # true, force handling all data as if followed by EOF marker.
+ def goahead(self, end):
+ rawdata = self.rawdata
+ i = 0
+ n = len(rawdata)
+ while i < n:
+ if self.nomoretags:
+ self.handle_data(rawdata[i:n])
+ i = n
+ break
+ match = interesting.search(rawdata, i)
+ if match: j = match.start()
+ else: j = n
+ if i < j:
+ self.handle_data(rawdata[i:j])
+ i = j
+ if i == n: break
+ if rawdata[i] == '<':
+ if starttagopen.match(rawdata, i):
+ if self.literal:
+ self.handle_data(rawdata[i])
+ i = i+1
+ continue
+ k = self.parse_starttag(i)
+ if k < 0: break
+ i = k
+ continue
+ if rawdata.startswith(" (i + 1):
+ self.handle_data("<")
+ i = i+1
+ else:
+ # incomplete
+ break
+ continue
+ if rawdata.startswith(" 'host[:port]', '/path'."""
+ global _hostprog
+ if _hostprog is None:
+ import re
+ _hostprog = re.compile('^//([^/?]*)(.*)$')
+
+ match = _hostprog.match(url)
+ if match: return match.group(1, 2)
+ return None, url
+
+
+from urllib import (unwrap, unquote, splittype, quote,
+ addinfourl, splitport,
+ splitattr, ftpwrapper, splituser, splitpasswd, splitvalue)
+
+# support for FileHandler, proxies via environment variables
+from urllib import localhost, url2pathname, getproxies
+
+from urllib2 import HTTPError, URLError
+
+import _request
+import _rfc3986
+import _sockettimeout
+
+from _clientcookie import CookieJar
+from _response import closeable_response
+
+
+# used in User-Agent header sent
+__version__ = sys.version[:3]
+
+_opener = None
+def urlopen(url, data=None, timeout=_sockettimeout._GLOBAL_DEFAULT_TIMEOUT):
+ global _opener
+ if _opener is None:
+ _opener = build_opener()
+ return _opener.open(url, data, timeout)
+
+def install_opener(opener):
+ global _opener
+ _opener = opener
+
+# copied from cookielib.py
+_cut_port_re = re.compile(r":\d+$")
+def request_host(request):
+ """Return request-host, as defined by RFC 2965.
+
+ Variation from RFC: returned value is lowercased, for convenient
+ comparison.
+
+ """
+ url = request.get_full_url()
+ host = urlparse.urlparse(url)[1]
+ if host == "":
+ host = request.get_header("Host", "")
+
+ # remove port, if present
+ host = _cut_port_re.sub("", host, 1)
+ return host.lower()
+
+class Request:
+
+ def __init__(self, url, data=None, headers={},
+ origin_req_host=None, unverifiable=False):
+ # unwrap('') --> 'type://host/path'
+ self.__original = unwrap(url)
+ self.type = None
+ # self.__r_type is what's left after doing the splittype
+ self.host = None
+ self.port = None
+ self._tunnel_host = None
+ self.data = data
+ self.headers = {}
+ for key, value in headers.items():
+ self.add_header(key, value)
+ self.unredirected_hdrs = {}
+ if origin_req_host is None:
+ origin_req_host = request_host(self)
+ self.origin_req_host = origin_req_host
+ self.unverifiable = unverifiable
+
+ def __getattr__(self, attr):
+ # XXX this is a fallback mechanism to guard against these
+ # methods getting called in a non-standard order. this may be
+ # too complicated and/or unnecessary.
+ # XXX should the __r_XXX attributes be public?
+ if attr[:12] == '_Request__r_':
+ name = attr[12:]
+ if hasattr(Request, 'get_' + name):
+ getattr(self, 'get_' + name)()
+ return getattr(self, attr)
+ raise AttributeError, attr
+
+ def get_method(self):
+ if self.has_data():
+ return "POST"
+ else:
+ return "GET"
+
+ # XXX these helper methods are lame
+
+ def add_data(self, data):
+ self.data = data
+
+ def has_data(self):
+ return self.data is not None
+
+ def get_data(self):
+ return self.data
+
+ def get_full_url(self):
+ return self.__original
+
+ def get_type(self):
+ if self.type is None:
+ self.type, self.__r_type = splittype(self.__original)
+ if self.type is None:
+ raise ValueError, "unknown url type: %s" % self.__original
+ return self.type
+
+ def get_host(self):
+ if self.host is None:
+ self.host, self.__r_host = splithost(self.__r_type)
+ if self.host:
+ self.host = unquote(self.host)
+ return self.host
+
+ def get_selector(self):
+ scheme, authority, path, query, fragment = _rfc3986.urlsplit(
+ self.__r_host)
+ if path == "":
+ path = "/" # RFC 2616, section 3.2.2
+ fragment = None # RFC 3986, section 3.5
+ return _rfc3986.urlunsplit([scheme, authority, path, query, fragment])
+
+ def set_proxy(self, host, type):
+ orig_host = self.get_host()
+ if self.get_type() == 'https' and not self._tunnel_host:
+ self._tunnel_host = orig_host
+ else:
+ self.type = type
+ self.__r_host = self.__original
+
+ self.host = host
+
+ def has_proxy(self):
+ """Private method."""
+ # has non-HTTPS proxy
+ return self.__r_host == self.__original
+
+ def get_origin_req_host(self):
+ return self.origin_req_host
+
+ def is_unverifiable(self):
+ return self.unverifiable
+
+ def add_header(self, key, val):
+ # useful for something like authentication
+ self.headers[key.capitalize()] = val
+
+ def add_unredirected_header(self, key, val):
+ # will not be added to a redirected request
+ self.unredirected_hdrs[key.capitalize()] = val
+
+ def has_header(self, header_name):
+ return (header_name in self.headers or
+ header_name in self.unredirected_hdrs)
+
+ def get_header(self, header_name, default=None):
+ return self.headers.get(
+ header_name,
+ self.unredirected_hdrs.get(header_name, default))
+
+ def header_items(self):
+ hdrs = self.unredirected_hdrs.copy()
+ hdrs.update(self.headers)
+ return hdrs.items()
+
+class OpenerDirector:
+ def __init__(self):
+ client_version = "Python-urllib/%s" % __version__
+ self.addheaders = [('User-agent', client_version)]
+ # manage the individual handlers
+ self.handlers = []
+ self.handle_open = {}
+ self.handle_error = {}
+ self.process_response = {}
+ self.process_request = {}
+
+ def add_handler(self, handler):
+ if not hasattr(handler, "add_parent"):
+ raise TypeError("expected BaseHandler instance, got %r" %
+ type(handler))
+
+ added = False
+ for meth in dir(handler):
+ if meth in ["redirect_request", "do_open", "proxy_open"]:
+ # oops, coincidental match
+ continue
+
+ i = meth.find("_")
+ protocol = meth[:i]
+ condition = meth[i+1:]
+
+ if condition.startswith("error"):
+ j = condition.find("_") + i + 1
+ kind = meth[j+1:]
+ try:
+ kind = int(kind)
+ except ValueError:
+ pass
+ lookup = self.handle_error.get(protocol, {})
+ self.handle_error[protocol] = lookup
+ elif condition == "open":
+ kind = protocol
+ lookup = self.handle_open
+ elif condition == "response":
+ kind = protocol
+ lookup = self.process_response
+ elif condition == "request":
+ kind = protocol
+ lookup = self.process_request
+ else:
+ continue
+
+ handlers = lookup.setdefault(kind, [])
+ if handlers:
+ bisect.insort(handlers, handler)
+ else:
+ handlers.append(handler)
+ added = True
+
+ if added:
+ # the handlers must work in an specific order, the order
+ # is specified in a Handler attribute
+ bisect.insort(self.handlers, handler)
+ handler.add_parent(self)
+
+ def close(self):
+ # Only exists for backwards compatibility.
+ pass
+
+ def _call_chain(self, chain, kind, meth_name, *args):
+ # Handlers raise an exception if no one else should try to handle
+ # the request, or return None if they can't but another handler
+ # could. Otherwise, they return the response.
+ handlers = chain.get(kind, ())
+ for handler in handlers:
+ func = getattr(handler, meth_name)
+
+ result = func(*args)
+ if result is not None:
+ return result
+
+ def _open(self, req, data=None):
+ result = self._call_chain(self.handle_open, 'default',
+ 'default_open', req)
+ if result:
+ return result
+
+ protocol = req.get_type()
+ result = self._call_chain(self.handle_open, protocol, protocol +
+ '_open', req)
+ if result:
+ return result
+
+ return self._call_chain(self.handle_open, 'unknown',
+ 'unknown_open', req)
+
+ def error(self, proto, *args):
+ if proto in ('http', 'https'):
+ # XXX http[s] protocols are special-cased
+ dict = self.handle_error['http'] # https is not different than http
+ proto = args[2] # YUCK!
+ meth_name = 'http_error_%s' % proto
+ http_err = 1
+ orig_args = args
+ else:
+ dict = self.handle_error
+ meth_name = proto + '_error'
+ http_err = 0
+ args = (dict, proto, meth_name) + args
+ result = self._call_chain(*args)
+ if result:
+ return result
+
+ if http_err:
+ args = (dict, 'default', 'http_error_default') + orig_args
+ return self._call_chain(*args)
+
+# XXX probably also want an abstract factory that knows when it makes
+# sense to skip a superclass in favor of a subclass and when it might
+# make sense to include both
+
+def build_opener(*handlers):
+ """Create an opener object from a list of handlers.
+
+ The opener will use several default handlers, including support
+ for HTTP, FTP and when applicable, HTTPS.
+
+ If any of the handlers passed as arguments are subclasses of the
+ default handlers, the default handlers will not be used.
+ """
+ import types
+ def isclass(obj):
+ return isinstance(obj, (types.ClassType, type))
+
+ opener = OpenerDirector()
+ default_classes = [ProxyHandler, UnknownHandler, HTTPHandler,
+ HTTPDefaultErrorHandler, HTTPRedirectHandler,
+ FTPHandler, FileHandler, HTTPErrorProcessor]
+ if hasattr(httplib, 'HTTPS'):
+ default_classes.append(HTTPSHandler)
+ skip = set()
+ for klass in default_classes:
+ for check in handlers:
+ if isclass(check):
+ if issubclass(check, klass):
+ skip.add(klass)
+ elif isinstance(check, klass):
+ skip.add(klass)
+ for klass in skip:
+ default_classes.remove(klass)
+
+ for klass in default_classes:
+ opener.add_handler(klass())
+
+ for h in handlers:
+ if isclass(h):
+ h = h()
+ opener.add_handler(h)
+ return opener
+
+class BaseHandler:
+ handler_order = 500
+
+ def add_parent(self, parent):
+ self.parent = parent
+
+ def close(self):
+ # Only exists for backwards compatibility
+ pass
+
+ def __lt__(self, other):
+ if not hasattr(other, "handler_order"):
+ # Try to preserve the old behavior of having custom classes
+ # inserted after default ones (works only for custom user
+ # classes which are not aware of handler_order).
+ return True
+ return self.handler_order < other.handler_order
+
+
+class HTTPErrorProcessor(BaseHandler):
+ """Process HTTP error responses.
+
+ The purpose of this handler is to to allow other response processors a
+ look-in by removing the call to parent.error() from
+ AbstractHTTPHandler.
+
+ For non-2xx error codes, this just passes the job on to the
+ Handler._error_ methods, via the OpenerDirector.error method.
+ Eventually, HTTPDefaultErrorHandler will raise an HTTPError if no other
+ handler handles the error.
+
+ """
+ handler_order = 1000 # after all other processors
+
+ def http_response(self, request, response):
+ code, msg, hdrs = response.code, response.msg, response.info()
+
+ # According to RFC 2616, "2xx" code indicates that the client's
+ # request was successfully received, understood, and accepted.
+ if not (200 <= code < 300):
+ # hardcoded http is NOT a bug
+ response = self.parent.error(
+ 'http', request, response, code, msg, hdrs)
+
+ return response
+
+ https_response = http_response
+
+class HTTPDefaultErrorHandler(BaseHandler):
+ def http_error_default(self, req, fp, code, msg, hdrs):
+ # why these error methods took the code, msg, headers args in the first
+ # place rather than a response object, I don't know, but to avoid
+ # multiple wrapping, we're discarding them
+
+ if isinstance(fp, HTTPError):
+ response = fp
+ else:
+ response = HTTPError(
+ req.get_full_url(), code, msg, hdrs, fp)
+ assert code == response.code
+ assert msg == response.msg
+ assert hdrs == response.hdrs
+ raise response
+
+class HTTPRedirectHandler(BaseHandler):
+ # maximum number of redirections to any single URL
+ # this is needed because of the state that cookies introduce
+ max_repeats = 4
+ # maximum total number of redirections (regardless of URL) before
+ # assuming we're in a loop
+ max_redirections = 10
+
+ # Implementation notes:
+
+ # To avoid the server sending us into an infinite loop, the request
+ # object needs to track what URLs we have already seen. Do this by
+ # adding a handler-specific attribute to the Request object. The value
+ # of the dict is used to count the number of times the same URL has
+ # been visited. This is needed because visiting the same URL twice
+ # does not necessarily imply a loop, thanks to state introduced by
+ # cookies.
+
+ # Always unhandled redirection codes:
+ # 300 Multiple Choices: should not handle this here.
+ # 304 Not Modified: no need to handle here: only of interest to caches
+ # that do conditional GETs
+ # 305 Use Proxy: probably not worth dealing with here
+ # 306 Unused: what was this for in the previous versions of protocol??
+
+ def redirect_request(self, req, fp, code, msg, headers, newurl):
+ """Return a Request or None in response to a redirect.
+
+ This is called by the http_error_30x methods when a
+ redirection response is received. If a redirection should
+ take place, return a new Request to allow http_error_30x to
+ perform the redirect. Otherwise, raise HTTPError if no-one
+ else should try to handle this url. Return None if you can't
+ but another Handler might.
+ """
+ m = req.get_method()
+ if (code in (301, 302, 303, 307, "refresh") and m in ("GET", "HEAD")
+ or code in (301, 302, 303, "refresh") and m == "POST"):
+ # Strictly (according to RFC 2616), 301 or 302 in response
+ # to a POST MUST NOT cause a redirection without confirmation
+ # from the user (of urllib2, in this case). In practice,
+ # essentially all clients do redirect in this case, so we do
+ # the same.
+ # TODO: really refresh redirections should be visiting; tricky to fix
+ new = _request.Request(
+ newurl,
+ headers=req.headers,
+ origin_req_host=req.get_origin_req_host(),
+ unverifiable=True,
+ visit=False,
+ timeout=req.timeout)
+ new._origin_req = getattr(req, "_origin_req", req)
+ return new
+ else:
+ raise HTTPError(req.get_full_url(), code, msg, headers, fp)
+
+ def http_error_302(self, req, fp, code, msg, headers):
+ # Some servers (incorrectly) return multiple Location headers
+ # (so probably same goes for URI). Use first header.
+ if 'location' in headers:
+ newurl = headers.getheaders('location')[0]
+ elif 'uri' in headers:
+ newurl = headers.getheaders('uri')[0]
+ else:
+ return
+ newurl = _rfc3986.clean_url(newurl, "latin-1")
+ newurl = _rfc3986.urljoin(req.get_full_url(), newurl)
+
+ # XXX Probably want to forget about the state of the current
+ # request, although that might interact poorly with other
+ # handlers that also use handler-specific request attributes
+ new = self.redirect_request(req, fp, code, msg, headers, newurl)
+ if new is None:
+ return
+
+ # loop detection
+ # .redirect_dict has a key url if url was previously visited.
+ if hasattr(req, 'redirect_dict'):
+ visited = new.redirect_dict = req.redirect_dict
+ if (visited.get(newurl, 0) >= self.max_repeats or
+ len(visited) >= self.max_redirections):
+ raise HTTPError(req.get_full_url(), code,
+ self.inf_msg + msg, headers, fp)
+ else:
+ visited = new.redirect_dict = req.redirect_dict = {}
+ visited[newurl] = visited.get(newurl, 0) + 1
+
+ # Don't close the fp until we are sure that we won't use it
+ # with HTTPError.
+ fp.read()
+ fp.close()
+
+ return self.parent.open(new)
+
+ http_error_301 = http_error_303 = http_error_307 = http_error_302
+ http_error_refresh = http_error_302
+
+ inf_msg = "The HTTP server returned a redirect error that would " \
+ "lead to an infinite loop.\n" \
+ "The last 30x error message was:\n"
+
+
+def _parse_proxy(proxy):
+ """Return (scheme, user, password, host/port) given a URL or an authority.
+
+ If a URL is supplied, it must have an authority (host:port) component.
+ According to RFC 3986, having an authority component means the URL must
+ have two slashes after the scheme:
+
+ >>> _parse_proxy('file:/ftp.example.com/')
+ Traceback (most recent call last):
+ ValueError: proxy URL with no authority: 'file:/ftp.example.com/'
+
+ The first three items of the returned tuple may be None.
+
+ Examples of authority parsing:
+
+ >>> _parse_proxy('proxy.example.com')
+ (None, None, None, 'proxy.example.com')
+ >>> _parse_proxy('proxy.example.com:3128')
+ (None, None, None, 'proxy.example.com:3128')
+
+ The authority component may optionally include userinfo (assumed to be
+ username:password):
+
+ >>> _parse_proxy('joe:password@proxy.example.com')
+ (None, 'joe', 'password', 'proxy.example.com')
+ >>> _parse_proxy('joe:password@proxy.example.com:3128')
+ (None, 'joe', 'password', 'proxy.example.com:3128')
+
+ Same examples, but with URLs instead:
+
+ >>> _parse_proxy('http://proxy.example.com/')
+ ('http', None, None, 'proxy.example.com')
+ >>> _parse_proxy('http://proxy.example.com:3128/')
+ ('http', None, None, 'proxy.example.com:3128')
+ >>> _parse_proxy('http://joe:password@proxy.example.com/')
+ ('http', 'joe', 'password', 'proxy.example.com')
+ >>> _parse_proxy('http://joe:password@proxy.example.com:3128')
+ ('http', 'joe', 'password', 'proxy.example.com:3128')
+
+ Everything after the authority is ignored:
+
+ >>> _parse_proxy('ftp://joe:password@proxy.example.com/rubbish:3128')
+ ('ftp', 'joe', 'password', 'proxy.example.com')
+
+ Test for no trailing '/' case:
+
+ >>> _parse_proxy('http://joe:password@proxy.example.com')
+ ('http', 'joe', 'password', 'proxy.example.com')
+
+ """
+ scheme, r_scheme = splittype(proxy)
+ if not r_scheme.startswith("/"):
+ # authority
+ scheme = None
+ authority = proxy
+ else:
+ # URL
+ if not r_scheme.startswith("//"):
+ raise ValueError("proxy URL with no authority: %r" % proxy)
+ # We have an authority, so for RFC 3986-compliant URLs (by ss 3.
+ # and 3.3.), path is empty or starts with '/'
+ end = r_scheme.find("/", 2)
+ if end == -1:
+ end = None
+ authority = r_scheme[2:end]
+ userinfo, hostport = splituser(authority)
+ if userinfo is not None:
+ user, password = splitpasswd(userinfo)
+ else:
+ user = password = None
+ return scheme, user, password, hostport
+
+class ProxyHandler(BaseHandler):
+ # Proxies must be in front
+ handler_order = 100
+
+ def __init__(self, proxies=None, proxy_bypass=None):
+ if proxies is None:
+ proxies = getproxies()
+
+ assert hasattr(proxies, 'has_key'), "proxies must be a mapping"
+ self.proxies = proxies
+ for type, url in proxies.items():
+ setattr(self, '%s_open' % type,
+ lambda r, proxy=url, type=type, meth=self.proxy_open: \
+ meth(r, proxy, type))
+ if proxy_bypass is None:
+ proxy_bypass = urllib.proxy_bypass
+ self._proxy_bypass = proxy_bypass
+
+ def proxy_open(self, req, proxy, type):
+ orig_type = req.get_type()
+ proxy_type, user, password, hostport = _parse_proxy(proxy)
+
+ if proxy_type is None:
+ proxy_type = orig_type
+
+ if req.get_host() and self._proxy_bypass(req.get_host()):
+ return None
+
+ if user and password:
+ user_pass = '%s:%s' % (unquote(user), unquote(password))
+ creds = base64.b64encode(user_pass).strip()
+ req.add_header('Proxy-authorization', 'Basic ' + creds)
+ hostport = unquote(hostport)
+ req.set_proxy(hostport, proxy_type)
+ if orig_type == proxy_type or orig_type == 'https':
+ # let other handlers take care of it
+ return None
+ else:
+ # need to start over, because the other handlers don't
+ # grok the proxy's URL type
+ # e.g. if we have a constructor arg proxies like so:
+ # {'http': 'ftp://proxy.example.com'}, we may end up turning
+ # a request for http://acme.example.com/a into one for
+ # ftp://proxy.example.com/a
+ return self.parent.open(req)
+
+
+class HTTPPasswordMgr:
+
+ def __init__(self):
+ self.passwd = {}
+
+ def add_password(self, realm, uri, user, passwd):
+ # uri could be a single URI or a sequence
+ if isinstance(uri, basestring):
+ uri = [uri]
+ if not realm in self.passwd:
+ self.passwd[realm] = {}
+ for default_port in True, False:
+ reduced_uri = tuple(
+ [self.reduce_uri(u, default_port) for u in uri])
+ self.passwd[realm][reduced_uri] = (user, passwd)
+
+ def find_user_password(self, realm, authuri):
+ domains = self.passwd.get(realm, {})
+ for default_port in True, False:
+ reduced_authuri = self.reduce_uri(authuri, default_port)
+ for uris, authinfo in domains.iteritems():
+ for uri in uris:
+ if self.is_suburi(uri, reduced_authuri):
+ return authinfo
+ return None, None
+
+ def reduce_uri(self, uri, default_port=True):
+ """Accept authority or URI and extract only the authority and path."""
+ # note HTTP URLs do not have a userinfo component
+ parts = urlparse.urlsplit(uri)
+ if parts[1]:
+ # URI
+ scheme = parts[0]
+ authority = parts[1]
+ path = parts[2] or '/'
+ else:
+ # host or host:port
+ scheme = None
+ authority = uri
+ path = '/'
+ host, port = splitport(authority)
+ if default_port and port is None and scheme is not None:
+ dport = {"http": 80,
+ "https": 443,
+ }.get(scheme)
+ if dport is not None:
+ authority = "%s:%d" % (host, dport)
+ return authority, path
+
+ def is_suburi(self, base, test):
+ """Check if test is below base in a URI tree
+
+ Both args must be URIs in reduced form.
+ """
+ if base == test:
+ return True
+ if base[0] != test[0]:
+ return False
+ common = posixpath.commonprefix((base[1], test[1]))
+ if len(common) == len(base[1]):
+ return True
+ return False
+
+
+class HTTPPasswordMgrWithDefaultRealm(HTTPPasswordMgr):
+
+ def find_user_password(self, realm, authuri):
+ user, password = HTTPPasswordMgr.find_user_password(self, realm,
+ authuri)
+ if user is not None:
+ return user, password
+ return HTTPPasswordMgr.find_user_password(self, None, authuri)
+
+
+class AbstractBasicAuthHandler:
+
+ # XXX this allows for multiple auth-schemes, but will stupidly pick
+ # the last one with a realm specified.
+
+ # allow for double- and single-quoted realm values
+ # (single quotes are a violation of the RFC, but appear in the wild)
+ rx = re.compile('(?:.*,)*[ \t]*([^ \t]+)[ \t]+'
+ 'realm=(["\'])(.*?)\\2', re.I)
+
+ # XXX could pre-emptively send auth info already accepted (RFC 2617,
+ # end of section 2, and section 1.2 immediately after "credentials"
+ # production).
+
+ def __init__(self, password_mgr=None):
+ if password_mgr is None:
+ password_mgr = HTTPPasswordMgr()
+ self.passwd = password_mgr
+ self.add_password = self.passwd.add_password
+
+ def http_error_auth_reqed(self, authreq, host, req, headers):
+ # host may be an authority (without userinfo) or a URL with an
+ # authority
+ # XXX could be multiple headers
+ authreq = headers.get(authreq, None)
+ if authreq:
+ mo = AbstractBasicAuthHandler.rx.search(authreq)
+ if mo:
+ scheme, quote, realm = mo.groups()
+ if scheme.lower() == 'basic':
+ return self.retry_http_basic_auth(host, req, realm)
+
+ def retry_http_basic_auth(self, host, req, realm):
+ user, pw = self.passwd.find_user_password(realm, host)
+ if pw is not None:
+ raw = "%s:%s" % (user, pw)
+ auth = 'Basic %s' % base64.b64encode(raw).strip()
+ if req.headers.get(self.auth_header, None) == auth:
+ return None
+ newreq = copy.copy(req)
+ newreq.add_header(self.auth_header, auth)
+ newreq.visit = False
+ return self.parent.open(newreq)
+ else:
+ return None
+
+
+class HTTPBasicAuthHandler(AbstractBasicAuthHandler, BaseHandler):
+
+ auth_header = 'Authorization'
+
+ def http_error_401(self, req, fp, code, msg, headers):
+ url = req.get_full_url()
+ return self.http_error_auth_reqed('www-authenticate',
+ url, req, headers)
+
+
+class ProxyBasicAuthHandler(AbstractBasicAuthHandler, BaseHandler):
+
+ auth_header = 'Proxy-authorization'
+
+ def http_error_407(self, req, fp, code, msg, headers):
+ # http_error_auth_reqed requires that there is no userinfo component in
+ # authority. Assume there isn't one, since urllib2 does not (and
+ # should not, RFC 3986 s. 3.2.1) support requests for URLs containing
+ # userinfo.
+ authority = req.get_host()
+ return self.http_error_auth_reqed('proxy-authenticate',
+ authority, req, headers)
+
+
+def randombytes(n):
+ """Return n random bytes."""
+ # Use /dev/urandom if it is available. Fall back to random module
+ # if not. It might be worthwhile to extend this function to use
+ # other platform-specific mechanisms for getting random bytes.
+ if os.path.exists("/dev/urandom"):
+ f = open("/dev/urandom")
+ s = f.read(n)
+ f.close()
+ return s
+ else:
+ L = [chr(random.randrange(0, 256)) for i in range(n)]
+ return "".join(L)
+
+class AbstractDigestAuthHandler:
+ # Digest authentication is specified in RFC 2617.
+
+ # XXX The client does not inspect the Authentication-Info header
+ # in a successful response.
+
+ # XXX It should be possible to test this implementation against
+ # a mock server that just generates a static set of challenges.
+
+ # XXX qop="auth-int" supports is shaky
+
+ def __init__(self, passwd=None):
+ if passwd is None:
+ passwd = HTTPPasswordMgr()
+ self.passwd = passwd
+ self.add_password = self.passwd.add_password
+ self.retried = 0
+ self.nonce_count = 0
+ self.last_nonce = None
+
+ def reset_retry_count(self):
+ self.retried = 0
+
+ def http_error_auth_reqed(self, auth_header, host, req, headers):
+ authreq = headers.get(auth_header, None)
+ if self.retried > 5:
+ # Don't fail endlessly - if we failed once, we'll probably
+ # fail a second time. Hm. Unless the Password Manager is
+ # prompting for the information. Crap. This isn't great
+ # but it's better than the current 'repeat until recursion
+ # depth exceeded' approach
+ raise HTTPError(req.get_full_url(), 401, "digest auth failed",
+ headers, None)
+ else:
+ self.retried += 1
+ if authreq:
+ scheme = authreq.split()[0]
+ if scheme.lower() == 'digest':
+ return self.retry_http_digest_auth(req, authreq)
+
+ def retry_http_digest_auth(self, req, auth):
+ token, challenge = auth.split(' ', 1)
+ chal = parse_keqv_list(parse_http_list(challenge))
+ auth = self.get_authorization(req, chal)
+ if auth:
+ auth_val = 'Digest %s' % auth
+ if req.headers.get(self.auth_header, None) == auth_val:
+ return None
+ newreq = copy.copy(req)
+ newreq.add_unredirected_header(self.auth_header, auth_val)
+ newreq.visit = False
+ return self.parent.open(newreq)
+
+ def get_cnonce(self, nonce):
+ # The cnonce-value is an opaque
+ # quoted string value provided by the client and used by both client
+ # and server to avoid chosen plaintext attacks, to provide mutual
+ # authentication, and to provide some message integrity protection.
+ # This isn't a fabulous effort, but it's probably Good Enough.
+ dig = sha1_digest("%s:%s:%s:%s" % (self.nonce_count, nonce,
+ time.ctime(), randombytes(8)))
+ return dig[:16]
+
+ def get_authorization(self, req, chal):
+ try:
+ realm = chal['realm']
+ nonce = chal['nonce']
+ qop = chal.get('qop')
+ algorithm = chal.get('algorithm', 'MD5')
+ # mod_digest doesn't send an opaque, even though it isn't
+ # supposed to be optional
+ opaque = chal.get('opaque', None)
+ except KeyError:
+ return None
+
+ H, KD = self.get_algorithm_impls(algorithm)
+ if H is None:
+ return None
+
+ user, pw = self.passwd.find_user_password(realm, req.get_full_url())
+ if user is None:
+ return None
+
+ # XXX not implemented yet
+ if req.has_data():
+ entdig = self.get_entity_digest(req.get_data(), chal)
+ else:
+ entdig = None
+
+ A1 = "%s:%s:%s" % (user, realm, pw)
+ A2 = "%s:%s" % (req.get_method(),
+ # XXX selector: what about proxies and full urls
+ req.get_selector())
+ if qop == 'auth':
+ if nonce == self.last_nonce:
+ self.nonce_count += 1
+ else:
+ self.nonce_count = 1
+ self.last_nonce = nonce
+
+ ncvalue = '%08x' % self.nonce_count
+ cnonce = self.get_cnonce(nonce)
+ noncebit = "%s:%s:%s:%s:%s" % (nonce, ncvalue, cnonce, qop, H(A2))
+ respdig = KD(H(A1), noncebit)
+ elif qop is None:
+ respdig = KD(H(A1), "%s:%s" % (nonce, H(A2)))
+ else:
+ # XXX handle auth-int.
+ logger = logging.getLogger("mechanize.auth")
+ logger.info("digest auth auth-int qop is not supported, not "
+ "handling digest authentication")
+ return None
+
+ # XXX should the partial digests be encoded too?
+
+ base = 'username="%s", realm="%s", nonce="%s", uri="%s", ' \
+ 'response="%s"' % (user, realm, nonce, req.get_selector(),
+ respdig)
+ if opaque:
+ base += ', opaque="%s"' % opaque
+ if entdig:
+ base += ', digest="%s"' % entdig
+ base += ', algorithm="%s"' % algorithm
+ if qop:
+ base += ', qop=auth, nc=%s, cnonce="%s"' % (ncvalue, cnonce)
+ return base
+
+ def get_algorithm_impls(self, algorithm):
+ # algorithm should be case-insensitive according to RFC2617
+ algorithm = algorithm.upper()
+ if algorithm == 'MD5':
+ H = md5_digest
+ elif algorithm == 'SHA':
+ H = sha1_digest
+ # XXX MD5-sess
+ KD = lambda s, d: H("%s:%s" % (s, d))
+ return H, KD
+
+ def get_entity_digest(self, data, chal):
+ # XXX not implemented yet
+ return None
+
+
+class HTTPDigestAuthHandler(BaseHandler, AbstractDigestAuthHandler):
+ """An authentication protocol defined by RFC 2069
+
+ Digest authentication improves on basic authentication because it
+ does not transmit passwords in the clear.
+ """
+
+ auth_header = 'Authorization'
+ handler_order = 490 # before Basic auth
+
+ def http_error_401(self, req, fp, code, msg, headers):
+ host = urlparse.urlparse(req.get_full_url())[1]
+ retry = self.http_error_auth_reqed('www-authenticate',
+ host, req, headers)
+ self.reset_retry_count()
+ return retry
+
+
+class ProxyDigestAuthHandler(BaseHandler, AbstractDigestAuthHandler):
+
+ auth_header = 'Proxy-Authorization'
+ handler_order = 490 # before Basic auth
+
+ def http_error_407(self, req, fp, code, msg, headers):
+ host = req.get_host()
+ retry = self.http_error_auth_reqed('proxy-authenticate',
+ host, req, headers)
+ self.reset_retry_count()
+ return retry
+
+class AbstractHTTPHandler(BaseHandler):
+
+ def __init__(self, debuglevel=0):
+ self._debuglevel = debuglevel
+
+ def set_http_debuglevel(self, level):
+ self._debuglevel = level
+
+ def do_request_(self, request):
+ host = request.get_host()
+ if not host:
+ raise URLError('no host given')
+
+ if request.has_data(): # POST
+ data = request.get_data()
+ if not request.has_header('Content-type'):
+ request.add_unredirected_header(
+ 'Content-type',
+ 'application/x-www-form-urlencoded')
+ if not request.has_header('Content-length'):
+ request.add_unredirected_header(
+ 'Content-length', '%d' % len(data))
+
+ sel_host = host
+ if request.has_proxy():
+ scheme, sel = splittype(request.get_selector())
+ sel_host, sel_path = splithost(sel)
+
+ if not request.has_header('Host'):
+ request.add_unredirected_header('Host', sel_host)
+ for name, value in self.parent.addheaders:
+ name = name.capitalize()
+ if not request.has_header(name):
+ request.add_unredirected_header(name, value)
+
+ return request
+
+ def do_open(self, http_class, req):
+ """Return an addinfourl object for the request, using http_class.
+
+ http_class must implement the HTTPConnection API from httplib.
+ The addinfourl return value is a file-like object. It also
+ has methods and attributes including:
+ - info(): return a mimetools.Message object for the headers
+ - geturl(): return the original request URL
+ - code: HTTP status code
+ """
+ host_port = req.get_host()
+ if not host_port:
+ raise URLError('no host given')
+
+ try:
+ h = http_class(host_port, timeout=req.timeout)
+ except TypeError:
+ # Python < 2.6, no per-connection timeout support
+ h = http_class(host_port)
+ h.set_debuglevel(self._debuglevel)
+
+ headers = dict(req.headers)
+ headers.update(req.unredirected_hdrs)
+ # We want to make an HTTP/1.1 request, but the addinfourl
+ # class isn't prepared to deal with a persistent connection.
+ # It will try to read all remaining data from the socket,
+ # which will block while the server waits for the next request.
+ # So make sure the connection gets closed after the (only)
+ # request.
+ headers["Connection"] = "close"
+ headers = dict(
+ (name.title(), val) for name, val in headers.items())
+
+ if req._tunnel_host:
+ if not hasattr(h, "set_tunnel"):
+ if not hasattr(h, "_set_tunnel"):
+ raise URLError("HTTPS through proxy not supported "
+ "(Python >= 2.6.4 required)")
+ else:
+ # python 2.6
+ set_tunnel = h._set_tunnel
+ else:
+ set_tunnel = h.set_tunnel
+ set_tunnel(req._tunnel_host)
+
+ try:
+ h.request(req.get_method(), req.get_selector(), req.data, headers)
+ r = h.getresponse()
+ except socket.error, err: # XXX what error?
+ raise URLError(err)
+
+ # Pick apart the HTTPResponse object to get the addinfourl
+ # object initialized properly.
+
+ # Wrap the HTTPResponse object in socket's file object adapter
+ # for Windows. That adapter calls recv(), so delegate recv()
+ # to read(). This weird wrapping allows the returned object to
+ # have readline() and readlines() methods.
+
+ # XXX It might be better to extract the read buffering code
+ # out of socket._fileobject() and into a base class.
+
+ r.recv = r.read
+ fp = create_readline_wrapper(r)
+
+ resp = closeable_response(fp, r.msg, req.get_full_url(),
+ r.status, r.reason)
+ return resp
+
+
+class HTTPHandler(AbstractHTTPHandler):
+
+ def http_open(self, req):
+ return self.do_open(httplib.HTTPConnection, req)
+
+ http_request = AbstractHTTPHandler.do_request_
+
+if hasattr(httplib, 'HTTPS'):
+
+ class HTTPSConnectionFactory:
+ def __init__(self, key_file, cert_file):
+ self._key_file = key_file
+ self._cert_file = cert_file
+ def __call__(self, hostport):
+ return httplib.HTTPSConnection(
+ hostport,
+ key_file=self._key_file, cert_file=self._cert_file)
+
+ class HTTPSHandler(AbstractHTTPHandler):
+
+ def __init__(self, client_cert_manager=None):
+ AbstractHTTPHandler.__init__(self)
+ self.client_cert_manager = client_cert_manager
+
+ def https_open(self, req):
+ if self.client_cert_manager is not None:
+ key_file, cert_file = self.client_cert_manager.find_key_cert(
+ req.get_full_url())
+ conn_factory = HTTPSConnectionFactory(key_file, cert_file)
+ else:
+ conn_factory = httplib.HTTPSConnection
+ return self.do_open(conn_factory, req)
+
+ https_request = AbstractHTTPHandler.do_request_
+
+class HTTPCookieProcessor(BaseHandler):
+ """Handle HTTP cookies.
+
+ Public attributes:
+
+ cookiejar: CookieJar instance
+
+ """
+ def __init__(self, cookiejar=None):
+ if cookiejar is None:
+ cookiejar = CookieJar()
+ self.cookiejar = cookiejar
+
+ def http_request(self, request):
+ self.cookiejar.add_cookie_header(request)
+ return request
+
+ def http_response(self, request, response):
+ self.cookiejar.extract_cookies(response, request)
+ return response
+
+ https_request = http_request
+ https_response = http_response
+
+class UnknownHandler(BaseHandler):
+ def unknown_open(self, req):
+ type = req.get_type()
+ raise URLError('unknown url type: %s' % type)
+
+def parse_keqv_list(l):
+ """Parse list of key=value strings where keys are not duplicated."""
+ parsed = {}
+ for elt in l:
+ k, v = elt.split('=', 1)
+ if v[0] == '"' and v[-1] == '"':
+ v = v[1:-1]
+ parsed[k] = v
+ return parsed
+
+def parse_http_list(s):
+ """Parse lists as described by RFC 2068 Section 2.
+
+ In particular, parse comma-separated lists where the elements of
+ the list may include quoted-strings. A quoted-string could
+ contain a comma. A non-quoted string could have quotes in the
+ middle. Neither commas nor quotes count if they are escaped.
+ Only double-quotes count, not single-quotes.
+ """
+ res = []
+ part = ''
+
+ escape = quote = False
+ for cur in s:
+ if escape:
+ part += cur
+ escape = False
+ continue
+ if quote:
+ if cur == '\\':
+ escape = True
+ continue
+ elif cur == '"':
+ quote = False
+ part += cur
+ continue
+
+ if cur == ',':
+ res.append(part)
+ part = ''
+ continue
+
+ if cur == '"':
+ quote = True
+
+ part += cur
+
+ # append last part
+ if part:
+ res.append(part)
+
+ return [part.strip() for part in res]
+
+class FileHandler(BaseHandler):
+ # Use local file or FTP depending on form of URL
+ def file_open(self, req):
+ url = req.get_selector()
+ if url[:2] == '//' and url[2:3] != '/':
+ req.type = 'ftp'
+ return self.parent.open(req)
+ else:
+ return self.open_local_file(req)
+
+ # names for the localhost
+ names = None
+ def get_names(self):
+ if FileHandler.names is None:
+ try:
+ FileHandler.names = (socket.gethostbyname('localhost'),
+ socket.gethostbyname(socket.gethostname()))
+ except socket.gaierror:
+ FileHandler.names = (socket.gethostbyname('localhost'),)
+ return FileHandler.names
+
+ # not entirely sure what the rules are here
+ def open_local_file(self, req):
+ try:
+ import email.utils as emailutils
+ except ImportError:
+ # python 2.4
+ import email.Utils as emailutils
+ import mimetypes
+ host = req.get_host()
+ file = req.get_selector()
+ localfile = url2pathname(file)
+ try:
+ stats = os.stat(localfile)
+ size = stats.st_size
+ modified = emailutils.formatdate(stats.st_mtime, usegmt=True)
+ mtype = mimetypes.guess_type(file)[0]
+ headers = mimetools.Message(StringIO(
+ 'Content-type: %s\nContent-length: %d\nLast-modified: %s\n' %
+ (mtype or 'text/plain', size, modified)))
+ if host:
+ host, port = splitport(host)
+ if not host or \
+ (not port and socket.gethostbyname(host) in self.get_names()):
+ return addinfourl(open(localfile, 'rb'),
+ headers, 'file:'+file)
+ except OSError, msg:
+ # urllib2 users shouldn't expect OSErrors coming from urlopen()
+ raise URLError(msg)
+ raise URLError('file not on local host')
+
+class FTPHandler(BaseHandler):
+ def ftp_open(self, req):
+ import ftplib
+ import mimetypes
+ host = req.get_host()
+ if not host:
+ raise URLError('ftp error: no host given')
+ host, port = splitport(host)
+ if port is None:
+ port = ftplib.FTP_PORT
+ else:
+ port = int(port)
+
+ # username/password handling
+ user, host = splituser(host)
+ if user:
+ user, passwd = splitpasswd(user)
+ else:
+ passwd = None
+ host = unquote(host)
+ user = unquote(user or '')
+ passwd = unquote(passwd or '')
+
+ try:
+ host = socket.gethostbyname(host)
+ except socket.error, msg:
+ raise URLError(msg)
+ path, attrs = splitattr(req.get_selector())
+ dirs = path.split('/')
+ dirs = map(unquote, dirs)
+ dirs, file = dirs[:-1], dirs[-1]
+ if dirs and not dirs[0]:
+ dirs = dirs[1:]
+ try:
+ fw = self.connect_ftp(user, passwd, host, port, dirs, req.timeout)
+ type = file and 'I' or 'D'
+ for attr in attrs:
+ attr, value = splitvalue(attr)
+ if attr.lower() == 'type' and \
+ value in ('a', 'A', 'i', 'I', 'd', 'D'):
+ type = value.upper()
+ fp, retrlen = fw.retrfile(file, type)
+ headers = ""
+ mtype = mimetypes.guess_type(req.get_full_url())[0]
+ if mtype:
+ headers += "Content-type: %s\n" % mtype
+ if retrlen is not None and retrlen >= 0:
+ headers += "Content-length: %d\n" % retrlen
+ sf = StringIO(headers)
+ headers = mimetools.Message(sf)
+ return addinfourl(fp, headers, req.get_full_url())
+ except ftplib.all_errors, msg:
+ raise URLError, ('ftp error: %s' % msg), sys.exc_info()[2]
+
+ def connect_ftp(self, user, passwd, host, port, dirs, timeout):
+ try:
+ fw = ftpwrapper(user, passwd, host, port, dirs, timeout)
+ except TypeError:
+ # Python < 2.6, no per-connection timeout support
+ fw = ftpwrapper(user, passwd, host, port, dirs)
+## fw.ftp.set_debuglevel(1)
+ return fw
+
+class CacheFTPHandler(FTPHandler):
+ # XXX would be nice to have pluggable cache strategies
+ # XXX this stuff is definitely not thread safe
+ def __init__(self):
+ self.cache = {}
+ self.timeout = {}
+ self.soonest = 0
+ self.delay = 60
+ self.max_conns = 16
+
+ def setTimeout(self, t):
+ self.delay = t
+
+ def setMaxConns(self, m):
+ self.max_conns = m
+
+ def connect_ftp(self, user, passwd, host, port, dirs, timeout):
+ key = user, host, port, '/'.join(dirs), timeout
+ if key in self.cache:
+ self.timeout[key] = time.time() + self.delay
+ else:
+ self.cache[key] = ftpwrapper(user, passwd, host, port, dirs, timeout)
+ self.timeout[key] = time.time() + self.delay
+ self.check_cache()
+ return self.cache[key]
+
+ def check_cache(self):
+ # first check for old ones
+ t = time.time()
+ if self.soonest <= t:
+ for k, v in self.timeout.items():
+ if v < t:
+ self.cache[k].close()
+ del self.cache[k]
+ del self.timeout[k]
+ self.soonest = min(self.timeout.values())
+
+ # then check the size
+ if len(self.cache) == self.max_conns:
+ for k, v in self.timeout.items():
+ if v == self.soonest:
+ del self.cache[k]
+ del self.timeout[k]
+ break
+ self.soonest = min(self.timeout.values())
diff --git a/couchpotato/core/media/movie/providers/trailer/mechanize/_useragent.py b/couchpotato/core/media/movie/providers/trailer/mechanize/_useragent.py
new file mode 100644
index 0000000000..ac28bdd7bb
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/mechanize/_useragent.py
@@ -0,0 +1,367 @@
+"""Convenient HTTP UserAgent class.
+
+This is a subclass of urllib2.OpenerDirector.
+
+
+Copyright 2003-2006 John J. Lee
+
+This code is free software; you can redistribute it and/or modify it under
+the terms of the BSD or ZPL 2.1 licenses (see the file COPYING.txt
+included with the distribution).
+
+"""
+
+import warnings
+
+import _auth
+import _gzip
+import _opener
+import _response
+import _sockettimeout
+import _urllib2
+
+
+class UserAgentBase(_opener.OpenerDirector):
+ """Convenient user-agent class.
+
+ Do not use .add_handler() to add a handler for something already dealt with
+ by this code.
+
+ The only reason at present for the distinction between UserAgent and
+ UserAgentBase is so that classes that depend on .seek()able responses
+ (e.g. mechanize.Browser) can inherit from UserAgentBase. The subclass
+ UserAgent exposes a .set_seekable_responses() method that allows switching
+ off the adding of a .seek() method to responses.
+
+ Public attributes:
+
+ addheaders: list of (name, value) pairs specifying headers to send with
+ every request, unless they are overridden in the Request instance.
+
+ >>> ua = UserAgentBase()
+ >>> ua.addheaders = [
+ ... ("User-agent", "Mozilla/5.0 (compatible)"),
+ ... ("From", "responsible.person@example.com")]
+
+ """
+
+ handler_classes = {
+ # scheme handlers
+ "http": _urllib2.HTTPHandler,
+ # CacheFTPHandler is buggy, at least in 2.3, so we don't use it
+ "ftp": _urllib2.FTPHandler,
+ "file": _urllib2.FileHandler,
+
+ # other handlers
+ "_unknown": _urllib2.UnknownHandler,
+ # HTTP{S,}Handler depend on HTTPErrorProcessor too
+ "_http_error": _urllib2.HTTPErrorProcessor,
+ "_http_default_error": _urllib2.HTTPDefaultErrorHandler,
+
+ # feature handlers
+ "_basicauth": _urllib2.HTTPBasicAuthHandler,
+ "_digestauth": _urllib2.HTTPDigestAuthHandler,
+ "_redirect": _urllib2.HTTPRedirectHandler,
+ "_cookies": _urllib2.HTTPCookieProcessor,
+ "_refresh": _urllib2.HTTPRefreshProcessor,
+ "_equiv": _urllib2.HTTPEquivProcessor,
+ "_proxy": _urllib2.ProxyHandler,
+ "_proxy_basicauth": _urllib2.ProxyBasicAuthHandler,
+ "_proxy_digestauth": _urllib2.ProxyDigestAuthHandler,
+ "_robots": _urllib2.HTTPRobotRulesProcessor,
+ "_gzip": _gzip.HTTPGzipProcessor, # experimental!
+
+ # debug handlers
+ "_debug_redirect": _urllib2.HTTPRedirectDebugProcessor,
+ "_debug_response_body": _urllib2.HTTPResponseDebugProcessor,
+ }
+
+ default_schemes = ["http", "ftp", "file"]
+ default_others = ["_unknown", "_http_error", "_http_default_error"]
+ default_features = ["_redirect", "_cookies",
+ "_refresh", "_equiv",
+ "_basicauth", "_digestauth",
+ "_proxy", "_proxy_basicauth", "_proxy_digestauth",
+ "_robots",
+ ]
+ if hasattr(_urllib2, 'HTTPSHandler'):
+ handler_classes["https"] = _urllib2.HTTPSHandler
+ default_schemes.append("https")
+
+ def __init__(self):
+ _opener.OpenerDirector.__init__(self)
+
+ ua_handlers = self._ua_handlers = {}
+ for scheme in (self.default_schemes+
+ self.default_others+
+ self.default_features):
+ klass = self.handler_classes[scheme]
+ ua_handlers[scheme] = klass()
+ for handler in ua_handlers.itervalues():
+ self.add_handler(handler)
+
+ # Yuck.
+ # Ensure correct default constructor args were passed to
+ # HTTPRefreshProcessor and HTTPEquivProcessor.
+ if "_refresh" in ua_handlers:
+ self.set_handle_refresh(True)
+ if "_equiv" in ua_handlers:
+ self.set_handle_equiv(True)
+ # Ensure default password managers are installed.
+ pm = ppm = None
+ if "_basicauth" in ua_handlers or "_digestauth" in ua_handlers:
+ pm = _urllib2.HTTPPasswordMgrWithDefaultRealm()
+ if ("_proxy_basicauth" in ua_handlers or
+ "_proxy_digestauth" in ua_handlers):
+ ppm = _auth.HTTPProxyPasswordMgr()
+ self.set_password_manager(pm)
+ self.set_proxy_password_manager(ppm)
+ # set default certificate manager
+ if "https" in ua_handlers:
+ cm = _urllib2.HTTPSClientCertMgr()
+ self.set_client_cert_manager(cm)
+
+ def close(self):
+ _opener.OpenerDirector.close(self)
+ self._ua_handlers = None
+
+ # XXX
+## def set_timeout(self, timeout):
+## self._timeout = timeout
+## def set_http_connection_cache(self, conn_cache):
+## self._http_conn_cache = conn_cache
+## def set_ftp_connection_cache(self, conn_cache):
+## # XXX ATM, FTP has cache as part of handler; should it be separate?
+## self._ftp_conn_cache = conn_cache
+
+ def set_handled_schemes(self, schemes):
+ """Set sequence of URL scheme (protocol) strings.
+
+ For example: ua.set_handled_schemes(["http", "ftp"])
+
+ If this fails (with ValueError) because you've passed an unknown
+ scheme, the set of handled schemes will not be changed.
+
+ """
+ want = {}
+ for scheme in schemes:
+ if scheme.startswith("_"):
+ raise ValueError("not a scheme '%s'" % scheme)
+ if scheme not in self.handler_classes:
+ raise ValueError("unknown scheme '%s'")
+ want[scheme] = None
+
+ # get rid of scheme handlers we don't want
+ for scheme, oldhandler in self._ua_handlers.items():
+ if scheme.startswith("_"): continue # not a scheme handler
+ if scheme not in want:
+ self._replace_handler(scheme, None)
+ else:
+ del want[scheme] # already got it
+ # add the scheme handlers that are missing
+ for scheme in want.keys():
+ self._set_handler(scheme, True)
+
+ def set_cookiejar(self, cookiejar):
+ """Set a mechanize.CookieJar, or None."""
+ self._set_handler("_cookies", obj=cookiejar)
+
+ # XXX could use Greg Stein's httpx for some of this instead?
+ # or httplib2??
+ def set_proxies(self, proxies=None, proxy_bypass=None):
+ """Configure proxy settings.
+
+ proxies: dictionary mapping URL scheme to proxy specification. None
+ means use the default system-specific settings.
+ proxy_bypass: function taking hostname, returning whether proxy should
+ be used. None means use the default system-specific settings.
+
+ The default is to try to obtain proxy settings from the system (see the
+ documentation for urllib.urlopen for information about the
+ system-specific methods used -- note that's urllib, not urllib2).
+
+ To avoid all use of proxies, pass an empty proxies dict.
+
+ >>> ua = UserAgentBase()
+ >>> def proxy_bypass(hostname):
+ ... return hostname == "noproxy.com"
+ >>> ua.set_proxies(
+ ... {"http": "joe:password@myproxy.example.com:3128",
+ ... "ftp": "proxy.example.com"},
+ ... proxy_bypass)
+
+ """
+ self._set_handler("_proxy", True,
+ constructor_kwds=dict(proxies=proxies,
+ proxy_bypass=proxy_bypass))
+
+ def add_password(self, url, user, password, realm=None):
+ self._password_manager.add_password(realm, url, user, password)
+ def add_proxy_password(self, user, password, hostport=None, realm=None):
+ self._proxy_password_manager.add_password(
+ realm, hostport, user, password)
+
+ def add_client_certificate(self, url, key_file, cert_file):
+ """Add an SSL client certificate, for HTTPS client auth.
+
+ key_file and cert_file must be filenames of the key and certificate
+ files, in PEM format. You can use e.g. OpenSSL to convert a p12 (PKCS
+ 12) file to PEM format:
+
+ openssl pkcs12 -clcerts -nokeys -in cert.p12 -out cert.pem
+ openssl pkcs12 -nocerts -in cert.p12 -out key.pem
+
+
+ Note that client certificate password input is very inflexible ATM. At
+ the moment this seems to be console only, which is presumably the
+ default behaviour of libopenssl. In future mechanize may support
+ third-party libraries that (I assume) allow more options here.
+
+ """
+ self._client_cert_manager.add_key_cert(url, key_file, cert_file)
+
+ # the following are rarely useful -- use add_password / add_proxy_password
+ # instead
+ def set_password_manager(self, password_manager):
+ """Set a mechanize.HTTPPasswordMgrWithDefaultRealm, or None."""
+ self._password_manager = password_manager
+ self._set_handler("_basicauth", obj=password_manager)
+ self._set_handler("_digestauth", obj=password_manager)
+ def set_proxy_password_manager(self, password_manager):
+ """Set a mechanize.HTTPProxyPasswordMgr, or None."""
+ self._proxy_password_manager = password_manager
+ self._set_handler("_proxy_basicauth", obj=password_manager)
+ self._set_handler("_proxy_digestauth", obj=password_manager)
+ def set_client_cert_manager(self, cert_manager):
+ """Set a mechanize.HTTPClientCertMgr, or None."""
+ self._client_cert_manager = cert_manager
+ handler = self._ua_handlers["https"]
+ handler.client_cert_manager = cert_manager
+
+ # these methods all take a boolean parameter
+ def set_handle_robots(self, handle):
+ """Set whether to observe rules from robots.txt."""
+ self._set_handler("_robots", handle)
+ def set_handle_redirect(self, handle):
+ """Set whether to handle HTTP 30x redirections."""
+ self._set_handler("_redirect", handle)
+ def set_handle_refresh(self, handle, max_time=None, honor_time=True):
+ """Set whether to handle HTTP Refresh headers."""
+ self._set_handler("_refresh", handle, constructor_kwds=
+ {"max_time": max_time, "honor_time": honor_time})
+ def set_handle_equiv(self, handle, head_parser_class=None):
+ """Set whether to treat HTML http-equiv headers like HTTP headers.
+
+ Response objects may be .seek()able if this is set (currently returned
+ responses are, raised HTTPError exception responses are not).
+
+ """
+ if head_parser_class is not None:
+ constructor_kwds = {"head_parser_class": head_parser_class}
+ else:
+ constructor_kwds={}
+ self._set_handler("_equiv", handle, constructor_kwds=constructor_kwds)
+ def set_handle_gzip(self, handle):
+ """Handle gzip transfer encoding.
+
+ """
+ if handle:
+ warnings.warn(
+ "gzip transfer encoding is experimental!", stacklevel=2)
+ self._set_handler("_gzip", handle)
+ def set_debug_redirects(self, handle):
+ """Log information about HTTP redirects (including refreshes).
+
+ Logging is performed using module logging. The logger name is
+ "mechanize.http_redirects". To actually print some debug output,
+ eg:
+
+ import sys, logging
+ logger = logging.getLogger("mechanize.http_redirects")
+ logger.addHandler(logging.StreamHandler(sys.stdout))
+ logger.setLevel(logging.INFO)
+
+ Other logger names relevant to this module:
+
+ "mechanize.http_responses"
+ "mechanize.cookies"
+
+ To turn on everything:
+
+ import sys, logging
+ logger = logging.getLogger("mechanize")
+ logger.addHandler(logging.StreamHandler(sys.stdout))
+ logger.setLevel(logging.INFO)
+
+ """
+ self._set_handler("_debug_redirect", handle)
+ def set_debug_responses(self, handle):
+ """Log HTTP response bodies.
+
+ See docstring for .set_debug_redirects() for details of logging.
+
+ Response objects may be .seek()able if this is set (currently returned
+ responses are, raised HTTPError exception responses are not).
+
+ """
+ self._set_handler("_debug_response_body", handle)
+ def set_debug_http(self, handle):
+ """Print HTTP headers to sys.stdout."""
+ level = int(bool(handle))
+ for scheme in "http", "https":
+ h = self._ua_handlers.get(scheme)
+ if h is not None:
+ h.set_http_debuglevel(level)
+
+ def _set_handler(self, name, handle=None, obj=None,
+ constructor_args=(), constructor_kwds={}):
+ if handle is None:
+ handle = obj is not None
+ if handle:
+ handler_class = self.handler_classes[name]
+ if obj is not None:
+ newhandler = handler_class(obj)
+ else:
+ newhandler = handler_class(
+ *constructor_args, **constructor_kwds)
+ else:
+ newhandler = None
+ self._replace_handler(name, newhandler)
+
+ def _replace_handler(self, name, newhandler=None):
+ # first, if handler was previously added, remove it
+ if name is not None:
+ handler = self._ua_handlers.get(name)
+ if handler:
+ try:
+ self.handlers.remove(handler)
+ except ValueError:
+ pass
+ # then add the replacement, if any
+ if newhandler is not None:
+ self.add_handler(newhandler)
+ self._ua_handlers[name] = newhandler
+
+
+class UserAgent(UserAgentBase):
+
+ def __init__(self):
+ UserAgentBase.__init__(self)
+ self._seekable = False
+
+ def set_seekable_responses(self, handle):
+ """Make response objects .seek()able."""
+ self._seekable = bool(handle)
+
+ def open(self, fullurl, data=None,
+ timeout=_sockettimeout._GLOBAL_DEFAULT_TIMEOUT):
+ if self._seekable:
+ def bound_open(fullurl, data=None,
+ timeout=_sockettimeout._GLOBAL_DEFAULT_TIMEOUT):
+ return UserAgentBase.open(self, fullurl, data, timeout)
+ response = _opener.wrapped_open(
+ bound_open, _response.seek_wrapped_response, fullurl, data,
+ timeout)
+ else:
+ response = UserAgentBase.open(self, fullurl, data)
+ return response
diff --git a/couchpotato/core/media/movie/providers/trailer/mechanize/_util.py b/couchpotato/core/media/movie/providers/trailer/mechanize/_util.py
new file mode 100644
index 0000000000..0a5ebb1f31
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/mechanize/_util.py
@@ -0,0 +1,305 @@
+"""Utility functions and date/time routines.
+
+ Copyright 2002-2006 John J Lee
+
+This code is free software; you can redistribute it and/or modify it
+under the terms of the BSD or ZPL 2.1 licenses (see the file
+COPYING.txt included with the distribution).
+"""
+
+import re
+import time
+import warnings
+
+
+class ExperimentalWarning(UserWarning):
+ pass
+
+def experimental(message):
+ warnings.warn(message, ExperimentalWarning, stacklevel=3)
+def hide_experimental_warnings():
+ warnings.filterwarnings("ignore", category=ExperimentalWarning)
+def reset_experimental_warnings():
+ warnings.filterwarnings("default", category=ExperimentalWarning)
+
+def deprecation(message):
+ warnings.warn(message, DeprecationWarning, stacklevel=3)
+def hide_deprecations():
+ warnings.filterwarnings("ignore", category=DeprecationWarning)
+def reset_deprecations():
+ warnings.filterwarnings("default", category=DeprecationWarning)
+
+
+def write_file(filename, data):
+ f = open(filename, "wb")
+ try:
+ f.write(data)
+ finally:
+ f.close()
+
+
+def get1(sequence):
+ assert len(sequence) == 1
+ return sequence[0]
+
+
+def isstringlike(x):
+ try: x+""
+ except: return False
+ else: return True
+
+## def caller():
+## try:
+## raise SyntaxError
+## except:
+## import sys
+## return sys.exc_traceback.tb_frame.f_back.f_back.f_code.co_name
+
+
+from calendar import timegm
+
+# Date/time conversion routines for formats used by the HTTP protocol.
+
+EPOCH = 1970
+def my_timegm(tt):
+ year, month, mday, hour, min, sec = tt[:6]
+ if ((year >= EPOCH) and (1 <= month <= 12) and (1 <= mday <= 31) and
+ (0 <= hour <= 24) and (0 <= min <= 59) and (0 <= sec <= 61)):
+ return timegm(tt)
+ else:
+ return None
+
+days = ["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"]
+months = ["Jan", "Feb", "Mar", "Apr", "May", "Jun",
+ "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"]
+months_lower = []
+for month in months: months_lower.append(month.lower())
+
+
+def time2isoz(t=None):
+ """Return a string representing time in seconds since epoch, t.
+
+ If the function is called without an argument, it will use the current
+ time.
+
+ The format of the returned string is like "YYYY-MM-DD hh:mm:ssZ",
+ representing Universal Time (UTC, aka GMT). An example of this format is:
+
+ 1994-11-24 08:49:37Z
+
+ """
+ if t is None: t = time.time()
+ year, mon, mday, hour, min, sec = time.gmtime(t)[:6]
+ return "%04d-%02d-%02d %02d:%02d:%02dZ" % (
+ year, mon, mday, hour, min, sec)
+
+def time2netscape(t=None):
+ """Return a string representing time in seconds since epoch, t.
+
+ If the function is called without an argument, it will use the current
+ time.
+
+ The format of the returned string is like this:
+
+ Wed, DD-Mon-YYYY HH:MM:SS GMT
+
+ """
+ if t is None: t = time.time()
+ year, mon, mday, hour, min, sec, wday = time.gmtime(t)[:7]
+ return "%s %02d-%s-%04d %02d:%02d:%02d GMT" % (
+ days[wday], mday, months[mon-1], year, hour, min, sec)
+
+
+UTC_ZONES = {"GMT": None, "UTC": None, "UT": None, "Z": None}
+
+timezone_re = re.compile(r"^([-+])?(\d\d?):?(\d\d)?$")
+def offset_from_tz_string(tz):
+ offset = None
+ if UTC_ZONES.has_key(tz):
+ offset = 0
+ else:
+ m = timezone_re.search(tz)
+ if m:
+ offset = 3600 * int(m.group(2))
+ if m.group(3):
+ offset = offset + 60 * int(m.group(3))
+ if m.group(1) == '-':
+ offset = -offset
+ return offset
+
+def _str2time(day, mon, yr, hr, min, sec, tz):
+ # translate month name to number
+ # month numbers start with 1 (January)
+ try:
+ mon = months_lower.index(mon.lower())+1
+ except ValueError:
+ # maybe it's already a number
+ try:
+ imon = int(mon)
+ except ValueError:
+ return None
+ if 1 <= imon <= 12:
+ mon = imon
+ else:
+ return None
+
+ # make sure clock elements are defined
+ if hr is None: hr = 0
+ if min is None: min = 0
+ if sec is None: sec = 0
+
+ yr = int(yr)
+ day = int(day)
+ hr = int(hr)
+ min = int(min)
+ sec = int(sec)
+
+ if yr < 1000:
+ # find "obvious" year
+ cur_yr = time.localtime(time.time())[0]
+ m = cur_yr % 100
+ tmp = yr
+ yr = yr + cur_yr - m
+ m = m - tmp
+ if abs(m) > 50:
+ if m > 0: yr = yr + 100
+ else: yr = yr - 100
+
+ # convert UTC time tuple to seconds since epoch (not timezone-adjusted)
+ t = my_timegm((yr, mon, day, hr, min, sec, tz))
+
+ if t is not None:
+ # adjust time using timezone string, to get absolute time since epoch
+ if tz is None:
+ tz = "UTC"
+ tz = tz.upper()
+ offset = offset_from_tz_string(tz)
+ if offset is None:
+ return None
+ t = t - offset
+
+ return t
+
+
+strict_re = re.compile(r"^[SMTWF][a-z][a-z], (\d\d) ([JFMASOND][a-z][a-z]) "
+ r"(\d\d\d\d) (\d\d):(\d\d):(\d\d) GMT$")
+wkday_re = re.compile(
+ r"^(?:Sun|Mon|Tue|Wed|Thu|Fri|Sat)[a-z]*,?\s*", re.I)
+loose_http_re = re.compile(
+ r"""^
+ (\d\d?) # day
+ (?:\s+|[-\/])
+ (\w+) # month
+ (?:\s+|[-\/])
+ (\d+) # year
+ (?:
+ (?:\s+|:) # separator before clock
+ (\d\d?):(\d\d) # hour:min
+ (?::(\d\d))? # optional seconds
+ )? # optional clock
+ \s*
+ ([-+]?\d{2,4}|(?![APap][Mm]\b)[A-Za-z]+)? # timezone
+ \s*
+ (?:\(\w+\))? # ASCII representation of timezone in parens.
+ \s*$""", re.X)
+def http2time(text):
+ """Returns time in seconds since epoch of time represented by a string.
+
+ Return value is an integer.
+
+ None is returned if the format of str is unrecognized, the time is outside
+ the representable range, or the timezone string is not recognized. If the
+ string contains no timezone, UTC is assumed.
+
+ The timezone in the string may be numerical (like "-0800" or "+0100") or a
+ string timezone (like "UTC", "GMT", "BST" or "EST"). Currently, only the
+ timezone strings equivalent to UTC (zero offset) are known to the function.
+
+ The function loosely parses the following formats:
+
+ Wed, 09 Feb 1994 22:23:32 GMT -- HTTP format
+ Tuesday, 08-Feb-94 14:15:29 GMT -- old rfc850 HTTP format
+ Tuesday, 08-Feb-1994 14:15:29 GMT -- broken rfc850 HTTP format
+ 09 Feb 1994 22:23:32 GMT -- HTTP format (no weekday)
+ 08-Feb-94 14:15:29 GMT -- rfc850 format (no weekday)
+ 08-Feb-1994 14:15:29 GMT -- broken rfc850 format (no weekday)
+
+ The parser ignores leading and trailing whitespace. The time may be
+ absent.
+
+ If the year is given with only 2 digits, the function will select the
+ century that makes the year closest to the current date.
+
+ """
+ # fast exit for strictly conforming string
+ m = strict_re.search(text)
+ if m:
+ g = m.groups()
+ mon = months_lower.index(g[1].lower()) + 1
+ tt = (int(g[2]), mon, int(g[0]),
+ int(g[3]), int(g[4]), float(g[5]))
+ return my_timegm(tt)
+
+ # No, we need some messy parsing...
+
+ # clean up
+ text = text.lstrip()
+ text = wkday_re.sub("", text, 1) # Useless weekday
+
+ # tz is time zone specifier string
+ day, mon, yr, hr, min, sec, tz = [None]*7
+
+ # loose regexp parse
+ m = loose_http_re.search(text)
+ if m is not None:
+ day, mon, yr, hr, min, sec, tz = m.groups()
+ else:
+ return None # bad format
+
+ return _str2time(day, mon, yr, hr, min, sec, tz)
+
+
+iso_re = re.compile(
+ """^
+ (\d{4}) # year
+ [-\/]?
+ (\d\d?) # numerical month
+ [-\/]?
+ (\d\d?) # day
+ (?:
+ (?:\s+|[-:Tt]) # separator before clock
+ (\d\d?):?(\d\d) # hour:min
+ (?::?(\d\d(?:\.\d*)?))? # optional seconds (and fractional)
+ )? # optional clock
+ \s*
+ ([-+]?\d\d?:?(:?\d\d)?
+ |Z|z)? # timezone (Z is "zero meridian", i.e. GMT)
+ \s*$""", re.X)
+def iso2time(text):
+ """
+ As for http2time, but parses the ISO 8601 formats:
+
+ 1994-02-03 14:15:29 -0100 -- ISO 8601 format
+ 1994-02-03 14:15:29 -- zone is optional
+ 1994-02-03 -- only date
+ 1994-02-03T14:15:29 -- Use T as separator
+ 19940203T141529Z -- ISO 8601 compact format
+ 19940203 -- only date
+
+ """
+ # clean up
+ text = text.lstrip()
+
+ # tz is time zone specifier string
+ day, mon, yr, hr, min, sec, tz = [None]*7
+
+ # loose regexp parse
+ m = iso_re.search(text)
+ if m is not None:
+ # XXX there's an extra bit of the timezone I'm ignoring here: is
+ # this the right thing to do?
+ yr, mon, day, hr, min, sec, tz, _ = m.groups()
+ else:
+ return None # bad format
+
+ return _str2time(day, mon, yr, hr, min, sec, tz)
diff --git a/couchpotato/core/media/movie/providers/trailer/mechanize/_version.py b/couchpotato/core/media/movie/providers/trailer/mechanize/_version.py
new file mode 100644
index 0000000000..ab5b07b714
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/mechanize/_version.py
@@ -0,0 +1,2 @@
+"0.2.5"
+__version__ = (0, 2, 5, None, None)
diff --git a/couchpotato/core/media/movie/providers/trailer/vftrailers.py b/couchpotato/core/media/movie/providers/trailer/vftrailers.py
new file mode 100644
index 0000000000..3ea1d1f8d0
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/vftrailers.py
@@ -0,0 +1,473 @@
+# -*- coding: latin-1 -*-
+import subprocess
+import youtube_dl
+import time
+import sys
+import os.path
+import unicodedata
+import glob
+import shutil
+import mechanize
+import re
+import urllib2
+import urllib
+from bs4 import BeautifulSoup
+from allocine import allocine
+from couchpotato.core.media.movie.providers.trailer.base import VFTrailerProvider
+from couchpotato.core.helpers.variable import getTitle
+from couchpotato.core.logger import CPLog
+log = CPLog(__name__)
+
+autoload = 'vftrailers'
+api = allocine()
+api.configure('100043982026','29d185d98c984a359e6e6f26a0474269')
+rootDir = os.path.dirname(os.path.abspath(__file__))
+try:
+ _DEV_NULL = subprocess.DEVNULL
+except AttributeError:
+ _DEV_NULL = open(os.devnull, 'wb')
+class vftrailers(VFTrailerProvider):
+
+ def logg(self,string,debug=None):
+ log.info(string)
+ return True
+
+ def cleantitle(self,title):
+ specialchars=['(',')',',','.',';','!','?','-',':','_','[',']','|',' ',' ',' ']
+ try:
+ title=unicodedata.normalize('NFKD',title).encode('ascii','ignore')
+ except:
+ title=title
+ for chars in specialchars:
+ title=title.replace(chars,' ')
+ return title.lower()
+
+ def controltitle(self,title,moviename):
+ realtitle=self.cleantitle(moviename[:-5].decode('unicode-escape'))
+ year=moviename[len(moviename)-4:]
+ listcommonwords=['youtube','dailymotion','vf','francais','francaise','vo','vost','version','annonce','bande','bande-annonce','trailer',
+ 'vostfr','fr','bandeannonce','video','ba','hd','hq','720p','1080p','film','official','#1','#2',
+ '#4','#6','#7','du','and','premiere','n╟1','n╟2','n╟3','n╟4','n╟5','officielle']
+ wordsleft=[]
+ cleantitles=self.cleantitle(title)
+ for word in cleantitles.split():
+ if word not in listcommonwords and word not in realtitle.split() and word<>year:
+ wordsleft.append(word)
+ if len(wordsleft)==0:
+ return True
+ else:
+ return False
+
+ def controltitle2(self,title,moviename):
+ realtitle=self.cleantitle(moviename[:-5].decode('unicode-escape'))
+ year=moviename[len(moviename)-4:]
+ wordsleft=[]
+ cleantitles=self.cleantitle(title)
+ for word in realtitle.split():
+ if word not in cleantitles.split():
+ wordsleft.append(word)
+ self.logg(title+' \\\\mots non trouves//// '+str(wordsleft),True)
+ if len(wordsleft)==0:
+ return True
+ else:
+ return False
+ def cleandic(self,dict,moviename):
+ series=['2','3','4','5','6','7','8']
+ titlenames=dict.keys()
+ listkeysvf=[]
+ listkeysvostfr=[]
+ listkeysvo=[]
+ year=moviename[len(moviename)-4:]
+ for titledict in titlenames:
+ testcontinue=self.controltitle(titledict,moviename)
+ if testcontinue==False:
+ continue
+ testcontinue2=self.controltitle2(titledict,moviename)
+ if testcontinue2==False:
+ continue
+ cleandict=self.cleantitle(titledict)
+ if not '3d' in cleandict and ('vf' in cleandict or 'francais' in cleandict or ' fr ' in cleandict) and not ' vo ' in cleandict :
+ if year in cleandict:
+ listkeysvf.append(titledict)
+ else:
+ compteur=0
+ for x in series:
+ if x in cleandict and not x in moviename[:-5]:
+ compteur+=1
+ if compteur==0:
+ listkeysvf.append(titledict)
+ elif not '3d' in cleandict and ('vost' in cleandict):
+ if year in cleandict:
+ listkeysvostfr.append(titledict)
+ else:
+ compteur=0
+ for x in series:
+ if x in cleandict and not x in moviename[:-5]:
+ compteur+=1
+ if compteur==0:
+ listkeysvostfr.append(titledict)
+ elif not '3d' in cleandict:
+ if year in cleandict:
+ listkeysvo.append(titledict)
+ else:
+ compteur=0
+ for x in series:
+ if x in cleandict and not x in moviename[:-5]:
+ compteur+=1
+ if compteur==0:
+ listkeysvo.append(titledict)
+ urllistvf=[]
+ urllistvostfr=[]
+ urllistvo=[]
+ for listkey in listkeysvf:
+ urllistvf.append(dict[listkey])
+ for listkey in listkeysvostfr:
+ urllistvostfr.append(dict[listkey])
+ for listkey in listkeysvo:
+ urllistvo.append(dict[listkey])
+ self.logg(str(len(urllistvf)) + ' liens de bandes annonces VF trouves sur google')
+ self.logg(str(len(urllistvostfr)) + ' liens de bandes annonces VOSTFR trouves sur google')
+ self.logg(str(len(urllistvo)) + ' liens de bandes annonces VO trouves sur google')
+ return urllistvf,urllistvostfr,urllistvo
+
+ def googlesearch(self,searchstringori):
+ uploadtoignore=['UniversalMoviesFR','ParamountmoviesFR']
+ searchstring=searchstringori[:-5].replace(' ','+')
+ urldic={}
+ regexurl ="url(?!.*url).*?&"
+ patternurl = re.compile(regexurl)
+
+ regextitle='">(?!.*">).*?<\/a'
+ patterntitle= re.compile(regextitle)
+
+ br=mechanize.Browser()
+ br.set_handle_robots(False)
+ br.addheaders=[('User-agent','chrome')]
+
+ query="https://www.google.fr/search?num=100&q=bande-annonce+OR+bande+OR+annonce+"+'"'+searchstring+'"'+"+VF+HD+site:http://www.youtube.com+OR+site:http://www.dailymotion.com&ie=latin-1&oe=latin-1&aq=t&rls=org.mozilla:fr:official&client=firefox-a&channel=np&source=hp&gfe_rd=cr&ei=MW9lU_vDIK2A0AXbroCADw"
+ self.logg('En train de rechercher sur google : ' +searchstring)
+ self.logg('Query : ' +query,True)
+ htmltext=br.open(query).read()
+ soup=BeautifulSoup(htmltext,'html5lib')
+ list_items=soup.findAll('div',attrs={'id':'search'})[0].findAll('li')
+ x=0
+ urldic={}
+ for li in list_items:
+ try:
+ doweignore=0
+ soup2 = list_items[x]
+ for toignore in uploadtoignore:
+ if toignore in str(soup2):
+ doweignore+=1
+ if doweignore<>0:
+ x+=1
+ continue
+ links= soup2.findAll('a')
+ if not 'webcache' in str(links):
+ source_link=links[0]
+ source_url = str(re.findall(patternurl,str(source_link))[0]).replace('url?q=','').replace('&','').replace('%3F','?').replace('%3D','=')
+ source_title= str(re.findall(patterntitle,str(source_link))[0]).replace('">','').replace('','').replace('','').decode("utf-8")
+ urldic.update({source_title:source_url})
+
+ except:
+ x+=1
+ continue
+ x+=1
+ self.logg(str(len(urldic))+ ' resultats trouves sur google')
+ return urldic
+
+ def allocinesearch(self,moviename):
+ series=['2','3','4','5','6','7','8']
+ listallovostfr=[]
+ listallovo=[]
+ listallovf=[]
+ self.logg('Tentative de recherche sur Allocine de ' +moviename[:-5])
+ try:
+ search = api.search(moviename[:-5], "movie")
+ for result in search['feed']['movie']:
+ countseries=0
+ ficheresult=api.movie(result['code'])
+ ficheresulttitle=self.cleantitle(ficheresult['movie']['title'])
+ ficheresulttitleori=self.cleantitle(ficheresult['movie']['originalTitle'])
+ yearresult=ficheresult['movie']['productionYear']
+ if not yearresult:
+ yearresult=0
+ for x in series:
+ if (x in ficheresulttitle or x in ficheresulttitleori) and (not '3d' in ficheresulttitle and not '3d' in ficheresulttitleori):
+ if x not in moviename[:-5]:
+ countseries+=1
+ if self.cleantitle(moviename[:-5].decode('unicode-escape')) in ficheresulttitle and countseries==0 and int(moviename[len(moviename)-4:])+2>yearresult and int(moviename[len(moviename)-4:])-2=481:
+ self.logg('Bande annonce vf et HD trouve sur Allocine jarrete de chercher')
+ break
+ else:
+ self.logg('Bande annonce vf non HD trouve sur Allocine je continue de chercher')
+ elif 'annonce' in str(lien).lower() and 'vost' in str(lien).lower():
+ lienid=lien['href'][:lien['href'].find('&')].replace('/video/player_gen_cmedia=','')
+ self.logg("Potentiel code de bande annonce [{0}] en VOSTFR".format(lienid))
+ trailerallo = api.trailer(lienid)
+ long=len(trailerallo['media']['rendition'])
+ bestba=trailerallo['media']['rendition'][long-1]
+ linkallo=trailerallo['media']['rendition'][long-1]['href']
+ heightbaallo=bestba['height']
+ longadr=len(linkallo)
+ extallo=linkallo[longadr-3:]
+
+ listallovostfr.append({'link':linkallo,'ext':extallo,'height':heightbaallo})
+ self.logg('Bande annonce vostfr trouve sur Allocine je continue de chercher')
+ elif 'annonce' in str(lien).lower() and ' VO' in str(lien):
+ lienid=lien['href'][:lien['href'].find('&')].replace('/video/player_gen_cmedia=','')
+ trailerallo = api.trailer(lienid)
+ long=len(trailerallo['media']['rendition'])
+ bestba=trailerallo['media']['rendition'][long-1]
+ linkallo=trailerallo['media']['rendition'][long-1]['href']
+ heightbaallo=bestba['height']
+ longadr=len(linkallo)
+ extallo=linkallo[longadr-3:]
+ if hasattr(trailerallo['media'],'subtitles') and trailerallo['media']['subtitles']['$'].lower().replace('Г','c') ==u'francais':
+ self.logg("Potentiel code de bande annonce [{0}] en VOSTFR".format(lienid))
+ listallovostfr.append({'link':linkallo,'ext':extallo,'height':heightbaallo})
+ self.logg('Bande annonce vostfr trouve sur Allocine je continue de chercher')
+ else:
+ self.logg("Potentiel code de bande annonce [{0}] en VO".format(lienid))
+ listallovo.append({'link':linkallo,'ext':extallo,'height':heightbaallo})
+ self.logg('Bande annonce vo trouve sur Allocine je continue de chercher')
+
+ else:
+ continue
+ except Exception,e:
+ print e
+ continue
+ self.logg(str(len(listallovf)) +" bandes annonces en VF trouvees sur allocine")
+ self.logg(str(len(listallovostfr)) +" bandes annonces en VOSTFR trouvees sur allocine")
+ self.logg(str(len(listallovo)) +" bandes annonces en VO trouvees sur allocine")
+ return listallovf,listallovostfr,listallovo
+ except :
+ self.logg(str(len(listallovf)) +" bandes annonces en VF trouvees sur allocine")
+ self.logg(str(len(listallovostfr)) +" bandes annonces en VOSTFR trouvees sur allocine")
+ self.logg(str(len(listallovo)) +" bandes annonces en VO trouvees sur allocine")
+ return listallovf,listallovostfr,listallovo
+
+ def quacontrol(self,url):
+ quallist=[]
+ p=subprocess.Popen([sys.executable, 'youtube_dl/__main__.py', '-F',url],cwd=rootDir, shell=True, stdout=subprocess.PIPE,stderr=subprocess.PIPE)
+ while p.poll() is None:
+ l = p.stdout.readline()
+ quallist.append(l)
+ (out, err) = p.communicate()
+ for qual in quallist:
+ if 'best' in qual and ('720' in qual or '1080' in qual):
+ return True
+ else:
+ continue
+ return False
+
+ def quacontrolallo(self,listallo,type):
+ bestqualallo=0
+ for linkvf in listallo:
+ if bestqualallo=481:
+ self.videodl(listvfallo,filename,moviename,destination,True,maxqual)
+ return True
+ else:
+ self.logg('Bande annonce en VF non HD trouvee sur Allocine tentative de recherche dune meilleure qualite sur google')
+ else:
+ self.logg('Rien trouve sur Allocine en VF tentative de recherche sur google')
+ urldic=self.googlesearch(moviename)
+ listgooglevf, listgooglevostfr,listgooglevo=self.cleandic(urldic,moviename)
+ if listvfallo:
+ maxqual=self.quacontrolallo(listvfallo,'vf')
+ if listgooglevf:
+ self.logg('Jai trouve des bandes annonces VF sur google, controlons leur qualite')
+ cleanlistvf,listlowqvf=self.totqualcontrol(listgooglevf,'vf')
+ if cleanlistvf:
+ self.logg('Si jen crois google jai trouve mieux que la bande annonce allocine . Lets go')
+ self.videodl(cleanlistvf,filename,moviename,destination)
+ return True
+ else:
+ self.logg('Rien trouve de mieux sur google pour : '+moviename+' je telecharge donc la bande annonce non HD vf Allocine')
+ maxqual=self.quacontrolallo(listvfallo,'vf')
+ self.videodl(listvfallo,filename,moviename,destination,True,maxqual)
+ return True
+ else:
+ self.logg('Rien trouve de mieux sur google pour : '+moviename+' je telecharge donc la bande annonce non HD vf Allocine')
+ maxqual=self.quacontrolallo(listvfallo,'vf')
+ self.videodl(listvfallo,filename,moviename,destination,True,maxqual)
+ return True
+
+ elif listgooglevf:
+ cleanlistvf,listlowqvf=self.totqualcontrol(listgooglevf,'vf')
+ if cleanlistvf:
+ self.logg('Si jen crois google jai trouve mieux que la bande annonce allocine . Lets go')
+ self.videodl(cleanlistvf,filename,moviename,destination)
+ return True
+ elif listlowqvf:
+ self.logg('Rien trouve sur Allocine pour : ' +moviename+' je recupere donc une bande annonce non HD vf trouve sur google')
+ self.videodl(listlowqvf,filename,moviename,destination)
+ return True
+ elif listvostfrallo:
+ maxqual=self.quacontrolallo(listvostfrallo,'vostfr')
+ if maxqual>=481:
+ self.videodl(cleanlistvf,filename,moviename,destination,True,maxqual)
+ return True
+ else:
+ if listgooglevostfr:
+ cleanlistvostfr,listlowqvostfr=self.totqualcontrol(listgooglevostfr,'vostfr')
+ if cleanlistvostfr:
+ self.logg('Si jen crois google jai trouve mieux que la bande annonce allocine . Lets go')
+ self.videodl(cleanlistvostfr,filename,moviename,destination)
+ return True
+ else:
+ self.logg('Rien trouve de mieux sur google pour : '+moviename+' je telecharge donc la bande annonce non HD vostfr Allocine')
+ self.videodl(listvostfrallo,filename,moviename,destination,True,maxqual)
+ return True
+ else:
+ self.logg('Rien trouve de mieux sur google pour : '+moviename+' je telecharge donc la bande annonce non HD vostfr Allocine')
+ self.videodl(listvostfrallo,filename,moviename,destination,True,maxqual)
+ return True
+
+ elif listgooglevostfr:
+ cleanlistvostfr,listlowqvostfr=self.totqualcontrol(listgooglevostfr,'vostfr')
+ if cleanlistvostfr:
+ self.logg('Si jen crois google jai trouve mieux que la bande annonce allocine . Lets go')
+ self.videodl(cleanlistvostfr,filename,moviename,destination)
+ return True
+ elif listlowqvostfr:
+ self.logg('Rien trouve sur Allocine pour : ' +moviename+' je recupere donc une bande annonce non HD vostfr trouve sur google')
+ self.videodl(listlowqvostfr,filename,moviename,destination)
+ return True
+ elif listvoallo:
+ maxqual=self.quacontrolallo(listvoallo,'vo')
+ if maxqual>=481:
+ self.videodl(listvoallo,filename,moviename,destination,True,maxqual)
+ return True
+ else:
+ if listgooglevo:
+ cleanlistvo,listlowqvo=self.totqualcontrol(listgooglevo,'vo')
+ if cleanlistvo:
+ self.logg('Si jen crois google jai trouve mieux que la bande annonce allocine . Lets go')
+ self.videodl(cleanlistvo,filename,moviename,destination)
+ return True
+ else:
+ self.logg('Rien trouve de mieux sur google pour : '+moviename+' je telecharge donc la bande annonce non HD vo Allocine')
+ self.videodl(listvoallo,filename,moviename,destination,True,maxqual)
+ return True
+ else:
+ self.logg('Rien trouve de mieux sur google pour : '+moviename+' je telecharge donc la bande annonce non HD vo Allocine')
+ self.videodl(listvoallo,filename,moviename,destination,True,maxqual)
+ return True
+
+ elif listgooglevo:
+ cleanlistvo,listlowqvo=self.totqualcontrol(listgooglevo,'vos')
+ if cleanlistvo:
+ self.logg('Si jen crois google jai trouve mieux que la bande annonce allocine . Lets go')
+ self.videodl(cleanlistvo,filename,moviename,destination)
+ return True
+ elif listlowqvo:
+ self.logg('Rien trouve sur Allocine pour : ' +moviename+' je recupere donc une bande annonce non HD vo trouve sur google')
+ self.videodl(listlowqvo,filename,moviename,destination)
+ return True
+ else:
+ self.logg('Snifff encore un film pourri pas de bande annonce trouve pour ' + moviename)
+ return False
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/FileDownloader.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/FileDownloader.py
new file mode 100644
index 0000000000..5c8e676a20
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/FileDownloader.py
@@ -0,0 +1,12 @@
+# Legacy file for backwards compatibility, use youtube_dl.downloader instead!
+from .downloader import FileDownloader as RealFileDownloader
+from .downloader import get_suitable_downloader
+
+
+# This class reproduces the old behaviour of FileDownloader
+class FileDownloader(RealFileDownloader):
+ def _do_download(self, filename, info_dict):
+ real_fd = get_suitable_downloader(info_dict)(self.ydl, self.params)
+ for ph in self._progress_hooks:
+ real_fd.add_progress_hook(ph)
+ return real_fd.download(filename, info_dict)
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/YoutubeDL.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/YoutubeDL.py
new file mode 100644
index 0000000000..e2b823f667
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/YoutubeDL.py
@@ -0,0 +1,1480 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from __future__ import absolute_import, unicode_literals
+
+import collections
+import datetime
+import errno
+import io
+import itertools
+import json
+import locale
+import os
+import platform
+import re
+import shutil
+import subprocess
+import socket
+import sys
+import time
+import traceback
+
+if os.name == 'nt':
+ import ctypes
+
+from .compat import (
+ compat_cookiejar,
+ compat_expanduser,
+ compat_http_client,
+ compat_kwargs,
+ compat_str,
+ compat_urllib_error,
+ compat_urllib_request,
+)
+from .utils import (
+ escape_url,
+ ContentTooShortError,
+ date_from_str,
+ DateRange,
+ DEFAULT_OUTTMPL,
+ determine_ext,
+ DownloadError,
+ encodeFilename,
+ ExtractorError,
+ format_bytes,
+ formatSeconds,
+ get_term_width,
+ locked_file,
+ make_HTTPS_handler,
+ MaxDownloadsReached,
+ PagedList,
+ PostProcessingError,
+ platform_name,
+ preferredencoding,
+ SameFileError,
+ sanitize_filename,
+ subtitles_filename,
+ takewhile_inclusive,
+ UnavailableVideoError,
+ url_basename,
+ write_json_file,
+ write_string,
+ YoutubeDLHandler,
+ prepend_extension,
+ args_to_str,
+)
+from .cache import Cache
+from .extractor import get_info_extractor, gen_extractors
+from .downloader import get_suitable_downloader
+from .downloader.rtmp import rtmpdump_version
+from .postprocessor import (
+ FFmpegMergerPP,
+ FFmpegPostProcessor,
+ get_postprocessor,
+)
+from .version import __version__
+
+
+class YoutubeDL(object):
+ """YoutubeDL class.
+
+ YoutubeDL objects are the ones responsible of downloading the
+ actual video file and writing it to disk if the user has requested
+ it, among some other tasks. In most cases there should be one per
+ program. As, given a video URL, the downloader doesn't know how to
+ extract all the needed information, task that InfoExtractors do, it
+ has to pass the URL to one of them.
+
+ For this, YoutubeDL objects have a method that allows
+ InfoExtractors to be registered in a given order. When it is passed
+ a URL, the YoutubeDL object handles it to the first InfoExtractor it
+ finds that reports being able to handle it. The InfoExtractor extracts
+ all the information about the video or videos the URL refers to, and
+ YoutubeDL process the extracted information, possibly using a File
+ Downloader to download the video.
+
+ YoutubeDL objects accept a lot of parameters. In order not to saturate
+ the object constructor with arguments, it receives a dictionary of
+ options instead. These options are available through the params
+ attribute for the InfoExtractors to use. The YoutubeDL also
+ registers itself as the downloader in charge for the InfoExtractors
+ that are added to it, so this is a "mutual registration".
+
+ Available options:
+
+ username: Username for authentication purposes.
+ password: Password for authentication purposes.
+ videopassword: Password for acces a video.
+ usenetrc: Use netrc for authentication instead.
+ verbose: Print additional info to stdout.
+ quiet: Do not print messages to stdout.
+ no_warnings: Do not print out anything for warnings.
+ forceurl: Force printing final URL.
+ forcetitle: Force printing title.
+ forceid: Force printing ID.
+ forcethumbnail: Force printing thumbnail URL.
+ forcedescription: Force printing description.
+ forcefilename: Force printing final filename.
+ forceduration: Force printing duration.
+ forcejson: Force printing info_dict as JSON.
+ dump_single_json: Force printing the info_dict of the whole playlist
+ (or video) as a single JSON line.
+ simulate: Do not download the video files.
+ format: Video format code. See options.py for more information.
+ format_limit: Highest quality format to try.
+ outtmpl: Template for output names.
+ restrictfilenames: Do not allow "&" and spaces in file names
+ ignoreerrors: Do not stop on download errors.
+ nooverwrites: Prevent overwriting files.
+ playliststart: Playlist item to start at.
+ playlistend: Playlist item to end at.
+ playlistreverse: Download playlist items in reverse order.
+ matchtitle: Download only matching titles.
+ rejecttitle: Reject downloads for matching titles.
+ logger: Log messages to a logging.Logger instance.
+ logtostderr: Log messages to stderr instead of stdout.
+ writedescription: Write the video description to a .description file
+ writeinfojson: Write the video description to a .info.json file
+ writeannotations: Write the video annotations to a .annotations.xml file
+ writethumbnail: Write the thumbnail image to a file
+ writesubtitles: Write the video subtitles to a file
+ writeautomaticsub: Write the automatic subtitles to a file
+ allsubtitles: Downloads all the subtitles of the video
+ (requires writesubtitles or writeautomaticsub)
+ listsubtitles: Lists all available subtitles for the video
+ subtitlesformat: Subtitle format [srt/sbv/vtt] (default=srt)
+ subtitleslangs: List of languages of the subtitles to download
+ keepvideo: Keep the video file after post-processing
+ daterange: A DateRange object, download only if the upload_date is in the range.
+ skip_download: Skip the actual download of the video file
+ cachedir: Location of the cache files in the filesystem.
+ False to disable filesystem cache.
+ noplaylist: Download single video instead of a playlist if in doubt.
+ age_limit: An integer representing the user's age in years.
+ Unsuitable videos for the given age are skipped.
+ min_views: An integer representing the minimum view count the video
+ must have in order to not be skipped.
+ Videos without view count information are always
+ downloaded. None for no limit.
+ max_views: An integer representing the maximum view count.
+ Videos that are more popular than that are not
+ downloaded.
+ Videos without view count information are always
+ downloaded. None for no limit.
+ download_archive: File name of a file where all downloads are recorded.
+ Videos already present in the file are not downloaded
+ again.
+ cookiefile: File name where cookies should be read from and dumped to.
+ nocheckcertificate:Do not verify SSL certificates
+ prefer_insecure: Use HTTP instead of HTTPS to retrieve information.
+ At the moment, this is only supported by YouTube.
+ proxy: URL of the proxy server to use
+ socket_timeout: Time to wait for unresponsive hosts, in seconds
+ bidi_workaround: Work around buggy terminals without bidirectional text
+ support, using fridibi
+ debug_printtraffic:Print out sent and received HTTP traffic
+ include_ads: Download ads as well
+ default_search: Prepend this string if an input url is not valid.
+ 'auto' for elaborate guessing
+ encoding: Use this encoding instead of the system-specified.
+ extract_flat: Do not resolve URLs, return the immediate result.
+ Pass in 'in_playlist' to only show this behavior for
+ playlist items.
+ postprocessors: A list of dictionaries, each with an entry
+ * key: The name of the postprocessor. See
+ youtube_dl/postprocessor/__init__.py for a list.
+ as well as any further keyword arguments for the
+ postprocessor.
+ progress_hooks: A list of functions that get called on download
+ progress, with a dictionary with the entries
+ * filename: The final filename
+ * status: One of "downloading" and "finished"
+
+ The dict may also have some of the following entries:
+
+ * downloaded_bytes: Bytes on disk
+ * total_bytes: Size of the whole file, None if unknown
+ * tmpfilename: The filename we're currently writing to
+ * eta: The estimated time in seconds, None if unknown
+ * speed: The download speed in bytes/second, None if
+ unknown
+
+ Progress hooks are guaranteed to be called at least once
+ (with status "finished") if the download is successful.
+
+
+ The following parameters are not used by YoutubeDL itself, they are used by
+ the FileDownloader:
+ nopart, updatetime, buffersize, ratelimit, min_filesize, max_filesize, test,
+ noresizebuffer, retries, continuedl, noprogress, consoletitle
+
+ The following options are used by the post processors:
+ prefer_ffmpeg: If True, use ffmpeg instead of avconv if both are available,
+ otherwise prefer avconv.
+ exec_cmd: Arbitrary command to run after downloading
+ """
+
+ params = None
+ _ies = []
+ _pps = []
+ _download_retcode = None
+ _num_downloads = None
+ _screen_file = None
+
+ def __init__(self, params=None, auto_init=True):
+ """Create a FileDownloader object with the given options."""
+ if params is None:
+ params = {}
+ self._ies = []
+ self._ies_instances = {}
+ self._pps = []
+ self._progress_hooks = []
+ self._download_retcode = 0
+ self._num_downloads = 0
+ self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)]
+ self._err_file = sys.stderr
+ self.params = params
+ self.cache = Cache(self)
+
+ if params.get('bidi_workaround', False):
+ try:
+ import pty
+ master, slave = pty.openpty()
+ width = get_term_width()
+ if width is None:
+ width_args = []
+ else:
+ width_args = ['-w', str(width)]
+ sp_kwargs = dict(
+ stdin=subprocess.PIPE,
+ stdout=slave,
+ stderr=self._err_file)
+ try:
+ self._output_process = subprocess.Popen(
+ ['bidiv'] + width_args, **sp_kwargs
+ )
+ except OSError:
+ self._output_process = subprocess.Popen(
+ ['fribidi', '-c', 'UTF-8'] + width_args, **sp_kwargs)
+ self._output_channel = os.fdopen(master, 'rb')
+ except OSError as ose:
+ if ose.errno == 2:
+ self.report_warning('Could not find fribidi executable, ignoring --bidi-workaround . Make sure that fribidi is an executable file in one of the directories in your $PATH.')
+ else:
+ raise
+
+ if (sys.version_info >= (3,) and sys.platform != 'win32' and
+ sys.getfilesystemencoding() in ['ascii', 'ANSI_X3.4-1968']
+ and not params.get('restrictfilenames', False)):
+ # On Python 3, the Unicode filesystem API will throw errors (#1474)
+ self.report_warning(
+ 'Assuming --restrict-filenames since file system encoding '
+ 'cannot encode all characters. '
+ 'Set the LC_ALL environment variable to fix this.')
+ self.params['restrictfilenames'] = True
+
+ if '%(stitle)s' in self.params.get('outtmpl', ''):
+ self.report_warning('%(stitle)s is deprecated. Use the %(title)s and the --restrict-filenames flag(which also secures %(uploader)s et al) instead.')
+
+ self._setup_opener()
+
+ if auto_init:
+ self.print_debug_header()
+ self.add_default_info_extractors()
+
+ for pp_def_raw in self.params.get('postprocessors', []):
+ pp_class = get_postprocessor(pp_def_raw['key'])
+ pp_def = dict(pp_def_raw)
+ del pp_def['key']
+ pp = pp_class(self, **compat_kwargs(pp_def))
+ self.add_post_processor(pp)
+
+ for ph in self.params.get('progress_hooks', []):
+ self.add_progress_hook(ph)
+
+ def warn_if_short_id(self, argv):
+ # short YouTube ID starting with dash?
+ idxs = [
+ i for i, a in enumerate(argv)
+ if re.match(r'^-[0-9A-Za-z_-]{10}$', a)]
+ if idxs:
+ correct_argv = (
+ ['youtube-dl'] +
+ [a for i, a in enumerate(argv) if i not in idxs] +
+ ['--'] + [argv[i] for i in idxs]
+ )
+ self.report_warning(
+ 'Long argument string detected. '
+ 'Use -- to separate parameters and URLs, like this:\n%s\n' %
+ args_to_str(correct_argv))
+
+ def add_info_extractor(self, ie):
+ """Add an InfoExtractor object to the end of the list."""
+ self._ies.append(ie)
+ self._ies_instances[ie.ie_key()] = ie
+ ie.set_downloader(self)
+
+ def get_info_extractor(self, ie_key):
+ """
+ Get an instance of an IE with name ie_key, it will try to get one from
+ the _ies list, if there's no instance it will create a new one and add
+ it to the extractor list.
+ """
+ ie = self._ies_instances.get(ie_key)
+ if ie is None:
+ ie = get_info_extractor(ie_key)()
+ self.add_info_extractor(ie)
+ return ie
+
+ def add_default_info_extractors(self):
+ """
+ Add the InfoExtractors returned by gen_extractors to the end of the list
+ """
+ for ie in gen_extractors():
+ self.add_info_extractor(ie)
+
+ def add_post_processor(self, pp):
+ """Add a PostProcessor object to the end of the chain."""
+ self._pps.append(pp)
+ pp.set_downloader(self)
+
+ def add_progress_hook(self, ph):
+ """Add the progress hook (currently only for the file downloader)"""
+ self._progress_hooks.append(ph)
+
+ def _bidi_workaround(self, message):
+ if not hasattr(self, '_output_channel'):
+ return message
+
+ assert hasattr(self, '_output_process')
+ assert isinstance(message, compat_str)
+ line_count = message.count('\n') + 1
+ self._output_process.stdin.write((message + '\n').encode('utf-8'))
+ self._output_process.stdin.flush()
+ res = ''.join(self._output_channel.readline().decode('utf-8')
+ for _ in range(line_count))
+ return res[:-len('\n')]
+
+ def to_screen(self, message, skip_eol=False):
+ """Print message to stdout if not in quiet mode."""
+ return self.to_stdout(message, skip_eol, check_quiet=True)
+
+ def _write_string(self, s, out=None):
+ write_string(s, out=out, encoding=self.params.get('encoding'))
+
+ def to_stdout(self, message, skip_eol=False, check_quiet=False):
+ """Print message to stdout if not in quiet mode."""
+ if self.params.get('logger'):
+ self.params['logger'].debug(message)
+ elif not check_quiet or not self.params.get('quiet', False):
+ message = self._bidi_workaround(message)
+ terminator = ['\n', ''][skip_eol]
+ output = message + terminator
+
+ self._write_string(output, self._screen_file)
+
+ def to_stderr(self, message):
+ """Print message to stderr."""
+ assert isinstance(message, compat_str)
+ if self.params.get('logger'):
+ self.params['logger'].error(message)
+ else:
+ message = self._bidi_workaround(message)
+ output = message + '\n'
+ self._write_string(output, self._err_file)
+
+ def to_console_title(self, message):
+ if not self.params.get('consoletitle', False):
+ return
+ if os.name == 'nt' and ctypes.windll.kernel32.GetConsoleWindow():
+ # c_wchar_p() might not be necessary if `message` is
+ # already of type unicode()
+ ctypes.windll.kernel32.SetConsoleTitleW(ctypes.c_wchar_p(message))
+ elif 'TERM' in os.environ:
+ self._write_string('\033]0;%s\007' % message, self._screen_file)
+
+ def save_console_title(self):
+ if not self.params.get('consoletitle', False):
+ return
+ if 'TERM' in os.environ:
+ # Save the title on stack
+ self._write_string('\033[22;0t', self._screen_file)
+
+ def restore_console_title(self):
+ if not self.params.get('consoletitle', False):
+ return
+ if 'TERM' in os.environ:
+ # Restore the title from stack
+ self._write_string('\033[23;0t', self._screen_file)
+
+ def __enter__(self):
+ self.save_console_title()
+ return self
+
+ def __exit__(self, *args):
+ self.restore_console_title()
+
+ if self.params.get('cookiefile') is not None:
+ self.cookiejar.save()
+
+ def trouble(self, message=None, tb=None):
+ """Determine action to take when a download problem appears.
+
+ Depending on if the downloader has been configured to ignore
+ download errors or not, this method may throw an exception or
+ not when errors are found, after printing the message.
+
+ tb, if given, is additional traceback information.
+ """
+ if message is not None:
+ self.to_stderr(message)
+ if self.params.get('verbose'):
+ if tb is None:
+ if sys.exc_info()[0]: # if .trouble has been called from an except block
+ tb = ''
+ if hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
+ tb += ''.join(traceback.format_exception(*sys.exc_info()[1].exc_info))
+ tb += compat_str(traceback.format_exc())
+ else:
+ tb_data = traceback.format_list(traceback.extract_stack())
+ tb = ''.join(tb_data)
+ self.to_stderr(tb)
+ if not self.params.get('ignoreerrors', False):
+ if sys.exc_info()[0] and hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
+ exc_info = sys.exc_info()[1].exc_info
+ else:
+ exc_info = sys.exc_info()
+ raise DownloadError(message, exc_info)
+ self._download_retcode = 1
+
+ def report_warning(self, message):
+ '''
+ Print the message to stderr, it will be prefixed with 'WARNING:'
+ If stderr is a tty file the 'WARNING:' will be colored
+ '''
+ if self.params.get('logger') is not None:
+ self.params['logger'].warning(message)
+ else:
+ if self.params.get('no_warnings'):
+ return
+ if self._err_file.isatty() and os.name != 'nt':
+ _msg_header = '\033[0;33mWARNING:\033[0m'
+ else:
+ _msg_header = 'WARNING:'
+ warning_message = '%s %s' % (_msg_header, message)
+ self.to_stderr(warning_message)
+
+ def report_error(self, message, tb=None):
+ '''
+ Do the same as trouble, but prefixes the message with 'ERROR:', colored
+ in red if stderr is a tty file.
+ '''
+ if self._err_file.isatty() and os.name != 'nt':
+ _msg_header = '\033[0;31mERROR:\033[0m'
+ else:
+ _msg_header = 'ERROR:'
+ error_message = '%s %s' % (_msg_header, message)
+ self.trouble(error_message, tb)
+
+ def report_file_already_downloaded(self, file_name):
+ """Report file has already been fully downloaded."""
+ try:
+ self.to_screen('[download] %s has already been downloaded' % file_name)
+ except UnicodeEncodeError:
+ self.to_screen('[download] The file has already been downloaded')
+
+ def prepare_filename(self, info_dict):
+ """Generate the output filename."""
+ try:
+ template_dict = dict(info_dict)
+
+ template_dict['epoch'] = int(time.time())
+ autonumber_size = self.params.get('autonumber_size')
+ if autonumber_size is None:
+ autonumber_size = 5
+ autonumber_templ = '%0' + str(autonumber_size) + 'd'
+ template_dict['autonumber'] = autonumber_templ % self._num_downloads
+ if template_dict.get('playlist_index') is not None:
+ template_dict['playlist_index'] = '%0*d' % (len(str(template_dict['n_entries'])), template_dict['playlist_index'])
+ if template_dict.get('resolution') is None:
+ if template_dict.get('width') and template_dict.get('height'):
+ template_dict['resolution'] = '%dx%d' % (template_dict['width'], template_dict['height'])
+ elif template_dict.get('height'):
+ template_dict['resolution'] = '%sp' % template_dict['height']
+ elif template_dict.get('width'):
+ template_dict['resolution'] = '?x%d' % template_dict['width']
+
+ sanitize = lambda k, v: sanitize_filename(
+ compat_str(v),
+ restricted=self.params.get('restrictfilenames'),
+ is_id=(k == 'id'))
+ template_dict = dict((k, sanitize(k, v))
+ for k, v in template_dict.items()
+ if v is not None)
+ template_dict = collections.defaultdict(lambda: 'NA', template_dict)
+
+ outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL)
+ tmpl = compat_expanduser(outtmpl)
+ filename = tmpl % template_dict
+ return filename
+ except ValueError as err:
+ self.report_error('Error in output template: ' + str(err) + ' (encoding: ' + repr(preferredencoding()) + ')')
+ return None
+
+ def _match_entry(self, info_dict):
+ """ Returns None iff the file should be downloaded """
+
+ video_title = info_dict.get('title', info_dict.get('id', 'video'))
+ if 'title' in info_dict:
+ # This can happen when we're just evaluating the playlist
+ title = info_dict['title']
+ matchtitle = self.params.get('matchtitle', False)
+ if matchtitle:
+ if not re.search(matchtitle, title, re.IGNORECASE):
+ return '"' + title + '" title did not match pattern "' + matchtitle + '"'
+ rejecttitle = self.params.get('rejecttitle', False)
+ if rejecttitle:
+ if re.search(rejecttitle, title, re.IGNORECASE):
+ return '"' + title + '" title matched reject pattern "' + rejecttitle + '"'
+ date = info_dict.get('upload_date', None)
+ if date is not None:
+ dateRange = self.params.get('daterange', DateRange())
+ if date not in dateRange:
+ return '%s upload date is not in range %s' % (date_from_str(date).isoformat(), dateRange)
+ view_count = info_dict.get('view_count', None)
+ if view_count is not None:
+ min_views = self.params.get('min_views')
+ if min_views is not None and view_count < min_views:
+ return 'Skipping %s, because it has not reached minimum view count (%d/%d)' % (video_title, view_count, min_views)
+ max_views = self.params.get('max_views')
+ if max_views is not None and view_count > max_views:
+ return 'Skipping %s, because it has exceeded the maximum view count (%d/%d)' % (video_title, view_count, max_views)
+ age_limit = self.params.get('age_limit')
+ if age_limit is not None:
+ actual_age_limit = info_dict.get('age_limit')
+ if actual_age_limit is None:
+ actual_age_limit = 0
+ if age_limit < actual_age_limit:
+ return 'Skipping "' + title + '" because it is age restricted'
+ if self.in_download_archive(info_dict):
+ return '%s has already been recorded in archive' % video_title
+ return None
+
+ @staticmethod
+ def add_extra_info(info_dict, extra_info):
+ '''Set the keys from extra_info in info dict if they are missing'''
+ for key, value in extra_info.items():
+ info_dict.setdefault(key, value)
+
+ def extract_info(self, url, download=True, ie_key=None, extra_info={},
+ process=True):
+ '''
+ Returns a list with a dictionary for each video we find.
+ If 'download', also downloads the videos.
+ extra_info is a dict containing the extra values to add to each result
+ '''
+
+ if ie_key:
+ ies = [self.get_info_extractor(ie_key)]
+ else:
+ ies = self._ies
+
+ for ie in ies:
+ if not ie.suitable(url):
+ continue
+
+ if not ie.working():
+ self.report_warning('The program functionality for this site has been marked as broken, '
+ 'and will probably not work.')
+
+ try:
+ ie_result = ie.extract(url)
+ if ie_result is None: # Finished already (backwards compatibility; listformats and friends should be moved here)
+ break
+ if isinstance(ie_result, list):
+ # Backwards compatibility: old IE result format
+ ie_result = {
+ '_type': 'compat_list',
+ 'entries': ie_result,
+ }
+ self.add_default_extra_info(ie_result, ie, url)
+ if process:
+ return self.process_ie_result(ie_result, download, extra_info)
+ else:
+ return ie_result
+ except ExtractorError as de: # An error we somewhat expected
+ self.report_error(compat_str(de), de.format_traceback())
+ break
+ except MaxDownloadsReached:
+ raise
+ except Exception as e:
+ if self.params.get('ignoreerrors', False):
+ self.report_error(compat_str(e), tb=compat_str(traceback.format_exc()))
+ break
+ else:
+ raise
+ else:
+ self.report_error('no suitable InfoExtractor for URL %s' % url)
+
+ def add_default_extra_info(self, ie_result, ie, url):
+ self.add_extra_info(ie_result, {
+ 'extractor': ie.IE_NAME,
+ 'webpage_url': url,
+ 'webpage_url_basename': url_basename(url),
+ 'extractor_key': ie.ie_key(),
+ })
+
+ def process_ie_result(self, ie_result, download=True, extra_info={}):
+ """
+ Take the result of the ie(may be modified) and resolve all unresolved
+ references (URLs, playlist items).
+
+ It will also download the videos if 'download'.
+ Returns the resolved ie_result.
+ """
+
+ result_type = ie_result.get('_type', 'video')
+
+ if result_type in ('url', 'url_transparent'):
+ extract_flat = self.params.get('extract_flat', False)
+ if ((extract_flat == 'in_playlist' and 'playlist' in extra_info) or
+ extract_flat is True):
+ if self.params.get('forcejson', False):
+ self.to_stdout(json.dumps(ie_result))
+ return ie_result
+
+ if result_type == 'video':
+ self.add_extra_info(ie_result, extra_info)
+ return self.process_video_result(ie_result, download=download)
+ elif result_type == 'url':
+ # We have to add extra_info to the results because it may be
+ # contained in a playlist
+ return self.extract_info(ie_result['url'],
+ download,
+ ie_key=ie_result.get('ie_key'),
+ extra_info=extra_info)
+ elif result_type == 'url_transparent':
+ # Use the information from the embedding page
+ info = self.extract_info(
+ ie_result['url'], ie_key=ie_result.get('ie_key'),
+ extra_info=extra_info, download=False, process=False)
+
+ force_properties = dict(
+ (k, v) for k, v in ie_result.items() if v is not None)
+ for f in ('_type', 'url'):
+ if f in force_properties:
+ del force_properties[f]
+ new_result = info.copy()
+ new_result.update(force_properties)
+
+ assert new_result.get('_type') != 'url_transparent'
+
+ return self.process_ie_result(
+ new_result, download=download, extra_info=extra_info)
+ elif result_type == 'playlist' or result_type == 'multi_video':
+ # We process each entry in the playlist
+ playlist = ie_result.get('title', None) or ie_result.get('id', None)
+ self.to_screen('[download] Downloading playlist: %s' % playlist)
+
+ playlist_results = []
+
+ playliststart = self.params.get('playliststart', 1) - 1
+ playlistend = self.params.get('playlistend', None)
+ # For backwards compatibility, interpret -1 as whole list
+ if playlistend == -1:
+ playlistend = None
+
+ ie_entries = ie_result['entries']
+ if isinstance(ie_entries, list):
+ n_all_entries = len(ie_entries)
+ entries = ie_entries[playliststart:playlistend]
+ n_entries = len(entries)
+ self.to_screen(
+ "[%s] playlist %s: Collected %d video ids (downloading %d of them)" %
+ (ie_result['extractor'], playlist, n_all_entries, n_entries))
+ elif isinstance(ie_entries, PagedList):
+ entries = ie_entries.getslice(
+ playliststart, playlistend)
+ n_entries = len(entries)
+ self.to_screen(
+ "[%s] playlist %s: Downloading %d videos" %
+ (ie_result['extractor'], playlist, n_entries))
+ else: # iterable
+ entries = list(itertools.islice(
+ ie_entries, playliststart, playlistend))
+ n_entries = len(entries)
+ self.to_screen(
+ "[%s] playlist %s: Downloading %d videos" %
+ (ie_result['extractor'], playlist, n_entries))
+
+ if self.params.get('playlistreverse', False):
+ entries = entries[::-1]
+
+ for i, entry in enumerate(entries, 1):
+ self.to_screen('[download] Downloading video %s of %s' % (i, n_entries))
+ extra = {
+ 'n_entries': n_entries,
+ 'playlist': playlist,
+ 'playlist_id': ie_result.get('id'),
+ 'playlist_title': ie_result.get('title'),
+ 'playlist_index': i + playliststart,
+ 'extractor': ie_result['extractor'],
+ 'webpage_url': ie_result['webpage_url'],
+ 'webpage_url_basename': url_basename(ie_result['webpage_url']),
+ 'extractor_key': ie_result['extractor_key'],
+ }
+
+ reason = self._match_entry(entry)
+ if reason is not None:
+ self.to_screen('[download] ' + reason)
+ continue
+
+ entry_result = self.process_ie_result(entry,
+ download=download,
+ extra_info=extra)
+ playlist_results.append(entry_result)
+ ie_result['entries'] = playlist_results
+ return ie_result
+ elif result_type == 'compat_list':
+ self.report_warning(
+ 'Extractor %s returned a compat_list result. '
+ 'It needs to be updated.' % ie_result.get('extractor'))
+
+ def _fixup(r):
+ self.add_extra_info(
+ r,
+ {
+ 'extractor': ie_result['extractor'],
+ 'webpage_url': ie_result['webpage_url'],
+ 'webpage_url_basename': url_basename(ie_result['webpage_url']),
+ 'extractor_key': ie_result['extractor_key'],
+ }
+ )
+ return r
+ ie_result['entries'] = [
+ self.process_ie_result(_fixup(r), download, extra_info)
+ for r in ie_result['entries']
+ ]
+ return ie_result
+ else:
+ raise Exception('Invalid result type: %s' % result_type)
+
+ def select_format(self, format_spec, available_formats):
+ if format_spec == 'best' or format_spec is None:
+ return available_formats[-1]
+ elif format_spec == 'worst':
+ return available_formats[0]
+ elif format_spec == 'bestaudio':
+ audio_formats = [
+ f for f in available_formats
+ if f.get('vcodec') == 'none']
+ if audio_formats:
+ return audio_formats[-1]
+ elif format_spec == 'worstaudio':
+ audio_formats = [
+ f for f in available_formats
+ if f.get('vcodec') == 'none']
+ if audio_formats:
+ return audio_formats[0]
+ elif format_spec == 'bestvideo':
+ video_formats = [
+ f for f in available_formats
+ if f.get('acodec') == 'none']
+ if video_formats:
+ return video_formats[-1]
+ elif format_spec == 'worstvideo':
+ video_formats = [
+ f for f in available_formats
+ if f.get('acodec') == 'none']
+ if video_formats:
+ return video_formats[0]
+ else:
+ extensions = ['mp4', 'flv', 'webm', '3gp', 'm4a']
+ if format_spec in extensions:
+ filter_f = lambda f: f['ext'] == format_spec
+ else:
+ filter_f = lambda f: f['format_id'] == format_spec
+ matches = list(filter(filter_f, available_formats))
+ if matches:
+ return matches[-1]
+ return None
+
+ def process_video_result(self, info_dict, download=True):
+ assert info_dict.get('_type', 'video') == 'video'
+
+ if 'id' not in info_dict:
+ raise ExtractorError('Missing "id" field in extractor result')
+ if 'title' not in info_dict:
+ raise ExtractorError('Missing "title" field in extractor result')
+
+ if 'playlist' not in info_dict:
+ # It isn't part of a playlist
+ info_dict['playlist'] = None
+ info_dict['playlist_index'] = None
+
+ thumbnails = info_dict.get('thumbnails')
+ if thumbnails:
+ thumbnails.sort(key=lambda t: (
+ t.get('width'), t.get('height'), t.get('url')))
+ for t in thumbnails:
+ if 'width' in t and 'height' in t:
+ t['resolution'] = '%dx%d' % (t['width'], t['height'])
+
+ if thumbnails and 'thumbnail' not in info_dict:
+ info_dict['thumbnail'] = thumbnails[-1]['url']
+
+ if 'display_id' not in info_dict and 'id' in info_dict:
+ info_dict['display_id'] = info_dict['id']
+
+ if info_dict.get('upload_date') is None and info_dict.get('timestamp') is not None:
+ # Working around negative timestamps in Windows
+ # (see http://bugs.python.org/issue1646728)
+ if info_dict['timestamp'] < 0 and os.name == 'nt':
+ info_dict['timestamp'] = 0
+ upload_date = datetime.datetime.utcfromtimestamp(
+ info_dict['timestamp'])
+ info_dict['upload_date'] = upload_date.strftime('%Y%m%d')
+
+ # This extractors handle format selection themselves
+ if info_dict['extractor'] in ['Youku']:
+ if download:
+ self.process_info(info_dict)
+ return info_dict
+
+ # We now pick which formats have to be downloaded
+ if info_dict.get('formats') is None:
+ # There's only one format available
+ formats = [info_dict]
+ else:
+ formats = info_dict['formats']
+
+ if not formats:
+ raise ExtractorError('No video formats found!')
+
+ # We check that all the formats have the format and format_id fields
+ for i, format in enumerate(formats):
+ if 'url' not in format:
+ raise ExtractorError('Missing "url" key in result (index %d)' % i)
+
+ if format.get('format_id') is None:
+ format['format_id'] = compat_str(i)
+ if format.get('format') is None:
+ format['format'] = '{id} - {res}{note}'.format(
+ id=format['format_id'],
+ res=self.format_resolution(format),
+ note=' ({0})'.format(format['format_note']) if format.get('format_note') is not None else '',
+ )
+ # Automatically determine file extension if missing
+ if 'ext' not in format:
+ format['ext'] = determine_ext(format['url']).lower()
+
+ format_limit = self.params.get('format_limit', None)
+ if format_limit:
+ formats = list(takewhile_inclusive(
+ lambda f: f['format_id'] != format_limit, formats
+ ))
+
+ # TODO Central sorting goes here
+
+ if formats[0] is not info_dict:
+ # only set the 'formats' fields if the original info_dict list them
+ # otherwise we end up with a circular reference, the first (and unique)
+ # element in the 'formats' field in info_dict is info_dict itself,
+ # wich can't be exported to json
+ info_dict['formats'] = formats
+ if self.params.get('listformats', None):
+ self.list_formats(info_dict)
+ return
+
+ req_format = self.params.get('format')
+ if req_format is None:
+ req_format = 'best'
+ formats_to_download = []
+ # The -1 is for supporting YoutubeIE
+ if req_format in ('-1', 'all'):
+ formats_to_download = formats
+ else:
+ for rfstr in req_format.split(','):
+ # We can accept formats requested in the format: 34/5/best, we pick
+ # the first that is available, starting from left
+ req_formats = rfstr.split('/')
+ for rf in req_formats:
+ if re.match(r'.+?\+.+?', rf) is not None:
+ # Two formats have been requested like '137+139'
+ format_1, format_2 = rf.split('+')
+ formats_info = (self.select_format(format_1, formats),
+ self.select_format(format_2, formats))
+ if all(formats_info):
+ # The first format must contain the video and the
+ # second the audio
+ if formats_info[0].get('vcodec') == 'none':
+ self.report_error('The first format must '
+ 'contain the video, try using '
+ '"-f %s+%s"' % (format_2, format_1))
+ return
+ selected_format = {
+ 'requested_formats': formats_info,
+ 'format': rf,
+ 'ext': formats_info[0]['ext'],
+ }
+ else:
+ selected_format = None
+ else:
+ selected_format = self.select_format(rf, formats)
+ if selected_format is not None:
+ formats_to_download.append(selected_format)
+ break
+ if not formats_to_download:
+ raise ExtractorError('requested format not available',
+ expected=True)
+
+ if download:
+ if len(formats_to_download) > 1:
+ self.to_screen('[info] %s: downloading video in %s formats' % (info_dict['id'], len(formats_to_download)))
+ for format in formats_to_download:
+ new_info = dict(info_dict)
+ new_info.update(format)
+ self.process_info(new_info)
+ # We update the info dict with the best quality format (backwards compatibility)
+ info_dict.update(formats_to_download[-1])
+ return info_dict
+
+ def process_info(self, info_dict):
+ """Process a single resolved IE result."""
+
+ assert info_dict.get('_type', 'video') == 'video'
+
+ max_downloads = self.params.get('max_downloads')
+ if max_downloads is not None:
+ if self._num_downloads >= int(max_downloads):
+ raise MaxDownloadsReached()
+
+ info_dict['fulltitle'] = info_dict['title']
+ if len(info_dict['title']) > 200:
+ info_dict['title'] = info_dict['title'][:197] + '...'
+
+ # Keep for backwards compatibility
+ info_dict['stitle'] = info_dict['title']
+
+ if 'format' not in info_dict:
+ info_dict['format'] = info_dict['ext']
+
+ reason = self._match_entry(info_dict)
+ if reason is not None:
+ self.to_screen('[download] ' + reason)
+ return
+
+ self._num_downloads += 1
+
+ filename = self.prepare_filename(info_dict)
+
+ # Forced printings
+ if self.params.get('forcetitle', False):
+ self.to_stdout(info_dict['fulltitle'])
+ if self.params.get('forceid', False):
+ self.to_stdout(info_dict['id'])
+ if self.params.get('forceurl', False):
+ if info_dict.get('requested_formats') is not None:
+ for f in info_dict['requested_formats']:
+ self.to_stdout(f['url'] + f.get('play_path', ''))
+ else:
+ # For RTMP URLs, also include the playpath
+ self.to_stdout(info_dict['url'] + info_dict.get('play_path', ''))
+ if self.params.get('forcethumbnail', False) and info_dict.get('thumbnail') is not None:
+ self.to_stdout(info_dict['thumbnail'])
+ if self.params.get('forcedescription', False) and info_dict.get('description') is not None:
+ self.to_stdout(info_dict['description'])
+ if self.params.get('forcefilename', False) and filename is not None:
+ self.to_stdout(filename)
+ if self.params.get('forceduration', False) and info_dict.get('duration') is not None:
+ self.to_stdout(formatSeconds(info_dict['duration']))
+ if self.params.get('forceformat', False):
+ self.to_stdout(info_dict['format'])
+ if self.params.get('forcejson', False):
+ info_dict['_filename'] = filename
+ self.to_stdout(json.dumps(info_dict))
+ if self.params.get('dump_single_json', False):
+ info_dict['_filename'] = filename
+
+ # Do nothing else if in simulate mode
+ if self.params.get('simulate', False):
+ return
+
+ if filename is None:
+ return
+
+ try:
+ dn = os.path.dirname(encodeFilename(filename))
+ if dn and not os.path.exists(dn):
+ os.makedirs(dn)
+ except (OSError, IOError) as err:
+ self.report_error('unable to create directory ' + compat_str(err))
+ return
+
+ if self.params.get('writedescription', False):
+ descfn = filename + '.description'
+ if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(descfn)):
+ self.to_screen('[info] Video description is already present')
+ elif info_dict.get('description') is None:
+ self.report_warning('There\'s no description to write.')
+ else:
+ try:
+ self.to_screen('[info] Writing video description to: ' + descfn)
+ with io.open(encodeFilename(descfn), 'w', encoding='utf-8') as descfile:
+ descfile.write(info_dict['description'])
+ except (OSError, IOError):
+ self.report_error('Cannot write description file ' + descfn)
+ return
+
+ if self.params.get('writeannotations', False):
+ annofn = filename + '.annotations.xml'
+ if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(annofn)):
+ self.to_screen('[info] Video annotations are already present')
+ else:
+ try:
+ self.to_screen('[info] Writing video annotations to: ' + annofn)
+ with io.open(encodeFilename(annofn), 'w', encoding='utf-8') as annofile:
+ annofile.write(info_dict['annotations'])
+ except (KeyError, TypeError):
+ self.report_warning('There are no annotations to write.')
+ except (OSError, IOError):
+ self.report_error('Cannot write annotations file: ' + annofn)
+ return
+
+ subtitles_are_requested = any([self.params.get('writesubtitles', False),
+ self.params.get('writeautomaticsub')])
+
+ if subtitles_are_requested and 'subtitles' in info_dict and info_dict['subtitles']:
+ # subtitles download errors are already managed as troubles in relevant IE
+ # that way it will silently go on when used with unsupporting IE
+ subtitles = info_dict['subtitles']
+ sub_format = self.params.get('subtitlesformat', 'srt')
+ for sub_lang in subtitles.keys():
+ sub = subtitles[sub_lang]
+ if sub is None:
+ continue
+ try:
+ sub_filename = subtitles_filename(filename, sub_lang, sub_format)
+ if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(sub_filename)):
+ self.to_screen('[info] Video subtitle %s.%s is already_present' % (sub_lang, sub_format))
+ else:
+ self.to_screen('[info] Writing video subtitles to: ' + sub_filename)
+ with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8') as subfile:
+ subfile.write(sub)
+ except (OSError, IOError):
+ self.report_error('Cannot write subtitles file ' + sub_filename)
+ return
+
+ if self.params.get('writeinfojson', False):
+ infofn = os.path.splitext(filename)[0] + '.info.json'
+ if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(infofn)):
+ self.to_screen('[info] Video description metadata is already present')
+ else:
+ self.to_screen('[info] Writing video description metadata as JSON to: ' + infofn)
+ try:
+ write_json_file(info_dict, infofn)
+ except (OSError, IOError):
+ self.report_error('Cannot write metadata to JSON file ' + infofn)
+ return
+
+ if self.params.get('writethumbnail', False):
+ if info_dict.get('thumbnail') is not None:
+ thumb_format = determine_ext(info_dict['thumbnail'], 'jpg')
+ thumb_filename = os.path.splitext(filename)[0] + '.' + thumb_format
+ if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(thumb_filename)):
+ self.to_screen('[%s] %s: Thumbnail is already present' %
+ (info_dict['extractor'], info_dict['id']))
+ else:
+ self.to_screen('[%s] %s: Downloading thumbnail ...' %
+ (info_dict['extractor'], info_dict['id']))
+ try:
+ uf = self.urlopen(info_dict['thumbnail'])
+ with open(thumb_filename, 'wb') as thumbf:
+ shutil.copyfileobj(uf, thumbf)
+ self.to_screen('[%s] %s: Writing thumbnail to: %s' %
+ (info_dict['extractor'], info_dict['id'], thumb_filename))
+ except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
+ self.report_warning('Unable to download thumbnail "%s": %s' %
+ (info_dict['thumbnail'], compat_str(err)))
+
+ if not self.params.get('skip_download', False):
+ if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(filename)):
+ success = True
+ else:
+ try:
+ def dl(name, info):
+ fd = get_suitable_downloader(info)(self, self.params)
+ for ph in self._progress_hooks:
+ fd.add_progress_hook(ph)
+ if self.params.get('verbose'):
+ self.to_stdout('[debug] Invoking downloader on %r' % info.get('url'))
+ return fd.download(name, info)
+ if info_dict.get('requested_formats') is not None:
+ downloaded = []
+ success = True
+ merger = FFmpegMergerPP(self, not self.params.get('keepvideo'))
+ if not merger._executable:
+ postprocessors = []
+ self.report_warning('You have requested multiple '
+ 'formats but ffmpeg or avconv are not installed.'
+ ' The formats won\'t be merged')
+ else:
+ postprocessors = [merger]
+ for f in info_dict['requested_formats']:
+ new_info = dict(info_dict)
+ new_info.update(f)
+ fname = self.prepare_filename(new_info)
+ fname = prepend_extension(fname, 'f%s' % f['format_id'])
+ downloaded.append(fname)
+ partial_success = dl(fname, new_info)
+ success = success and partial_success
+ info_dict['__postprocessors'] = postprocessors
+ info_dict['__files_to_merge'] = downloaded
+ else:
+ # Just a single file
+ success = dl(filename, info_dict)
+ except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
+ self.report_error('unable to download video data: %s' % str(err))
+ return
+ except (OSError, IOError) as err:
+ raise UnavailableVideoError(err)
+ except (ContentTooShortError, ) as err:
+ self.report_error('content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
+ return
+
+ if success:
+ try:
+ self.post_process(filename, info_dict)
+ except (PostProcessingError) as err:
+ self.report_error('postprocessing: %s' % str(err))
+ return
+ self.record_download_archive(info_dict)
+
+ def download(self, url_list):
+ """Download a given list of URLs."""
+ outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL)
+ if (len(url_list) > 1 and
+ '%' not in outtmpl
+ and self.params.get('max_downloads') != 1):
+ raise SameFileError(outtmpl)
+
+ for url in url_list:
+ try:
+ # It also downloads the videos
+ res = self.extract_info(url)
+ except UnavailableVideoError:
+ self.report_error('unable to download video')
+ except MaxDownloadsReached:
+ self.to_screen('[info] Maximum number of downloaded files reached.')
+ raise
+ else:
+ if self.params.get('dump_single_json', False):
+ self.to_stdout(json.dumps(res))
+
+ return self._download_retcode
+
+ def download_with_info_file(self, info_filename):
+ with io.open(info_filename, 'r', encoding='utf-8') as f:
+ info = json.load(f)
+ try:
+ self.process_ie_result(info, download=True)
+ except DownloadError:
+ webpage_url = info.get('webpage_url')
+ if webpage_url is not None:
+ self.report_warning('The info failed to download, trying with "%s"' % webpage_url)
+ return self.download([webpage_url])
+ else:
+ raise
+ return self._download_retcode
+
+ def post_process(self, filename, ie_info):
+ """Run all the postprocessors on the given file."""
+ info = dict(ie_info)
+ info['filepath'] = filename
+ keep_video = None
+ pps_chain = []
+ if ie_info.get('__postprocessors') is not None:
+ pps_chain.extend(ie_info['__postprocessors'])
+ pps_chain.extend(self._pps)
+ for pp in pps_chain:
+ try:
+ keep_video_wish, new_info = pp.run(info)
+ if keep_video_wish is not None:
+ if keep_video_wish:
+ keep_video = keep_video_wish
+ elif keep_video is None:
+ # No clear decision yet, let IE decide
+ keep_video = keep_video_wish
+ except PostProcessingError as e:
+ self.report_error(e.msg)
+ if keep_video is False and not self.params.get('keepvideo', False):
+ try:
+ self.to_screen('Deleting original file %s (pass -k to keep)' % filename)
+ os.remove(encodeFilename(filename))
+ except (IOError, OSError):
+ self.report_warning('Unable to remove downloaded video file')
+
+ def _make_archive_id(self, info_dict):
+ # Future-proof against any change in case
+ # and backwards compatibility with prior versions
+ extractor = info_dict.get('extractor_key')
+ if extractor is None:
+ if 'id' in info_dict:
+ extractor = info_dict.get('ie_key') # key in a playlist
+ if extractor is None:
+ return None # Incomplete video information
+ return extractor.lower() + ' ' + info_dict['id']
+
+ def in_download_archive(self, info_dict):
+ fn = self.params.get('download_archive')
+ if fn is None:
+ return False
+
+ vid_id = self._make_archive_id(info_dict)
+ if vid_id is None:
+ return False # Incomplete video information
+
+ try:
+ with locked_file(fn, 'r', encoding='utf-8') as archive_file:
+ for line in archive_file:
+ if line.strip() == vid_id:
+ return True
+ except IOError as ioe:
+ if ioe.errno != errno.ENOENT:
+ raise
+ return False
+
+ def record_download_archive(self, info_dict):
+ fn = self.params.get('download_archive')
+ if fn is None:
+ return
+ vid_id = self._make_archive_id(info_dict)
+ assert vid_id
+ with locked_file(fn, 'a', encoding='utf-8') as archive_file:
+ archive_file.write(vid_id + '\n')
+
+ @staticmethod
+ def format_resolution(format, default='unknown'):
+ if format.get('vcodec') == 'none':
+ return 'audio only'
+ if format.get('resolution') is not None:
+ return format['resolution']
+ if format.get('height') is not None:
+ if format.get('width') is not None:
+ res = '%sx%s' % (format['width'], format['height'])
+ else:
+ res = '%sp' % format['height']
+ elif format.get('width') is not None:
+ res = '?x%d' % format['width']
+ else:
+ res = default
+ return res
+
+ def _format_note(self, fdict):
+ res = ''
+ if fdict.get('ext') in ['f4f', 'f4m']:
+ res += '(unsupported) '
+ if fdict.get('format_note') is not None:
+ res += fdict['format_note'] + ' '
+ if fdict.get('tbr') is not None:
+ res += '%4dk ' % fdict['tbr']
+ if fdict.get('container') is not None:
+ if res:
+ res += ', '
+ res += '%s container' % fdict['container']
+ if (fdict.get('vcodec') is not None and
+ fdict.get('vcodec') != 'none'):
+ if res:
+ res += ', '
+ res += fdict['vcodec']
+ if fdict.get('vbr') is not None:
+ res += '@'
+ elif fdict.get('vbr') is not None and fdict.get('abr') is not None:
+ res += 'video@'
+ if fdict.get('vbr') is not None:
+ res += '%4dk' % fdict['vbr']
+ if fdict.get('fps') is not None:
+ res += ', %sfps' % fdict['fps']
+ if fdict.get('acodec') is not None:
+ if res:
+ res += ', '
+ if fdict['acodec'] == 'none':
+ res += 'video only'
+ else:
+ res += '%-5s' % fdict['acodec']
+ elif fdict.get('abr') is not None:
+ if res:
+ res += ', '
+ res += 'audio'
+ if fdict.get('abr') is not None:
+ res += '@%3dk' % fdict['abr']
+ if fdict.get('asr') is not None:
+ res += ' (%5dHz)' % fdict['asr']
+ if fdict.get('filesize') is not None:
+ if res:
+ res += ', '
+ res += format_bytes(fdict['filesize'])
+ elif fdict.get('filesize_approx') is not None:
+ if res:
+ res += ', '
+ res += '~' + format_bytes(fdict['filesize_approx'])
+ return res
+
+ def list_formats(self, info_dict):
+ def line(format, idlen=20):
+ return (('%-' + compat_str(idlen + 1) + 's%-10s%-12s%s') % (
+ format['format_id'],
+ format['ext'],
+ self.format_resolution(format),
+ self._format_note(format),
+ ))
+
+ formats = info_dict.get('formats', [info_dict])
+ idlen = max(len('format code'),
+ max(len(f['format_id']) for f in formats))
+ formats_s = [line(f, idlen) for f in formats]
+ if len(formats) > 1:
+ formats_s[0] += (' ' if self._format_note(formats[0]) else '') + '(worst)'
+ formats_s[-1] += (' ' if self._format_note(formats[-1]) else '') + '(best)'
+
+ header_line = line({
+ 'format_id': 'format code', 'ext': 'extension',
+ 'resolution': 'resolution', 'format_note': 'note'}, idlen=idlen)
+ self.to_screen('[info] Available formats for %s:\n%s\n%s' %
+ (info_dict['id'], header_line, '\n'.join(formats_s)))
+
+ def urlopen(self, req):
+ """ Start an HTTP download """
+
+ # According to RFC 3986, URLs can not contain non-ASCII characters, however this is not
+ # always respected by websites, some tend to give out URLs with non percent-encoded
+ # non-ASCII characters (see telemb.py, ard.py [#3412])
+ # urllib chokes on URLs with non-ASCII characters (see http://bugs.python.org/issue3991)
+ # To work around aforementioned issue we will replace request's original URL with
+ # percent-encoded one
+ req_is_string = isinstance(req, basestring if sys.version_info < (3, 0) else compat_str)
+ url = req if req_is_string else req.get_full_url()
+ url_escaped = escape_url(url)
+
+ # Substitute URL if any change after escaping
+ if url != url_escaped:
+ if req_is_string:
+ req = url_escaped
+ else:
+ req = compat_urllib_request.Request(
+ url_escaped, data=req.data, headers=req.headers,
+ origin_req_host=req.origin_req_host, unverifiable=req.unverifiable)
+
+ return self._opener.open(req, timeout=self._socket_timeout)
+
+ def print_debug_header(self):
+ if not self.params.get('verbose'):
+ return
+
+ if type('') is not compat_str:
+ # Python 2.6 on SLES11 SP1 (https://github.com/rg3/youtube-dl/issues/3326)
+ self.report_warning(
+ 'Your Python is broken! Update to a newer and supported version')
+
+ stdout_encoding = getattr(
+ sys.stdout, 'encoding', 'missing (%s)' % type(sys.stdout).__name__)
+ encoding_str = (
+ '[debug] Encodings: locale %s, fs %s, out %s, pref %s\n' % (
+ locale.getpreferredencoding(),
+ sys.getfilesystemencoding(),
+ stdout_encoding,
+ self.get_encoding()))
+ write_string(encoding_str, encoding=None)
+
+ self._write_string('[debug] youtube-dl version ' + __version__ + '\n')
+ try:
+ sp = subprocess.Popen(
+ ['git', 'rev-parse', '--short', 'HEAD'],
+ stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+ cwd=os.path.dirname(os.path.abspath(__file__)))
+ out, err = sp.communicate()
+ out = out.decode().strip()
+ if re.match('[0-9a-f]+', out):
+ self._write_string('[debug] Git HEAD: ' + out + '\n')
+ except:
+ try:
+ sys.exc_clear()
+ except:
+ pass
+ self._write_string('[debug] Python version %s - %s\n' % (
+ platform.python_version(), platform_name()))
+
+ exe_versions = FFmpegPostProcessor.get_versions()
+ exe_versions['rtmpdump'] = rtmpdump_version()
+ exe_str = ', '.join(
+ '%s %s' % (exe, v)
+ for exe, v in sorted(exe_versions.items())
+ if v
+ )
+ if not exe_str:
+ exe_str = 'none'
+ self._write_string('[debug] exe versions: %s\n' % exe_str)
+
+ proxy_map = {}
+ for handler in self._opener.handlers:
+ if hasattr(handler, 'proxies'):
+ proxy_map.update(handler.proxies)
+ self._write_string('[debug] Proxy map: ' + compat_str(proxy_map) + '\n')
+
+ def _setup_opener(self):
+ timeout_val = self.params.get('socket_timeout')
+ self._socket_timeout = 600 if timeout_val is None else float(timeout_val)
+
+ opts_cookiefile = self.params.get('cookiefile')
+ opts_proxy = self.params.get('proxy')
+
+ if opts_cookiefile is None:
+ self.cookiejar = compat_cookiejar.CookieJar()
+ else:
+ self.cookiejar = compat_cookiejar.MozillaCookieJar(
+ opts_cookiefile)
+ if os.access(opts_cookiefile, os.R_OK):
+ self.cookiejar.load()
+
+ cookie_processor = compat_urllib_request.HTTPCookieProcessor(
+ self.cookiejar)
+ if opts_proxy is not None:
+ if opts_proxy == '':
+ proxies = {}
+ else:
+ proxies = {'http': opts_proxy, 'https': opts_proxy}
+ else:
+ proxies = compat_urllib_request.getproxies()
+ # Set HTTPS proxy to HTTP one if given (https://github.com/rg3/youtube-dl/issues/805)
+ if 'http' in proxies and 'https' not in proxies:
+ proxies['https'] = proxies['http']
+ proxy_handler = compat_urllib_request.ProxyHandler(proxies)
+
+ debuglevel = 1 if self.params.get('debug_printtraffic') else 0
+ https_handler = make_HTTPS_handler(
+ self.params.get('nocheckcertificate', False), debuglevel=debuglevel)
+ ydlh = YoutubeDLHandler(debuglevel=debuglevel)
+ opener = compat_urllib_request.build_opener(
+ https_handler, proxy_handler, cookie_processor, ydlh)
+ # Delete the default user-agent header, which would otherwise apply in
+ # cases where our custom HTTP handler doesn't come into play
+ # (See https://github.com/rg3/youtube-dl/issues/1309 for details)
+ opener.addheaders = []
+ self._opener = opener
+
+ def encode(self, s):
+ if isinstance(s, bytes):
+ return s # Already encoded
+
+ try:
+ return s.encode(self.get_encoding())
+ except UnicodeEncodeError as err:
+ err.reason = err.reason + '. Check your system encoding configuration or use the --encoding option.'
+ raise
+
+ def get_encoding(self):
+ encoding = self.params.get('encoding')
+ if encoding is None:
+ encoding = preferredencoding()
+ return encoding
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/__init__.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/__init__.py
new file mode 100644
index 0000000000..e793203232
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/__init__.py
@@ -0,0 +1,367 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from __future__ import unicode_literals
+
+__license__ = 'Public Domain'
+
+import codecs
+import io
+import os
+import random
+import sys
+
+
+from .options import (
+ parseOpts,
+)
+from .compat import (
+ compat_expanduser,
+ compat_getpass,
+ compat_print,
+ workaround_optparse_bug9161,
+)
+from .utils import (
+ DateRange,
+ DEFAULT_OUTTMPL,
+ decodeOption,
+ DownloadError,
+ MaxDownloadsReached,
+ preferredencoding,
+ read_batch_urls,
+ SameFileError,
+ setproctitle,
+ std_headers,
+ write_string,
+)
+from .update import update_self
+from .downloader import (
+ FileDownloader,
+)
+from .extractor import gen_extractors
+from .YoutubeDL import YoutubeDL
+
+
+def _real_main(argv=None):
+ # Compatibility fixes for Windows
+ if sys.platform == 'win32':
+ # https://github.com/rg3/youtube-dl/issues/820
+ codecs.register(lambda name: codecs.lookup('utf-8') if name == 'cp65001' else None)
+
+ workaround_optparse_bug9161()
+
+ setproctitle('youtube-dl')
+
+ parser, opts, args = parseOpts(argv)
+
+ # Set user agent
+ if opts.user_agent is not None:
+ std_headers['User-Agent'] = opts.user_agent
+
+ # Set referer
+ if opts.referer is not None:
+ std_headers['Referer'] = opts.referer
+
+ # Custom HTTP headers
+ if opts.headers is not None:
+ for h in opts.headers:
+ if h.find(':', 1) < 0:
+ parser.error('wrong header formatting, it should be key:value, not "%s"' % h)
+ key, value = h.split(':', 2)
+ if opts.verbose:
+ write_string('[debug] Adding header from command line option %s:%s\n' % (key, value))
+ std_headers[key] = value
+
+ # Dump user agent
+ if opts.dump_user_agent:
+ compat_print(std_headers['User-Agent'])
+ sys.exit(0)
+
+ # Batch file verification
+ batch_urls = []
+ if opts.batchfile is not None:
+ try:
+ if opts.batchfile == '-':
+ batchfd = sys.stdin
+ else:
+ batchfd = io.open(opts.batchfile, 'r', encoding='utf-8', errors='ignore')
+ batch_urls = read_batch_urls(batchfd)
+ if opts.verbose:
+ write_string('[debug] Batch file urls: ' + repr(batch_urls) + '\n')
+ except IOError:
+ sys.exit('ERROR: batch file could not be read')
+ all_urls = batch_urls + args
+ all_urls = [url.strip() for url in all_urls]
+ _enc = preferredencoding()
+ all_urls = [url.decode(_enc, 'ignore') if isinstance(url, bytes) else url for url in all_urls]
+
+ extractors = gen_extractors()
+
+ if opts.list_extractors:
+ for ie in sorted(extractors, key=lambda ie: ie.IE_NAME.lower()):
+ compat_print(ie.IE_NAME + (' (CURRENTLY BROKEN)' if not ie._WORKING else ''))
+ matchedUrls = [url for url in all_urls if ie.suitable(url)]
+ for mu in matchedUrls:
+ compat_print(' ' + mu)
+ sys.exit(0)
+ if opts.list_extractor_descriptions:
+ for ie in sorted(extractors, key=lambda ie: ie.IE_NAME.lower()):
+ if not ie._WORKING:
+ continue
+ desc = getattr(ie, 'IE_DESC', ie.IE_NAME)
+ if desc is False:
+ continue
+ if hasattr(ie, 'SEARCH_KEY'):
+ _SEARCHES = ('cute kittens', 'slithering pythons', 'falling cat', 'angry poodle', 'purple fish', 'running tortoise', 'sleeping bunny')
+ _COUNTS = ('', '5', '10', 'all')
+ desc += ' (Example: "%s%s:%s" )' % (ie.SEARCH_KEY, random.choice(_COUNTS), random.choice(_SEARCHES))
+ compat_print(desc)
+ sys.exit(0)
+
+ # Conflicting, missing and erroneous options
+ if opts.usenetrc and (opts.username is not None or opts.password is not None):
+ parser.error('using .netrc conflicts with giving username/password')
+ if opts.password is not None and opts.username is None:
+ parser.error('account username missing\n')
+ if opts.outtmpl is not None and (opts.usetitle or opts.autonumber or opts.useid):
+ parser.error('using output template conflicts with using title, video ID or auto number')
+ if opts.usetitle and opts.useid:
+ parser.error('using title conflicts with using video ID')
+ if opts.username is not None and opts.password is None:
+ opts.password = compat_getpass('Type account password and press [Return]: ')
+ if opts.ratelimit is not None:
+ numeric_limit = FileDownloader.parse_bytes(opts.ratelimit)
+ if numeric_limit is None:
+ parser.error('invalid rate limit specified')
+ opts.ratelimit = numeric_limit
+ if opts.min_filesize is not None:
+ numeric_limit = FileDownloader.parse_bytes(opts.min_filesize)
+ if numeric_limit is None:
+ parser.error('invalid min_filesize specified')
+ opts.min_filesize = numeric_limit
+ if opts.max_filesize is not None:
+ numeric_limit = FileDownloader.parse_bytes(opts.max_filesize)
+ if numeric_limit is None:
+ parser.error('invalid max_filesize specified')
+ opts.max_filesize = numeric_limit
+ if opts.retries is not None:
+ try:
+ opts.retries = int(opts.retries)
+ except (TypeError, ValueError):
+ parser.error('invalid retry count specified')
+ if opts.buffersize is not None:
+ numeric_buffersize = FileDownloader.parse_bytes(opts.buffersize)
+ if numeric_buffersize is None:
+ parser.error('invalid buffer size specified')
+ opts.buffersize = numeric_buffersize
+ if opts.playliststart <= 0:
+ raise ValueError('Playlist start must be positive')
+ if opts.playlistend not in (-1, None) and opts.playlistend < opts.playliststart:
+ raise ValueError('Playlist end must be greater than playlist start')
+ if opts.extractaudio:
+ if opts.audioformat not in ['best', 'aac', 'mp3', 'm4a', 'opus', 'vorbis', 'wav']:
+ parser.error('invalid audio format specified')
+ if opts.audioquality:
+ opts.audioquality = opts.audioquality.strip('k').strip('K')
+ if not opts.audioquality.isdigit():
+ parser.error('invalid audio quality specified')
+ if opts.recodevideo is not None:
+ if opts.recodevideo not in ['mp4', 'flv', 'webm', 'ogg', 'mkv']:
+ parser.error('invalid video recode format specified')
+ if opts.date is not None:
+ date = DateRange.day(opts.date)
+ else:
+ date = DateRange(opts.dateafter, opts.datebefore)
+
+ # Do not download videos when there are audio-only formats
+ if opts.extractaudio and not opts.keepvideo and opts.format is None:
+ opts.format = 'bestaudio/best'
+
+ # --all-sub automatically sets --write-sub if --write-auto-sub is not given
+ # this was the old behaviour if only --all-sub was given.
+ if opts.allsubtitles and not opts.writeautomaticsub:
+ opts.writesubtitles = True
+
+ if sys.version_info < (3,):
+ # In Python 2, sys.argv is a bytestring (also note http://bugs.python.org/issue2128 for Windows systems)
+ if opts.outtmpl is not None:
+ opts.outtmpl = opts.outtmpl.decode(preferredencoding())
+ outtmpl = ((opts.outtmpl is not None and opts.outtmpl)
+ or (opts.format == '-1' and opts.usetitle and '%(title)s-%(id)s-%(format)s.%(ext)s')
+ or (opts.format == '-1' and '%(id)s-%(format)s.%(ext)s')
+ or (opts.usetitle and opts.autonumber and '%(autonumber)s-%(title)s-%(id)s.%(ext)s')
+ or (opts.usetitle and '%(title)s-%(id)s.%(ext)s')
+ or (opts.useid and '%(id)s.%(ext)s')
+ or (opts.autonumber and '%(autonumber)s-%(id)s.%(ext)s')
+ or DEFAULT_OUTTMPL)
+ if not os.path.splitext(outtmpl)[1] and opts.extractaudio:
+ parser.error('Cannot download a video and extract audio into the same'
+ ' file! Use "{0}.%(ext)s" instead of "{0}" as the output'
+ ' template'.format(outtmpl))
+
+ any_printing = opts.geturl or opts.gettitle or opts.getid or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat or opts.getduration or opts.dumpjson or opts.dump_single_json
+ download_archive_fn = compat_expanduser(opts.download_archive) if opts.download_archive is not None else opts.download_archive
+
+ # PostProcessors
+ postprocessors = []
+ # Add the metadata pp first, the other pps will copy it
+ if opts.addmetadata:
+ postprocessors.append({'key': 'FFmpegMetadata'})
+ if opts.extractaudio:
+ postprocessors.append({
+ 'key': 'FFmpegExtractAudio',
+ 'preferredcodec': opts.audioformat,
+ 'preferredquality': opts.audioquality,
+ 'nopostoverwrites': opts.nopostoverwrites,
+ })
+ if opts.recodevideo:
+ postprocessors.append({
+ 'key': 'FFmpegVideoConvertor',
+ 'preferedformat': opts.recodevideo,
+ })
+ if opts.embedsubtitles:
+ postprocessors.append({
+ 'key': 'FFmpegEmbedSubtitle',
+ 'subtitlesformat': opts.subtitlesformat,
+ })
+ if opts.xattrs:
+ postprocessors.append({'key': 'XAttrMetadata'})
+ if opts.embedthumbnail:
+ if not opts.addmetadata:
+ postprocessors.append({'key': 'FFmpegAudioFix'})
+ postprocessors.append({'key': 'AtomicParsley'})
+ # Please keep ExecAfterDownload towards the bottom as it allows the user to modify the final file in any way.
+ # So if the user is able to remove the file before your postprocessor runs it might cause a few problems.
+ if opts.exec_cmd:
+ postprocessors.append({
+ 'key': 'ExecAfterDownload',
+ 'verboseOutput': opts.verbose,
+ 'exec_cmd': opts.exec_cmd,
+ })
+
+ ydl_opts = {
+ 'usenetrc': opts.usenetrc,
+ 'username': opts.username,
+ 'password': opts.password,
+ 'twofactor': opts.twofactor,
+ 'videopassword': opts.videopassword,
+ 'quiet': (opts.quiet or any_printing),
+ 'no_warnings': opts.no_warnings,
+ 'forceurl': opts.geturl,
+ 'forcetitle': opts.gettitle,
+ 'forceid': opts.getid,
+ 'forcethumbnail': opts.getthumbnail,
+ 'forcedescription': opts.getdescription,
+ 'forceduration': opts.getduration,
+ 'forcefilename': opts.getfilename,
+ 'forceformat': opts.getformat,
+ 'forcejson': opts.dumpjson,
+ 'dump_single_json': opts.dump_single_json,
+ 'simulate': opts.simulate or any_printing,
+ 'skip_download': opts.skip_download,
+ 'format': opts.format,
+ 'format_limit': opts.format_limit,
+ 'listformats': opts.listformats,
+ 'outtmpl': outtmpl,
+ 'autonumber_size': opts.autonumber_size,
+ 'restrictfilenames': opts.restrictfilenames,
+ 'ignoreerrors': opts.ignoreerrors,
+ 'ratelimit': opts.ratelimit,
+ 'nooverwrites': opts.nooverwrites,
+ 'retries': opts.retries,
+ 'buffersize': opts.buffersize,
+ 'noresizebuffer': opts.noresizebuffer,
+ 'continuedl': opts.continue_dl,
+ 'noprogress': opts.noprogress,
+ 'progress_with_newline': opts.progress_with_newline,
+ 'playliststart': opts.playliststart,
+ 'playlistend': opts.playlistend,
+ 'playlistreverse': opts.playlist_reverse,
+ 'noplaylist': opts.noplaylist,
+ 'logtostderr': opts.outtmpl == '-',
+ 'consoletitle': opts.consoletitle,
+ 'nopart': opts.nopart,
+ 'updatetime': opts.updatetime,
+ 'writedescription': opts.writedescription,
+ 'writeannotations': opts.writeannotations,
+ 'writeinfojson': opts.writeinfojson,
+ 'writethumbnail': opts.writethumbnail,
+ 'writesubtitles': opts.writesubtitles,
+ 'writeautomaticsub': opts.writeautomaticsub,
+ 'allsubtitles': opts.allsubtitles,
+ 'listsubtitles': opts.listsubtitles,
+ 'subtitlesformat': opts.subtitlesformat,
+ 'subtitleslangs': opts.subtitleslangs,
+ 'matchtitle': decodeOption(opts.matchtitle),
+ 'rejecttitle': decodeOption(opts.rejecttitle),
+ 'max_downloads': opts.max_downloads,
+ 'prefer_free_formats': opts.prefer_free_formats,
+ 'verbose': opts.verbose,
+ 'dump_intermediate_pages': opts.dump_intermediate_pages,
+ 'write_pages': opts.write_pages,
+ 'test': opts.test,
+ 'keepvideo': opts.keepvideo,
+ 'min_filesize': opts.min_filesize,
+ 'max_filesize': opts.max_filesize,
+ 'min_views': opts.min_views,
+ 'max_views': opts.max_views,
+ 'daterange': date,
+ 'cachedir': opts.cachedir,
+ 'youtube_print_sig_code': opts.youtube_print_sig_code,
+ 'age_limit': opts.age_limit,
+ 'download_archive': download_archive_fn,
+ 'cookiefile': opts.cookiefile,
+ 'nocheckcertificate': opts.no_check_certificate,
+ 'prefer_insecure': opts.prefer_insecure,
+ 'proxy': opts.proxy,
+ 'socket_timeout': opts.socket_timeout,
+ 'bidi_workaround': opts.bidi_workaround,
+ 'debug_printtraffic': opts.debug_printtraffic,
+ 'prefer_ffmpeg': opts.prefer_ffmpeg,
+ 'include_ads': opts.include_ads,
+ 'default_search': opts.default_search,
+ 'youtube_include_dash_manifest': opts.youtube_include_dash_manifest,
+ 'encoding': opts.encoding,
+ 'exec_cmd': opts.exec_cmd,
+ 'extract_flat': opts.extract_flat,
+ 'postprocessors': postprocessors,
+ }
+
+ with YoutubeDL(ydl_opts) as ydl:
+ # Update version
+ if opts.update_self:
+ update_self(ydl.to_screen, opts.verbose)
+
+ # Remove cache dir
+ if opts.rm_cachedir:
+ ydl.cache.remove()
+
+ # Maybe do nothing
+ if (len(all_urls) < 1) and (opts.load_info_filename is None):
+ if opts.update_self or opts.rm_cachedir:
+ sys.exit()
+
+ ydl.warn_if_short_id(sys.argv[1:] if argv is None else argv)
+ parser.error('you must provide at least one URL')
+
+ try:
+ if opts.load_info_filename is not None:
+ retcode = ydl.download_with_info_file(opts.load_info_filename)
+ else:
+ retcode = ydl.download(all_urls)
+ except MaxDownloadsReached:
+ ydl.to_screen('--max-download limit reached, aborting.')
+ retcode = 101
+
+ sys.exit(retcode)
+
+
+def main(argv=None):
+ try:
+ _real_main(argv)
+ except DownloadError:
+ sys.exit(1)
+ except SameFileError:
+ sys.exit('ERROR: fixed output name but more than one file to download')
+ except KeyboardInterrupt:
+ sys.exit('\nERROR: Interrupted by user')
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/__main__.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/__main__.py
new file mode 100644
index 0000000000..65a0f891c5
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/__main__.py
@@ -0,0 +1,19 @@
+#!/usr/bin/env python
+from __future__ import unicode_literals
+
+# Execute with
+# $ python youtube_dl/__main__.py (2.6+)
+# $ python -m youtube_dl (2.7+)
+
+import sys
+
+if __package__ is None and not hasattr(sys, "frozen"):
+ # direct call of __main__.py
+ import os.path
+ path = os.path.realpath(os.path.abspath(__file__))
+ sys.path.append(os.path.dirname(os.path.dirname(path)))
+
+import youtube_dl
+
+if __name__ == '__main__':
+ youtube_dl.main()
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/aes.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/aes.py
new file mode 100644
index 0000000000..5efd0f836b
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/aes.py
@@ -0,0 +1,331 @@
+from __future__ import unicode_literals
+
+__all__ = ['aes_encrypt', 'key_expansion', 'aes_ctr_decrypt', 'aes_cbc_decrypt', 'aes_decrypt_text']
+
+import base64
+from math import ceil
+
+from .utils import bytes_to_intlist, intlist_to_bytes
+
+BLOCK_SIZE_BYTES = 16
+
+
+def aes_ctr_decrypt(data, key, counter):
+ """
+ Decrypt with aes in counter mode
+
+ @param {int[]} data cipher
+ @param {int[]} key 16/24/32-Byte cipher key
+ @param {instance} counter Instance whose next_value function (@returns {int[]} 16-Byte block)
+ returns the next counter block
+ @returns {int[]} decrypted data
+ """
+ expanded_key = key_expansion(key)
+ block_count = int(ceil(float(len(data)) / BLOCK_SIZE_BYTES))
+
+ decrypted_data = []
+ for i in range(block_count):
+ counter_block = counter.next_value()
+ block = data[i * BLOCK_SIZE_BYTES: (i + 1) * BLOCK_SIZE_BYTES]
+ block += [0] * (BLOCK_SIZE_BYTES - len(block))
+
+ cipher_counter_block = aes_encrypt(counter_block, expanded_key)
+ decrypted_data += xor(block, cipher_counter_block)
+ decrypted_data = decrypted_data[:len(data)]
+
+ return decrypted_data
+
+
+def aes_cbc_decrypt(data, key, iv):
+ """
+ Decrypt with aes in CBC mode
+
+ @param {int[]} data cipher
+ @param {int[]} key 16/24/32-Byte cipher key
+ @param {int[]} iv 16-Byte IV
+ @returns {int[]} decrypted data
+ """
+ expanded_key = key_expansion(key)
+ block_count = int(ceil(float(len(data)) / BLOCK_SIZE_BYTES))
+
+ decrypted_data = []
+ previous_cipher_block = iv
+ for i in range(block_count):
+ block = data[i * BLOCK_SIZE_BYTES: (i + 1) * BLOCK_SIZE_BYTES]
+ block += [0] * (BLOCK_SIZE_BYTES - len(block))
+
+ decrypted_block = aes_decrypt(block, expanded_key)
+ decrypted_data += xor(decrypted_block, previous_cipher_block)
+ previous_cipher_block = block
+ decrypted_data = decrypted_data[:len(data)]
+
+ return decrypted_data
+
+
+def key_expansion(data):
+ """
+ Generate key schedule
+
+ @param {int[]} data 16/24/32-Byte cipher key
+ @returns {int[]} 176/208/240-Byte expanded key
+ """
+ data = data[:] # copy
+ rcon_iteration = 1
+ key_size_bytes = len(data)
+ expanded_key_size_bytes = (key_size_bytes // 4 + 7) * BLOCK_SIZE_BYTES
+
+ while len(data) < expanded_key_size_bytes:
+ temp = data[-4:]
+ temp = key_schedule_core(temp, rcon_iteration)
+ rcon_iteration += 1
+ data += xor(temp, data[-key_size_bytes: 4 - key_size_bytes])
+
+ for _ in range(3):
+ temp = data[-4:]
+ data += xor(temp, data[-key_size_bytes: 4 - key_size_bytes])
+
+ if key_size_bytes == 32:
+ temp = data[-4:]
+ temp = sub_bytes(temp)
+ data += xor(temp, data[-key_size_bytes: 4 - key_size_bytes])
+
+ for _ in range(3 if key_size_bytes == 32 else 2 if key_size_bytes == 24 else 0):
+ temp = data[-4:]
+ data += xor(temp, data[-key_size_bytes: 4 - key_size_bytes])
+ data = data[:expanded_key_size_bytes]
+
+ return data
+
+
+def aes_encrypt(data, expanded_key):
+ """
+ Encrypt one block with aes
+
+ @param {int[]} data 16-Byte state
+ @param {int[]} expanded_key 176/208/240-Byte expanded key
+ @returns {int[]} 16-Byte cipher
+ """
+ rounds = len(expanded_key) // BLOCK_SIZE_BYTES - 1
+
+ data = xor(data, expanded_key[:BLOCK_SIZE_BYTES])
+ for i in range(1, rounds + 1):
+ data = sub_bytes(data)
+ data = shift_rows(data)
+ if i != rounds:
+ data = mix_columns(data)
+ data = xor(data, expanded_key[i * BLOCK_SIZE_BYTES: (i + 1) * BLOCK_SIZE_BYTES])
+
+ return data
+
+
+def aes_decrypt(data, expanded_key):
+ """
+ Decrypt one block with aes
+
+ @param {int[]} data 16-Byte cipher
+ @param {int[]} expanded_key 176/208/240-Byte expanded key
+ @returns {int[]} 16-Byte state
+ """
+ rounds = len(expanded_key) // BLOCK_SIZE_BYTES - 1
+
+ for i in range(rounds, 0, -1):
+ data = xor(data, expanded_key[i * BLOCK_SIZE_BYTES: (i + 1) * BLOCK_SIZE_BYTES])
+ if i != rounds:
+ data = mix_columns_inv(data)
+ data = shift_rows_inv(data)
+ data = sub_bytes_inv(data)
+ data = xor(data, expanded_key[:BLOCK_SIZE_BYTES])
+
+ return data
+
+
+def aes_decrypt_text(data, password, key_size_bytes):
+ """
+ Decrypt text
+ - The first 8 Bytes of decoded 'data' are the 8 high Bytes of the counter
+ - The cipher key is retrieved by encrypting the first 16 Byte of 'password'
+ with the first 'key_size_bytes' Bytes from 'password' (if necessary filled with 0's)
+ - Mode of operation is 'counter'
+
+ @param {str} data Base64 encoded string
+ @param {str,unicode} password Password (will be encoded with utf-8)
+ @param {int} key_size_bytes Possible values: 16 for 128-Bit, 24 for 192-Bit or 32 for 256-Bit
+ @returns {str} Decrypted data
+ """
+ NONCE_LENGTH_BYTES = 8
+
+ data = bytes_to_intlist(base64.b64decode(data))
+ password = bytes_to_intlist(password.encode('utf-8'))
+
+ key = password[:key_size_bytes] + [0] * (key_size_bytes - len(password))
+ key = aes_encrypt(key[:BLOCK_SIZE_BYTES], key_expansion(key)) * (key_size_bytes // BLOCK_SIZE_BYTES)
+
+ nonce = data[:NONCE_LENGTH_BYTES]
+ cipher = data[NONCE_LENGTH_BYTES:]
+
+ class Counter:
+ __value = nonce + [0] * (BLOCK_SIZE_BYTES - NONCE_LENGTH_BYTES)
+
+ def next_value(self):
+ temp = self.__value
+ self.__value = inc(self.__value)
+ return temp
+
+ decrypted_data = aes_ctr_decrypt(cipher, key, Counter())
+ plaintext = intlist_to_bytes(decrypted_data)
+
+ return plaintext
+
+RCON = (0x8d, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1b, 0x36)
+SBOX = (0x63, 0x7C, 0x77, 0x7B, 0xF2, 0x6B, 0x6F, 0xC5, 0x30, 0x01, 0x67, 0x2B, 0xFE, 0xD7, 0xAB, 0x76,
+ 0xCA, 0x82, 0xC9, 0x7D, 0xFA, 0x59, 0x47, 0xF0, 0xAD, 0xD4, 0xA2, 0xAF, 0x9C, 0xA4, 0x72, 0xC0,
+ 0xB7, 0xFD, 0x93, 0x26, 0x36, 0x3F, 0xF7, 0xCC, 0x34, 0xA5, 0xE5, 0xF1, 0x71, 0xD8, 0x31, 0x15,
+ 0x04, 0xC7, 0x23, 0xC3, 0x18, 0x96, 0x05, 0x9A, 0x07, 0x12, 0x80, 0xE2, 0xEB, 0x27, 0xB2, 0x75,
+ 0x09, 0x83, 0x2C, 0x1A, 0x1B, 0x6E, 0x5A, 0xA0, 0x52, 0x3B, 0xD6, 0xB3, 0x29, 0xE3, 0x2F, 0x84,
+ 0x53, 0xD1, 0x00, 0xED, 0x20, 0xFC, 0xB1, 0x5B, 0x6A, 0xCB, 0xBE, 0x39, 0x4A, 0x4C, 0x58, 0xCF,
+ 0xD0, 0xEF, 0xAA, 0xFB, 0x43, 0x4D, 0x33, 0x85, 0x45, 0xF9, 0x02, 0x7F, 0x50, 0x3C, 0x9F, 0xA8,
+ 0x51, 0xA3, 0x40, 0x8F, 0x92, 0x9D, 0x38, 0xF5, 0xBC, 0xB6, 0xDA, 0x21, 0x10, 0xFF, 0xF3, 0xD2,
+ 0xCD, 0x0C, 0x13, 0xEC, 0x5F, 0x97, 0x44, 0x17, 0xC4, 0xA7, 0x7E, 0x3D, 0x64, 0x5D, 0x19, 0x73,
+ 0x60, 0x81, 0x4F, 0xDC, 0x22, 0x2A, 0x90, 0x88, 0x46, 0xEE, 0xB8, 0x14, 0xDE, 0x5E, 0x0B, 0xDB,
+ 0xE0, 0x32, 0x3A, 0x0A, 0x49, 0x06, 0x24, 0x5C, 0xC2, 0xD3, 0xAC, 0x62, 0x91, 0x95, 0xE4, 0x79,
+ 0xE7, 0xC8, 0x37, 0x6D, 0x8D, 0xD5, 0x4E, 0xA9, 0x6C, 0x56, 0xF4, 0xEA, 0x65, 0x7A, 0xAE, 0x08,
+ 0xBA, 0x78, 0x25, 0x2E, 0x1C, 0xA6, 0xB4, 0xC6, 0xE8, 0xDD, 0x74, 0x1F, 0x4B, 0xBD, 0x8B, 0x8A,
+ 0x70, 0x3E, 0xB5, 0x66, 0x48, 0x03, 0xF6, 0x0E, 0x61, 0x35, 0x57, 0xB9, 0x86, 0xC1, 0x1D, 0x9E,
+ 0xE1, 0xF8, 0x98, 0x11, 0x69, 0xD9, 0x8E, 0x94, 0x9B, 0x1E, 0x87, 0xE9, 0xCE, 0x55, 0x28, 0xDF,
+ 0x8C, 0xA1, 0x89, 0x0D, 0xBF, 0xE6, 0x42, 0x68, 0x41, 0x99, 0x2D, 0x0F, 0xB0, 0x54, 0xBB, 0x16)
+SBOX_INV = (0x52, 0x09, 0x6a, 0xd5, 0x30, 0x36, 0xa5, 0x38, 0xbf, 0x40, 0xa3, 0x9e, 0x81, 0xf3, 0xd7, 0xfb,
+ 0x7c, 0xe3, 0x39, 0x82, 0x9b, 0x2f, 0xff, 0x87, 0x34, 0x8e, 0x43, 0x44, 0xc4, 0xde, 0xe9, 0xcb,
+ 0x54, 0x7b, 0x94, 0x32, 0xa6, 0xc2, 0x23, 0x3d, 0xee, 0x4c, 0x95, 0x0b, 0x42, 0xfa, 0xc3, 0x4e,
+ 0x08, 0x2e, 0xa1, 0x66, 0x28, 0xd9, 0x24, 0xb2, 0x76, 0x5b, 0xa2, 0x49, 0x6d, 0x8b, 0xd1, 0x25,
+ 0x72, 0xf8, 0xf6, 0x64, 0x86, 0x68, 0x98, 0x16, 0xd4, 0xa4, 0x5c, 0xcc, 0x5d, 0x65, 0xb6, 0x92,
+ 0x6c, 0x70, 0x48, 0x50, 0xfd, 0xed, 0xb9, 0xda, 0x5e, 0x15, 0x46, 0x57, 0xa7, 0x8d, 0x9d, 0x84,
+ 0x90, 0xd8, 0xab, 0x00, 0x8c, 0xbc, 0xd3, 0x0a, 0xf7, 0xe4, 0x58, 0x05, 0xb8, 0xb3, 0x45, 0x06,
+ 0xd0, 0x2c, 0x1e, 0x8f, 0xca, 0x3f, 0x0f, 0x02, 0xc1, 0xaf, 0xbd, 0x03, 0x01, 0x13, 0x8a, 0x6b,
+ 0x3a, 0x91, 0x11, 0x41, 0x4f, 0x67, 0xdc, 0xea, 0x97, 0xf2, 0xcf, 0xce, 0xf0, 0xb4, 0xe6, 0x73,
+ 0x96, 0xac, 0x74, 0x22, 0xe7, 0xad, 0x35, 0x85, 0xe2, 0xf9, 0x37, 0xe8, 0x1c, 0x75, 0xdf, 0x6e,
+ 0x47, 0xf1, 0x1a, 0x71, 0x1d, 0x29, 0xc5, 0x89, 0x6f, 0xb7, 0x62, 0x0e, 0xaa, 0x18, 0xbe, 0x1b,
+ 0xfc, 0x56, 0x3e, 0x4b, 0xc6, 0xd2, 0x79, 0x20, 0x9a, 0xdb, 0xc0, 0xfe, 0x78, 0xcd, 0x5a, 0xf4,
+ 0x1f, 0xdd, 0xa8, 0x33, 0x88, 0x07, 0xc7, 0x31, 0xb1, 0x12, 0x10, 0x59, 0x27, 0x80, 0xec, 0x5f,
+ 0x60, 0x51, 0x7f, 0xa9, 0x19, 0xb5, 0x4a, 0x0d, 0x2d, 0xe5, 0x7a, 0x9f, 0x93, 0xc9, 0x9c, 0xef,
+ 0xa0, 0xe0, 0x3b, 0x4d, 0xae, 0x2a, 0xf5, 0xb0, 0xc8, 0xeb, 0xbb, 0x3c, 0x83, 0x53, 0x99, 0x61,
+ 0x17, 0x2b, 0x04, 0x7e, 0xba, 0x77, 0xd6, 0x26, 0xe1, 0x69, 0x14, 0x63, 0x55, 0x21, 0x0c, 0x7d)
+MIX_COLUMN_MATRIX = ((0x2, 0x3, 0x1, 0x1),
+ (0x1, 0x2, 0x3, 0x1),
+ (0x1, 0x1, 0x2, 0x3),
+ (0x3, 0x1, 0x1, 0x2))
+MIX_COLUMN_MATRIX_INV = ((0xE, 0xB, 0xD, 0x9),
+ (0x9, 0xE, 0xB, 0xD),
+ (0xD, 0x9, 0xE, 0xB),
+ (0xB, 0xD, 0x9, 0xE))
+RIJNDAEL_EXP_TABLE = (0x01, 0x03, 0x05, 0x0F, 0x11, 0x33, 0x55, 0xFF, 0x1A, 0x2E, 0x72, 0x96, 0xA1, 0xF8, 0x13, 0x35,
+ 0x5F, 0xE1, 0x38, 0x48, 0xD8, 0x73, 0x95, 0xA4, 0xF7, 0x02, 0x06, 0x0A, 0x1E, 0x22, 0x66, 0xAA,
+ 0xE5, 0x34, 0x5C, 0xE4, 0x37, 0x59, 0xEB, 0x26, 0x6A, 0xBE, 0xD9, 0x70, 0x90, 0xAB, 0xE6, 0x31,
+ 0x53, 0xF5, 0x04, 0x0C, 0x14, 0x3C, 0x44, 0xCC, 0x4F, 0xD1, 0x68, 0xB8, 0xD3, 0x6E, 0xB2, 0xCD,
+ 0x4C, 0xD4, 0x67, 0xA9, 0xE0, 0x3B, 0x4D, 0xD7, 0x62, 0xA6, 0xF1, 0x08, 0x18, 0x28, 0x78, 0x88,
+ 0x83, 0x9E, 0xB9, 0xD0, 0x6B, 0xBD, 0xDC, 0x7F, 0x81, 0x98, 0xB3, 0xCE, 0x49, 0xDB, 0x76, 0x9A,
+ 0xB5, 0xC4, 0x57, 0xF9, 0x10, 0x30, 0x50, 0xF0, 0x0B, 0x1D, 0x27, 0x69, 0xBB, 0xD6, 0x61, 0xA3,
+ 0xFE, 0x19, 0x2B, 0x7D, 0x87, 0x92, 0xAD, 0xEC, 0x2F, 0x71, 0x93, 0xAE, 0xE9, 0x20, 0x60, 0xA0,
+ 0xFB, 0x16, 0x3A, 0x4E, 0xD2, 0x6D, 0xB7, 0xC2, 0x5D, 0xE7, 0x32, 0x56, 0xFA, 0x15, 0x3F, 0x41,
+ 0xC3, 0x5E, 0xE2, 0x3D, 0x47, 0xC9, 0x40, 0xC0, 0x5B, 0xED, 0x2C, 0x74, 0x9C, 0xBF, 0xDA, 0x75,
+ 0x9F, 0xBA, 0xD5, 0x64, 0xAC, 0xEF, 0x2A, 0x7E, 0x82, 0x9D, 0xBC, 0xDF, 0x7A, 0x8E, 0x89, 0x80,
+ 0x9B, 0xB6, 0xC1, 0x58, 0xE8, 0x23, 0x65, 0xAF, 0xEA, 0x25, 0x6F, 0xB1, 0xC8, 0x43, 0xC5, 0x54,
+ 0xFC, 0x1F, 0x21, 0x63, 0xA5, 0xF4, 0x07, 0x09, 0x1B, 0x2D, 0x77, 0x99, 0xB0, 0xCB, 0x46, 0xCA,
+ 0x45, 0xCF, 0x4A, 0xDE, 0x79, 0x8B, 0x86, 0x91, 0xA8, 0xE3, 0x3E, 0x42, 0xC6, 0x51, 0xF3, 0x0E,
+ 0x12, 0x36, 0x5A, 0xEE, 0x29, 0x7B, 0x8D, 0x8C, 0x8F, 0x8A, 0x85, 0x94, 0xA7, 0xF2, 0x0D, 0x17,
+ 0x39, 0x4B, 0xDD, 0x7C, 0x84, 0x97, 0xA2, 0xFD, 0x1C, 0x24, 0x6C, 0xB4, 0xC7, 0x52, 0xF6, 0x01)
+RIJNDAEL_LOG_TABLE = (0x00, 0x00, 0x19, 0x01, 0x32, 0x02, 0x1a, 0xc6, 0x4b, 0xc7, 0x1b, 0x68, 0x33, 0xee, 0xdf, 0x03,
+ 0x64, 0x04, 0xe0, 0x0e, 0x34, 0x8d, 0x81, 0xef, 0x4c, 0x71, 0x08, 0xc8, 0xf8, 0x69, 0x1c, 0xc1,
+ 0x7d, 0xc2, 0x1d, 0xb5, 0xf9, 0xb9, 0x27, 0x6a, 0x4d, 0xe4, 0xa6, 0x72, 0x9a, 0xc9, 0x09, 0x78,
+ 0x65, 0x2f, 0x8a, 0x05, 0x21, 0x0f, 0xe1, 0x24, 0x12, 0xf0, 0x82, 0x45, 0x35, 0x93, 0xda, 0x8e,
+ 0x96, 0x8f, 0xdb, 0xbd, 0x36, 0xd0, 0xce, 0x94, 0x13, 0x5c, 0xd2, 0xf1, 0x40, 0x46, 0x83, 0x38,
+ 0x66, 0xdd, 0xfd, 0x30, 0xbf, 0x06, 0x8b, 0x62, 0xb3, 0x25, 0xe2, 0x98, 0x22, 0x88, 0x91, 0x10,
+ 0x7e, 0x6e, 0x48, 0xc3, 0xa3, 0xb6, 0x1e, 0x42, 0x3a, 0x6b, 0x28, 0x54, 0xfa, 0x85, 0x3d, 0xba,
+ 0x2b, 0x79, 0x0a, 0x15, 0x9b, 0x9f, 0x5e, 0xca, 0x4e, 0xd4, 0xac, 0xe5, 0xf3, 0x73, 0xa7, 0x57,
+ 0xaf, 0x58, 0xa8, 0x50, 0xf4, 0xea, 0xd6, 0x74, 0x4f, 0xae, 0xe9, 0xd5, 0xe7, 0xe6, 0xad, 0xe8,
+ 0x2c, 0xd7, 0x75, 0x7a, 0xeb, 0x16, 0x0b, 0xf5, 0x59, 0xcb, 0x5f, 0xb0, 0x9c, 0xa9, 0x51, 0xa0,
+ 0x7f, 0x0c, 0xf6, 0x6f, 0x17, 0xc4, 0x49, 0xec, 0xd8, 0x43, 0x1f, 0x2d, 0xa4, 0x76, 0x7b, 0xb7,
+ 0xcc, 0xbb, 0x3e, 0x5a, 0xfb, 0x60, 0xb1, 0x86, 0x3b, 0x52, 0xa1, 0x6c, 0xaa, 0x55, 0x29, 0x9d,
+ 0x97, 0xb2, 0x87, 0x90, 0x61, 0xbe, 0xdc, 0xfc, 0xbc, 0x95, 0xcf, 0xcd, 0x37, 0x3f, 0x5b, 0xd1,
+ 0x53, 0x39, 0x84, 0x3c, 0x41, 0xa2, 0x6d, 0x47, 0x14, 0x2a, 0x9e, 0x5d, 0x56, 0xf2, 0xd3, 0xab,
+ 0x44, 0x11, 0x92, 0xd9, 0x23, 0x20, 0x2e, 0x89, 0xb4, 0x7c, 0xb8, 0x26, 0x77, 0x99, 0xe3, 0xa5,
+ 0x67, 0x4a, 0xed, 0xde, 0xc5, 0x31, 0xfe, 0x18, 0x0d, 0x63, 0x8c, 0x80, 0xc0, 0xf7, 0x70, 0x07)
+
+
+def sub_bytes(data):
+ return [SBOX[x] for x in data]
+
+
+def sub_bytes_inv(data):
+ return [SBOX_INV[x] for x in data]
+
+
+def rotate(data):
+ return data[1:] + [data[0]]
+
+
+def key_schedule_core(data, rcon_iteration):
+ data = rotate(data)
+ data = sub_bytes(data)
+ data[0] = data[0] ^ RCON[rcon_iteration]
+
+ return data
+
+
+def xor(data1, data2):
+ return [x ^ y for x, y in zip(data1, data2)]
+
+
+def rijndael_mul(a, b):
+ if(a == 0 or b == 0):
+ return 0
+ return RIJNDAEL_EXP_TABLE[(RIJNDAEL_LOG_TABLE[a] + RIJNDAEL_LOG_TABLE[b]) % 0xFF]
+
+
+def mix_column(data, matrix):
+ data_mixed = []
+ for row in range(4):
+ mixed = 0
+ for column in range(4):
+ # xor is (+) and (-)
+ mixed ^= rijndael_mul(data[column], matrix[row][column])
+ data_mixed.append(mixed)
+ return data_mixed
+
+
+def mix_columns(data, matrix=MIX_COLUMN_MATRIX):
+ data_mixed = []
+ for i in range(4):
+ column = data[i * 4: (i + 1) * 4]
+ data_mixed += mix_column(column, matrix)
+ return data_mixed
+
+
+def mix_columns_inv(data):
+ return mix_columns(data, MIX_COLUMN_MATRIX_INV)
+
+
+def shift_rows(data):
+ data_shifted = []
+ for column in range(4):
+ for row in range(4):
+ data_shifted.append(data[((column + row) & 0b11) * 4 + row])
+ return data_shifted
+
+
+def shift_rows_inv(data):
+ data_shifted = []
+ for column in range(4):
+ for row in range(4):
+ data_shifted.append(data[((column - row) & 0b11) * 4 + row])
+ return data_shifted
+
+
+def inc(data):
+ data = data[:] # copy
+ for i in range(len(data) - 1, -1, -1):
+ if data[i] == 255:
+ data[i] = 0
+ else:
+ data[i] = data[i] + 1
+ break
+ return data
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/cache.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/cache.py
new file mode 100644
index 0000000000..5fe839eb12
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/cache.py
@@ -0,0 +1,93 @@
+from __future__ import unicode_literals
+
+import errno
+import io
+import json
+import os
+import re
+import shutil
+import traceback
+
+from .compat import compat_expanduser, compat_getenv
+from .utils import write_json_file
+
+
+class Cache(object):
+ def __init__(self, ydl):
+ self._ydl = ydl
+
+ def _get_root_dir(self):
+ res = self._ydl.params.get('cachedir')
+ if res is None:
+ cache_root = compat_getenv('XDG_CACHE_HOME', '~/.cache')
+ res = os.path.join(cache_root, 'youtube-dl')
+ return compat_expanduser(res)
+
+ def _get_cache_fn(self, section, key, dtype):
+ assert re.match(r'^[a-zA-Z0-9_.-]+$', section), \
+ 'invalid section %r' % section
+ assert re.match(r'^[a-zA-Z0-9_.-]+$', key), 'invalid key %r' % key
+ return os.path.join(
+ self._get_root_dir(), section, '%s.%s' % (key, dtype))
+
+ @property
+ def enabled(self):
+ return self._ydl.params.get('cachedir') is not False
+
+ def store(self, section, key, data, dtype='json'):
+ assert dtype in ('json',)
+
+ if not self.enabled:
+ return
+
+ fn = self._get_cache_fn(section, key, dtype)
+ try:
+ try:
+ os.makedirs(os.path.dirname(fn))
+ except OSError as ose:
+ if ose.errno != errno.EEXIST:
+ raise
+ write_json_file(data, fn)
+ except Exception:
+ tb = traceback.format_exc()
+ self._ydl.report_warning(
+ 'Writing cache to %r failed: %s' % (fn, tb))
+
+ def load(self, section, key, dtype='json', default=None):
+ assert dtype in ('json',)
+
+ if not self.enabled:
+ return default
+
+ cache_fn = self._get_cache_fn(section, key, dtype)
+ try:
+ try:
+ with io.open(cache_fn, 'r', encoding='utf-8') as cachef:
+ return json.load(cachef)
+ except ValueError:
+ try:
+ file_size = os.path.getsize(cache_fn)
+ except (OSError, IOError) as oe:
+ file_size = str(oe)
+ self._ydl.report_warning(
+ 'Cache retrieval from %s failed (%s)' % (cache_fn, file_size))
+ except IOError:
+ pass # No cache available
+
+ return default
+
+ def remove(self):
+ if not self.enabled:
+ self._ydl.to_screen('Cache is disabled (Did you combine --no-cache-dir and --rm-cache-dir?)')
+ return
+
+ cachedir = self._get_root_dir()
+ if not any((term in cachedir) for term in ('cache', 'tmp')):
+ raise Exception('Not removing directory %s - this does not look like a cache dir' % cachedir)
+
+ self._ydl.to_screen(
+ 'Removing cache dir %s .' % cachedir, skip_eol=True)
+ if os.path.exists(cachedir):
+ self._ydl.to_screen('.', skip_eol=True)
+ shutil.rmtree(cachedir)
+ self._ydl.to_screen('.')
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/compat.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/compat.py
new file mode 100644
index 0000000000..46d4388467
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/compat.py
@@ -0,0 +1,358 @@
+from __future__ import unicode_literals
+
+import getpass
+import optparse
+import os
+import re
+import subprocess
+import sys
+
+
+try:
+ import urllib.request as compat_urllib_request
+except ImportError: # Python 2
+ import urllib2 as compat_urllib_request
+
+try:
+ import urllib.error as compat_urllib_error
+except ImportError: # Python 2
+ import urllib2 as compat_urllib_error
+
+try:
+ import urllib.parse as compat_urllib_parse
+except ImportError: # Python 2
+ import urllib as compat_urllib_parse
+
+try:
+ from urllib.parse import urlparse as compat_urllib_parse_urlparse
+except ImportError: # Python 2
+ from urlparse import urlparse as compat_urllib_parse_urlparse
+
+try:
+ import urllib.parse as compat_urlparse
+except ImportError: # Python 2
+ import urlparse as compat_urlparse
+
+try:
+ import http.cookiejar as compat_cookiejar
+except ImportError: # Python 2
+ import cookielib as compat_cookiejar
+
+try:
+ import html.entities as compat_html_entities
+except ImportError: # Python 2
+ import htmlentitydefs as compat_html_entities
+
+try:
+ import html.parser as compat_html_parser
+except ImportError: # Python 2
+ import HTMLParser as compat_html_parser
+
+try:
+ import http.client as compat_http_client
+except ImportError: # Python 2
+ import httplib as compat_http_client
+
+try:
+ from urllib.error import HTTPError as compat_HTTPError
+except ImportError: # Python 2
+ from urllib2 import HTTPError as compat_HTTPError
+
+try:
+ from urllib.request import urlretrieve as compat_urlretrieve
+except ImportError: # Python 2
+ from urllib import urlretrieve as compat_urlretrieve
+
+
+try:
+ from subprocess import DEVNULL
+ compat_subprocess_get_DEVNULL = lambda: DEVNULL
+except ImportError:
+ compat_subprocess_get_DEVNULL = lambda: open(os.path.devnull, 'w')
+
+try:
+ from urllib.parse import unquote as compat_urllib_parse_unquote
+except ImportError:
+ def compat_urllib_parse_unquote(string, encoding='utf-8', errors='replace'):
+ if string == '':
+ return string
+ res = string.split('%')
+ if len(res) == 1:
+ return string
+ if encoding is None:
+ encoding = 'utf-8'
+ if errors is None:
+ errors = 'replace'
+ # pct_sequence: contiguous sequence of percent-encoded bytes, decoded
+ pct_sequence = b''
+ string = res[0]
+ for item in res[1:]:
+ try:
+ if not item:
+ raise ValueError
+ pct_sequence += item[:2].decode('hex')
+ rest = item[2:]
+ if not rest:
+ # This segment was just a single percent-encoded character.
+ # May be part of a sequence of code units, so delay decoding.
+ # (Stored in pct_sequence).
+ continue
+ except ValueError:
+ rest = '%' + item
+ # Encountered non-percent-encoded characters. Flush the current
+ # pct_sequence.
+ string += pct_sequence.decode(encoding, errors) + rest
+ pct_sequence = b''
+ if pct_sequence:
+ # Flush the final pct_sequence
+ string += pct_sequence.decode(encoding, errors)
+ return string
+
+
+try:
+ from urllib.parse import parse_qs as compat_parse_qs
+except ImportError: # Python 2
+ # HACK: The following is the correct parse_qs implementation from cpython 3's stdlib.
+ # Python 2's version is apparently totally broken
+
+ def _parse_qsl(qs, keep_blank_values=False, strict_parsing=False,
+ encoding='utf-8', errors='replace'):
+ qs, _coerce_result = qs, unicode
+ pairs = [s2 for s1 in qs.split('&') for s2 in s1.split(';')]
+ r = []
+ for name_value in pairs:
+ if not name_value and not strict_parsing:
+ continue
+ nv = name_value.split('=', 1)
+ if len(nv) != 2:
+ if strict_parsing:
+ raise ValueError("bad query field: %r" % (name_value,))
+ # Handle case of a control-name with no equal sign
+ if keep_blank_values:
+ nv.append('')
+ else:
+ continue
+ if len(nv[1]) or keep_blank_values:
+ name = nv[0].replace('+', ' ')
+ name = compat_urllib_parse_unquote(
+ name, encoding=encoding, errors=errors)
+ name = _coerce_result(name)
+ value = nv[1].replace('+', ' ')
+ value = compat_urllib_parse_unquote(
+ value, encoding=encoding, errors=errors)
+ value = _coerce_result(value)
+ r.append((name, value))
+ return r
+
+ def compat_parse_qs(qs, keep_blank_values=False, strict_parsing=False,
+ encoding='utf-8', errors='replace'):
+ parsed_result = {}
+ pairs = _parse_qsl(qs, keep_blank_values, strict_parsing,
+ encoding=encoding, errors=errors)
+ for name, value in pairs:
+ if name in parsed_result:
+ parsed_result[name].append(value)
+ else:
+ parsed_result[name] = [value]
+ return parsed_result
+
+try:
+ compat_str = unicode # Python 2
+except NameError:
+ compat_str = str
+
+try:
+ compat_chr = unichr # Python 2
+except NameError:
+ compat_chr = chr
+
+try:
+ from xml.etree.ElementTree import ParseError as compat_xml_parse_error
+except ImportError: # Python 2.6
+ from xml.parsers.expat import ExpatError as compat_xml_parse_error
+
+try:
+ from shlex import quote as shlex_quote
+except ImportError: # Python < 3.3
+ def shlex_quote(s):
+ if re.match(r'^[-_\w./]+$', s):
+ return s
+ else:
+ return "'" + s.replace("'", "'\"'\"'") + "'"
+
+
+def compat_ord(c):
+ if type(c) is int:
+ return c
+ else:
+ return ord(c)
+
+
+if sys.version_info >= (3, 0):
+ compat_getenv = os.getenv
+ compat_expanduser = os.path.expanduser
+else:
+ # Environment variables should be decoded with filesystem encoding.
+ # Otherwise it will fail if any non-ASCII characters present (see #3854 #3217 #2918)
+
+ def compat_getenv(key, default=None):
+ from .utils import get_filesystem_encoding
+ env = os.getenv(key, default)
+ if env:
+ env = env.decode(get_filesystem_encoding())
+ return env
+
+ # HACK: The default implementations of os.path.expanduser from cpython do not decode
+ # environment variables with filesystem encoding. We will work around this by
+ # providing adjusted implementations.
+ # The following are os.path.expanduser implementations from cpython 2.7.8 stdlib
+ # for different platforms with correct environment variables decoding.
+
+ if os.name == 'posix':
+ def compat_expanduser(path):
+ """Expand ~ and ~user constructions. If user or $HOME is unknown,
+ do nothing."""
+ if not path.startswith('~'):
+ return path
+ i = path.find('/', 1)
+ if i < 0:
+ i = len(path)
+ if i == 1:
+ if 'HOME' not in os.environ:
+ import pwd
+ userhome = pwd.getpwuid(os.getuid()).pw_dir
+ else:
+ userhome = compat_getenv('HOME')
+ else:
+ import pwd
+ try:
+ pwent = pwd.getpwnam(path[1:i])
+ except KeyError:
+ return path
+ userhome = pwent.pw_dir
+ userhome = userhome.rstrip('/')
+ return (userhome + path[i:]) or '/'
+ elif os.name == 'nt' or os.name == 'ce':
+ def compat_expanduser(path):
+ """Expand ~ and ~user constructs.
+
+ If user or $HOME is unknown, do nothing."""
+ if path[:1] != '~':
+ return path
+ i, n = 1, len(path)
+ while i < n and path[i] not in '/\\':
+ i = i + 1
+
+ if 'HOME' in os.environ:
+ userhome = compat_getenv('HOME')
+ elif 'USERPROFILE' in os.environ:
+ userhome = compat_getenv('USERPROFILE')
+ elif 'HOMEPATH' not in os.environ:
+ return path
+ else:
+ try:
+ drive = compat_getenv('HOMEDRIVE')
+ except KeyError:
+ drive = ''
+ userhome = os.path.join(drive, compat_getenv('HOMEPATH'))
+
+ if i != 1: # ~user
+ userhome = os.path.join(os.path.dirname(userhome), path[1:i])
+
+ return userhome + path[i:]
+ else:
+ compat_expanduser = os.path.expanduser
+
+
+if sys.version_info < (3, 0):
+ def compat_print(s):
+ from .utils import preferredencoding
+ print(s.encode(preferredencoding(), 'xmlcharrefreplace'))
+else:
+ def compat_print(s):
+ assert isinstance(s, compat_str)
+ print(s)
+
+
+try:
+ subprocess_check_output = subprocess.check_output
+except AttributeError:
+ def subprocess_check_output(*args, **kwargs):
+ assert 'input' not in kwargs
+ p = subprocess.Popen(*args, stdout=subprocess.PIPE, **kwargs)
+ output, _ = p.communicate()
+ ret = p.poll()
+ if ret:
+ raise subprocess.CalledProcessError(ret, p.args, output=output)
+ return output
+
+if sys.version_info < (3, 0) and sys.platform == 'win32':
+ def compat_getpass(prompt, *args, **kwargs):
+ if isinstance(prompt, compat_str):
+ from .utils import preferredencoding
+ prompt = prompt.encode(preferredencoding())
+ return getpass.getpass(prompt, *args, **kwargs)
+else:
+ compat_getpass = getpass.getpass
+
+# Old 2.6 and 2.7 releases require kwargs to be bytes
+try:
+ def _testfunc(x):
+ pass
+ _testfunc(**{'x': 0})
+except TypeError:
+ def compat_kwargs(kwargs):
+ return dict((bytes(k), v) for k, v in kwargs.items())
+else:
+ compat_kwargs = lambda kwargs: kwargs
+
+
+# Fix https://github.com/rg3/youtube-dl/issues/4223
+# See http://bugs.python.org/issue9161 for what is broken
+def workaround_optparse_bug9161():
+ op = optparse.OptionParser()
+ og = optparse.OptionGroup(op, 'foo')
+ try:
+ og.add_option('-t')
+ except TypeError:
+ real_add_option = optparse.OptionGroup.add_option
+
+ def _compat_add_option(self, *args, **kwargs):
+ enc = lambda v: (
+ v.encode('ascii', 'replace') if isinstance(v, compat_str)
+ else v)
+ bargs = [enc(a) for a in args]
+ bkwargs = dict(
+ (k, enc(v)) for k, v in kwargs.items())
+ return real_add_option(self, *bargs, **bkwargs)
+ optparse.OptionGroup.add_option = _compat_add_option
+
+
+__all__ = [
+ 'compat_HTTPError',
+ 'compat_chr',
+ 'compat_cookiejar',
+ 'compat_expanduser',
+ 'compat_getenv',
+ 'compat_getpass',
+ 'compat_html_entities',
+ 'compat_html_parser',
+ 'compat_http_client',
+ 'compat_kwargs',
+ 'compat_ord',
+ 'compat_parse_qs',
+ 'compat_print',
+ 'compat_str',
+ 'compat_subprocess_get_DEVNULL',
+ 'compat_urllib_error',
+ 'compat_urllib_parse',
+ 'compat_urllib_parse_unquote',
+ 'compat_urllib_parse_urlparse',
+ 'compat_urllib_request',
+ 'compat_urlparse',
+ 'compat_urlretrieve',
+ 'compat_xml_parse_error',
+ 'shlex_quote',
+ 'subprocess_check_output',
+ 'workaround_optparse_bug9161',
+]
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/downloader/__init__.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/downloader/__init__.py
new file mode 100644
index 0000000000..31e28df58e
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/downloader/__init__.py
@@ -0,0 +1,37 @@
+from __future__ import unicode_literals
+
+from .common import FileDownloader
+from .hls import HlsFD
+from .hls import NativeHlsFD
+from .http import HttpFD
+from .mplayer import MplayerFD
+from .rtmp import RtmpFD
+from .f4m import F4mFD
+
+from ..utils import (
+ determine_ext,
+)
+
+
+def get_suitable_downloader(info_dict):
+ """Get the downloader class that can handle the info dict."""
+ url = info_dict['url']
+ protocol = info_dict.get('protocol')
+
+ if url.startswith('rtmp'):
+ return RtmpFD
+ if protocol == 'm3u8_native':
+ return NativeHlsFD
+ if (protocol == 'm3u8') or (protocol is None and determine_ext(url) == 'm3u8'):
+ return HlsFD
+ if url.startswith('mms') or url.startswith('rtsp'):
+ return MplayerFD
+ if determine_ext(url) == 'f4m':
+ return F4mFD
+ else:
+ return HttpFD
+
+__all__ = [
+ 'get_suitable_downloader',
+ 'FileDownloader',
+]
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/downloader/common.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/downloader/common.py
new file mode 100644
index 0000000000..584bde732f
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/downloader/common.py
@@ -0,0 +1,310 @@
+from __future__ import unicode_literals
+
+import os
+import re
+import sys
+import time
+
+from ..compat import compat_str
+from ..utils import (
+ encodeFilename,
+ format_bytes,
+ timeconvert,
+)
+
+
+class FileDownloader(object):
+ """File Downloader class.
+
+ File downloader objects are the ones responsible of downloading the
+ actual video file and writing it to disk.
+
+ File downloaders accept a lot of parameters. In order not to saturate
+ the object constructor with arguments, it receives a dictionary of
+ options instead.
+
+ Available options:
+
+ verbose: Print additional info to stdout.
+ quiet: Do not print messages to stdout.
+ ratelimit: Download speed limit, in bytes/sec.
+ retries: Number of times to retry for HTTP error 5xx
+ buffersize: Size of download buffer in bytes.
+ noresizebuffer: Do not automatically resize the download buffer.
+ continuedl: Try to continue downloads if possible.
+ noprogress: Do not print the progress bar.
+ logtostderr: Log messages to stderr instead of stdout.
+ consoletitle: Display progress in console window's titlebar.
+ nopart: Do not use temporary .part files.
+ updatetime: Use the Last-modified header to set output file timestamps.
+ test: Download only first bytes to test the downloader.
+ min_filesize: Skip files smaller than this size
+ max_filesize: Skip files larger than this size
+
+ Subclasses of this one must re-define the real_download method.
+ """
+
+ _TEST_FILE_SIZE = 10241
+ params = None
+
+ def __init__(self, ydl, params):
+ """Create a FileDownloader object with the given options."""
+ self.ydl = ydl
+ self._progress_hooks = []
+ self.params = params
+
+ @staticmethod
+ def format_seconds(seconds):
+ (mins, secs) = divmod(seconds, 60)
+ (hours, mins) = divmod(mins, 60)
+ if hours > 99:
+ return '--:--:--'
+ if hours == 0:
+ return '%02d:%02d' % (mins, secs)
+ else:
+ return '%02d:%02d:%02d' % (hours, mins, secs)
+
+ @staticmethod
+ def calc_percent(byte_counter, data_len):
+ if data_len is None:
+ return None
+ return float(byte_counter) / float(data_len) * 100.0
+
+ @staticmethod
+ def format_percent(percent):
+ if percent is None:
+ return '---.-%'
+ return '%6s' % ('%3.1f%%' % percent)
+
+ @staticmethod
+ def calc_eta(start, now, total, current):
+ if total is None:
+ return None
+ if now is None:
+ now = time.time()
+ dif = now - start
+ if current == 0 or dif < 0.001: # One millisecond
+ return None
+ rate = float(current) / dif
+ return int((float(total) - float(current)) / rate)
+
+ @staticmethod
+ def format_eta(eta):
+ if eta is None:
+ return '--:--'
+ return FileDownloader.format_seconds(eta)
+
+ @staticmethod
+ def calc_speed(start, now, bytes):
+ dif = now - start
+ if bytes == 0 or dif < 0.001: # One millisecond
+ return None
+ return float(bytes) / dif
+
+ @staticmethod
+ def format_speed(speed):
+ if speed is None:
+ return '%10s' % '---b/s'
+ return '%10s' % ('%s/s' % format_bytes(speed))
+
+ @staticmethod
+ def best_block_size(elapsed_time, bytes):
+ new_min = max(bytes / 2.0, 1.0)
+ new_max = min(max(bytes * 2.0, 1.0), 4194304) # Do not surpass 4 MB
+ if elapsed_time < 0.001:
+ return int(new_max)
+ rate = bytes / elapsed_time
+ if rate > new_max:
+ return int(new_max)
+ if rate < new_min:
+ return int(new_min)
+ return int(rate)
+
+ @staticmethod
+ def parse_bytes(bytestr):
+ """Parse a string indicating a byte quantity into an integer."""
+ matchobj = re.match(r'(?i)^(\d+(?:\.\d+)?)([kMGTPEZY]?)$', bytestr)
+ if matchobj is None:
+ return None
+ number = float(matchobj.group(1))
+ multiplier = 1024.0 ** 'bkmgtpezy'.index(matchobj.group(2).lower())
+ return int(round(number * multiplier))
+
+ def to_screen(self, *args, **kargs):
+ self.ydl.to_screen(*args, **kargs)
+
+ def to_stderr(self, message):
+ self.ydl.to_screen(message)
+
+ def to_console_title(self, message):
+ self.ydl.to_console_title(message)
+
+ def trouble(self, *args, **kargs):
+ self.ydl.trouble(*args, **kargs)
+
+ def report_warning(self, *args, **kargs):
+ self.ydl.report_warning(*args, **kargs)
+
+ def report_error(self, *args, **kargs):
+ self.ydl.report_error(*args, **kargs)
+
+ def slow_down(self, start_time, now, byte_counter):
+ """Sleep if the download speed is over the rate limit."""
+ rate_limit = self.params.get('ratelimit', None)
+ if rate_limit is None or byte_counter == 0:
+ return
+ if now is None:
+ now = time.time()
+ elapsed = now - start_time
+ if elapsed <= 0.0:
+ return
+ speed = float(byte_counter) / elapsed
+ if speed > rate_limit:
+ time.sleep(max((byte_counter // rate_limit) - elapsed, 0))
+
+ def temp_name(self, filename):
+ """Returns a temporary filename for the given filename."""
+ if self.params.get('nopart', False) or filename == '-' or \
+ (os.path.exists(encodeFilename(filename)) and not os.path.isfile(encodeFilename(filename))):
+ return filename
+ return filename + '.part'
+
+ def undo_temp_name(self, filename):
+ if filename.endswith('.part'):
+ return filename[:-len('.part')]
+ return filename
+
+ def try_rename(self, old_filename, new_filename):
+ try:
+ if old_filename == new_filename:
+ return
+ os.rename(encodeFilename(old_filename), encodeFilename(new_filename))
+ except (IOError, OSError) as err:
+ self.report_error('unable to rename file: %s' % compat_str(err))
+
+ def try_utime(self, filename, last_modified_hdr):
+ """Try to set the last-modified time of the given file."""
+ if last_modified_hdr is None:
+ return
+ if not os.path.isfile(encodeFilename(filename)):
+ return
+ timestr = last_modified_hdr
+ if timestr is None:
+ return
+ filetime = timeconvert(timestr)
+ if filetime is None:
+ return filetime
+ # Ignore obviously invalid dates
+ if filetime == 0:
+ return
+ try:
+ os.utime(filename, (time.time(), filetime))
+ except:
+ pass
+ return filetime
+
+ def report_destination(self, filename):
+ """Report destination filename."""
+ self.to_screen('[download] Destination: ' + filename)
+
+ def _report_progress_status(self, msg, is_last_line=False):
+ fullmsg = '[download] ' + msg
+ if self.params.get('progress_with_newline', False):
+ self.to_screen(fullmsg)
+ else:
+ if os.name == 'nt':
+ prev_len = getattr(self, '_report_progress_prev_line_length',
+ 0)
+ if prev_len > len(fullmsg):
+ fullmsg += ' ' * (prev_len - len(fullmsg))
+ self._report_progress_prev_line_length = len(fullmsg)
+ clear_line = '\r'
+ else:
+ clear_line = ('\r\x1b[K' if sys.stderr.isatty() else '\r')
+ self.to_screen(clear_line + fullmsg, skip_eol=not is_last_line)
+ self.to_console_title('youtube-dl ' + msg)
+
+ def report_progress(self, percent, data_len_str, speed, eta):
+ """Report download progress."""
+ if self.params.get('noprogress', False):
+ return
+ if eta is not None:
+ eta_str = self.format_eta(eta)
+ else:
+ eta_str = 'Unknown ETA'
+ if percent is not None:
+ percent_str = self.format_percent(percent)
+ else:
+ percent_str = 'Unknown %'
+ speed_str = self.format_speed(speed)
+
+ msg = ('%s of %s at %s ETA %s' %
+ (percent_str, data_len_str, speed_str, eta_str))
+ self._report_progress_status(msg)
+
+ def report_progress_live_stream(self, downloaded_data_len, speed, elapsed):
+ if self.params.get('noprogress', False):
+ return
+ downloaded_str = format_bytes(downloaded_data_len)
+ speed_str = self.format_speed(speed)
+ elapsed_str = FileDownloader.format_seconds(elapsed)
+ msg = '%s at %s (%s)' % (downloaded_str, speed_str, elapsed_str)
+ self._report_progress_status(msg)
+
+ def report_finish(self, data_len_str, tot_time):
+ """Report download finished."""
+ if self.params.get('noprogress', False):
+ self.to_screen('[download] Download completed')
+ else:
+ self._report_progress_status(
+ ('100%% of %s in %s' %
+ (data_len_str, self.format_seconds(tot_time))),
+ is_last_line=True)
+
+ def report_resuming_byte(self, resume_len):
+ """Report attempt to resume at given byte."""
+ self.to_screen('[download] Resuming download at byte %s' % resume_len)
+
+ def report_retry(self, count, retries):
+ """Report retry in case of HTTP error 5xx"""
+ self.to_screen('[download] Got server HTTP error. Retrying (attempt %d of %d)...' % (count, retries))
+
+ def report_file_already_downloaded(self, file_name):
+ """Report file has already been fully downloaded."""
+ try:
+ self.to_screen('[download] %s has already been downloaded' % file_name)
+ except UnicodeEncodeError:
+ self.to_screen('[download] The file has already been downloaded')
+
+ def report_unable_to_resume(self):
+ """Report it was impossible to resume download."""
+ self.to_screen('[download] Unable to resume')
+
+ def download(self, filename, info_dict):
+ """Download to a filename using the info from info_dict
+ Return True on success and False otherwise
+ """
+ # Check file already present
+ if filename != '-' and self.params.get('continuedl', False) and os.path.isfile(encodeFilename(filename)) and not self.params.get('nopart', False):
+ self.report_file_already_downloaded(filename)
+ self._hook_progress({
+ 'filename': filename,
+ 'status': 'finished',
+ 'total_bytes': os.path.getsize(encodeFilename(filename)),
+ })
+ return True
+
+ return self.real_download(filename, info_dict)
+
+ def real_download(self, filename, info_dict):
+ """Real download process. Redefine in subclasses."""
+ raise NotImplementedError('This method must be implemented by subclasses')
+
+ def _hook_progress(self, status):
+ for ph in self._progress_hooks:
+ ph(status)
+
+ def add_progress_hook(self, ph):
+ # See YoutubeDl.py (search for progress_hooks) for a description of
+ # this interface
+ self._progress_hooks.append(ph)
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/downloader/f4m.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/downloader/f4m.py
new file mode 100644
index 0000000000..f9f6f3e734
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/downloader/f4m.py
@@ -0,0 +1,339 @@
+from __future__ import unicode_literals
+
+import base64
+import io
+import itertools
+import os
+import time
+import xml.etree.ElementTree as etree
+
+from .common import FileDownloader
+from .http import HttpFD
+from ..compat import (
+ compat_urlparse,
+)
+from ..utils import (
+ struct_pack,
+ struct_unpack,
+ format_bytes,
+ encodeFilename,
+ sanitize_open,
+ xpath_text,
+)
+
+
+class FlvReader(io.BytesIO):
+ """
+ Reader for Flv files
+ The file format is documented in https://www.adobe.com/devnet/f4v.html
+ """
+
+ # Utility functions for reading numbers and strings
+ def read_unsigned_long_long(self):
+ return struct_unpack('!Q', self.read(8))[0]
+
+ def read_unsigned_int(self):
+ return struct_unpack('!I', self.read(4))[0]
+
+ def read_unsigned_char(self):
+ return struct_unpack('!B', self.read(1))[0]
+
+ def read_string(self):
+ res = b''
+ while True:
+ char = self.read(1)
+ if char == b'\x00':
+ break
+ res += char
+ return res
+
+ def read_box_info(self):
+ """
+ Read a box and return the info as a tuple: (box_size, box_type, box_data)
+ """
+ real_size = size = self.read_unsigned_int()
+ box_type = self.read(4)
+ header_end = 8
+ if size == 1:
+ real_size = self.read_unsigned_long_long()
+ header_end = 16
+ return real_size, box_type, self.read(real_size - header_end)
+
+ def read_asrt(self):
+ # version
+ self.read_unsigned_char()
+ # flags
+ self.read(3)
+ quality_entry_count = self.read_unsigned_char()
+ # QualityEntryCount
+ for i in range(quality_entry_count):
+ self.read_string()
+
+ segment_run_count = self.read_unsigned_int()
+ segments = []
+ for i in range(segment_run_count):
+ first_segment = self.read_unsigned_int()
+ fragments_per_segment = self.read_unsigned_int()
+ segments.append((first_segment, fragments_per_segment))
+
+ return {
+ 'segment_run': segments,
+ }
+
+ def read_afrt(self):
+ # version
+ self.read_unsigned_char()
+ # flags
+ self.read(3)
+ # time scale
+ self.read_unsigned_int()
+
+ quality_entry_count = self.read_unsigned_char()
+ # QualitySegmentUrlModifiers
+ for i in range(quality_entry_count):
+ self.read_string()
+
+ fragments_count = self.read_unsigned_int()
+ fragments = []
+ for i in range(fragments_count):
+ first = self.read_unsigned_int()
+ first_ts = self.read_unsigned_long_long()
+ duration = self.read_unsigned_int()
+ if duration == 0:
+ discontinuity_indicator = self.read_unsigned_char()
+ else:
+ discontinuity_indicator = None
+ fragments.append({
+ 'first': first,
+ 'ts': first_ts,
+ 'duration': duration,
+ 'discontinuity_indicator': discontinuity_indicator,
+ })
+
+ return {
+ 'fragments': fragments,
+ }
+
+ def read_abst(self):
+ # version
+ self.read_unsigned_char()
+ # flags
+ self.read(3)
+
+ self.read_unsigned_int() # BootstrapinfoVersion
+ # Profile,Live,Update,Reserved
+ self.read(1)
+ # time scale
+ self.read_unsigned_int()
+ # CurrentMediaTime
+ self.read_unsigned_long_long()
+ # SmpteTimeCodeOffset
+ self.read_unsigned_long_long()
+
+ self.read_string() # MovieIdentifier
+ server_count = self.read_unsigned_char()
+ # ServerEntryTable
+ for i in range(server_count):
+ self.read_string()
+ quality_count = self.read_unsigned_char()
+ # QualityEntryTable
+ for i in range(quality_count):
+ self.read_string()
+ # DrmData
+ self.read_string()
+ # MetaData
+ self.read_string()
+
+ segments_count = self.read_unsigned_char()
+ segments = []
+ for i in range(segments_count):
+ box_size, box_type, box_data = self.read_box_info()
+ assert box_type == b'asrt'
+ segment = FlvReader(box_data).read_asrt()
+ segments.append(segment)
+ fragments_run_count = self.read_unsigned_char()
+ fragments = []
+ for i in range(fragments_run_count):
+ box_size, box_type, box_data = self.read_box_info()
+ assert box_type == b'afrt'
+ fragments.append(FlvReader(box_data).read_afrt())
+
+ return {
+ 'segments': segments,
+ 'fragments': fragments,
+ }
+
+ def read_bootstrap_info(self):
+ total_size, box_type, box_data = self.read_box_info()
+ assert box_type == b'abst'
+ return FlvReader(box_data).read_abst()
+
+
+def read_bootstrap_info(bootstrap_bytes):
+ return FlvReader(bootstrap_bytes).read_bootstrap_info()
+
+
+def build_fragments_list(boot_info):
+ """ Return a list of (segment, fragment) for each fragment in the video """
+ res = []
+ segment_run_table = boot_info['segments'][0]
+ # I've only found videos with one segment
+ segment_run_entry = segment_run_table['segment_run'][0]
+ n_frags = segment_run_entry[1]
+ fragment_run_entry_table = boot_info['fragments'][0]['fragments']
+ first_frag_number = fragment_run_entry_table[0]['first']
+ for (i, frag_number) in zip(range(1, n_frags + 1), itertools.count(first_frag_number)):
+ res.append((1, frag_number))
+ return res
+
+
+def write_flv_header(stream, metadata):
+ """Writes the FLV header and the metadata to stream"""
+ # FLV header
+ stream.write(b'FLV\x01')
+ stream.write(b'\x05')
+ stream.write(b'\x00\x00\x00\x09')
+ # FLV File body
+ stream.write(b'\x00\x00\x00\x00')
+ # FLVTAG
+ # Script data
+ stream.write(b'\x12')
+ # Size of the metadata with 3 bytes
+ stream.write(struct_pack('!L', len(metadata))[1:])
+ stream.write(b'\x00\x00\x00\x00\x00\x00\x00')
+ stream.write(metadata)
+ # Magic numbers extracted from the output files produced by AdobeHDS.php
+ # (https://github.com/K-S-V/Scripts)
+ stream.write(b'\x00\x00\x01\x73')
+
+
+def _add_ns(prop):
+ return '{http://ns.adobe.com/f4m/1.0}%s' % prop
+
+
+class HttpQuietDownloader(HttpFD):
+ def to_screen(self, *args, **kargs):
+ pass
+
+
+class F4mFD(FileDownloader):
+ """
+ A downloader for f4m manifests or AdobeHDS.
+ """
+
+ def real_download(self, filename, info_dict):
+ man_url = info_dict['url']
+ requested_bitrate = info_dict.get('tbr')
+ self.to_screen('[download] Downloading f4m manifest')
+ manifest = self.ydl.urlopen(man_url).read()
+ self.report_destination(filename)
+ http_dl = HttpQuietDownloader(
+ self.ydl,
+ {
+ 'continuedl': True,
+ 'quiet': True,
+ 'noprogress': True,
+ 'ratelimit': self.params.get('ratelimit', None),
+ 'test': self.params.get('test', False),
+ }
+ )
+
+ doc = etree.fromstring(manifest)
+ formats = [(int(f.attrib.get('bitrate', -1)), f) for f in doc.findall(_add_ns('media'))]
+ if requested_bitrate is None:
+ # get the best format
+ formats = sorted(formats, key=lambda f: f[0])
+ rate, media = formats[-1]
+ else:
+ rate, media = list(filter(
+ lambda f: int(f[0]) == requested_bitrate, formats))[0]
+
+ base_url = compat_urlparse.urljoin(man_url, media.attrib['url'])
+ bootstrap_node = doc.find(_add_ns('bootstrapInfo'))
+ if bootstrap_node.text is None:
+ bootstrap_url = compat_urlparse.urljoin(
+ base_url, bootstrap_node.attrib['url'])
+ bootstrap = self.ydl.urlopen(bootstrap_url).read()
+ else:
+ bootstrap = base64.b64decode(bootstrap_node.text)
+ metadata = base64.b64decode(media.find(_add_ns('metadata')).text)
+ boot_info = read_bootstrap_info(bootstrap)
+
+ fragments_list = build_fragments_list(boot_info)
+ if self.params.get('test', False):
+ # We only download the first fragment
+ fragments_list = fragments_list[:1]
+ total_frags = len(fragments_list)
+ # For some akamai manifests we'll need to add a query to the fragment url
+ akamai_pv = xpath_text(doc, _add_ns('pv-2.0'))
+
+ tmpfilename = self.temp_name(filename)
+ (dest_stream, tmpfilename) = sanitize_open(tmpfilename, 'wb')
+ write_flv_header(dest_stream, metadata)
+
+ # This dict stores the download progress, it's updated by the progress
+ # hook
+ state = {
+ 'downloaded_bytes': 0,
+ 'frag_counter': 0,
+ }
+ start = time.time()
+
+ def frag_progress_hook(status):
+ frag_total_bytes = status.get('total_bytes', 0)
+ estimated_size = (state['downloaded_bytes'] +
+ (total_frags - state['frag_counter']) * frag_total_bytes)
+ if status['status'] == 'finished':
+ state['downloaded_bytes'] += frag_total_bytes
+ state['frag_counter'] += 1
+ progress = self.calc_percent(state['frag_counter'], total_frags)
+ byte_counter = state['downloaded_bytes']
+ else:
+ frag_downloaded_bytes = status['downloaded_bytes']
+ byte_counter = state['downloaded_bytes'] + frag_downloaded_bytes
+ frag_progress = self.calc_percent(frag_downloaded_bytes,
+ frag_total_bytes)
+ progress = self.calc_percent(state['frag_counter'], total_frags)
+ progress += frag_progress / float(total_frags)
+
+ eta = self.calc_eta(start, time.time(), estimated_size, byte_counter)
+ self.report_progress(progress, format_bytes(estimated_size),
+ status.get('speed'), eta)
+ http_dl.add_progress_hook(frag_progress_hook)
+
+ frags_filenames = []
+ for (seg_i, frag_i) in fragments_list:
+ name = 'Seg%d-Frag%d' % (seg_i, frag_i)
+ url = base_url + name
+ if akamai_pv:
+ url += '?' + akamai_pv.strip(';')
+ frag_filename = '%s-%s' % (tmpfilename, name)
+ success = http_dl.download(frag_filename, {'url': url})
+ if not success:
+ return False
+ with open(frag_filename, 'rb') as down:
+ down_data = down.read()
+ reader = FlvReader(down_data)
+ while True:
+ _, box_type, box_data = reader.read_box_info()
+ if box_type == b'mdat':
+ dest_stream.write(box_data)
+ break
+ frags_filenames.append(frag_filename)
+
+ dest_stream.close()
+ self.report_finish(format_bytes(state['downloaded_bytes']), time.time() - start)
+
+ self.try_rename(tmpfilename, filename)
+ for frag_file in frags_filenames:
+ os.remove(frag_file)
+
+ fsize = os.path.getsize(encodeFilename(filename))
+ self._hook_progress({
+ 'downloaded_bytes': fsize,
+ 'total_bytes': fsize,
+ 'filename': filename,
+ 'status': 'finished',
+ })
+
+ return True
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/downloader/hls.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/downloader/hls.py
new file mode 100644
index 0000000000..5bb0f3cfd1
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/downloader/hls.py
@@ -0,0 +1,109 @@
+from __future__ import unicode_literals
+
+import os
+import re
+import subprocess
+
+from ..postprocessor.ffmpeg import FFmpegPostProcessor
+from .common import FileDownloader
+from ..compat import (
+ compat_urlparse,
+ compat_urllib_request,
+)
+from ..utils import (
+ check_executable,
+ encodeFilename,
+)
+
+
+class HlsFD(FileDownloader):
+ def real_download(self, filename, info_dict):
+ url = info_dict['url']
+ self.report_destination(filename)
+ tmpfilename = self.temp_name(filename)
+
+ args = [
+ '-y', '-i', url, '-f', 'mp4', '-c', 'copy',
+ '-bsf:a', 'aac_adtstoasc',
+ encodeFilename(tmpfilename, for_subprocess=True)]
+
+ for program in ['avconv', 'ffmpeg']:
+ if check_executable(program, ['-version']):
+ break
+ else:
+ self.report_error('m3u8 download detected but ffmpeg or avconv could not be found. Please install one.')
+ return False
+ cmd = [program] + args
+
+ ffpp = FFmpegPostProcessor(downloader=self)
+ ffpp.check_version()
+
+ retval = subprocess.call(cmd)
+ if retval == 0:
+ fsize = os.path.getsize(encodeFilename(tmpfilename))
+ self.to_screen('\r[%s] %s bytes' % (cmd[0], fsize))
+ self.try_rename(tmpfilename, filename)
+ self._hook_progress({
+ 'downloaded_bytes': fsize,
+ 'total_bytes': fsize,
+ 'filename': filename,
+ 'status': 'finished',
+ })
+ return True
+ else:
+ self.to_stderr('\n')
+ self.report_error('%s exited with code %d' % (program, retval))
+ return False
+
+
+class NativeHlsFD(FileDownloader):
+ """ A more limited implementation that does not require ffmpeg """
+
+ def real_download(self, filename, info_dict):
+ url = info_dict['url']
+ self.report_destination(filename)
+ tmpfilename = self.temp_name(filename)
+
+ self.to_screen(
+ '[hlsnative] %s: Downloading m3u8 manifest' % info_dict['id'])
+ data = self.ydl.urlopen(url).read()
+ s = data.decode('utf-8', 'ignore')
+ segment_urls = []
+ for line in s.splitlines():
+ line = line.strip()
+ if line and not line.startswith('#'):
+ segment_url = (
+ line
+ if re.match(r'^https?://', line)
+ else compat_urlparse.urljoin(url, line))
+ segment_urls.append(segment_url)
+
+ is_test = self.params.get('test', False)
+ remaining_bytes = self._TEST_FILE_SIZE if is_test else None
+ byte_counter = 0
+ with open(tmpfilename, 'wb') as outf:
+ for i, segurl in enumerate(segment_urls):
+ self.to_screen(
+ '[hlsnative] %s: Downloading segment %d / %d' %
+ (info_dict['id'], i + 1, len(segment_urls)))
+ seg_req = compat_urllib_request.Request(segurl)
+ if remaining_bytes is not None:
+ seg_req.add_header('Range', 'bytes=0-%d' % (remaining_bytes - 1))
+
+ segment = self.ydl.urlopen(seg_req).read()
+ if remaining_bytes is not None:
+ segment = segment[:remaining_bytes]
+ remaining_bytes -= len(segment)
+ outf.write(segment)
+ byte_counter += len(segment)
+ if remaining_bytes is not None and remaining_bytes <= 0:
+ break
+
+ self._hook_progress({
+ 'downloaded_bytes': byte_counter,
+ 'total_bytes': byte_counter,
+ 'filename': filename,
+ 'status': 'finished',
+ })
+ self.try_rename(tmpfilename, filename)
+ return True
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/downloader/http.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/downloader/http.py
new file mode 100644
index 0000000000..e68f20c9f4
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/downloader/http.py
@@ -0,0 +1,228 @@
+from __future__ import unicode_literals
+
+import os
+import time
+
+from .common import FileDownloader
+from ..compat import (
+ compat_urllib_request,
+ compat_urllib_error,
+)
+from ..utils import (
+ ContentTooShortError,
+ encodeFilename,
+ sanitize_open,
+ format_bytes,
+)
+
+
+class HttpFD(FileDownloader):
+ def real_download(self, filename, info_dict):
+ url = info_dict['url']
+ tmpfilename = self.temp_name(filename)
+ stream = None
+
+ # Do not include the Accept-Encoding header
+ headers = {'Youtubedl-no-compression': 'True'}
+ if 'user_agent' in info_dict:
+ headers['Youtubedl-user-agent'] = info_dict['user_agent']
+ if 'http_referer' in info_dict:
+ headers['Referer'] = info_dict['http_referer']
+ add_headers = info_dict.get('http_headers')
+ if add_headers:
+ headers.update(add_headers)
+ data = info_dict.get('http_post_data')
+ http_method = info_dict.get('http_method')
+ basic_request = compat_urllib_request.Request(url, data, headers)
+ request = compat_urllib_request.Request(url, data, headers)
+ if http_method is not None:
+ basic_request.get_method = lambda: http_method
+ request.get_method = lambda: http_method
+
+ is_test = self.params.get('test', False)
+
+ if is_test:
+ request.add_header('Range', 'bytes=0-%s' % str(self._TEST_FILE_SIZE - 1))
+
+ # Establish possible resume length
+ if os.path.isfile(encodeFilename(tmpfilename)):
+ resume_len = os.path.getsize(encodeFilename(tmpfilename))
+ else:
+ resume_len = 0
+
+ open_mode = 'wb'
+ if resume_len != 0:
+ if self.params.get('continuedl', False):
+ self.report_resuming_byte(resume_len)
+ request.add_header('Range', 'bytes=%d-' % resume_len)
+ open_mode = 'ab'
+ else:
+ resume_len = 0
+
+ count = 0
+ retries = self.params.get('retries', 0)
+ while count <= retries:
+ # Establish connection
+ try:
+ data = self.ydl.urlopen(request)
+ break
+ except (compat_urllib_error.HTTPError, ) as err:
+ if (err.code < 500 or err.code >= 600) and err.code != 416:
+ # Unexpected HTTP error
+ raise
+ elif err.code == 416:
+ # Unable to resume (requested range not satisfiable)
+ try:
+ # Open the connection again without the range header
+ data = self.ydl.urlopen(basic_request)
+ content_length = data.info()['Content-Length']
+ except (compat_urllib_error.HTTPError, ) as err:
+ if err.code < 500 or err.code >= 600:
+ raise
+ else:
+ # Examine the reported length
+ if (content_length is not None and
+ (resume_len - 100 < int(content_length) < resume_len + 100)):
+ # The file had already been fully downloaded.
+ # Explanation to the above condition: in issue #175 it was revealed that
+ # YouTube sometimes adds or removes a few bytes from the end of the file,
+ # changing the file size slightly and causing problems for some users. So
+ # I decided to implement a suggested change and consider the file
+ # completely downloaded if the file size differs less than 100 bytes from
+ # the one in the hard drive.
+ self.report_file_already_downloaded(filename)
+ self.try_rename(tmpfilename, filename)
+ self._hook_progress({
+ 'filename': filename,
+ 'status': 'finished',
+ })
+ return True
+ else:
+ # The length does not match, we start the download over
+ self.report_unable_to_resume()
+ resume_len = 0
+ open_mode = 'wb'
+ break
+ # Retry
+ count += 1
+ if count <= retries:
+ self.report_retry(count, retries)
+
+ if count > retries:
+ self.report_error('giving up after %s retries' % retries)
+ return False
+
+ data_len = data.info().get('Content-length', None)
+
+ # Range HTTP header may be ignored/unsupported by a webserver
+ # (e.g. extractor/scivee.py, extractor/bambuser.py).
+ # However, for a test we still would like to download just a piece of a file.
+ # To achieve this we limit data_len to _TEST_FILE_SIZE and manually control
+ # block size when downloading a file.
+ if is_test and (data_len is None or int(data_len) > self._TEST_FILE_SIZE):
+ data_len = self._TEST_FILE_SIZE
+
+ if data_len is not None:
+ data_len = int(data_len) + resume_len
+ min_data_len = self.params.get("min_filesize", None)
+ max_data_len = self.params.get("max_filesize", None)
+ if min_data_len is not None and data_len < min_data_len:
+ self.to_screen('\r[download] File is smaller than min-filesize (%s bytes < %s bytes). Aborting.' % (data_len, min_data_len))
+ return False
+ if max_data_len is not None and data_len > max_data_len:
+ self.to_screen('\r[download] File is larger than max-filesize (%s bytes > %s bytes). Aborting.' % (data_len, max_data_len))
+ return False
+
+ data_len_str = format_bytes(data_len)
+ byte_counter = 0 + resume_len
+ block_size = self.params.get('buffersize', 1024)
+ start = time.time()
+
+ # measure time over whole while-loop, so slow_down() and best_block_size() work together properly
+ now = None # needed for slow_down() in the first loop run
+ before = start # start measuring
+ while True:
+
+ # Download and write
+ data_block = data.read(block_size if not is_test else min(block_size, data_len - byte_counter))
+ byte_counter += len(data_block)
+
+ # exit loop when download is finished
+ if len(data_block) == 0:
+ break
+
+ # Open destination file just in time
+ if stream is None:
+ try:
+ (stream, tmpfilename) = sanitize_open(tmpfilename, open_mode)
+ assert stream is not None
+ filename = self.undo_temp_name(tmpfilename)
+ self.report_destination(filename)
+ except (OSError, IOError) as err:
+ self.report_error('unable to open for writing: %s' % str(err))
+ return False
+ try:
+ stream.write(data_block)
+ except (IOError, OSError) as err:
+ self.to_stderr('\n')
+ self.report_error('unable to write data: %s' % str(err))
+ return False
+
+ # Apply rate limit
+ self.slow_down(start, now, byte_counter - resume_len)
+
+ # end measuring of one loop run
+ now = time.time()
+ after = now
+
+ # Adjust block size
+ if not self.params.get('noresizebuffer', False):
+ block_size = self.best_block_size(after - before, len(data_block))
+
+ before = after
+
+ # Progress message
+ speed = self.calc_speed(start, now, byte_counter - resume_len)
+ if data_len is None:
+ eta = percent = None
+ else:
+ percent = self.calc_percent(byte_counter, data_len)
+ eta = self.calc_eta(start, time.time(), data_len - resume_len, byte_counter - resume_len)
+ self.report_progress(percent, data_len_str, speed, eta)
+
+ self._hook_progress({
+ 'downloaded_bytes': byte_counter,
+ 'total_bytes': data_len,
+ 'tmpfilename': tmpfilename,
+ 'filename': filename,
+ 'status': 'downloading',
+ 'eta': eta,
+ 'speed': speed,
+ })
+
+ if is_test and byte_counter == data_len:
+ break
+
+ if stream is None:
+ self.to_stderr('\n')
+ self.report_error('Did not get any data blocks')
+ return False
+ if tmpfilename != '-':
+ stream.close()
+ self.report_finish(data_len_str, (time.time() - start))
+ if data_len is not None and byte_counter != data_len:
+ raise ContentTooShortError(byte_counter, int(data_len))
+ self.try_rename(tmpfilename, filename)
+
+ # Update file modification time
+ if self.params.get('updatetime', True):
+ info_dict['filetime'] = self.try_utime(filename, data.info().get('last-modified', None))
+
+ self._hook_progress({
+ 'downloaded_bytes': byte_counter,
+ 'total_bytes': byte_counter,
+ 'filename': filename,
+ 'status': 'finished',
+ })
+
+ return True
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/downloader/mplayer.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/downloader/mplayer.py
new file mode 100644
index 0000000000..c53195da0c
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/downloader/mplayer.py
@@ -0,0 +1,47 @@
+from __future__ import unicode_literals
+
+import os
+import subprocess
+
+from .common import FileDownloader
+from ..compat import compat_subprocess_get_DEVNULL
+from ..utils import (
+ encodeFilename,
+)
+
+
+class MplayerFD(FileDownloader):
+ def real_download(self, filename, info_dict):
+ url = info_dict['url']
+ self.report_destination(filename)
+ tmpfilename = self.temp_name(filename)
+
+ args = [
+ 'mplayer', '-really-quiet', '-vo', 'null', '-vc', 'dummy',
+ '-dumpstream', '-dumpfile', tmpfilename, url]
+ # Check for mplayer first
+ try:
+ subprocess.call(
+ ['mplayer', '-h'],
+ stdout=compat_subprocess_get_DEVNULL(), stderr=subprocess.STDOUT)
+ except (OSError, IOError):
+ self.report_error('MMS or RTSP download detected but "%s" could not be run' % args[0])
+ return False
+
+ # Download using mplayer.
+ retval = subprocess.call(args)
+ if retval == 0:
+ fsize = os.path.getsize(encodeFilename(tmpfilename))
+ self.to_screen('\r[%s] %s bytes' % (args[0], fsize))
+ self.try_rename(tmpfilename, filename)
+ self._hook_progress({
+ 'downloaded_bytes': fsize,
+ 'total_bytes': fsize,
+ 'filename': filename,
+ 'status': 'finished',
+ })
+ return True
+ else:
+ self.to_stderr('\n')
+ self.report_error('mplayer exited with code %d' % retval)
+ return False
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/downloader/rtmp.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/downloader/rtmp.py
new file mode 100644
index 0000000000..5346cb9a0a
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/downloader/rtmp.py
@@ -0,0 +1,207 @@
+from __future__ import unicode_literals
+
+import os
+import re
+import subprocess
+import sys
+import time
+
+from .common import FileDownloader
+from ..compat import compat_str
+from ..utils import (
+ check_executable,
+ encodeFilename,
+ format_bytes,
+ get_exe_version,
+)
+
+
+def rtmpdump_version():
+ return get_exe_version(
+ 'rtmpdump', ['--help'], r'(?i)RTMPDump\s*v?([0-9a-zA-Z._-]+)')
+
+
+class RtmpFD(FileDownloader):
+ def real_download(self, filename, info_dict):
+ def run_rtmpdump(args):
+ start = time.time()
+ resume_percent = None
+ resume_downloaded_data_len = None
+ proc = subprocess.Popen(args, stderr=subprocess.PIPE)
+ cursor_in_new_line = True
+ proc_stderr_closed = False
+ while not proc_stderr_closed:
+ # read line from stderr
+ line = ''
+ while True:
+ char = proc.stderr.read(1)
+ if not char:
+ proc_stderr_closed = True
+ break
+ if char in [b'\r', b'\n']:
+ break
+ line += char.decode('ascii', 'replace')
+ if not line:
+ # proc_stderr_closed is True
+ continue
+ mobj = re.search(r'([0-9]+\.[0-9]{3}) kB / [0-9]+\.[0-9]{2} sec \(([0-9]{1,2}\.[0-9])%\)', line)
+ if mobj:
+ downloaded_data_len = int(float(mobj.group(1)) * 1024)
+ percent = float(mobj.group(2))
+ if not resume_percent:
+ resume_percent = percent
+ resume_downloaded_data_len = downloaded_data_len
+ eta = self.calc_eta(start, time.time(), 100 - resume_percent, percent - resume_percent)
+ speed = self.calc_speed(start, time.time(), downloaded_data_len - resume_downloaded_data_len)
+ data_len = None
+ if percent > 0:
+ data_len = int(downloaded_data_len * 100 / percent)
+ data_len_str = '~' + format_bytes(data_len)
+ self.report_progress(percent, data_len_str, speed, eta)
+ cursor_in_new_line = False
+ self._hook_progress({
+ 'downloaded_bytes': downloaded_data_len,
+ 'total_bytes': data_len,
+ 'tmpfilename': tmpfilename,
+ 'filename': filename,
+ 'status': 'downloading',
+ 'eta': eta,
+ 'speed': speed,
+ })
+ else:
+ # no percent for live streams
+ mobj = re.search(r'([0-9]+\.[0-9]{3}) kB / [0-9]+\.[0-9]{2} sec', line)
+ if mobj:
+ downloaded_data_len = int(float(mobj.group(1)) * 1024)
+ time_now = time.time()
+ speed = self.calc_speed(start, time_now, downloaded_data_len)
+ self.report_progress_live_stream(downloaded_data_len, speed, time_now - start)
+ cursor_in_new_line = False
+ self._hook_progress({
+ 'downloaded_bytes': downloaded_data_len,
+ 'tmpfilename': tmpfilename,
+ 'filename': filename,
+ 'status': 'downloading',
+ 'speed': speed,
+ })
+ elif self.params.get('verbose', False):
+ if not cursor_in_new_line:
+ self.to_screen('')
+ cursor_in_new_line = True
+ self.to_screen('[rtmpdump] ' + line)
+ proc.wait()
+ if not cursor_in_new_line:
+ self.to_screen('')
+ return proc.returncode
+
+ url = info_dict['url']
+ player_url = info_dict.get('player_url', None)
+ page_url = info_dict.get('page_url', None)
+ app = info_dict.get('app', None)
+ play_path = info_dict.get('play_path', None)
+ tc_url = info_dict.get('tc_url', None)
+ flash_version = info_dict.get('flash_version', None)
+ live = info_dict.get('rtmp_live', False)
+ conn = info_dict.get('rtmp_conn', None)
+ protocol = info_dict.get('rtmp_protocol', None)
+
+ self.report_destination(filename)
+ tmpfilename = self.temp_name(filename)
+ test = self.params.get('test', False)
+
+ # Check for rtmpdump first
+ if not check_executable('rtmpdump', ['-h']):
+ self.report_error('RTMP download detected but "rtmpdump" could not be run. Please install it.')
+ return False
+
+ # Download using rtmpdump. rtmpdump returns exit code 2 when
+ # the connection was interrumpted and resuming appears to be
+ # possible. This is part of rtmpdump's normal usage, AFAIK.
+ basic_args = ['rtmpdump', '--verbose', '-r', url, '-o', tmpfilename]
+ if player_url is not None:
+ basic_args += ['--swfVfy', player_url]
+ if page_url is not None:
+ basic_args += ['--pageUrl', page_url]
+ if app is not None:
+ basic_args += ['--app', app]
+ if play_path is not None:
+ basic_args += ['--playpath', play_path]
+ if tc_url is not None:
+ basic_args += ['--tcUrl', url]
+ if test:
+ basic_args += ['--stop', '1']
+ if flash_version is not None:
+ basic_args += ['--flashVer', flash_version]
+ if live:
+ basic_args += ['--live']
+ if isinstance(conn, list):
+ for entry in conn:
+ basic_args += ['--conn', entry]
+ elif isinstance(conn, compat_str):
+ basic_args += ['--conn', conn]
+ if protocol is not None:
+ basic_args += ['--protocol', protocol]
+ args = basic_args + [[], ['--resume', '--skip', '1']][not live and self.params.get('continuedl', False)]
+
+ if sys.platform == 'win32' and sys.version_info < (3, 0):
+ # Windows subprocess module does not actually support Unicode
+ # on Python 2.x
+ # See http://stackoverflow.com/a/9951851/35070
+ subprocess_encoding = sys.getfilesystemencoding()
+ args = [a.encode(subprocess_encoding, 'ignore') for a in args]
+ else:
+ subprocess_encoding = None
+
+ if self.params.get('verbose', False):
+ if subprocess_encoding:
+ str_args = [
+ a.decode(subprocess_encoding) if isinstance(a, bytes) else a
+ for a in args]
+ else:
+ str_args = args
+ try:
+ import pipes
+ shell_quote = lambda args: ' '.join(map(pipes.quote, str_args))
+ except ImportError:
+ shell_quote = repr
+ self.to_screen('[debug] rtmpdump command line: ' + shell_quote(str_args))
+
+ RD_SUCCESS = 0
+ RD_FAILED = 1
+ RD_INCOMPLETE = 2
+ RD_NO_CONNECT = 3
+
+ retval = run_rtmpdump(args)
+
+ if retval == RD_NO_CONNECT:
+ self.report_error('[rtmpdump] Could not connect to RTMP server.')
+ return False
+
+ while (retval == RD_INCOMPLETE or retval == RD_FAILED) and not test and not live:
+ prevsize = os.path.getsize(encodeFilename(tmpfilename))
+ self.to_screen('[rtmpdump] %s bytes' % prevsize)
+ time.sleep(5.0) # This seems to be needed
+ retval = run_rtmpdump(basic_args + ['-e'] + [[], ['-k', '1']][retval == RD_FAILED])
+ cursize = os.path.getsize(encodeFilename(tmpfilename))
+ if prevsize == cursize and retval == RD_FAILED:
+ break
+ # Some rtmp streams seem abort after ~ 99.8%. Don't complain for those
+ if prevsize == cursize and retval == RD_INCOMPLETE and cursize > 1024:
+ self.to_screen('[rtmpdump] Could not download the whole video. This can happen for some advertisements.')
+ retval = RD_SUCCESS
+ break
+ if retval == RD_SUCCESS or (test and retval == RD_INCOMPLETE):
+ fsize = os.path.getsize(encodeFilename(tmpfilename))
+ self.to_screen('[rtmpdump] %s bytes' % fsize)
+ self.try_rename(tmpfilename, filename)
+ self._hook_progress({
+ 'downloaded_bytes': fsize,
+ 'total_bytes': fsize,
+ 'filename': filename,
+ 'status': 'finished',
+ })
+ return True
+ else:
+ self.to_stderr('\n')
+ self.report_error('rtmpdump exited with code %d' % retval)
+ return False
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/__init__.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/__init__.py
new file mode 100644
index 0000000000..92aca503c1
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/__init__.py
@@ -0,0 +1,564 @@
+from __future__ import unicode_literals
+
+from .abc import ABCIE
+from .academicearth import AcademicEarthCourseIE
+from .addanime import AddAnimeIE
+from .adobetv import AdobeTVIE
+from .adultswim import AdultSwimIE
+from .aftonbladet import AftonbladetIE
+from .aljazeera import AlJazeeraIE
+from .anitube import AnitubeIE
+from .anysex import AnySexIE
+from .aol import AolIE
+from .allocine import AllocineIE
+from .aparat import AparatIE
+from .appletrailers import AppleTrailersIE
+from .archiveorg import ArchiveOrgIE
+from .ard import ARDIE, ARDMediathekIE
+from .arte import (
+ ArteTvIE,
+ ArteTVPlus7IE,
+ ArteTVCreativeIE,
+ ArteTVConcertIE,
+ ArteTVFutureIE,
+ ArteTVDDCIE,
+ ArteTVEmbedIE,
+)
+from .audiomack import AudiomackIE
+from .auengine import AUEngineIE
+from .azubu import AzubuIE
+from .bambuser import BambuserIE, BambuserChannelIE
+from .bandcamp import BandcampIE, BandcampAlbumIE
+from .bbccouk import BBCCoUkIE
+from .beeg import BeegIE
+from .behindkink import BehindKinkIE
+from .bet import BetIE
+from .bild import BildIE
+from .bilibili import BiliBiliIE
+from .blinkx import BlinkxIE
+from .bliptv import BlipTVIE, BlipTVUserIE
+from .bloomberg import BloombergIE
+from .bpb import BpbIE
+from .br import BRIE
+from .breakcom import BreakIE
+from .brightcove import BrightcoveIE
+from .buzzfeed import BuzzFeedIE
+from .byutv import BYUtvIE
+from .c56 import C56IE
+from .canal13cl import Canal13clIE
+from .canalplus import CanalplusIE
+from .canalc2 import Canalc2IE
+from .cbs import CBSIE
+from .cbsnews import CBSNewsIE
+from .ceskatelevize import CeskaTelevizeIE
+from .channel9 import Channel9IE
+from .chilloutzone import ChilloutzoneIE
+from .cinchcast import CinchcastIE
+from .clipfish import ClipfishIE
+from .cliphunter import CliphunterIE
+from .clipsyndicate import ClipsyndicateIE
+from .cloudy import CloudyIE
+from .clubic import ClubicIE
+from .cmt import CMTIE
+from .cnet import CNETIE
+from .cnn import (
+ CNNIE,
+ CNNBlogsIE,
+)
+from .collegehumor import CollegeHumorIE
+from .comedycentral import ComedyCentralIE, ComedyCentralShowsIE
+from .comcarcoff import ComCarCoffIE
+from .condenast import CondeNastIE
+from .cracked import CrackedIE
+from .criterion import CriterionIE
+from .crunchyroll import (
+ CrunchyrollIE,
+ CrunchyrollShowPlaylistIE
+)
+from .cspan import CSpanIE
+from .dailymotion import (
+ DailymotionIE,
+ DailymotionPlaylistIE,
+ DailymotionUserIE,
+)
+from .daum import DaumIE
+from .dbtv import DBTVIE
+from .deezer import DeezerPlaylistIE
+from .dfb import DFBIE
+from .dotsub import DotsubIE
+from .dreisat import DreiSatIE
+from .drtuber import DrTuberIE
+from .drtv import DRTVIE
+from .dvtv import DVTVIE
+from .dump import DumpIE
+from .defense import DefenseGouvFrIE
+from .discovery import DiscoveryIE
+from .divxstage import DivxStageIE
+from .dropbox import DropboxIE
+from .ebaumsworld import EbaumsWorldIE
+from .ehow import EHowIE
+from .eighttracks import EightTracksIE
+from .einthusan import EinthusanIE
+from .eitb import EitbIE
+from .ellentv import (
+ EllenTVIE,
+ EllenTVClipsIE,
+)
+from .elpais import ElPaisIE
+from .empflix import EMPFlixIE
+from .engadget import EngadgetIE
+from .eporner import EpornerIE
+from .escapist import EscapistIE
+from .everyonesmixtape import EveryonesMixtapeIE
+from .exfm import ExfmIE
+from .expotv import ExpoTVIE
+from .extremetube import ExtremeTubeIE
+from .facebook import FacebookIE
+from .faz import FazIE
+from .fc2 import FC2IE
+from .firedrive import FiredriveIE
+from .firstpost import FirstpostIE
+from .firsttv import FirstTVIE
+from .fivemin import FiveMinIE
+from .fktv import (
+ FKTVIE,
+ FKTVPosteckeIE,
+)
+from .flickr import FlickrIE
+from .folketinget import FolketingetIE
+from .fourtube import FourTubeIE
+from .foxgay import FoxgayIE
+from .foxnews import FoxNewsIE
+from .franceculture import FranceCultureIE
+from .franceinter import FranceInterIE
+from .francetv import (
+ PluzzIE,
+ FranceTvInfoIE,
+ FranceTVIE,
+ GenerationQuoiIE,
+ CultureboxIE,
+)
+from .freesound import FreesoundIE
+from .freespeech import FreespeechIE
+from .freevideo import FreeVideoIE
+from .funnyordie import FunnyOrDieIE
+from .gamekings import GamekingsIE
+from .gameone import (
+ GameOneIE,
+ GameOnePlaylistIE,
+)
+from .gamespot import GameSpotIE
+from .gamestar import GameStarIE
+from .gametrailers import GametrailersIE
+from .gdcvault import GDCVaultIE
+from .generic import GenericIE
+from .giantbomb import GiantBombIE
+from .glide import GlideIE
+from .globo import GloboIE
+from .godtube import GodTubeIE
+from .goldenmoustache import GoldenMoustacheIE
+from .golem import GolemIE
+from .googleplus import GooglePlusIE
+from .googlesearch import GoogleSearchIE
+from .gorillavid import GorillaVidIE
+from .goshgay import GoshgayIE
+from .grooveshark import GroovesharkIE
+from .groupon import GrouponIE
+from .hark import HarkIE
+from .heise import HeiseIE
+from .helsinki import HelsinkiIE
+from .hentaistigma import HentaiStigmaIE
+from .hornbunny import HornBunnyIE
+from .hostingbulk import HostingBulkIE
+from .hotnewhiphop import HotNewHipHopIE
+from .howcast import HowcastIE
+from .howstuffworks import HowStuffWorksIE
+from .huffpost import HuffPostIE
+from .hypem import HypemIE
+from .iconosquare import IconosquareIE
+from .ign import IGNIE, OneUPIE
+from .imdb import (
+ ImdbIE,
+ ImdbListIE
+)
+from .ina import InaIE
+from .infoq import InfoQIE
+from .instagram import InstagramIE, InstagramUserIE
+from .internetvideoarchive import InternetVideoArchiveIE
+from .iprima import IPrimaIE
+from .ivi import (
+ IviIE,
+ IviCompilationIE
+)
+from .izlesene import IzleseneIE
+from .jadorecettepub import JadoreCettePubIE
+from .jeuxvideo import JeuxVideoIE
+from .jove import JoveIE
+from .jukebox import JukeboxIE
+from .jpopsukitv import JpopsukiIE
+from .kankan import KankanIE
+from .keezmovies import KeezMoviesIE
+from .khanacademy import KhanAcademyIE
+from .kickstarter import KickStarterIE
+from .keek import KeekIE
+from .kontrtube import KontrTubeIE
+from .krasview import KrasViewIE
+from .ku6 import Ku6IE
+from .la7 import LA7IE
+from .laola1tv import Laola1TvIE
+from .lifenews import LifeNewsIE
+from .liveleak import LiveLeakIE
+from .livestream import (
+ LivestreamIE,
+ LivestreamOriginalIE,
+ LivestreamShortenerIE,
+)
+from .lrt import LRTIE
+from .lynda import (
+ LyndaIE,
+ LyndaCourseIE
+)
+from .m6 import M6IE
+from .macgamestore import MacGameStoreIE
+from .mailru import MailRuIE
+from .malemotion import MalemotionIE
+from .mdr import MDRIE
+from .metacafe import MetacafeIE
+from .metacritic import MetacriticIE
+from .mgoon import MgoonIE
+from .minhateca import MinhatecaIE
+from .ministrygrid import MinistryGridIE
+from .mit import TechTVMITIE, MITIE, OCWMITIE
+from .mitele import MiTeleIE
+from .mixcloud import MixcloudIE
+from .mlb import MLBIE
+from .mpora import MporaIE
+from .moevideo import MoeVideoIE
+from .mofosex import MofosexIE
+from .mojvideo import MojvideoIE
+from .moniker import MonikerIE
+from .mooshare import MooshareIE
+from .morningstar import MorningstarIE
+from .motherless import MotherlessIE
+from .motorsport import MotorsportIE
+from .movieclips import MovieClipsIE
+from .moviezine import MoviezineIE
+from .movshare import MovShareIE
+from .mtv import (
+ MTVIE,
+ MTVServicesEmbeddedIE,
+ MTVIggyIE,
+)
+from .muenchentv import MuenchenTVIE
+from .musicplayon import MusicPlayOnIE
+from .musicvault import MusicVaultIE
+from .muzu import MuzuTVIE
+from .myspace import MySpaceIE, MySpaceAlbumIE
+from .myspass import MySpassIE
+from .myvideo import MyVideoIE
+from .myvidster import MyVidsterIE
+from .naver import NaverIE
+from .nba import NBAIE
+from .nbc import (
+ NBCIE,
+ NBCNewsIE,
+)
+from .ndr import NDRIE
+from .ndtv import NDTVIE
+from .nerdcubed import NerdCubedFeedIE
+from .newgrounds import NewgroundsIE
+from .newstube import NewstubeIE
+from .nfb import NFBIE
+from .nfl import NFLIE
+from .nhl import NHLIE, NHLVideocenterIE
+from .niconico import NiconicoIE, NiconicoPlaylistIE
+from .ninegag import NineGagIE
+from .noco import NocoIE
+from .normalboots import NormalbootsIE
+from .nosvideo import NosVideoIE
+from .novamov import NovaMovIE
+from .nowness import NownessIE
+from .nowvideo import NowVideoIE
+from .npo import (
+ NPOIE,
+ TegenlichtVproIE,
+)
+from .nrk import (
+ NRKIE,
+ NRKTVIE,
+)
+from .ntv import NTVIE
+from .nytimes import NYTimesIE
+from .nuvid import NuvidIE
+from .oktoberfesttv import OktoberfestTVIE
+from .ooyala import OoyalaIE
+from .orf import (
+ ORFTVthekIE,
+ ORFOE1IE,
+ ORFFM4IE,
+)
+from .parliamentliveuk import ParliamentLiveUKIE
+from .patreon import PatreonIE
+from .pbs import PBSIE
+from .phoenix import PhoenixIE
+from .photobucket import PhotobucketIE
+from .planetaplay import PlanetaPlayIE
+from .played import PlayedIE
+from .playfm import PlayFMIE
+from .playvid import PlayvidIE
+from .podomatic import PodomaticIE
+from .pornhd import PornHdIE
+from .pornhub import PornHubIE
+from .pornotube import PornotubeIE
+from .pornoxo import PornoXOIE
+from .promptfile import PromptFileIE
+from .prosiebensat1 import ProSiebenSat1IE
+from .pyvideo import PyvideoIE
+from .quickvid import QuickVidIE
+from .radiode import RadioDeIE
+from .radiofrance import RadioFranceIE
+from .rai import RaiIE
+from .rbmaradio import RBMARadioIE
+from .redtube import RedTubeIE
+from .restudy import RestudyIE
+from .reverbnation import ReverbNationIE
+from .ringtv import RingTVIE
+from .ro220 import Ro220IE
+from .rottentomatoes import RottenTomatoesIE
+from .roxwel import RoxwelIE
+from .rtbf import RTBFIE
+from .rtlnl import RtlXlIE
+from .rtlnow import RTLnowIE
+from .rtp import RTPIE
+from .rts import RTSIE
+from .rtve import RTVEALaCartaIE, RTVELiveIE
+from .ruhd import RUHDIE
+from .rutube import (
+ RutubeIE,
+ RutubeChannelIE,
+ RutubeMovieIE,
+ RutubePersonIE,
+)
+from .rutv import RUTVIE
+from .sapo import SapoIE
+from .savefrom import SaveFromIE
+from .sbs import SBSIE
+from .scivee import SciVeeIE
+from .screencast import ScreencastIE
+from .screencastomatic import ScreencastOMaticIE
+from .screenwavemedia import CinemassacreIE, ScreenwaveMediaIE, TeamFourIE
+from .servingsys import ServingSysIE
+from .sexu import SexuIE
+from .sexykarma import SexyKarmaIE
+from .shared import SharedIE
+from .sharesix import ShareSixIE
+from .sina import SinaIE
+from .slideshare import SlideshareIE
+from .slutload import SlutloadIE
+from .smotri import (
+ SmotriIE,
+ SmotriCommunityIE,
+ SmotriUserIE,
+ SmotriBroadcastIE,
+)
+from .snotr import SnotrIE
+from .sockshare import SockshareIE
+from .sohu import SohuIE
+from .soundcloud import (
+ SoundcloudIE,
+ SoundcloudSetIE,
+ SoundcloudUserIE,
+ SoundcloudPlaylistIE
+)
+from .soundgasm import SoundgasmIE
+from .southpark import (
+ SouthParkIE,
+ SouthparkDeIE,
+)
+from .space import SpaceIE
+from .spankwire import SpankwireIE
+from .spiegel import SpiegelIE, SpiegelArticleIE
+from .spiegeltv import SpiegeltvIE
+from .spike import SpikeIE
+from .sport5 import Sport5IE
+from .sportbox import SportBoxIE
+from .sportdeutschland import SportDeutschlandIE
+from .srmediathek import SRMediathekIE
+from .stanfordoc import StanfordOpenClassroomIE
+from .steam import SteamIE
+from .streamcloud import StreamcloudIE
+from .streamcz import StreamCZIE
+from .sunporno import SunPornoIE
+from .swrmediathek import SWRMediathekIE
+from .syfy import SyfyIE
+from .sztvhu import SztvHuIE
+from .tagesschau import TagesschauIE
+from .tapely import TapelyIE
+from .tass import TassIE
+from .teachertube import (
+ TeacherTubeIE,
+ TeacherTubeUserIE,
+)
+from .teachingchannel import TeachingChannelIE
+from .teamcoco import TeamcocoIE
+from .techtalks import TechTalksIE
+from .ted import TEDIE
+from .telebruxelles import TeleBruxellesIE
+from .telecinco import TelecincoIE
+from .telemb import TeleMBIE
+from .tenplay import TenPlayIE
+from .testurl import TestURLIE
+from .tf1 import TF1IE
+from .theonion import TheOnionIE
+from .theplatform import ThePlatformIE
+from .thesixtyone import TheSixtyOneIE
+from .thisav import ThisAVIE
+from .tinypic import TinyPicIE
+from .tlc import TlcIE, TlcDeIE
+from .tmz import TMZIE
+from .tnaflix import TNAFlixIE
+from .thvideo import (
+ THVideoIE,
+ THVideoPlaylistIE
+)
+from .toutv import TouTvIE
+from .toypics import ToypicsUserIE, ToypicsIE
+from .traileraddict import TrailerAddictIE
+from .trilulilu import TriluliluIE
+from .trutube import TruTubeIE
+from .tube8 import Tube8IE
+from .tudou import TudouIE
+from .tumblr import TumblrIE
+from .tunein import TuneInIE
+from .turbo import TurboIE
+from .tutv import TutvIE
+from .tvigle import TvigleIE
+from .tvp import TvpIE
+from .tvplay import TVPlayIE
+from .twentyfourvideo import TwentyFourVideoIE
+from .twitch import TwitchIE
+from .ubu import UbuIE
+from .udemy import (
+ UdemyIE,
+ UdemyCourseIE
+)
+from .unistra import UnistraIE
+from .urort import UrortIE
+from .ustream import UstreamIE, UstreamChannelIE
+from .vbox7 import Vbox7IE
+from .veehd import VeeHDIE
+from .veoh import VeohIE
+from .vesti import VestiIE
+from .vevo import VevoIE
+from .vgtv import VGTVIE
+from .vh1 import VH1IE
+from .vice import ViceIE
+from .viddler import ViddlerIE
+from .videobam import VideoBamIE
+from .videodetective import VideoDetectiveIE
+from .videolecturesnet import VideoLecturesNetIE
+from .videofyme import VideofyMeIE
+from .videomega import VideoMegaIE
+from .videopremium import VideoPremiumIE
+from .videott import VideoTtIE
+from .videoweed import VideoWeedIE
+from .vidme import VidmeIE
+from .vidzi import VidziIE
+from .vimeo import (
+ VimeoIE,
+ VimeoAlbumIE,
+ VimeoChannelIE,
+ VimeoGroupsIE,
+ VimeoLikesIE,
+ VimeoReviewIE,
+ VimeoUserIE,
+ VimeoWatchLaterIE,
+)
+from .vimple import VimpleIE
+from .vine import (
+ VineIE,
+ VineUserIE,
+)
+from .viki import VikiIE
+from .vk import (
+ VKIE,
+ VKUserVideosIE,
+)
+from .vodlocker import VodlockerIE
+from .vporn import VpornIE
+from .vrt import VRTIE
+from .vube import VubeIE
+from .vuclip import VuClipIE
+from .vulture import VultureIE
+from .walla import WallaIE
+from .washingtonpost import WashingtonPostIE
+from .wat import WatIE
+from .wayofthemaster import WayOfTheMasterIE
+from .wdr import (
+ WDRIE,
+ WDRMobileIE,
+ WDRMausIE,
+)
+from .weibo import WeiboIE
+from .wimp import WimpIE
+from .wistia import WistiaIE
+from .worldstarhiphop import WorldStarHipHopIE
+from .wrzuta import WrzutaIE
+from .xbef import XBefIE
+from .xboxclips import XboxClipsIE
+from .xhamster import XHamsterIE
+from .xminus import XMinusIE
+from .xnxx import XNXXIE
+from .xvideos import XVideosIE
+from .xtube import XTubeUserIE, XTubeIE
+from .yahoo import (
+ YahooIE,
+ YahooSearchIE,
+)
+from .yesjapan import YesJapanIE
+from .ynet import YnetIE
+from .youjizz import YouJizzIE
+from .youku import YoukuIE
+from .youporn import YouPornIE
+from .yourupload import YourUploadIE
+from .youtube import (
+ YoutubeIE,
+ YoutubeChannelIE,
+ YoutubeFavouritesIE,
+ YoutubeHistoryIE,
+ YoutubePlaylistIE,
+ YoutubeRecommendedIE,
+ YoutubeSearchDateIE,
+ YoutubeSearchIE,
+ YoutubeSearchURLIE,
+ YoutubeShowIE,
+ YoutubeSubscriptionsIE,
+ YoutubeTopListIE,
+ YoutubeTruncatedURLIE,
+ YoutubeUserIE,
+ YoutubeWatchLaterIE,
+)
+from .zdf import ZDFIE, ZDFChannelIE
+from .zingmp3 import (
+ ZingMp3SongIE,
+ ZingMp3AlbumIE,
+)
+
+_ALL_CLASSES = [
+ klass
+ for name, klass in globals().items()
+ if name.endswith('IE') and name != 'GenericIE'
+]
+_ALL_CLASSES.append(GenericIE)
+
+
+def gen_extractors():
+ """ Return a list of an instance of every supported extractor.
+ The order does matter; the first extractor matched is the one handling the URL.
+ """
+ return [klass() for klass in _ALL_CLASSES]
+
+
+def get_info_extractor(ie_name):
+ """Returns the info extractor class with the given ie_name"""
+ return globals()[ie_name + 'IE']
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/abc.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/abc.py
new file mode 100644
index 0000000000..dc0fb85d60
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/abc.py
@@ -0,0 +1,47 @@
+from __future__ import unicode_literals
+
+import re
+import json
+
+from .common import InfoExtractor
+
+
+class ABCIE(InfoExtractor):
+ IE_NAME = 'abc.net.au'
+ _VALID_URL = r'http://www\.abc\.net\.au/news/[^/]+/[^/]+/(?P\d+)'
+
+ _TEST = {
+ 'url': 'http://www.abc.net.au/news/2014-11-05/australia-to-staff-ebola-treatment-centre-in-sierra-leone/5868334',
+ 'md5': 'cb3dd03b18455a661071ee1e28344d9f',
+ 'info_dict': {
+ 'id': '5868334',
+ 'ext': 'mp4',
+ 'title': 'Australia to help staff Ebola treatment centre in Sierra Leone',
+ 'description': 'md5:809ad29c67a05f54eb41f2a105693a67',
+ },
+ }
+
+ def _real_extract(self, url):
+ video_id = self._match_id(url)
+ webpage = self._download_webpage(url, video_id)
+
+ urls_info_json = self._search_regex(
+ r'inlineVideoData\.push\((.*?)\);', webpage, 'video urls',
+ flags=re.DOTALL)
+ urls_info = json.loads(urls_info_json.replace('\'', '"'))
+ formats = [{
+ 'url': url_info['url'],
+ 'width': int(url_info['width']),
+ 'height': int(url_info['height']),
+ 'tbr': int(url_info['bitrate']),
+ 'filesize': int(url_info['filesize']),
+ } for url_info in urls_info]
+ self._sort_formats(formats)
+
+ return {
+ 'id': video_id,
+ 'title': self._og_search_title(webpage),
+ 'formats': formats,
+ 'description': self._og_search_description(webpage),
+ 'thumbnail': self._og_search_thumbnail(webpage),
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/academicearth.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/academicearth.py
new file mode 100644
index 0000000000..47313fba87
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/academicearth.py
@@ -0,0 +1,41 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+
+
+class AcademicEarthCourseIE(InfoExtractor):
+ _VALID_URL = r'^https?://(?:www\.)?academicearth\.org/playlists/(?P[^?#/]+)'
+ IE_NAME = 'AcademicEarth:Course'
+ _TEST = {
+ 'url': 'http://academicearth.org/playlists/laws-of-nature/',
+ 'info_dict': {
+ 'id': 'laws-of-nature',
+ 'title': 'Laws of Nature',
+ 'description': 'Introduce yourself to the laws of nature with these free online college lectures from Yale, Harvard, and MIT.',
+ },
+ 'playlist_count': 4,
+ }
+
+ def _real_extract(self, url):
+ playlist_id = self._match_id(url)
+
+ webpage = self._download_webpage(url, playlist_id)
+ title = self._html_search_regex(
+ r']*?>(.*?) ', webpage, 'title')
+ description = self._html_search_regex(
+ r']*?>(.*?)
',
+ webpage, 'description', fatal=False)
+ urls = re.findall(
+ r'\s*?',
+ webpage)
+ entries = [self.url_result(u) for u in urls]
+
+ return {
+ '_type': 'playlist',
+ 'id': playlist_id,
+ 'title': title,
+ 'description': description,
+ 'entries': entries,
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/addanime.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/addanime.py
new file mode 100644
index 0000000000..203936e54a
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/addanime.py
@@ -0,0 +1,87 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..compat import (
+ compat_HTTPError,
+ compat_str,
+ compat_urllib_parse,
+ compat_urllib_parse_urlparse,
+)
+from ..utils import (
+ ExtractorError,
+)
+
+
+class AddAnimeIE(InfoExtractor):
+ _VALID_URL = r'^http://(?:\w+\.)?add-anime\.net/watch_video\.php\?(?:.*?)v=(?P[\w_]+)(?:.*)'
+ _TEST = {
+ 'url': 'http://www.add-anime.net/watch_video.php?v=24MR3YO5SAS9',
+ 'md5': '72954ea10bc979ab5e2eb288b21425a0',
+ 'info_dict': {
+ 'id': '24MR3YO5SAS9',
+ 'ext': 'mp4',
+ 'description': 'One Piece 606',
+ 'title': 'One Piece 606',
+ }
+ }
+
+ def _real_extract(self, url):
+ video_id = self._match_id(url)
+
+ try:
+ webpage = self._download_webpage(url, video_id)
+ except ExtractorError as ee:
+ if not isinstance(ee.cause, compat_HTTPError) or \
+ ee.cause.code != 503:
+ raise
+
+ redir_webpage = ee.cause.read().decode('utf-8')
+ action = self._search_regex(
+ r'',
+ redir_webpage, 'redirect vc value')
+ av = re.search(
+ r'a\.value = ([0-9]+)[+]([0-9]+)[*]([0-9]+);',
+ redir_webpage)
+ if av is None:
+ raise ExtractorError('Cannot find redirect math task')
+ av_res = int(av.group(1)) + int(av.group(2)) * int(av.group(3))
+
+ parsed_url = compat_urllib_parse_urlparse(url)
+ av_val = av_res + len(parsed_url.netloc)
+ confirm_url = (
+ parsed_url.scheme + '://' + parsed_url.netloc +
+ action + '?' +
+ compat_urllib_parse.urlencode({
+ 'jschl_vc': vc, 'jschl_answer': compat_str(av_val)}))
+ self._download_webpage(
+ confirm_url, video_id,
+ note='Confirming after redirect')
+ webpage = self._download_webpage(url, video_id)
+
+ formats = []
+ for format_id in ('normal', 'hq'):
+ rex = r"var %s_video_file = '(.*?)';" % re.escape(format_id)
+ video_url = self._search_regex(rex, webpage, 'video file URLx',
+ fatal=False)
+ if not video_url:
+ continue
+ formats.append({
+ 'format_id': format_id,
+ 'url': video_url,
+ })
+ self._sort_formats(formats)
+ video_title = self._og_search_title(webpage)
+ video_description = self._og_search_description(webpage)
+
+ return {
+ '_type': 'video',
+ 'id': video_id,
+ 'formats': formats,
+ 'title': video_title,
+ 'description': video_description
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/adobetv.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/adobetv.py
new file mode 100644
index 0000000000..28e07f8b04
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/adobetv.py
@@ -0,0 +1,70 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+ parse_duration,
+ unified_strdate,
+ str_to_int,
+)
+
+
+class AdobeTVIE(InfoExtractor):
+ _VALID_URL = r'https?://tv\.adobe\.com/watch/[^/]+/(?P[^/]+)'
+
+ _TEST = {
+ 'url': 'http://tv.adobe.com/watch/the-complete-picture-with-julieanne-kost/quick-tip-how-to-draw-a-circle-around-an-object-in-photoshop/',
+ 'md5': '9bc5727bcdd55251f35ad311ca74fa1e',
+ 'info_dict': {
+ 'id': 'quick-tip-how-to-draw-a-circle-around-an-object-in-photoshop',
+ 'ext': 'mp4',
+ 'title': 'Quick Tip - How to Draw a Circle Around an Object in Photoshop',
+ 'description': 'md5:99ec318dc909d7ba2a1f2b038f7d2311',
+ 'thumbnail': 're:https?://.*\.jpg$',
+ 'upload_date': '20110914',
+ 'duration': 60,
+ 'view_count': int,
+ },
+ }
+
+ def _real_extract(self, url):
+ video_id = self._match_id(url)
+
+ webpage = self._download_webpage(url, video_id)
+
+ player = self._parse_json(
+ self._search_regex(r'html5player:\s*({.+?})\s*\n', webpage, 'player'),
+ video_id)
+
+ title = player.get('title') or self._search_regex(
+ r'data-title="([^"]+)"', webpage, 'title')
+ description = self._og_search_description(webpage)
+ thumbnail = self._og_search_thumbnail(webpage)
+
+ upload_date = unified_strdate(
+ self._html_search_meta('datepublished', webpage, 'upload date'))
+
+ duration = parse_duration(
+ self._html_search_meta('duration', webpage, 'duration')
+ or self._search_regex(r'Runtime:\s*(\d{2}:\d{2}:\d{2})', webpage, 'duration'))
+
+ view_count = str_to_int(self._search_regex(
+ r'\s*Views?:\s*([\d,.]+)\s*
',
+ webpage, 'view count'))
+
+ formats = [{
+ 'url': source['src'],
+ 'format_id': source.get('quality') or source['src'].split('-')[-1].split('.')[0] or None,
+ 'tbr': source.get('bitrate'),
+ } for source in player['sources']]
+ self._sort_formats(formats)
+
+ return {
+ 'id': video_id,
+ 'title': title,
+ 'description': description,
+ 'thumbnail': thumbnail,
+ 'upload_date': upload_date,
+ 'duration': duration,
+ 'view_count': view_count,
+ 'formats': formats,
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/adultswim.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/adultswim.py
new file mode 100644
index 0000000000..502a9c25ad
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/adultswim.py
@@ -0,0 +1,169 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+import json
+
+from .common import InfoExtractor
+from ..utils import (
+ ExtractorError,
+ xpath_text,
+ float_or_none,
+)
+
+
+class AdultSwimIE(InfoExtractor):
+ _VALID_URL = r'https?://(?:www\.)?adultswim\.com/videos/(?Pplaylists/)?(?P[^/]+)/(?P[^/?#]+)/?'
+
+ _TESTS = [{
+ 'url': 'http://adultswim.com/videos/rick-and-morty/pilot',
+ 'playlist': [
+ {
+ 'md5': '247572debc75c7652f253c8daa51a14d',
+ 'info_dict': {
+ 'id': 'rQxZvXQ4ROaSOqq-or2Mow-0',
+ 'ext': 'flv',
+ 'title': 'Rick and Morty - Pilot Part 1',
+ 'description': "Rick moves in with his daughter's family and establishes himself as a bad influence on his grandson, Morty. "
+ },
+ },
+ {
+ 'md5': '77b0e037a4b20ec6b98671c4c379f48d',
+ 'info_dict': {
+ 'id': 'rQxZvXQ4ROaSOqq-or2Mow-3',
+ 'ext': 'flv',
+ 'title': 'Rick and Morty - Pilot Part 4',
+ 'description': "Rick moves in with his daughter's family and establishes himself as a bad influence on his grandson, Morty. "
+ },
+ },
+ ],
+ 'info_dict': {
+ 'title': 'Rick and Morty - Pilot',
+ 'description': "Rick moves in with his daughter's family and establishes himself as a bad influence on his grandson, Morty. "
+ }
+ }, {
+ 'url': 'http://www.adultswim.com/videos/playlists/american-parenting/putting-francine-out-of-business/',
+ 'playlist': [
+ {
+ 'md5': '2eb5c06d0f9a1539da3718d897f13ec5',
+ 'info_dict': {
+ 'id': '-t8CamQlQ2aYZ49ItZCFog-0',
+ 'ext': 'flv',
+ 'title': 'American Dad - Putting Francine Out of Business',
+ 'description': 'Stan hatches a plan to get Francine out of the real estate business.Watch more American Dad on [adult swim].'
+ },
+ }
+ ],
+ 'info_dict': {
+ 'title': 'American Dad - Putting Francine Out of Business',
+ 'description': 'Stan hatches a plan to get Francine out of the real estate business.Watch more American Dad on [adult swim].'
+ },
+ }]
+
+ @staticmethod
+ def find_video_info(collection, slug):
+ for video in collection.get('videos'):
+ if video.get('slug') == slug:
+ return video
+
+ @staticmethod
+ def find_collection_by_linkURL(collections, linkURL):
+ for collection in collections:
+ if collection.get('linkURL') == linkURL:
+ return collection
+
+ @staticmethod
+ def find_collection_containing_video(collections, slug):
+ for collection in collections:
+ for video in collection.get('videos'):
+ if video.get('slug') == slug:
+ return collection, video
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ show_path = mobj.group('show_path')
+ episode_path = mobj.group('episode_path')
+ is_playlist = True if mobj.group('is_playlist') else False
+
+ webpage = self._download_webpage(url, episode_path)
+
+ # Extract the value of `bootstrappedData` from the Javascript in the page.
+ bootstrappedDataJS = self._search_regex(r'var bootstrappedData = ({.*});', webpage, episode_path)
+
+ try:
+ bootstrappedData = json.loads(bootstrappedDataJS)
+ except ValueError as ve:
+ errmsg = '%s: Failed to parse JSON ' % episode_path
+ raise ExtractorError(errmsg, cause=ve)
+
+ # Downloading videos from a /videos/playlist/ URL needs to be handled differently.
+ # NOTE: We are only downloading one video (the current one) not the playlist
+ if is_playlist:
+ collections = bootstrappedData['playlists']['collections']
+ collection = self.find_collection_by_linkURL(collections, show_path)
+ video_info = self.find_video_info(collection, episode_path)
+
+ show_title = video_info['showTitle']
+ segment_ids = [video_info['videoPlaybackID']]
+ else:
+ collections = bootstrappedData['show']['collections']
+ collection, video_info = self.find_collection_containing_video(collections, episode_path)
+
+ show = bootstrappedData['show']
+ show_title = show['title']
+ segment_ids = [clip['videoPlaybackID'] for clip in video_info['clips']]
+
+ episode_id = video_info['id']
+ episode_title = video_info['title']
+ episode_description = video_info['description']
+ episode_duration = video_info.get('duration')
+
+ entries = []
+ for part_num, segment_id in enumerate(segment_ids):
+ segment_url = 'http://www.adultswim.com/videos/api/v0/assets?id=%s&platform=mobile' % segment_id
+
+ segment_title = '%s - %s' % (show_title, episode_title)
+ if len(segment_ids) > 1:
+ segment_title += ' Part %d' % (part_num + 1)
+
+ idoc = self._download_xml(
+ segment_url, segment_title,
+ 'Downloading segment information', 'Unable to download segment information')
+
+ segment_duration = float_or_none(
+ xpath_text(idoc, './/trt', 'segment duration').strip())
+
+ formats = []
+ file_els = idoc.findall('.//files/file')
+
+ for file_el in file_els:
+ bitrate = file_el.attrib.get('bitrate')
+ ftype = file_el.attrib.get('type')
+
+ formats.append({
+ 'format_id': '%s_%s' % (bitrate, ftype),
+ 'url': file_el.text.strip(),
+ # The bitrate may not be a number (for example: 'iphone')
+ 'tbr': int(bitrate) if bitrate.isdigit() else None,
+ 'quality': 1 if ftype == 'hd' else -1
+ })
+
+ self._sort_formats(formats)
+
+ entries.append({
+ 'id': segment_id,
+ 'title': segment_title,
+ 'formats': formats,
+ 'duration': segment_duration,
+ 'description': episode_description
+ })
+
+ return {
+ '_type': 'playlist',
+ 'id': episode_id,
+ 'display_id': episode_path,
+ 'entries': entries,
+ 'title': '%s - %s' % (show_title, episode_title),
+ 'description': episode_description,
+ 'duration': episode_duration
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/aftonbladet.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/aftonbladet.py
new file mode 100644
index 0000000000..cfc7370ae4
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/aftonbladet.py
@@ -0,0 +1,66 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+
+
+class AftonbladetIE(InfoExtractor):
+ _VALID_URL = r'^http://tv\.aftonbladet\.se/webbtv.+?(?Particle[0-9]+)\.ab(?:$|[?#])'
+ _TEST = {
+ 'url': 'http://tv.aftonbladet.se/webbtv/nyheter/vetenskap/rymden/article36015.ab',
+ 'info_dict': {
+ 'id': 'article36015',
+ 'ext': 'mp4',
+ 'title': 'Vulkanutbrott i rymden - nu slц╓pper NASA bilderna',
+ 'description': 'Jupiters mц╔ne mest aktiv av alla himlakroppar',
+ 'timestamp': 1394142732,
+ 'upload_date': '20140306',
+ },
+ }
+
+ def _real_extract(self, url):
+ mobj = re.search(self._VALID_URL, url)
+
+ video_id = mobj.group('video_id')
+ webpage = self._download_webpage(url, video_id)
+
+ # find internal video meta data
+ meta_url = 'http://aftonbladet-play.drlib.aptoma.no/video/%s.json'
+ internal_meta_id = self._html_search_regex(
+ r'data-aptomaId="([\w\d]+)"', webpage, 'internal_meta_id')
+ internal_meta_url = meta_url % internal_meta_id
+ internal_meta_json = self._download_json(
+ internal_meta_url, video_id, 'Downloading video meta data')
+
+ # find internal video formats
+ format_url = 'http://aftonbladet-play.videodata.drvideo.aptoma.no/actions/video/?id=%s'
+ internal_video_id = internal_meta_json['videoId']
+ internal_formats_url = format_url % internal_video_id
+ internal_formats_json = self._download_json(
+ internal_formats_url, video_id, 'Downloading video formats')
+
+ formats = []
+ for fmt in internal_formats_json['formats']['http']['pseudostreaming']['mp4']:
+ p = fmt['paths'][0]
+ formats.append({
+ 'url': 'http://%s:%d/%s/%s' % (p['address'], p['port'], p['path'], p['filename']),
+ 'ext': 'mp4',
+ 'width': fmt['width'],
+ 'height': fmt['height'],
+ 'tbr': fmt['bitrate'],
+ 'protocol': 'http',
+ })
+ self._sort_formats(formats)
+
+ return {
+ 'id': video_id,
+ 'title': internal_meta_json['title'],
+ 'formats': formats,
+ 'thumbnail': internal_meta_json['imageUrl'],
+ 'description': internal_meta_json['shortPreamble'],
+ 'timestamp': internal_meta_json['timePublished'],
+ 'duration': internal_meta_json['duration'],
+ 'view_count': internal_meta_json['views'],
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/aljazeera.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/aljazeera.py
new file mode 100644
index 0000000000..612708e257
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/aljazeera.py
@@ -0,0 +1,35 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class AlJazeeraIE(InfoExtractor):
+ _VALID_URL = r'http://www\.aljazeera\.com/programmes/.*?/(?P[^/]+)\.html'
+
+ _TEST = {
+ 'url': 'http://www.aljazeera.com/programmes/the-slum/2014/08/deliverance-201482883754237240.html',
+ 'info_dict': {
+ 'id': '3792260579001',
+ 'ext': 'mp4',
+ 'title': 'The Slum - Episode 1: Deliverance',
+ 'description': 'As a birth attendant advocating for family planning, Remy is on the frontline of Tondo\'s battle with overcrowding.',
+ 'uploader': 'Al Jazeera English',
+ },
+ 'add_ie': ['Brightcove'],
+ }
+
+ def _real_extract(self, url):
+ program_name = self._match_id(url)
+ webpage = self._download_webpage(url, program_name)
+ brightcove_id = self._search_regex(
+ r'RenderPagesVideo\(\'(.+?)\'', webpage, 'brightcove id')
+
+ return {
+ '_type': 'url',
+ 'url': (
+ 'brightcove:'
+ 'playerKey=AQ~~%2CAAAAmtVJIFk~%2CTVGOQ5ZTwJbeMWnq5d_H4MOM57xfzApc'
+ '&%40videoPlayer={0}'.format(brightcove_id)
+ ),
+ 'ie_key': 'Brightcove',
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/allocine.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/allocine.py
new file mode 100644
index 0000000000..7d65b81931
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/allocine.py
@@ -0,0 +1,89 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+import re
+import json
+
+from .common import InfoExtractor
+from ..compat import compat_str
+from ..utils import (
+ qualities,
+)
+
+
+class AllocineIE(InfoExtractor):
+ _VALID_URL = r'https?://(?:www\.)?allocine\.fr/(?Particle|video|film)/(fichearticle_gen_carticle=|player_gen_cmedia=|fichefilm_gen_cfilm=|video-)(?P[0-9]+)(?:\.html)?'
+
+ _TESTS = [{
+ 'url': 'http://www.allocine.fr/article/fichearticle_gen_carticle=18635087.html',
+ 'md5': '0c9fcf59a841f65635fa300ac43d8269',
+ 'info_dict': {
+ 'id': '19546517',
+ 'ext': 'mp4',
+ 'title': 'Astц╘rix - Le Domaine des Dieux Teaser VF',
+ 'description': 'md5:abcd09ce503c6560512c14ebfdb720d2',
+ 'thumbnail': 're:http://.*\.jpg',
+ },
+ }, {
+ 'url': 'http://www.allocine.fr/video/player_gen_cmedia=19540403&cfilm=222257.html',
+ 'md5': 'd0cdce5d2b9522ce279fdfec07ff16e0',
+ 'info_dict': {
+ 'id': '19540403',
+ 'ext': 'mp4',
+ 'title': 'Planes 2 Bande-annonce VF',
+ 'description': 'md5:eeaffe7c2d634525e21159b93acf3b1e',
+ 'thumbnail': 're:http://.*\.jpg',
+ },
+ }, {
+ 'url': 'http://www.allocine.fr/film/fichefilm_gen_cfilm=181290.html',
+ 'md5': '101250fb127ef9ca3d73186ff22a47ce',
+ 'info_dict': {
+ 'id': '19544709',
+ 'ext': 'mp4',
+ 'title': 'Dragons 2 - Bande annonce finale VF',
+ 'description': 'md5:71742e3a74b0d692c7fce0dd2017a4ac',
+ 'thumbnail': 're:http://.*\.jpg',
+ },
+ }, {
+ 'url': 'http://www.allocine.fr/video/video-19550147/',
+ 'only_matching': True,
+ }]
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ typ = mobj.group('typ')
+ display_id = mobj.group('id')
+
+ webpage = self._download_webpage(url, display_id)
+
+ if typ == 'film':
+ video_id = self._search_regex(r'href="/video/player_gen_cmedia=([0-9]+).+"', webpage, 'video id')
+ else:
+ player = self._search_regex(r'data-player=\'([^\']+)\'>', webpage, 'data player')
+
+ player_data = json.loads(player)
+ video_id = compat_str(player_data['refMedia'])
+
+ xml = self._download_xml('http://www.allocine.fr/ws/AcVisiondataV4.ashx?media=%s' % video_id, display_id)
+
+ video = xml.find('.//AcVisionVideo').attrib
+ quality = qualities(['ld', 'md', 'hd'])
+
+ formats = []
+ for k, v in video.items():
+ if re.match(r'.+_path', k):
+ format_id = k.split('_')[0]
+ formats.append({
+ 'format_id': format_id,
+ 'quality': quality(format_id),
+ 'url': v,
+ })
+ self._sort_formats(formats)
+
+ return {
+ 'id': video_id,
+ 'title': video['videoTitle'],
+ 'thumbnail': self._og_search_thumbnail(webpage),
+ 'formats': formats,
+ 'description': self._og_search_description(webpage),
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/anitube.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/anitube.py
new file mode 100644
index 0000000000..31f0d417ce
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/anitube.py
@@ -0,0 +1,59 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+
+
+class AnitubeIE(InfoExtractor):
+ IE_NAME = 'anitube.se'
+ _VALID_URL = r'https?://(?:www\.)?anitube\.se/video/(?P\d+)'
+
+ _TEST = {
+ 'url': 'http://www.anitube.se/video/36621',
+ 'md5': '59d0eeae28ea0bc8c05e7af429998d43',
+ 'info_dict': {
+ 'id': '36621',
+ 'ext': 'mp4',
+ 'title': 'Recorder to Randoseru 01',
+ 'duration': 180.19,
+ },
+ 'skip': 'Blocked in the US',
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+
+ webpage = self._download_webpage(url, video_id)
+ key = self._html_search_regex(
+ r'http://www\.anitube\.se/embed/([A-Za-z0-9_-]*)', webpage, 'key')
+
+ config_xml = self._download_xml(
+ 'http://www.anitube.se/nuevo/econfig.php?key=%s' % key, key)
+
+ video_title = config_xml.find('title').text
+ thumbnail = config_xml.find('image').text
+ duration = float(config_xml.find('duration').text)
+
+ formats = []
+ video_url = config_xml.find('file')
+ if video_url is not None:
+ formats.append({
+ 'format_id': 'sd',
+ 'url': video_url.text,
+ })
+ video_url = config_xml.find('filehd')
+ if video_url is not None:
+ formats.append({
+ 'format_id': 'hd',
+ 'url': video_url.text,
+ })
+
+ return {
+ 'id': video_id,
+ 'title': video_title,
+ 'thumbnail': thumbnail,
+ 'duration': duration,
+ 'formats': formats
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/anysex.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/anysex.py
new file mode 100644
index 0000000000..ad86d6e58a
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/anysex.py
@@ -0,0 +1,61 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+ parse_duration,
+ int_or_none,
+)
+
+
+class AnySexIE(InfoExtractor):
+ _VALID_URL = r'https?://(?:www\.)?anysex\.com/(?P\d+)'
+ _TEST = {
+ 'url': 'http://anysex.com/156592/',
+ 'md5': '023e9fbb7f7987f5529a394c34ad3d3d',
+ 'info_dict': {
+ 'id': '156592',
+ 'ext': 'mp4',
+ 'title': 'Busty and sexy blondie in her bikini strips for you',
+ 'description': 'md5:de9e418178e2931c10b62966474e1383',
+ 'categories': ['Erotic'],
+ 'duration': 270,
+ 'age_limit': 18,
+ }
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+
+ webpage = self._download_webpage(url, video_id)
+
+ video_url = self._html_search_regex(r"video_url\s*:\s*'([^']+)'", webpage, 'video URL')
+
+ title = self._html_search_regex(r'(.*?) ', webpage, 'title')
+ description = self._html_search_regex(
+ r']*>([^<]+)
', webpage, 'description', fatal=False)
+ thumbnail = self._html_search_regex(
+ r'preview_url\s*:\s*\'(.*?)\'', webpage, 'thumbnail', fatal=False)
+
+ categories = re.findall(
+ r'([^<]+) ', webpage)
+
+ duration = parse_duration(self._search_regex(
+ r'Duration: (?:)?(\d+:\d+)', webpage, 'duration', fatal=False))
+ view_count = int_or_none(self._html_search_regex(
+ r'Views: (\d+)', webpage, 'view count', fatal=False))
+
+ return {
+ 'id': video_id,
+ 'url': video_url,
+ 'ext': 'mp4',
+ 'title': title,
+ 'description': description,
+ 'thumbnail': thumbnail,
+ 'categories': categories,
+ 'duration': duration,
+ 'view_count': view_count,
+ 'age_limit': 18,
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/aol.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/aol.py
new file mode 100644
index 0000000000..b51eafc459
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/aol.py
@@ -0,0 +1,70 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+
+
+class AolIE(InfoExtractor):
+ IE_NAME = 'on.aol.com'
+ _VALID_URL = r'''(?x)
+ (?:
+ aol-video:|
+ http://on\.aol\.com/
+ (?:
+ video/.*-|
+ playlist/(?P[^/?#]+?)-(?P[0-9]+)[?#].*_videoid=
+ )
+ )
+ (?P[0-9]+)
+ (?:$|\?)
+ '''
+
+ _TESTS = [{
+ 'url': 'http://on.aol.com/video/u-s--official-warns-of-largest-ever-irs-phone-scam-518167793?icid=OnHomepageC2Wide_MustSee_Img',
+ 'md5': '18ef68f48740e86ae94b98da815eec42',
+ 'info_dict': {
+ 'id': '518167793',
+ 'ext': 'mp4',
+ 'title': 'U.S. Official Warns Of \'Largest Ever\' IRS Phone Scam',
+ },
+ 'add_ie': ['FiveMin'],
+ }, {
+ 'url': 'http://on.aol.com/playlist/brace-yourself---todays-weirdest-news-152147?icid=OnHomepageC4_Omg_Img#_videoid=518184316',
+ 'info_dict': {
+ 'id': '152147',
+ 'title': 'Brace Yourself - Today\'s Weirdest News',
+ },
+ 'playlist_mincount': 10,
+ }]
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+ playlist_id = mobj.group('playlist_id')
+ if not playlist_id or self._downloader.params.get('noplaylist'):
+ return self.url_result('5min:%s' % video_id)
+
+ self.to_screen('Downloading playlist %s - add --no-playlist to just download video %s' % (playlist_id, video_id))
+
+ webpage = self._download_webpage(url, playlist_id)
+ title = self._html_search_regex(
+ r'(.+?) ', webpage, 'title')
+ playlist_html = self._search_regex(
+ r"(?s)(.*?)", webpage,
+ 'playlist HTML')
+ entries = [{
+ '_type': 'url',
+ 'url': 'aol-video:%s' % m.group('id'),
+ 'ie_key': 'Aol',
+ } for m in re.finditer(
+ r"[0-9]+)'\s+class='video-thumb'>",
+ playlist_html)]
+
+ return {
+ '_type': 'playlist',
+ 'id': playlist_id,
+ 'display_id': mobj.group('playlist_display_id'),
+ 'title': title,
+ 'entries': entries,
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/aparat.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/aparat.py
new file mode 100644
index 0000000000..15006336fa
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/aparat.py
@@ -0,0 +1,57 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+ ExtractorError,
+ HEADRequest,
+)
+
+
+class AparatIE(InfoExtractor):
+ _VALID_URL = r'^https?://(?:www\.)?aparat\.com/(?:v/|video/video/embed/videohash/)(?P[a-zA-Z0-9]+)'
+
+ _TEST = {
+ 'url': 'http://www.aparat.com/v/wP8On',
+ 'md5': '6714e0af7e0d875c5a39c4dc4ab46ad1',
+ 'info_dict': {
+ 'id': 'wP8On',
+ 'ext': 'mp4',
+ 'title': 'ь╙ш▄ы┘ з╞ы└з╘ьЁш▄ 11 - ь╡ы┬ы┘ш▄ь╙',
+ },
+ # 'skip': 'Extremely unreliable',
+ }
+
+ def _real_extract(self, url):
+ video_id = self._match_id(url)
+
+ # Note: There is an easier-to-parse configuration at
+ # http://www.aparat.com/video/video/config/videohash/%video_id
+ # but the URL in there does not work
+ embed_url = ('http://www.aparat.com/video/video/embed/videohash/' +
+ video_id + '/vt/frame')
+ webpage = self._download_webpage(embed_url, video_id)
+
+ video_urls = re.findall(r'fileList\[[0-9]+\]\s*=\s*"([^"]+)"', webpage)
+ for i, video_url in enumerate(video_urls):
+ req = HEADRequest(video_url)
+ res = self._request_webpage(
+ req, video_id, note='Testing video URL %d' % i, errnote=False)
+ if res:
+ break
+ else:
+ raise ExtractorError('No working video URLs found')
+
+ title = self._search_regex(r'\s+title:\s*"([^"]+)"', webpage, 'title')
+ thumbnail = self._search_regex(
+ r'\s+image:\s*"([^"]+)"', webpage, 'thumbnail', fatal=False)
+
+ return {
+ 'id': video_id,
+ 'title': title,
+ 'url': video_url,
+ 'ext': 'mp4',
+ 'thumbnail': thumbnail,
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/appletrailers.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/appletrailers.py
new file mode 100644
index 0000000000..7cd0482c75
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/appletrailers.py
@@ -0,0 +1,139 @@
+from __future__ import unicode_literals
+
+import re
+import json
+
+from .common import InfoExtractor
+from ..compat import compat_urlparse
+from ..utils import (
+ int_or_none,
+)
+
+
+class AppleTrailersIE(InfoExtractor):
+ _VALID_URL = r'https?://(?:www\.)?trailers\.apple\.com/trailers/(?P[^/]+)/(?P[^/]+)'
+ _TEST = {
+ "url": "http://trailers.apple.com/trailers/wb/manofsteel/",
+ "playlist": [
+ {
+ "md5": "d97a8e575432dbcb81b7c3acb741f8a8",
+ "info_dict": {
+ "id": "manofsteel-trailer4",
+ "ext": "mov",
+ "duration": 111,
+ "title": "Trailer 4",
+ "upload_date": "20130523",
+ "uploader_id": "wb",
+ },
+ },
+ {
+ "md5": "b8017b7131b721fb4e8d6f49e1df908c",
+ "info_dict": {
+ "id": "manofsteel-trailer3",
+ "ext": "mov",
+ "duration": 182,
+ "title": "Trailer 3",
+ "upload_date": "20130417",
+ "uploader_id": "wb",
+ },
+ },
+ {
+ "md5": "d0f1e1150989b9924679b441f3404d48",
+ "info_dict": {
+ "id": "manofsteel-trailer",
+ "ext": "mov",
+ "duration": 148,
+ "title": "Trailer",
+ "upload_date": "20121212",
+ "uploader_id": "wb",
+ },
+ },
+ {
+ "md5": "5fe08795b943eb2e757fa95cb6def1cb",
+ "info_dict": {
+ "id": "manofsteel-teaser",
+ "ext": "mov",
+ "duration": 93,
+ "title": "Teaser",
+ "upload_date": "20120721",
+ "uploader_id": "wb",
+ },
+ },
+ ]
+ }
+
+ _JSON_RE = r'iTunes.playURL\((.*?)\);'
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ movie = mobj.group('movie')
+ uploader_id = mobj.group('company')
+
+ playlist_url = compat_urlparse.urljoin(url, 'includes/playlists/itunes.inc')
+
+ def fix_html(s):
+ s = re.sub(r'(?s).*?', '', s)
+ s = re.sub(r'[0-9]+):(?P[0-9]{1,2})', runtime)
+ duration = None
+ if m:
+ duration = 60 * int(m.group('minutes')) + int(m.group('seconds'))
+
+ first_url = trailer_info['url']
+ trailer_id = first_url.split('/')[-1].rpartition('_')[0].lower()
+ settings_json_url = compat_urlparse.urljoin(url, 'includes/settings/%s.json' % trailer_id)
+ settings = self._download_json(settings_json_url, trailer_id, 'Downloading settings json')
+
+ formats = []
+ for format in settings['metadata']['sizes']:
+ # The src is a file pointing to the real video file
+ format_url = re.sub(r'_(\d*p.mov)', r'_h\1', format['src'])
+ formats.append({
+ 'url': format_url,
+ 'format': format['type'],
+ 'width': int_or_none(format['width']),
+ 'height': int_or_none(format['height']),
+ })
+
+ self._sort_formats(formats)
+
+ playlist.append({
+ '_type': 'video',
+ 'id': video_id,
+ 'title': title,
+ 'formats': formats,
+ 'title': title,
+ 'duration': duration,
+ 'thumbnail': thumbnail,
+ 'upload_date': upload_date,
+ 'uploader_id': uploader_id,
+ 'user_agent': 'QuickTime compatible (youtube-dl)',
+ })
+
+ return {
+ '_type': 'playlist',
+ 'id': movie,
+ 'entries': playlist,
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/archiveorg.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/archiveorg.py
new file mode 100644
index 0000000000..34ce8429b1
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/archiveorg.py
@@ -0,0 +1,61 @@
+from __future__ import unicode_literals
+
+import json
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+ unified_strdate,
+)
+
+
+class ArchiveOrgIE(InfoExtractor):
+ IE_NAME = 'archive.org'
+ IE_DESC = 'archive.org videos'
+ _VALID_URL = r'(?:https?://)?(?:www\.)?archive\.org/details/(?P[^?/]+)(?:[?].*)?$'
+ _TEST = {
+ "url": "http://archive.org/details/XD300-23_68HighlightsAResearchCntAugHumanIntellect",
+ 'file': 'XD300-23_68HighlightsAResearchCntAugHumanIntellect.ogv',
+ 'md5': '8af1d4cf447933ed3c7f4871162602db',
+ 'info_dict': {
+ "title": "1968 Demo - FJCC Conference Presentation Reel #1",
+ "description": "Reel 1 of 3: Also known as the \"Mother of All Demos\", Doug Engelbart's presentation at the Fall Joint Computer Conference in San Francisco, December 9, 1968 titled \"A Research Center for Augmenting Human Intellect.\" For this presentation, Doug and his team astonished the audience by not only relating their research, but demonstrating it live. This was the debut of the mouse, interactive computing, hypermedia, computer supported software engineering, video teleconferencing, etc. See also Doug's 1968 Demo page for more background, highlights, links, and the detailed paper published in this conference proceedings. Filmed on 3 reels: Reel 1 | Reel 2 | Reel 3 ",
+ "upload_date": "19681210",
+ "uploader": "SRI International"
+ }
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+
+ json_url = url + ('?' if '?' in url else '&') + 'output=json'
+ json_data = self._download_webpage(json_url, video_id)
+ data = json.loads(json_data)
+
+ title = data['metadata']['title'][0]
+ description = data['metadata']['description'][0]
+ uploader = data['metadata']['creator'][0]
+ upload_date = unified_strdate(data['metadata']['date'][0])
+
+ formats = [
+ {
+ 'format': fdata['format'],
+ 'url': 'http://' + data['server'] + data['dir'] + fn,
+ 'file_size': int(fdata['size']),
+ }
+ for fn, fdata in data['files'].items()
+ if 'Video' in fdata['format']]
+
+ self._sort_formats(formats)
+
+ return {
+ '_type': 'video',
+ 'id': video_id,
+ 'title': title,
+ 'formats': formats,
+ 'description': description,
+ 'uploader': uploader,
+ 'upload_date': upload_date,
+ 'thumbnail': data.get('misc', {}).get('image'),
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/ard.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/ard.py
new file mode 100644
index 0000000000..967bd865c5
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/ard.py
@@ -0,0 +1,194 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from .generic import GenericIE
+from ..utils import (
+ determine_ext,
+ ExtractorError,
+ qualities,
+ int_or_none,
+ parse_duration,
+ unified_strdate,
+ xpath_text,
+ parse_xml,
+)
+
+
+class ARDMediathekIE(InfoExtractor):
+ IE_NAME = 'ARD:mediathek'
+ _VALID_URL = r'^https?://(?:(?:www\.)?ardmediathek\.de|mediathek\.daserste\.de)/(?:.*/)(?P[0-9]+|[^0-9][^/\?]+)[^/\?]*(?:\?.*)?'
+
+ _TESTS = [{
+ 'url': 'http://mediathek.daserste.de/sendungen_a-z/328454_anne-will/22429276_vertrauen-ist-gut-spionieren-ist-besser-geht',
+ 'file': '22429276.mp4',
+ 'md5': '469751912f1de0816a9fc9df8336476c',
+ 'info_dict': {
+ 'title': 'Vertrauen ist gut, Spionieren ist besser - Geht so deutsch-amerikanische Freundschaft?',
+ 'description': 'Das Erste Mediathek [ARD]: Vertrauen ist gut, Spionieren ist besser - Geht so deutsch-amerikanische Freundschaft?, Anne Will, ц°ber die Spionage-Affц╓re diskutieren Clemens Binninger, Katrin Gц╤ring-Eckardt, Georg Mascolo, Andrew B. Denison und Constanze Kurz.. Das Video zur Sendung Anne Will am Mittwoch, 16.07.2014',
+ },
+ 'skip': 'Blocked outside of Germany',
+ }, {
+ 'url': 'http://www.ardmediathek.de/tv/Tatort/Das-Wunder-von-Wolbeck-Video-tgl-ab-20/Das-Erste/Video?documentId=22490580&bcastId=602916',
+ 'info_dict': {
+ 'id': '22490580',
+ 'ext': 'mp4',
+ 'title': 'Das Wunder von Wolbeck (Video tgl. ab 20 Uhr)',
+ 'description': 'Auf einem restaurierten Hof bei Wolbeck wird der Heilpraktiker Raffael Lembeck eines morgens von seiner Frau Stella tot aufgefunden. Das Opfer war offensichtlich in seiner Praxis zu Fall gekommen und ist dann verblutet, erklц╓rt Prof. Boerne am Tatort.',
+ },
+ 'skip': 'Blocked outside of Germany',
+ }]
+
+ def _real_extract(self, url):
+ # determine video id from url
+ m = re.match(self._VALID_URL, url)
+
+ numid = re.search(r'documentId=([0-9]+)', url)
+ if numid:
+ video_id = numid.group(1)
+ else:
+ video_id = m.group('video_id')
+
+ webpage = self._download_webpage(url, video_id)
+
+ if '>Der gewц╪nschte Beitrag ist nicht mehr verfц╪gbar.<' in webpage:
+ raise ExtractorError('Video %s is no longer available' % video_id, expected=True)
+
+ if re.search(r'[\?&]rss($|[=&])', url):
+ doc = parse_xml(webpage)
+ if doc.tag == 'rss':
+ return GenericIE()._extract_rss(url, video_id, doc)
+
+ title = self._html_search_regex(
+ [r'(.*?)',
+ r'(.*?) '],
+ webpage, 'title')
+ description = self._html_search_meta(
+ 'dcterms.abstract', webpage, 'description', default=None)
+ if description is None:
+ description = self._html_search_meta(
+ 'description', webpage, 'meta description')
+
+ # Thumbnail is sometimes not present.
+ # It is in the mobile version, but that seems to use a different URL
+ # structure altogether.
+ thumbnail = self._og_search_thumbnail(webpage, default=None)
+
+ media_streams = re.findall(r'''(?x)
+ mediaCollection\.addMediaStream\([0-9]+,\s*[0-9]+,\s*"[^"]*",\s*
+ "([^"]+)"''', webpage)
+
+ if media_streams:
+ QUALITIES = qualities(['lo', 'hi', 'hq'])
+ formats = []
+ for furl in set(media_streams):
+ if furl.endswith('.f4m'):
+ fid = 'f4m'
+ else:
+ fid_m = re.match(r'.*\.([^.]+)\.[^.]+$', furl)
+ fid = fid_m.group(1) if fid_m else None
+ formats.append({
+ 'quality': QUALITIES(fid),
+ 'format_id': fid,
+ 'url': furl,
+ })
+ else: # request JSON file
+ media_info = self._download_json(
+ 'http://www.ardmediathek.de/play/media/%s' % video_id, video_id)
+ # The second element of the _mediaArray contains the standard http urls
+ streams = media_info['_mediaArray'][1]['_mediaStreamArray']
+ if not streams:
+ if '"fsk"' in webpage:
+ raise ExtractorError('This video is only available after 20:00')
+
+ formats = []
+ for s in streams:
+ if type(s['_stream']) == list:
+ for index, url in enumerate(s['_stream'][::-1]):
+ quality = s['_quality'] + index
+ formats.append({
+ 'quality': quality,
+ 'url': url,
+ 'format_id': '%s-%s' % (determine_ext(url), quality)
+ })
+ continue
+
+ format = {
+ 'quality': s['_quality'],
+ 'url': s['_stream'],
+ }
+
+ format['format_id'] = '%s-%s' % (
+ determine_ext(format['url']), format['quality'])
+
+ formats.append(format)
+
+ self._sort_formats(formats)
+
+ return {
+ 'id': video_id,
+ 'title': title,
+ 'description': description,
+ 'formats': formats,
+ 'thumbnail': thumbnail,
+ }
+
+
+class ARDIE(InfoExtractor):
+ _VALID_URL = '(?Phttps?://(www\.)?daserste\.de/[^?#]+/videos/(?P[^/?#]+)-(?P[0-9]+))\.html'
+ _TEST = {
+ 'url': 'http://www.daserste.de/information/reportage-dokumentation/dokus/videos/die-story-im-ersten-mission-unter-falscher-flagge-100.html',
+ 'md5': 'd216c3a86493f9322545e045ddc3eb35',
+ 'info_dict': {
+ 'display_id': 'die-story-im-ersten-mission-unter-falscher-flagge',
+ 'id': '100',
+ 'ext': 'mp4',
+ 'duration': 2600,
+ 'title': 'Die Story im Ersten: Mission unter falscher Flagge',
+ 'upload_date': '20140804',
+ 'thumbnail': 're:^https?://.*\.jpg$',
+ }
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ display_id = mobj.group('display_id')
+
+ player_url = mobj.group('mainurl') + '~playerXml.xml'
+ doc = self._download_xml(player_url, display_id)
+ video_node = doc.find('./video')
+ upload_date = unified_strdate(xpath_text(
+ video_node, './broadcastDate'))
+ thumbnail = xpath_text(video_node, './/teaserImage//variant/url')
+
+ formats = []
+ for a in video_node.findall('.//asset'):
+ f = {
+ 'format_id': a.attrib['type'],
+ 'width': int_or_none(a.find('./frameWidth').text),
+ 'height': int_or_none(a.find('./frameHeight').text),
+ 'vbr': int_or_none(a.find('./bitrateVideo').text),
+ 'abr': int_or_none(a.find('./bitrateAudio').text),
+ 'vcodec': a.find('./codecVideo').text,
+ 'tbr': int_or_none(a.find('./totalBitrate').text),
+ }
+ if a.find('./serverPrefix').text:
+ f['url'] = a.find('./serverPrefix').text
+ f['playpath'] = a.find('./fileName').text
+ else:
+ f['url'] = a.find('./fileName').text
+ formats.append(f)
+ self._sort_formats(formats)
+
+ return {
+ 'id': mobj.group('id'),
+ 'formats': formats,
+ 'display_id': display_id,
+ 'title': video_node.find('./title').text,
+ 'duration': parse_duration(video_node.find('./duration').text),
+ 'upload_date': upload_date,
+ 'thumbnail': thumbnail,
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/arte.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/arte.py
new file mode 100644
index 0000000000..219631b9b0
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/arte.py
@@ -0,0 +1,252 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+ find_xpath_attr,
+ unified_strdate,
+ get_element_by_id,
+ get_element_by_attribute,
+ int_or_none,
+ qualities,
+)
+
+# There are different sources of video in arte.tv, the extraction process
+# is different for each one. The videos usually expire in 7 days, so we can't
+# add tests.
+
+
+class ArteTvIE(InfoExtractor):
+ _VALID_URL = r'http://videos\.arte\.tv/(?Pfr|de)/.*-(?P.*?)\.html'
+ IE_NAME = 'arte.tv'
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ lang = mobj.group('lang')
+ video_id = mobj.group('id')
+
+ ref_xml_url = url.replace('/videos/', '/do_delegate/videos/')
+ ref_xml_url = ref_xml_url.replace('.html', ',view,asPlayerXml.xml')
+ ref_xml_doc = self._download_xml(
+ ref_xml_url, video_id, note='Downloading metadata')
+ config_node = find_xpath_attr(ref_xml_doc, './/video', 'lang', lang)
+ config_xml_url = config_node.attrib['ref']
+ config = self._download_xml(
+ config_xml_url, video_id, note='Downloading configuration')
+
+ formats = [{
+ 'forma_id': q.attrib['quality'],
+ # The playpath starts at 'mp4:', if we don't manually
+ # split the url, rtmpdump will incorrectly parse them
+ 'url': q.text.split('mp4:', 1)[0],
+ 'play_path': 'mp4:' + q.text.split('mp4:', 1)[1],
+ 'ext': 'flv',
+ 'quality': 2 if q.attrib['quality'] == 'hd' else 1,
+ } for q in config.findall('./urls/url')]
+ self._sort_formats(formats)
+
+ title = config.find('.//name').text
+ thumbnail = config.find('.//firstThumbnailUrl').text
+ return {
+ 'id': video_id,
+ 'title': title,
+ 'thumbnail': thumbnail,
+ 'formats': formats,
+ }
+
+
+class ArteTVPlus7IE(InfoExtractor):
+ IE_NAME = 'arte.tv:+7'
+ _VALID_URL = r'https?://(?:www\.)?arte\.tv/guide/(?Pfr|de)/(?:(?:sendungen|emissions)/)?(?P.*?)/(?P.*?)(\?.*)?'
+
+ @classmethod
+ def _extract_url_info(cls, url):
+ mobj = re.match(cls._VALID_URL, url)
+ lang = mobj.group('lang')
+ # This is not a real id, it can be for example AJT for the news
+ # http://www.arte.tv/guide/fr/emissions/AJT/arte-journal
+ video_id = mobj.group('id')
+ return video_id, lang
+
+ def _real_extract(self, url):
+ video_id, lang = self._extract_url_info(url)
+ webpage = self._download_webpage(url, video_id)
+ return self._extract_from_webpage(webpage, video_id, lang)
+
+ def _extract_from_webpage(self, webpage, video_id, lang):
+ json_url = self._html_search_regex(
+ [r'arte_vp_url=["\'](.*?)["\']', r'data-url=["\']([^"]+)["\']'],
+ webpage, 'json vp url')
+ return self._extract_from_json_url(json_url, video_id, lang)
+
+ def _extract_from_json_url(self, json_url, video_id, lang):
+ info = self._download_json(json_url, video_id)
+ player_info = info['videoJsonPlayer']
+
+ upload_date_str = player_info.get('shootingDate')
+ if not upload_date_str:
+ upload_date_str = player_info.get('VDA', '').split(' ')[0]
+
+ title = player_info['VTI'].strip()
+ subtitle = player_info.get('VSU', '').strip()
+ if subtitle:
+ title += ' - %s' % subtitle
+
+ info_dict = {
+ 'id': player_info['VID'],
+ 'title': title,
+ 'description': player_info.get('VDE'),
+ 'upload_date': unified_strdate(upload_date_str),
+ 'thumbnail': player_info.get('programImage') or player_info.get('VTU', {}).get('IUR'),
+ }
+ qfunc = qualities(['HQ', 'MQ', 'EQ', 'SQ'])
+
+ formats = []
+ for format_id, format_dict in player_info['VSR'].items():
+ f = dict(format_dict)
+ versionCode = f.get('versionCode')
+
+ langcode = {
+ 'fr': 'F',
+ 'de': 'A',
+ }.get(lang, lang)
+ lang_rexs = [r'VO?%s' % langcode, r'VO?.-ST%s' % langcode]
+ lang_pref = (
+ None if versionCode is None else (
+ 10 if any(re.match(r, versionCode) for r in lang_rexs)
+ else -10))
+ source_pref = 0
+ if versionCode is not None:
+ # The original version with subtitles has lower relevance
+ if re.match(r'VO-ST(F|A)', versionCode):
+ source_pref -= 10
+ # The version with sourds/mal subtitles has also lower relevance
+ elif re.match(r'VO?(F|A)-STM\1', versionCode):
+ source_pref -= 9
+ format = {
+ 'format_id': format_id,
+ 'preference': -10 if f.get('videoFormat') == 'M3U8' else None,
+ 'language_preference': lang_pref,
+ 'format_note': '%s, %s' % (f.get('versionCode'), f.get('versionLibelle')),
+ 'width': int_or_none(f.get('width')),
+ 'height': int_or_none(f.get('height')),
+ 'tbr': int_or_none(f.get('bitrate')),
+ 'quality': qfunc(f['quality']),
+ 'source_preference': source_pref,
+ }
+
+ if f.get('mediaType') == 'rtmp':
+ format['url'] = f['streamer']
+ format['play_path'] = 'mp4:' + f['url']
+ format['ext'] = 'flv'
+ else:
+ format['url'] = f['url']
+
+ formats.append(format)
+
+ self._sort_formats(formats)
+
+ info_dict['formats'] = formats
+ return info_dict
+
+
+# It also uses the arte_vp_url url from the webpage to extract the information
+class ArteTVCreativeIE(ArteTVPlus7IE):
+ IE_NAME = 'arte.tv:creative'
+ _VALID_URL = r'https?://creative\.arte\.tv/(?Pfr|de)/(?:magazine?/)?(?P[^?#]+)'
+
+ _TESTS = [{
+ 'url': 'http://creative.arte.tv/de/magazin/agentur-amateur-corporate-design',
+ 'info_dict': {
+ 'id': '72176',
+ 'ext': 'mp4',
+ 'title': 'Folge 2 - Corporate Design',
+ 'upload_date': '20131004',
+ },
+ }, {
+ 'url': 'http://creative.arte.tv/fr/Monty-Python-Reunion',
+ 'info_dict': {
+ 'id': '160676',
+ 'ext': 'mp4',
+ 'title': 'Monty Python live (mostly)',
+ 'description': 'ц┴vц╘nement ! Quarante-cinqб═ans aprц╗s leurs premiers succц╗s, les lц╘gendaires Monty Python remontent sur scц╗ne.\n',
+ 'upload_date': '20140805',
+ }
+ }]
+
+
+class ArteTVFutureIE(ArteTVPlus7IE):
+ IE_NAME = 'arte.tv:future'
+ _VALID_URL = r'https?://future\.arte\.tv/(?Pfr|de)/(thema|sujet)/.*?#article-anchor-(?P\d+)'
+
+ _TEST = {
+ 'url': 'http://future.arte.tv/fr/sujet/info-sciences#article-anchor-7081',
+ 'info_dict': {
+ 'id': '5201',
+ 'ext': 'mp4',
+ 'title': 'Les champignons au secours de la planц╗te',
+ 'upload_date': '20131101',
+ },
+ }
+
+ def _real_extract(self, url):
+ anchor_id, lang = self._extract_url_info(url)
+ webpage = self._download_webpage(url, anchor_id)
+ row = get_element_by_id(anchor_id, webpage)
+ return self._extract_from_webpage(row, anchor_id, lang)
+
+
+class ArteTVDDCIE(ArteTVPlus7IE):
+ IE_NAME = 'arte.tv:ddc'
+ _VALID_URL = r'https?://ddc\.arte\.tv/(?Pemission|folge)/(?P.+)'
+
+ def _real_extract(self, url):
+ video_id, lang = self._extract_url_info(url)
+ if lang == 'folge':
+ lang = 'de'
+ elif lang == 'emission':
+ lang = 'fr'
+ webpage = self._download_webpage(url, video_id)
+ scriptElement = get_element_by_attribute('class', 'visu_video_block', webpage)
+ script_url = self._html_search_regex(r'src="(.*?)"', scriptElement, 'script url')
+ javascriptPlayerGenerator = self._download_webpage(script_url, video_id, 'Download javascript player generator')
+ json_url = self._search_regex(r"json_url=(.*)&rendering_place.*", javascriptPlayerGenerator, 'json url')
+ return self._extract_from_json_url(json_url, video_id, lang)
+
+
+class ArteTVConcertIE(ArteTVPlus7IE):
+ IE_NAME = 'arte.tv:concert'
+ _VALID_URL = r'https?://concert\.arte\.tv/(?Pde|fr)/(?P.+)'
+
+ _TEST = {
+ 'url': 'http://concert.arte.tv/de/notwist-im-pariser-konzertclub-divan-du-monde',
+ 'md5': '9ea035b7bd69696b67aa2ccaaa218161',
+ 'info_dict': {
+ 'id': '186',
+ 'ext': 'mp4',
+ 'title': 'The Notwist im Pariser Konzertclub "Divan du Monde"',
+ 'upload_date': '20140128',
+ 'description': 'md5:486eb08f991552ade77439fe6d82c305',
+ },
+ }
+
+
+class ArteTVEmbedIE(ArteTVPlus7IE):
+ IE_NAME = 'arte.tv:embed'
+ _VALID_URL = r'''(?x)
+ http://www\.arte\.tv
+ /playerv2/embed\.php\?json_url=
+ (?P
+ http://arte\.tv/papi/tvguide/videos/stream/player/
+ (?P[^/]+)/(?P[^/]+)[^&]*
+ )
+ '''
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+ lang = mobj.group('lang')
+ json_url = mobj.group('json_url')
+ return self._extract_from_json_url(json_url, video_id, lang)
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/audiomack.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/audiomack.py
new file mode 100644
index 0000000000..622b209899
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/audiomack.py
@@ -0,0 +1,69 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from .soundcloud import SoundcloudIE
+from ..utils import ExtractorError
+
+import time
+
+
+class AudiomackIE(InfoExtractor):
+ _VALID_URL = r'https?://(?:www\.)?audiomack\.com/song/(?P[\w/-]+)'
+ IE_NAME = 'audiomack'
+ _TESTS = [
+ # hosted on audiomack
+ {
+ 'url': 'http://www.audiomack.com/song/roosh-williams/extraordinary',
+ 'info_dict':
+ {
+ 'id': 'roosh-williams/extraordinary',
+ 'ext': 'mp3',
+ 'title': 'Roosh Williams - Extraordinary'
+ }
+ },
+ # hosted on soundcloud via audiomack
+ {
+ 'add_ie': ['Soundcloud'],
+ 'url': 'http://www.audiomack.com/song/xclusiveszone/take-kare',
+ 'info_dict': {
+ 'id': '172419696',
+ 'ext': 'mp3',
+ 'description': 'md5:1fc3272ed7a635cce5be1568c2822997',
+ 'title': 'Young Thug ft Lil Wayne - Take Kare',
+ 'uploader': 'Young Thug World',
+ 'upload_date': '20141016',
+ }
+ },
+ ]
+
+ def _real_extract(self, url):
+ video_id = self._match_id(url)
+
+ api_response = self._download_json(
+ "http://www.audiomack.com/api/music/url/song/%s?_=%d" % (
+ video_id, time.time()),
+ video_id)
+
+ if "url" not in api_response:
+ raise ExtractorError("Unable to deduce api url of song")
+ realurl = api_response["url"]
+
+ # Audiomack wraps a lot of soundcloud tracks in their branded wrapper
+ # - if so, pass the work off to the soundcloud extractor
+ if SoundcloudIE.suitable(realurl):
+ return {'_type': 'url', 'url': realurl, 'ie_key': 'Soundcloud'}
+
+ webpage = self._download_webpage(url, video_id)
+ artist = self._html_search_regex(
+ r'(.*?) ', webpage, "artist")
+ songtitle = self._html_search_regex(
+ r'.*? (.*?)[^&]+).*?'
+
+ _TEST = {
+ 'url': 'http://auengine.com/embed.php?file=lfvlytY6&w=650&h=370',
+ 'md5': '48972bdbcf1a3a2f5533e62425b41d4f',
+ 'info_dict': {
+ 'id': 'lfvlytY6',
+ 'ext': 'mp4',
+ 'title': '[Commie]The Legend of the Legendary Heroes - 03 - Replication Eye (Alpha Stigma)[F9410F5A]'
+ }
+ }
+
+ def _real_extract(self, url):
+ video_id = self._match_id(url)
+
+ webpage = self._download_webpage(url, video_id)
+ title = self._html_search_regex(r'(?P<title>.+?) ', webpage, 'title')
+ title = title.strip()
+ links = re.findall(r'\s(?:file|url):\s*["\']([^\'"]+)["\']', webpage)
+ links = map(compat_urllib_parse.unquote, links)
+
+ thumbnail = None
+ video_url = None
+ for link in links:
+ if link.endswith('.png'):
+ thumbnail = link
+ elif '/videos/' in link:
+ video_url = link
+ if not video_url:
+ raise ExtractorError('Could not find video URL')
+ ext = '.' + determine_ext(video_url)
+ if ext == title[-len(ext):]:
+ title = title[:-len(ext)]
+
+ return {
+ 'id': video_id,
+ 'url': video_url,
+ 'title': title,
+ 'thumbnail': thumbnail,
+ 'http_referer': 'http://www.auengine.com/flowplayer/flowplayer.commercial-3.2.14.swf',
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/azubu.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/azubu.py
new file mode 100644
index 0000000000..0961d339fd
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/azubu.py
@@ -0,0 +1,93 @@
+from __future__ import unicode_literals
+
+import json
+
+from .common import InfoExtractor
+from ..utils import float_or_none
+
+
+class AzubuIE(InfoExtractor):
+ _VALID_URL = r'https?://(?:www\.)?azubu\.tv/[^/]+#!/play/(?P\d+)'
+ _TESTS = [
+ {
+ 'url': 'http://www.azubu.tv/GSL#!/play/15575/2014-hot6-cup-last-big-match-ro8-day-1',
+ 'md5': 'a88b42fcf844f29ad6035054bd9ecaf4',
+ 'info_dict': {
+ 'id': '15575',
+ 'ext': 'mp4',
+ 'title': '2014 HOT6 CUP LAST BIG MATCH Ro8 Day 1',
+ 'description': 'md5:d06bdea27b8cc4388a90ad35b5c66c01',
+ 'thumbnail': 're:^https?://.*\.jpe?g',
+ 'timestamp': 1417523507.334,
+ 'upload_date': '20141202',
+ 'duration': 9988.7,
+ 'uploader': 'GSL',
+ 'uploader_id': 414310,
+ 'view_count': int,
+ },
+ },
+ {
+ 'url': 'http://www.azubu.tv/FnaticTV#!/play/9344/-fnatic-at-worlds-2014:-toyz---%22i-love-rekkles,-he-has-amazing-mechanics%22-',
+ 'md5': 'b72a871fe1d9f70bd7673769cdb3b925',
+ 'info_dict': {
+ 'id': '9344',
+ 'ext': 'mp4',
+ 'title': 'Fnatic at Worlds 2014: Toyz - "I love Rekkles, he has amazing mechanics"',
+ 'description': 'md5:4a649737b5f6c8b5c5be543e88dc62af',
+ 'thumbnail': 're:^https?://.*\.jpe?g',
+ 'timestamp': 1410530893.320,
+ 'upload_date': '20140912',
+ 'duration': 172.385,
+ 'uploader': 'FnaticTV',
+ 'uploader_id': 272749,
+ 'view_count': int,
+ },
+ },
+ ]
+
+ def _real_extract(self, url):
+ video_id = self._match_id(url)
+
+ data = self._download_json(
+ 'http://www.azubu.tv/api/video/%s' % video_id, video_id)['data']
+
+ title = data['title'].strip()
+ description = data['description']
+ thumbnail = data['thumbnail']
+ view_count = data['view_count']
+ uploader = data['user']['username']
+ uploader_id = data['user']['id']
+
+ stream_params = json.loads(data['stream_params'])
+
+ timestamp = float_or_none(stream_params['creationDate'], 1000)
+ duration = float_or_none(stream_params['length'], 1000)
+
+ renditions = stream_params.get('renditions') or []
+ video = stream_params.get('FLVFullLength') or stream_params.get('videoFullLength')
+ if video:
+ renditions.append(video)
+
+ formats = [{
+ 'url': fmt['url'],
+ 'width': fmt['frameWidth'],
+ 'height': fmt['frameHeight'],
+ 'vbr': float_or_none(fmt['encodingRate'], 1000),
+ 'filesize': fmt['size'],
+ 'vcodec': fmt['videoCodec'],
+ 'container': fmt['videoContainer'],
+ } for fmt in renditions if fmt['url']]
+ self._sort_formats(formats)
+
+ return {
+ 'id': video_id,
+ 'title': title,
+ 'description': description,
+ 'thumbnail': thumbnail,
+ 'timestamp': timestamp,
+ 'duration': duration,
+ 'uploader': uploader,
+ 'uploader_id': uploader_id,
+ 'view_count': view_count,
+ 'formats': formats,
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/bambuser.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/bambuser.py
new file mode 100644
index 0000000000..98e1443ab0
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/bambuser.py
@@ -0,0 +1,96 @@
+from __future__ import unicode_literals
+
+import re
+import json
+import itertools
+
+from .common import InfoExtractor
+from ..compat import (
+ compat_urllib_request,
+)
+
+
+class BambuserIE(InfoExtractor):
+ IE_NAME = 'bambuser'
+ _VALID_URL = r'https?://bambuser\.com/v/(?P\d+)'
+ _API_KEY = '005f64509e19a868399060af746a00aa'
+
+ _TEST = {
+ 'url': 'http://bambuser.com/v/4050584',
+ # MD5 seems to be flaky, see https://travis-ci.org/rg3/youtube-dl/jobs/14051016#L388
+ # 'md5': 'fba8f7693e48fd4e8641b3fd5539a641',
+ 'info_dict': {
+ 'id': '4050584',
+ 'ext': 'flv',
+ 'title': 'Education engineering days - lightning talks',
+ 'duration': 3741,
+ 'uploader': 'pixelversity',
+ 'uploader_id': '344706',
+ },
+ 'params': {
+ # It doesn't respect the 'Range' header, it would download the whole video
+ # caused the travis builds to fail: https://travis-ci.org/rg3/youtube-dl/jobs/14493845#L59
+ 'skip_download': True,
+ },
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+ info_url = ('http://player-c.api.bambuser.com/getVideo.json?'
+ '&api_key=%s&vid=%s' % (self._API_KEY, video_id))
+ info_json = self._download_webpage(info_url, video_id)
+ info = json.loads(info_json)['result']
+
+ return {
+ 'id': video_id,
+ 'title': info['title'],
+ 'url': info['url'],
+ 'thumbnail': info.get('preview'),
+ 'duration': int(info['length']),
+ 'view_count': int(info['views_total']),
+ 'uploader': info['username'],
+ 'uploader_id': info['uid'],
+ }
+
+
+class BambuserChannelIE(InfoExtractor):
+ IE_NAME = 'bambuser:channel'
+ _VALID_URL = r'https?://bambuser\.com/channel/(?P.*?)(?:/|#|\?|$)'
+ # The maximum number we can get with each request
+ _STEP = 50
+ _TEST = {
+ 'url': 'http://bambuser.com/channel/pixelversity',
+ 'info_dict': {
+ 'title': 'pixelversity',
+ },
+ 'playlist_mincount': 60,
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ user = mobj.group('user')
+ urls = []
+ last_id = ''
+ for i in itertools.count(1):
+ req_url = (
+ 'http://bambuser.com/xhr-api/index.php?username={user}'
+ '&sort=created&access_mode=0%2C1%2C2&limit={count}'
+ '&method=broadcast&format=json&vid_older_than={last}'
+ ).format(user=user, count=self._STEP, last=last_id)
+ req = compat_urllib_request.Request(req_url)
+ # Without setting this header, we wouldn't get any result
+ req.add_header('Referer', 'http://bambuser.com/channel/%s' % user)
+ data = self._download_json(
+ req, user, 'Downloading page %d' % i)
+ results = data['result']
+ if not results:
+ break
+ last_id = results[-1]['vid']
+ urls.extend(self.url_result(v['page'], 'Bambuser') for v in results)
+
+ return {
+ '_type': 'playlist',
+ 'title': user,
+ 'entries': urls,
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/bandcamp.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/bandcamp.py
new file mode 100644
index 0000000000..b45d68a614
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/bandcamp.py
@@ -0,0 +1,171 @@
+from __future__ import unicode_literals
+
+import json
+import re
+
+from .common import InfoExtractor
+from ..compat import (
+ compat_str,
+ compat_urlparse,
+)
+from ..utils import (
+ ExtractorError,
+)
+
+
+class BandcampIE(InfoExtractor):
+ _VALID_URL = r'https?://.*?\.bandcamp\.com/track/(?P.*)'
+ _TESTS = [{
+ 'url': 'http://youtube-dl.bandcamp.com/track/youtube-dl-test-song',
+ 'md5': 'c557841d5e50261777a6585648adf439',
+ 'info_dict': {
+ 'id': '1812978515',
+ 'ext': 'mp3',
+ 'title': "youtube-dl \"'/\\\u00e4\u21ad - youtube-dl test song \"'/\\\u00e4\u21ad",
+ 'duration': 9.8485,
+ },
+ '_skip': 'There is a limit of 200 free downloads / month for the test song'
+ }, {
+ 'url': 'http://benprunty.bandcamp.com/track/lanius-battle',
+ 'md5': '2b68e5851514c20efdff2afc5603b8b4',
+ 'info_dict': {
+ 'id': '2650410135',
+ 'ext': 'mp3',
+ 'title': 'Lanius (Battle)',
+ 'uploader': 'Ben Prunty Music',
+ },
+ }]
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ title = mobj.group('title')
+ webpage = self._download_webpage(url, title)
+ m_download = re.search(r'freeDownloadPage: "(.*?)"', webpage)
+ if not m_download:
+ m_trackinfo = re.search(r'trackinfo: (.+),\s*?\n', webpage)
+ if m_trackinfo:
+ json_code = m_trackinfo.group(1)
+ data = json.loads(json_code)[0]
+
+ formats = []
+ for format_id, format_url in data['file'].items():
+ ext, abr_str = format_id.split('-', 1)
+ formats.append({
+ 'format_id': format_id,
+ 'url': format_url,
+ 'ext': ext,
+ 'vcodec': 'none',
+ 'acodec': ext,
+ 'abr': int(abr_str),
+ })
+
+ self._sort_formats(formats)
+
+ return {
+ 'id': compat_str(data['id']),
+ 'title': data['title'],
+ 'formats': formats,
+ 'duration': float(data['duration']),
+ }
+ else:
+ raise ExtractorError('No free songs found')
+
+ download_link = m_download.group(1)
+ video_id = self._search_regex(
+ r'var TralbumData = {.*?id: (?P<id>\d+),?$',
+ webpage, 'video id', flags=re.MULTILINE | re.DOTALL)
+
+ download_webpage = self._download_webpage(download_link, video_id, 'Downloading free downloads page')
+ # We get the dictionary of the track from some javascript code
+ info = re.search(r'items: (.*?),$', download_webpage, re.MULTILINE).group(1)
+ info = json.loads(info)[0]
+ # We pick mp3-320 for now, until format selection can be easily implemented.
+ mp3_info = info['downloads']['mp3-320']
+ # If we try to use this url it says the link has expired
+ initial_url = mp3_info['url']
+ re_url = r'(?P<server>http://(.*?)\.bandcamp\.com)/download/track\?enc=mp3-320&fsig=(?P<fsig>.*?)&id=(?P<id>.*?)&ts=(?P<ts>.*)$'
+ m_url = re.match(re_url, initial_url)
+ # We build the url we will use to get the final track url
+ # This url is build in Bandcamp in the script download_bunde_*.js
+ request_url = '%s/statdownload/track?enc=mp3-320&fsig=%s&id=%s&ts=%s&.rand=665028774616&.vrs=1' % (m_url.group('server'), m_url.group('fsig'), video_id, m_url.group('ts'))
+ final_url_webpage = self._download_webpage(request_url, video_id, 'Requesting download url')
+ # If we could correctly generate the .rand field the url would be
+ # in the "download_url" key
+ final_url = re.search(r'"retry_url":"(.*?)"', final_url_webpage).group(1)
+
+ return {
+ 'id': video_id,
+ 'title': info['title'],
+ 'ext': 'mp3',
+ 'vcodec': 'none',
+ 'url': final_url,
+ 'thumbnail': info.get('thumb_url'),
+ 'uploader': info.get('artist'),
+ }
+
+
+class BandcampAlbumIE(InfoExtractor):
+ IE_NAME = 'Bandcamp:album'
+ _VALID_URL = r'https?://(?:(?P<subdomain>[^.]+)\.)?bandcamp\.com(?:/album/(?P<title>[^?#]+)|/?(?:$|[?#]))'
+
+ _TESTS = [{
+ 'url': 'http://blazo.bandcamp.com/album/jazz-format-mixtape-vol-1',
+ 'playlist': [
+ {
+ 'md5': '39bc1eded3476e927c724321ddf116cf',
+ 'info_dict': {
+ 'id': '1353101989',
+ 'ext': 'mp3',
+ 'title': 'Intro',
+ }
+ },
+ {
+ 'md5': '1a2c32e2691474643e912cc6cd4bffaa',
+ 'info_dict': {
+ 'id': '38097443',
+ 'ext': 'mp3',
+ 'title': 'Kero One - Keep It Alive (Blazo remix)',
+ }
+ },
+ ],
+ 'info_dict': {
+ 'title': 'Jazz Format Mixtape vol.1',
+ },
+ 'params': {
+ 'playlistend': 2
+ },
+ 'skip': 'Bandcamp imposes download limits. See test_playlists:test_bandcamp_album for the playlist test'
+ }, {
+ 'url': 'http://nightbringer.bandcamp.com/album/hierophany-of-the-open-grave',
+ 'info_dict': {
+ 'title': 'Hierophany of the Open Grave',
+ },
+ 'playlist_mincount': 9,
+ }, {
+ 'url': 'http://dotscale.bandcamp.com',
+ 'info_dict': {
+ 'title': 'Loom',
+ },
+ 'playlist_mincount': 7,
+ }]
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ playlist_id = mobj.group('subdomain')
+ title = mobj.group('title')
+ display_id = title or playlist_id
+ webpage = self._download_webpage(url, display_id)
+ tracks_paths = re.findall(r'<a href="(.*?)" itemprop="url">', webpage)
+ if not tracks_paths:
+ raise ExtractorError('The page doesn\'t contain any tracks')
+ entries = [
+ self.url_result(compat_urlparse.urljoin(url, t_path), ie=BandcampIE.ie_key())
+ for t_path in tracks_paths]
+ title = self._search_regex(r'album_title : "(.*?)"', webpage, 'title')
+ return {
+ '_type': 'playlist',
+ 'id': playlist_id,
+ 'display_id': display_id,
+ 'title': title,
+ 'entries': entries,
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/bbccouk.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/bbccouk.py
new file mode 100644
index 0000000000..2d2f742aee
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/bbccouk.py
@@ -0,0 +1,262 @@
+from __future__ import unicode_literals
+
+import xml.etree.ElementTree
+
+from .subtitles import SubtitlesInfoExtractor
+from ..utils import ExtractorError
+from ..compat import compat_HTTPError
+
+
+class BBCCoUkIE(SubtitlesInfoExtractor):
+ IE_NAME = 'bbc.co.uk'
+ IE_DESC = 'BBC iPlayer'
+ _VALID_URL = r'https?://(?:www\.)?bbc\.co\.uk/(?:programmes|iplayer/episode)/(?P<id>[\da-z]{8})'
+
+ _TESTS = [
+ {
+ 'url': 'http://www.bbc.co.uk/programmes/b039g8p7',
+ 'info_dict': {
+ 'id': 'b039d07m',
+ 'ext': 'flv',
+ 'title': 'Kaleidoscope: Leonard Cohen',
+ 'description': 'md5:db4755d7a665ae72343779f7dacb402c',
+ 'duration': 1740,
+ },
+ 'params': {
+ # rtmp download
+ 'skip_download': True,
+ }
+ },
+ {
+ 'url': 'http://www.bbc.co.uk/iplayer/episode/b00yng5w/The_Man_in_Black_Series_3_The_Printed_Name/',
+ 'info_dict': {
+ 'id': 'b00yng1d',
+ 'ext': 'flv',
+ 'title': 'The Man in Black: Series 3: The Printed Name',
+ 'description': "Mark Gatiss introduces Nicholas Pierpan's chilling tale of a writer's devilish pact with a mysterious man. Stars Ewan Bailey.",
+ 'duration': 1800,
+ },
+ 'params': {
+ # rtmp download
+ 'skip_download': True,
+ },
+ 'skip': 'Episode is no longer available on BBC iPlayer Radio',
+ },
+ {
+ 'url': 'http://www.bbc.co.uk/iplayer/episode/b03vhd1f/The_Voice_UK_Series_3_Blind_Auditions_5/',
+ 'info_dict': {
+ 'id': 'b00yng1d',
+ 'ext': 'flv',
+ 'title': 'The Voice UK: Series 3: Blind Auditions 5',
+ 'description': "Emma Willis and Marvin Humes present the fifth set of blind auditions in the singing competition, as the coaches continue to build their teams based on voice alone.",
+ 'duration': 5100,
+ },
+ 'params': {
+ # rtmp download
+ 'skip_download': True,
+ },
+ 'skip': 'Currently BBC iPlayer TV programmes are available to play in the UK only',
+ },
+ {
+ 'url': 'http://www.bbc.co.uk/iplayer/episode/p026c7jt/tomorrows-worlds-the-unearthly-history-of-science-fiction-2-invasion',
+ 'info_dict': {
+ 'id': 'b03k3pb7',
+ 'ext': 'flv',
+ 'title': "Tomorrow's Worlds: The Unearthly History of Science Fiction",
+ 'description': '2. Invasion',
+ 'duration': 3600,
+ },
+ 'params': {
+ # rtmp download
+ 'skip_download': True,
+ },
+ 'skip': 'Currently BBC iPlayer TV programmes are available to play in the UK only',
+ },
+ ]
+
+ def _extract_asx_playlist(self, connection, programme_id):
+ asx = self._download_xml(connection.get('href'), programme_id, 'Downloading ASX playlist')
+ return [ref.get('href') for ref in asx.findall('./Entry/ref')]
+
+ def _extract_connection(self, connection, programme_id):
+ formats = []
+ protocol = connection.get('protocol')
+ supplier = connection.get('supplier')
+ if protocol == 'http':
+ href = connection.get('href')
+ # ASX playlist
+ if supplier == 'asx':
+ for i, ref in enumerate(self._extract_asx_playlist(connection, programme_id)):
+ formats.append({
+ 'url': ref,
+ 'format_id': 'ref%s_%s' % (i, supplier),
+ })
+ # Direct link
+ else:
+ formats.append({
+ 'url': href,
+ 'format_id': supplier,
+ })
+ elif protocol == 'rtmp':
+ application = connection.get('application', 'ondemand')
+ auth_string = connection.get('authString')
+ identifier = connection.get('identifier')
+ server = connection.get('server')
+ formats.append({
+ 'url': '%s://%s/%s?%s' % (protocol, server, application, auth_string),
+ 'play_path': identifier,
+ 'app': '%s?%s' % (application, auth_string),
+ 'page_url': 'http://www.bbc.co.uk',
+ 'player_url': 'http://www.bbc.co.uk/emp/releases/iplayer/revisions/617463_618125_4/617463_618125_4_emp.swf',
+ 'rtmp_live': False,
+ 'ext': 'flv',
+ 'format_id': supplier,
+ })
+ return formats
+
+ def _extract_items(self, playlist):
+ return playlist.findall('./{http://bbc.co.uk/2008/emp/playlist}item')
+
+ def _extract_medias(self, media_selection):
+ error = media_selection.find('./{http://bbc.co.uk/2008/mp/mediaselection}error')
+ if error is not None:
+ raise ExtractorError(
+ '%s returned error: %s' % (self.IE_NAME, error.get('id')), expected=True)
+ return media_selection.findall('./{http://bbc.co.uk/2008/mp/mediaselection}media')
+
+ def _extract_connections(self, media):
+ return media.findall('./{http://bbc.co.uk/2008/mp/mediaselection}connection')
+
+ def _extract_video(self, media, programme_id):
+ formats = []
+ vbr = int(media.get('bitrate'))
+ vcodec = media.get('encoding')
+ service = media.get('service')
+ width = int(media.get('width'))
+ height = int(media.get('height'))
+ file_size = int(media.get('media_file_size'))
+ for connection in self._extract_connections(media):
+ conn_formats = self._extract_connection(connection, programme_id)
+ for format in conn_formats:
+ format.update({
+ 'format_id': '%s_%s' % (service, format['format_id']),
+ 'width': width,
+ 'height': height,
+ 'vbr': vbr,
+ 'vcodec': vcodec,
+ 'filesize': file_size,
+ })
+ formats.extend(conn_formats)
+ return formats
+
+ def _extract_audio(self, media, programme_id):
+ formats = []
+ abr = int(media.get('bitrate'))
+ acodec = media.get('encoding')
+ service = media.get('service')
+ for connection in self._extract_connections(media):
+ conn_formats = self._extract_connection(connection, programme_id)
+ for format in conn_formats:
+ format.update({
+ 'format_id': '%s_%s' % (service, format['format_id']),
+ 'abr': abr,
+ 'acodec': acodec,
+ })
+ formats.extend(conn_formats)
+ return formats
+
+ def _extract_captions(self, media, programme_id):
+ subtitles = {}
+ for connection in self._extract_connections(media):
+ captions = self._download_xml(connection.get('href'), programme_id, 'Downloading captions')
+ lang = captions.get('{http://www.w3.org/XML/1998/namespace}lang', 'en')
+ ps = captions.findall('./{0}body/{0}div/{0}p'.format('{http://www.w3.org/2006/10/ttaf1}'))
+ srt = ''
+ for pos, p in enumerate(ps):
+ srt += '%s\r\n%s --> %s\r\n%s\r\n\r\n' % (str(pos), p.get('begin'), p.get('end'),
+ p.text.strip() if p.text is not None else '')
+ subtitles[lang] = srt
+ return subtitles
+
+ def _download_media_selector(self, programme_id):
+ try:
+ media_selection = self._download_xml(
+ 'http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/pc/vpid/%s' % programme_id,
+ programme_id, 'Downloading media selection XML')
+ except ExtractorError as ee:
+ if isinstance(ee.cause, compat_HTTPError) and ee.cause.code == 403:
+ media_selection = xml.etree.ElementTree.fromstring(ee.cause.read().encode('utf-8'))
+ else:
+ raise
+
+ formats = []
+ subtitles = None
+
+ for media in self._extract_medias(media_selection):
+ kind = media.get('kind')
+ if kind == 'audio':
+ formats.extend(self._extract_audio(media, programme_id))
+ elif kind == 'video':
+ formats.extend(self._extract_video(media, programme_id))
+ elif kind == 'captions':
+ subtitles = self._extract_captions(media, programme_id)
+
+ return formats, subtitles
+
+ def _real_extract(self, url):
+ group_id = self._match_id(url)
+
+ webpage = self._download_webpage(url, group_id, 'Downloading video page')
+
+ programme_id = self._search_regex(
+ r'"vpid"\s*:\s*"([\da-z]{8})"', webpage, 'vpid', fatal=False, default=None)
+ if programme_id:
+ player = self._download_json(
+ 'http://www.bbc.co.uk/iplayer/episode/%s.json' % group_id,
+ group_id)['jsConf']['player']
+ title = player['title']
+ description = player['subtitle']
+ duration = player['duration']
+ formats, subtitles = self._download_media_selector(programme_id)
+ else:
+ playlist = self._download_xml(
+ 'http://www.bbc.co.uk/iplayer/playlist/%s' % group_id,
+ group_id, 'Downloading playlist XML')
+
+ no_items = playlist.find('./{http://bbc.co.uk/2008/emp/playlist}noItems')
+ if no_items is not None:
+ reason = no_items.get('reason')
+ if reason == 'preAvailability':
+ msg = 'Episode %s is not yet available' % group_id
+ elif reason == 'postAvailability':
+ msg = 'Episode %s is no longer available' % group_id
+ elif reason == 'noMedia':
+ msg = 'Episode %s is not currently available' % group_id
+ else:
+ msg = 'Episode %s is not available: %s' % (group_id, reason)
+ raise ExtractorError(msg, expected=True)
+
+ for item in self._extract_items(playlist):
+ kind = item.get('kind')
+ if kind != 'programme' and kind != 'radioProgramme':
+ continue
+ title = playlist.find('./{http://bbc.co.uk/2008/emp/playlist}title').text
+ description = playlist.find('./{http://bbc.co.uk/2008/emp/playlist}summary').text
+ programme_id = item.get('identifier')
+ duration = int(item.get('duration'))
+ formats, subtitles = self._download_media_selector(programme_id)
+
+ if self._downloader.params.get('listsubtitles', False):
+ self._list_available_subtitles(programme_id, subtitles)
+ return
+
+ self._sort_formats(formats)
+
+ return {
+ 'id': programme_id,
+ 'title': title,
+ 'description': description,
+ 'duration': duration,
+ 'formats': formats,
+ 'subtitles': subtitles,
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/beeg.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/beeg.py
new file mode 100644
index 0000000000..4e79fea8f0
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/beeg.py
@@ -0,0 +1,65 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+
+
+class BeegIE(InfoExtractor):
+ _VALID_URL = r'https?://(?:www\.)?beeg\.com/(?P<id>\d+)'
+ _TEST = {
+ 'url': 'http://beeg.com/5416503',
+ 'md5': '634526ae978711f6b748fe0dd6c11f57',
+ 'info_dict': {
+ 'id': '5416503',
+ 'ext': 'mp4',
+ 'title': 'Sultry Striptease',
+ 'description': 'md5:6db3c6177972822aaba18652ff59c773',
+ 'categories': list, # NSFW
+ 'thumbnail': 're:https?://.*\.jpg$',
+ 'age_limit': 18,
+ }
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+
+ webpage = self._download_webpage(url, video_id)
+
+ quality_arr = self._search_regex(
+ r'(?s)var\s+qualityArr\s*=\s*{\s*(.+?)\s*}', webpage, 'quality formats')
+
+ formats = [{
+ 'url': fmt[1],
+ 'format_id': fmt[0],
+ 'height': int(fmt[0][:-1]),
+ } for fmt in re.findall(r"'([^']+)'\s*:\s*'([^']+)'", quality_arr)]
+
+ self._sort_formats(formats)
+
+ title = self._html_search_regex(
+ r'<title>([^<]+)\s*-\s*beeg\.? ', webpage, 'title')
+
+ description = self._html_search_regex(
+ r'[0-9]{2})/(?P[0-9]{2})/(?P[^/#?_]+)'
+ _TEST = {
+ 'url': 'http://www.behindkink.com/2014/12/05/what-are-you-passionate-about-marley-blaze/',
+ 'md5': '507b57d8fdcd75a41a9a7bdb7989c762',
+ 'info_dict': {
+ 'id': '37127',
+ 'ext': 'mp4',
+ 'title': 'What are you passionate about Б─⌠ Marley Blaze',
+ 'description': 'md5:aee8e9611b4ff70186f752975d9b94b4',
+ 'upload_date': '20141205',
+ 'thumbnail': 'http://www.behindkink.com/wp-content/uploads/2014/12/blaze-1.jpg',
+ 'age_limit': 18,
+ }
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ display_id = mobj.group('id')
+
+ webpage = self._download_webpage(url, display_id)
+
+ video_url = self._search_regex(
+ r'.+?)\.html'
+ _TESTS = [
+ {
+ 'url': 'http://www.bet.com/news/politics/2014/12/08/in-bet-exclusive-obama-talks-race-and-racism.html',
+ 'info_dict': {
+ 'id': '417cd61c-c793-4e8e-b006-e445ecc45add',
+ 'display_id': 'in-bet-exclusive-obama-talks-race-and-racism',
+ 'ext': 'flv',
+ 'title': 'BET News Presents: A Conversation With President Obama',
+ 'description': 'md5:5a88d8ae912c1b33e090290af7ec33c6',
+ 'duration': 1534,
+ 'timestamp': 1418075340,
+ 'upload_date': '20141208',
+ 'uploader': 'admin',
+ 'thumbnail': 're:(?i)^https?://.*\.jpg$',
+ },
+ 'params': {
+ # rtmp download
+ 'skip_download': True,
+ },
+ },
+ {
+ 'url': 'http://www.bet.com/video/news/national/2014/justice-for-ferguson-a-community-reacts.html',
+ 'info_dict': {
+ 'id': '4160e53b-ad41-43b1-980f-8d85f63121f4',
+ 'display_id': 'justice-for-ferguson-a-community-reacts',
+ 'ext': 'flv',
+ 'title': 'Justice for Ferguson: A Community Reacts',
+ 'description': 'A BET News special.',
+ 'duration': 1696,
+ 'timestamp': 1416942360,
+ 'upload_date': '20141125',
+ 'uploader': 'admin',
+ 'thumbnail': 're:(?i)^https?://.*\.jpg$',
+ },
+ 'params': {
+ # rtmp download
+ 'skip_download': True,
+ },
+ }
+ ]
+
+ def _real_extract(self, url):
+ display_id = self._match_id(url)
+
+ webpage = self._download_webpage(url, display_id)
+
+ media_url = compat_urllib_parse.unquote(self._search_regex(
+ [r'mediaURL\s*:\s*"([^"]+)"', r"var\s+mrssMediaUrl\s*=\s*'([^']+)'"],
+ webpage, 'media URL'))
+
+ mrss = self._download_xml(media_url, display_id)
+
+ item = mrss.find('./channel/item')
+
+ NS_MAP = {
+ 'dc': 'http://purl.org/dc/elements/1.1/',
+ 'media': 'http://search.yahoo.com/mrss/',
+ 'ka': 'http://kickapps.com/karss',
+ }
+
+ title = xpath_text(item, './title', 'title')
+ description = xpath_text(
+ item, './description', 'description', fatal=False)
+
+ video_id = xpath_text(item, './guid', 'video id', fatal=False)
+
+ timestamp = parse_iso8601(xpath_text(
+ item, xpath_with_ns('./dc:date', NS_MAP),
+ 'upload date', fatal=False))
+ uploader = xpath_text(
+ item, xpath_with_ns('./dc:creator', NS_MAP),
+ 'uploader', fatal=False)
+
+ media_content = item.find(
+ xpath_with_ns('./media:content', NS_MAP))
+ duration = int_or_none(media_content.get('duration'))
+ smil_url = media_content.get('url')
+
+ thumbnail = media_content.find(
+ xpath_with_ns('./media:thumbnail', NS_MAP)).get('url')
+
+ formats = self._extract_smil_formats(smil_url, display_id)
+
+ return {
+ 'id': video_id,
+ 'display_id': display_id,
+ 'title': title,
+ 'description': description,
+ 'thumbnail': thumbnail,
+ 'timestamp': timestamp,
+ 'uploader': uploader,
+ 'duration': duration,
+ 'formats': formats,
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/bild.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/bild.py
new file mode 100644
index 0000000000..77b562d996
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/bild.py
@@ -0,0 +1,39 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import int_or_none
+
+
+class BildIE(InfoExtractor):
+ _VALID_URL = r'https?://(?:www\.)?bild\.de/(?:[^/]+/)+(?P[^/]+)-(?P\d+)(?:,auto=true)?\.bild\.html'
+ IE_DESC = 'Bild.de'
+ _TEST = {
+ 'url': 'http://www.bild.de/video/clip/apple-ipad-air/das-koennen-die-neuen-ipads-38184146.bild.html',
+ 'md5': 'dd495cbd99f2413502a1713a1156ac8a',
+ 'info_dict': {
+ 'id': '38184146',
+ 'ext': 'mp4',
+ 'title': 'BILD hat sie getestet',
+ 'thumbnail': 'http://bilder.bild.de/fotos/stand-das-koennen-die-neuen-ipads-38184138/Bild/1.bild.jpg',
+ 'duration': 196,
+ 'description': 'Mit dem iPad Air 2 und dem iPad Mini 3 hat Apple zwei neue Tablet-Modelle prц╓sentiert. BILD-Reporter Sven Stein durfte die Gerц╓te bereits testen. ',
+ }
+ }
+
+ def _real_extract(self, url):
+ video_id = self._match_id(url)
+
+ xml_url = url.split(".bild.html")[0] + ",view=xml.bild.xml"
+ doc = self._download_xml(xml_url, video_id)
+
+ duration = int_or_none(doc.attrib.get('duration'), scale=1000)
+
+ return {
+ 'id': video_id,
+ 'title': doc.attrib['ueberschrift'],
+ 'description': doc.attrib.get('text'),
+ 'url': doc.attrib['src'],
+ 'thumbnail': doc.attrib.get('img'),
+ 'duration': duration,
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/bilibili.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/bilibili.py
new file mode 100644
index 0000000000..241b904a9e
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/bilibili.py
@@ -0,0 +1,105 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..compat import compat_parse_qs
+from ..utils import (
+ ExtractorError,
+ int_or_none,
+ unified_strdate,
+)
+
+
+class BiliBiliIE(InfoExtractor):
+ _VALID_URL = r'http://www\.bilibili\.(?:tv|com)/video/av(?P[0-9]+)/'
+
+ _TEST = {
+ 'url': 'http://www.bilibili.tv/video/av1074402/',
+ 'md5': '2c301e4dab317596e837c3e7633e7d86',
+ 'info_dict': {
+ 'id': '1074402',
+ 'ext': 'flv',
+ 'title': 'Ц─░И┤▒Е²╥Е·┐Ц─▒И┤▒ФЁ║Ф╡╚',
+ 'duration': 308,
+ 'upload_date': '20140420',
+ 'thumbnail': 're:^https?://.+\.jpg',
+ },
+ }
+
+ def _real_extract(self, url):
+ video_id = self._match_id(url)
+ webpage = self._download_webpage(url, video_id)
+
+ video_code = self._search_regex(
+ r'(?s)(.*?)
', webpage, 'video code')
+
+ title = self._html_search_meta(
+ 'media:title', video_code, 'title', fatal=True)
+ duration_str = self._html_search_meta(
+ 'duration', video_code, 'duration')
+ if duration_str is None:
+ duration = None
+ else:
+ duration_mobj = re.match(
+ r'^T(?:(?P[0-9]+)H)?(?P[0-9]+)M(?P[0-9]+)S$',
+ duration_str)
+ duration = (
+ int_or_none(duration_mobj.group('hours'), default=0) * 3600 +
+ int(duration_mobj.group('minutes')) * 60 +
+ int(duration_mobj.group('seconds')))
+ upload_date = unified_strdate(self._html_search_meta(
+ 'uploadDate', video_code, fatal=False))
+ thumbnail = self._html_search_meta(
+ 'thumbnailUrl', video_code, 'thumbnail', fatal=False)
+
+ player_params = compat_parse_qs(self._html_search_regex(
+ r'',
+ start_page, 'xml root', default=None)
+ if xml_root is None:
+ # Probably need to authenticate
+ login_res = self._login(webpage_url, video_id)
+ if login_res is None:
+ self.report_warning('Could not login.')
+ else:
+ start_page = login_res
+ # Grab the url from the authenticated page
+ xml_root = self._html_search_regex(
+ r'',
+ start_page, 'xml filename', default=None)
+ if xml_name is None:
+ # Fallback to the older format
+ xml_name = self._html_search_regex(r'', webpage)
+ if mobj:
+ embedded_url = mobj.group(1)
+ return self.url_result(embedded_url)
+
+ video_title = self._html_search_regex(r']*>([^<]+)', webpage, 'title')
+ video_url = compat_urllib_parse.unquote(self._html_search_regex(r'video_url=(.+?)&', webpage, 'video_url'))
+ if 'encrypted=true' in webpage:
+ password = self._html_search_regex(r'video_title=(.+?)&', webpage, 'password')
+ video_url = aes_decrypt_text(video_url, password, 32).decode('utf-8')
+ path = compat_urllib_parse_urlparse(video_url).path
+ extension = os.path.splitext(path)[1][1:]
+ format = path.split('/')[4].split('_')[:2]
+ format = "-".join(format)
+
+ age_limit = self._rta_search(webpage)
+
+ return {
+ 'id': video_id,
+ 'title': video_title,
+ 'url': video_url,
+ 'ext': extension,
+ 'format': format,
+ 'format_id': format,
+ 'age_limit': age_limit,
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/khanacademy.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/khanacademy.py
new file mode 100644
index 0000000000..408d00944c
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/khanacademy.py
@@ -0,0 +1,80 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+ unified_strdate,
+)
+
+
+class KhanAcademyIE(InfoExtractor):
+ _VALID_URL = r'^https?://(?:(?:www|api)\.)?khanacademy\.org/(?P[^/]+)/(?:[^/]+/){,2}(?P[^?#/]+)(?:$|[?#])'
+ IE_NAME = 'KhanAcademy'
+
+ _TESTS = [{
+ 'url': 'http://www.khanacademy.org/video/one-time-pad',
+ 'md5': '7021db7f2d47d4fff89b13177cb1e8f4',
+ 'info_dict': {
+ 'id': 'one-time-pad',
+ 'ext': 'mp4',
+ 'title': 'The one-time pad',
+ 'description': 'The perfect cipher',
+ 'duration': 176,
+ 'uploader': 'Brit Cruise',
+ 'upload_date': '20120411',
+ }
+ }, {
+ 'url': 'https://www.khanacademy.org/math/applied-math/cryptography',
+ 'info_dict': {
+ 'id': 'cryptography',
+ 'title': 'Journey into cryptography',
+ 'description': 'How have humans protected their secret messages through history? What has changed today?',
+ },
+ 'playlist_mincount': 3,
+ }]
+
+ def _real_extract(self, url):
+ m = re.match(self._VALID_URL, url)
+ video_id = m.group('id')
+
+ if m.group('key') == 'video':
+ data = self._download_json(
+ 'http://api.khanacademy.org/api/v1/videos/' + video_id,
+ video_id, 'Downloading video info')
+
+ upload_date = unified_strdate(data['date_added'])
+ uploader = ', '.join(data['author_names'])
+ return {
+ '_type': 'url_transparent',
+ 'url': data['url'],
+ 'id': video_id,
+ 'title': data['title'],
+ 'thumbnail': data['image_url'],
+ 'duration': data['duration'],
+ 'description': data['description'],
+ 'uploader': uploader,
+ 'upload_date': upload_date,
+ }
+ else:
+ # topic
+ data = self._download_json(
+ 'http://api.khanacademy.org/api/v1/topic/' + video_id,
+ video_id, 'Downloading topic info')
+
+ entries = [
+ {
+ '_type': 'url',
+ 'url': c['url'],
+ 'id': c['id'],
+ 'title': c['title'],
+ }
+ for c in data['children'] if c['kind'] in ('Video', 'Topic')]
+
+ return {
+ '_type': 'playlist',
+ 'id': video_id,
+ 'title': data['title'],
+ 'description': data['description'],
+ 'entries': entries,
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/kickstarter.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/kickstarter.py
new file mode 100644
index 0000000000..7d4b570565
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/kickstarter.py
@@ -0,0 +1,57 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class KickStarterIE(InfoExtractor):
+ _VALID_URL = r'https?://www\.kickstarter\.com/projects/(?P[^/]*)/.*'
+ _TESTS = [{
+ 'url': 'https://www.kickstarter.com/projects/1404461844/intersection-the-story-of-josh-grant?ref=home_location',
+ 'md5': 'c81addca81327ffa66c642b5d8b08cab',
+ 'info_dict': {
+ 'id': '1404461844',
+ 'ext': 'mp4',
+ 'title': 'Intersection: The Story of Josh Grant by Kyle Cowling',
+ 'description': (
+ 'A unique motocross documentary that examines the '
+ 'life and mind of one of sports most elite athletes: Josh Grant.'
+ ),
+ },
+ }, {
+ 'note': 'Embedded video (not using the native kickstarter video service)',
+ 'url': 'https://www.kickstarter.com/projects/597507018/pebble-e-paper-watch-for-iphone-and-android/posts/659178',
+ 'info_dict': {
+ 'id': '78704821',
+ 'ext': 'mp4',
+ 'uploader_id': 'pebble',
+ 'uploader': 'Pebble Technology',
+ 'title': 'Pebble iOS Notifications',
+ }
+ }]
+
+ def _real_extract(self, url):
+ video_id = self._match_id(url)
+ webpage = self._download_webpage(url, video_id)
+
+ title = self._html_search_regex(
+ r'\s*(.*?)(?:\s*— Kickstarter)?\s* ',
+ webpage, 'title')
+ video_url = self._search_regex(
+ r'data-video-url="(.*?)"',
+ webpage, 'video URL', default=None)
+ if video_url is None: # No native kickstarter, look for embedded videos
+ return {
+ '_type': 'url_transparent',
+ 'ie_key': 'Generic',
+ 'url': url,
+ 'title': title,
+ }
+
+ return {
+ 'id': video_id,
+ 'url': video_url,
+ 'title': title,
+ 'description': self._og_search_description(webpage),
+ 'thumbnail': self._og_search_thumbnail(webpage),
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/kontrtube.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/kontrtube.py
new file mode 100644
index 0000000000..41fd62009a
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/kontrtube.py
@@ -0,0 +1,66 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import int_or_none
+
+
+class KontrTubeIE(InfoExtractor):
+ IE_NAME = 'kontrtube'
+ IE_DESC = 'KontrTube.ru - п╒я─я┐п╠п╟ п╥п╬п╡я▒я┌'
+ _VALID_URL = r'http://(?:www\.)?kontrtube\.ru/videos/(?P\d+)/.+'
+
+ _TEST = {
+ 'url': 'http://www.kontrtube.ru/videos/2678/nad-olimpiyskoy-derevney-v-sochi-podnyat-rossiyskiy-flag/',
+ 'md5': '975a991a4926c9a85f383a736a2e6b80',
+ 'info_dict': {
+ 'id': '2678',
+ 'ext': 'mp4',
+ 'title': 'п²п╟п╢ п╬п╩п╦п╪п©п╦п╧я│п╨п╬п╧ п╢п╣я─п╣п╡п╫п╣п╧ п╡ п║п╬я┤п╦ п©п╬п╢п╫я▐я┌ я─п╬я│я│п╦п╧я│п╨п╦п╧ я└п╩п╟пЁ',
+ 'description': 'md5:80edc4c613d5887ae8ccf1d59432be41',
+ 'thumbnail': 'http://www.kontrtube.ru/contents/videos_screenshots/2000/2678/preview.mp4.jpg',
+ 'duration': 270,
+ }
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+
+ webpage = self._download_webpage(url, video_id, 'Downloading page')
+
+ video_url = self._html_search_regex(r"video_url: '(.+?)/?',", webpage, 'video URL')
+ thumbnail = self._html_search_regex(r"preview_url: '(.+?)/?',", webpage, 'video thumbnail', fatal=False)
+ title = self._html_search_regex(
+ r'(.+?) ', webpage, 'video title')
+ description = self._html_search_meta('description', webpage, 'video description')
+
+ mobj = re.search(
+ r'п■п╩п╦я┌п╣п╩я▄п╫п╬я│я┌я▄: (?P\d+)п╪:(?P\d+)я│
', webpage)
+ duration = int(mobj.group('minutes')) * 60 + int(mobj.group('seconds')) if mobj else None
+
+ view_count = self._html_search_regex(
+ r'п÷я─п╬я│п╪п╬я┌я─п╬п╡: (\d+)
', webpage, 'view count', fatal=False)
+
+ comment_count = None
+ comment_str = self._html_search_regex(
+ r'п п╬п╪п╪п╣п╫я┌п╟я─п╦п╦: ([^<]+) ', webpage, 'comment count', fatal=False)
+ if comment_str.startswith('п╨п╬п╪п╪п╣п╫я┌п╟я─п╦п╣п╡ п╫п╣я┌'):
+ comment_count = 0
+ else:
+ mobj = re.search(r'\d+ п╦п╥ (?P\d+) п╨п╬п╪п╪п╣п╫я┌п╟я─п╦п╣п╡', comment_str)
+ if mobj:
+ comment_count = mobj.group('total')
+
+ return {
+ 'id': video_id,
+ 'url': video_url,
+ 'thumbnail': thumbnail,
+ 'title': title,
+ 'description': description,
+ 'duration': duration,
+ 'view_count': int_or_none(view_count),
+ 'comment_count': int_or_none(comment_count),
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/krasview.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/krasview.py
new file mode 100644
index 0000000000..6f3d2345b6
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/krasview.py
@@ -0,0 +1,59 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+import json
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+ int_or_none,
+ unescapeHTML,
+)
+
+
+class KrasViewIE(InfoExtractor):
+ IE_DESC = 'п я─п╟я│п╡я▄я▌'
+ _VALID_URL = r'https?://krasview\.ru/video/(?P\d+)'
+
+ _TEST = {
+ 'url': 'http://krasview.ru/video/512228',
+ 'md5': '3b91003cf85fc5db277870c8ebd98eae',
+ 'info_dict': {
+ 'id': '512228',
+ 'ext': 'mp4',
+ 'title': 'п║п╫п╣пЁ, п╩я▒п╢, п╥п╟п╫п╬я│я▀',
+ 'description': 'п║п╫я▐я┌п╬ п╡ пЁп╬я─п╬п╢п╣ п²я▐пЁп╟п╫я▄, п╡ п╔п╟п╫я┌я▀-п°п╟п╫я│п╦п╧я│п╨п╬п╪ п╟п╡я┌п╬п╫п╬п╪п╫п╬п╪ п╬п╨я─я┐пЁп╣.',
+ 'duration': 27,
+ 'thumbnail': 're:^https?://.*\.jpg',
+ },
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+
+ webpage = self._download_webpage(url, video_id)
+
+ flashvars = json.loads(self._search_regex(
+ r'flashvars\s*:\s*({.+?})\s*}\);', webpage, 'flashvars'))
+
+ video_url = flashvars['url']
+ title = unescapeHTML(flashvars['title'])
+ description = unescapeHTML(flashvars.get('subtitle') or self._og_search_description(webpage, default=None))
+ thumbnail = flashvars['image']
+ duration = int(flashvars['duration'])
+ filesize = int(flashvars['size'])
+ width = int_or_none(self._og_search_property('video:width', webpage, 'video width'))
+ height = int_or_none(self._og_search_property('video:height', webpage, 'video height'))
+
+ return {
+ 'id': video_id,
+ 'url': video_url,
+ 'title': title,
+ 'description': description,
+ 'thumbnail': thumbnail,
+ 'duration': duration,
+ 'filesize': filesize,
+ 'width': width,
+ 'height': height,
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/ku6.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/ku6.py
new file mode 100644
index 0000000000..a602980a14
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/ku6.py
@@ -0,0 +1,32 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class Ku6IE(InfoExtractor):
+ _VALID_URL = r'http://v\.ku6\.com/show/(?P[a-zA-Z0-9\-\_]+)(?:\.)*html'
+ _TEST = {
+ 'url': 'http://v.ku6.com/show/JG-8yS14xzBr4bCn1pu0xw...html',
+ 'md5': '01203549b9efbb45f4b87d55bdea1ed1',
+ 'info_dict': {
+ 'id': 'JG-8yS14xzBr4bCn1pu0xw',
+ 'ext': 'f4v',
+ 'title': 'techniques test',
+ }
+ }
+
+ def _real_extract(self, url):
+ video_id = self._match_id(url)
+ webpage = self._download_webpage(url, video_id)
+
+ title = self._html_search_regex(
+ r'(.*?) ', webpage, 'title')
+ dataUrl = 'http://v.ku6.com/fetchVideo4Player/%s.html' % video_id
+ jsonData = self._download_json(dataUrl, video_id)
+ downloadUrl = jsonData['data']['f']
+
+ return {
+ 'id': video_id,
+ 'title': title,
+ 'url': downloadUrl
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/la7.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/la7.py
new file mode 100644
index 0000000000..db2028e9f5
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/la7.py
@@ -0,0 +1,63 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+ parse_duration,
+)
+
+
+class LA7IE(InfoExtractor):
+ IE_NAME = 'la7.tv'
+ _VALID_URL = r'''(?x)
+ https?://(?:www\.)?la7\.tv/
+ (?:
+ richplayer/\?assetid=|
+ \?contentId=
+ )
+ (?P[0-9]+)'''
+
+ _TEST = {
+ 'url': 'http://www.la7.tv/richplayer/?assetid=50355319',
+ 'file': '50355319.mp4',
+ 'md5': 'ec7d1f0224d20ba293ab56cf2259651f',
+ 'info_dict': {
+ 'title': 'IL DIVO',
+ 'description': 'Un film di Paolo Sorrentino con Toni Servillo, Anna Bonaiuto, Giulio Bosetti e Flavio Bucci',
+ 'duration': 6254,
+ },
+ 'skip': 'Blocked in the US',
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+
+ xml_url = 'http://www.la7.tv/repliche/content/index.php?contentId=%s' % video_id
+ doc = self._download_xml(xml_url, video_id)
+
+ video_title = doc.find('title').text
+ description = doc.find('description').text
+ duration = parse_duration(doc.find('duration').text)
+ thumbnail = doc.find('img').text
+ view_count = int(doc.find('views').text)
+
+ prefix = doc.find('.//fqdn').text.strip().replace('auto:', 'http:')
+
+ formats = [{
+ 'format': vnode.find('quality').text,
+ 'tbr': int(vnode.find('quality').text),
+ 'url': vnode.find('fms').text.strip().replace('mp4:', prefix),
+ } for vnode in doc.findall('.//videos/video')]
+ self._sort_formats(formats)
+
+ return {
+ 'id': video_id,
+ 'title': video_title,
+ 'description': description,
+ 'thumbnail': thumbnail,
+ 'duration': duration,
+ 'formats': formats,
+ 'view_count': view_count,
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/laola1tv.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/laola1tv.py
new file mode 100644
index 0000000000..2fd3b4699d
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/laola1tv.py
@@ -0,0 +1,77 @@
+from __future__ import unicode_literals
+
+import random
+import re
+
+from .common import InfoExtractor
+from ..utils import ExtractorError
+
+
+class Laola1TvIE(InfoExtractor):
+ _VALID_URL = r'https?://(?:www\.)?laola1\.tv/(?P[a-z]+)-(?P[a-z]+)/.*?/(?P[0-9]+)\.html'
+ _TEST = {
+ 'url': 'http://www.laola1.tv/de-de/live/bwf-bitburger-open-grand-prix-gold-court-1/250019.html',
+ 'info_dict': {
+ 'id': '250019',
+ 'ext': 'mp4',
+ 'title': 'Bitburger Open Grand Prix Gold - Court 1',
+ 'categories': ['Badminton'],
+ 'uploader': 'BWF - Badminton World Federation',
+ 'is_live': True,
+ },
+ 'params': {
+ 'skip_download': True,
+ }
+ }
+
+ _BROKEN = True # Not really - extractor works fine, but f4m downloader does not support live streams yet.
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+ lang = mobj.group('lang')
+ portal = mobj.group('portal')
+
+ webpage = self._download_webpage(url, video_id)
+ iframe_url = self._search_regex(
+ r']*?class="main_tv_player"[^>]*?src="([^"]+)"',
+ webpage, 'iframe URL')
+
+ iframe = self._download_webpage(
+ iframe_url, video_id, note='Downloading iframe')
+ flashvars_m = re.findall(
+ r'flashvars\.([_a-zA-Z0-9]+)\s*=\s*"([^"]*)";', iframe)
+ flashvars = dict((m[0], m[1]) for m in flashvars_m)
+
+ xml_url = ('http://www.laola1.tv/server/hd_video.php?' +
+ 'play=%s&partner=1&portal=%s&v5ident=&lang=%s' % (
+ video_id, portal, lang))
+ hd_doc = self._download_xml(xml_url, video_id)
+
+ title = hd_doc.find('.//video/title').text
+ flash_url = hd_doc.find('.//video/url').text
+ categories = hd_doc.find('.//video/meta_sports').text.split(',')
+ uploader = hd_doc.find('.//video/meta_organistation').text
+
+ ident = random.randint(10000000, 99999999)
+ token_url = '%s&ident=%s&klub=0&unikey=0×tamp=%s&auth=%s' % (
+ flash_url, ident, flashvars['timestamp'], flashvars['auth'])
+
+ token_doc = self._download_xml(
+ token_url, video_id, note='Downloading token')
+ token_attrib = token_doc.find('.//token').attrib
+ if token_attrib.get('auth') == 'blocked':
+ raise ExtractorError('Token error: ' % token_attrib.get('comment'))
+
+ video_url = '%s?hdnea=%s&hdcore=3.2.0' % (
+ token_attrib['url'], token_attrib['auth'])
+
+ return {
+ 'id': video_id,
+ 'is_live': True,
+ 'title': title,
+ 'url': video_url,
+ 'uploader': uploader,
+ 'categories': categories,
+ 'ext': 'mp4',
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/lifenews.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/lifenews.py
new file mode 100644
index 0000000000..1dfe7f77f4
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/lifenews.py
@@ -0,0 +1,74 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+ int_or_none,
+ unified_strdate,
+ ExtractorError,
+)
+
+
+class LifeNewsIE(InfoExtractor):
+ IE_NAME = 'lifenews'
+ IE_DESC = 'LIFE | NEWS'
+ _VALID_URL = r'http://lifenews\.ru/(?:mobile/)?news/(?P\d+)'
+
+ _TEST = {
+ 'url': 'http://lifenews.ru/news/126342',
+ 'md5': 'e1b50a5c5fb98a6a544250f2e0db570a',
+ 'info_dict': {
+ 'id': '126342',
+ 'ext': 'mp4',
+ 'title': 'п°п▓п■ я─п╟п╥я▀я│п╨п╦п╡п╟п╣я┌ п╪я┐п╤я┤п╦п╫, п╬я│я┌п╟п╡п╦п╡я┬п╦я┘ п╡ IKEA я│я┐п╪п╨я┐ я│ п╟п╡я┌п╬п╪п╟я┌п╬п╪',
+ 'description': 'п п╟п╪п╣я─я▀ п╫п╟п╠п╩я▌п╢п╣п╫п╦я▐ пЁп╦п©п╣я─п╪п╟я─п╨п╣я┌п╟ п╥п╟я└п╦п╨я│п╦я─п╬п╡п╟п╩п╦ я┌я─п╬п╦я┘ п╪я┐п╤я┤п╦п╫, я│п©я─я▐я┌п╟п╡я┬п╦я┘ п╬я─я┐п╤п╣п╧п╫я▀п╧ п╟я─я│п╣п╫п╟п╩ п╡ п╨п╟п╪п╣я─п╣ я┘я─п╟п╫п╣п╫п╦я▐.',
+ 'thumbnail': 're:http://.*\.jpg',
+ 'upload_date': '20140130',
+ }
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+
+ webpage = self._download_webpage('http://lifenews.ru/news/%s' % video_id, video_id, 'Downloading page')
+
+ videos = re.findall(r'[^"]+)".*?src="(?P[^"]+)".*?> ', webpage)
+ if not videos:
+ raise ExtractorError('No media links available for %s' % video_id)
+
+ title = self._og_search_title(webpage)
+ TITLE_SUFFIX = ' - п÷п╣я─п╡я▀п╧ п©п╬ я│я─п╬я┤п╫я▀п╪ п╫п╬п╡п╬я│я┌я▐п╪ Б─■ LIFE | NEWS'
+ if title.endswith(TITLE_SUFFIX):
+ title = title[:-len(TITLE_SUFFIX)]
+
+ description = self._og_search_description(webpage)
+
+ view_count = self._html_search_regex(
+ r'(\d+)
', webpage, 'view count', fatal=False)
+ comment_count = self._html_search_regex(
+ r'', webpage, 'data fields')
+ uploader = self._html_search_regex(
+ r'(.*?) ', data_div, 'uploader', fatal=False)
+ title = self._html_search_regex(
+ r'(.*?)', data_div, 'title')
+ upload_date = unified_strdate(self._html_search_regex(
+ r'(.*?)', data_div, 'uploader', fatal=False))
+ location = self._html_search_regex(
+ r'(.*?)', data_div, 'location', fatal=False)
+
+ duration = parse_duration(self._html_search_meta('duration', webpage))
+
+ VIDEO_URL_TEMPLATE = 'http://cdnapi.kaltura.com/p/%(uid)s/sp/%(wid)s/playManifest/entryId/%(entry_id)s/format/url/protocol/http'
+ kaltura_id = self._search_regex(
+ r'\d+)'
+ IE_NAME = 'muzu.tv'
+
+ _TEST = {
+ 'url': 'http://www.muzu.tv/defected/marcashken-featuring-sos-cat-walk-original-mix-music-video/1981454/',
+ 'md5': '98f8b2c7bc50578d6a0364fff2bfb000',
+ 'info_dict': {
+ 'id': '1981454',
+ 'ext': 'mp4',
+ 'title': 'Cat Walk (Original Mix)',
+ 'description': 'md5:90e868994de201b2570e4e5854e19420',
+ 'uploader': 'MarcAshken featuring SOS',
+ },
+ }
+
+ def _real_extract(self, url):
+ video_id = self._match_id(url)
+
+ info_data = compat_urllib_parse.urlencode({
+ 'format': 'json',
+ 'url': url,
+ })
+ info = self._download_json(
+ 'http://www.muzu.tv/api/oembed/?%s' % info_data,
+ video_id, 'Downloading video info')
+
+ player_info = self._download_json(
+ 'http://player.muzu.tv/player/playerInit?ai=%s' % video_id,
+ video_id, 'Downloading player info')
+ video_info = player_info['videos'][0]
+ for quality in ['1080', '720', '480', '360']:
+ if video_info.get('v%s' % quality):
+ break
+
+ data = compat_urllib_parse.urlencode({
+ 'ai': video_id,
+ # Even if each time you watch a video the hash changes,
+ # it seems to work for different videos, and it will work
+ # even if you use any non empty string as a hash
+ 'viewhash': 'VBNff6djeV4HV5TRPW5kOHub2k',
+ 'device': 'web',
+ 'qv': quality,
+ })
+ video_url_info = self._download_json(
+ 'http://player.muzu.tv/player/requestVideo?%s' % data,
+ video_id, 'Downloading video url')
+ video_url = video_url_info['url']
+
+ return {
+ 'id': video_id,
+ 'title': info['title'],
+ 'url': video_url,
+ 'thumbnail': info['thumbnail_url'],
+ 'description': info['description'],
+ 'uploader': info['author_name'],
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/myspace.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/myspace.py
new file mode 100644
index 0000000000..83414a2325
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/myspace.py
@@ -0,0 +1,180 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+import re
+import json
+
+from .common import InfoExtractor
+from ..compat import (
+ compat_str,
+)
+from ..utils import ExtractorError
+
+
+class MySpaceIE(InfoExtractor):
+ _VALID_URL = r'https?://myspace\.com/([^/]+)/(?P
video/[^/]+/|music/song/.*?)(?P\d+)'
+
+ _TESTS = [
+ {
+ 'url': 'https://myspace.com/fiveminutestothestage/video/little-big-town/109594919',
+ 'info_dict': {
+ 'id': '109594919',
+ 'ext': 'flv',
+ 'title': 'Little Big Town',
+ 'description': 'This country quartet was all smilesб═whileб═playing a sold out show at the Pacificб═Amphitheatreб═in Orange County, California.',
+ 'uploader': 'Five Minutes to the Stage',
+ 'uploader_id': 'fiveminutestothestage',
+ },
+ 'params': {
+ # rtmp download
+ 'skip_download': True,
+ },
+ },
+ # songs
+ {
+ 'url': 'https://myspace.com/killsorrow/music/song/of-weakened-soul...-93388656-103880681',
+ 'info_dict': {
+ 'id': '93388656',
+ 'ext': 'flv',
+ 'title': 'Of weakened soul...',
+ 'uploader': 'Killsorrow',
+ 'uploader_id': 'killsorrow',
+ },
+ 'params': {
+ # rtmp download
+ 'skip_download': True,
+ },
+ }, {
+ 'add_ie': ['Vevo'],
+ 'url': 'https://myspace.com/threedaysgrace/music/song/animal-i-have-become-28400208-28218041',
+ 'info_dict': {
+ 'id': 'USZM20600099',
+ 'ext': 'mp4',
+ 'title': 'Animal I Have Become',
+ 'uploader': 'Three Days Grace',
+ 'timestamp': int,
+ 'upload_date': '20060502',
+ },
+ 'skip': 'VEVO is only available in some countries',
+ }, {
+ 'add_ie': ['Youtube'],
+ 'url': 'https://myspace.com/starset2/music/song/first-light-95799905-106964426',
+ 'info_dict': {
+ 'id': 'ypWvQgnJrSU',
+ 'ext': 'mp4',
+ 'title': 'Starset - First Light',
+ 'description': 'md5:2d5db6c9d11d527683bcda818d332414',
+ 'uploader': 'Jacob Soren',
+ 'uploader_id': 'SorenPromotions',
+ 'upload_date': '20140725',
+ }
+ },
+ ]
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+ webpage = self._download_webpage(url, video_id)
+ player_url = self._search_regex(
+ r'playerSwf":"([^"?]*)', webpage, 'player URL')
+
+ if mobj.group('mediatype').startswith('music/song'):
+ # songs don't store any useful info in the 'context' variable
+ song_data = self._search_regex(
+ r'''.*?)\1''' % name,
+ song_data, name, default='', group='data')
+ streamUrl = search_data('stream-url')
+ if not streamUrl:
+ vevo_id = search_data('vevo-id')
+ youtube_id = search_data('youtube-id')
+ if vevo_id:
+ self.to_screen('Vevo video detected: %s' % vevo_id)
+ return self.url_result('vevo:%s' % vevo_id, ie='Vevo')
+ elif youtube_id:
+ self.to_screen('Youtube video detected: %s' % youtube_id)
+ return self.url_result(youtube_id, ie='Youtube')
+ else:
+ raise ExtractorError(
+ 'Found song but don\'t know how to download it')
+ info = {
+ 'id': video_id,
+ 'title': self._og_search_title(webpage),
+ 'uploader': search_data('artist-name'),
+ 'uploader_id': search_data('artist-username'),
+ 'thumbnail': self._og_search_thumbnail(webpage),
+ }
+ else:
+ context = json.loads(self._search_regex(
+ r'context = ({.*?});', webpage, 'context'))
+ video = context['video']
+ streamUrl = video['streamUrl']
+ info = {
+ 'id': compat_str(video['mediaId']),
+ 'title': video['title'],
+ 'description': video['description'],
+ 'thumbnail': video['imageUrl'],
+ 'uploader': video['artistName'],
+ 'uploader_id': video['artistUsername'],
+ }
+
+ rtmp_url, play_path = streamUrl.split(';', 1)
+ info.update({
+ 'url': rtmp_url,
+ 'play_path': play_path,
+ 'player_url': player_url,
+ 'ext': 'flv',
+ })
+ return info
+
+
+class MySpaceAlbumIE(InfoExtractor):
+ IE_NAME = 'MySpace:album'
+ _VALID_URL = r'https?://myspace\.com/([^/]+)/music/album/(?P.*-)(?P<id>\d+)'
+
+ _TESTS = [{
+ 'url': 'https://myspace.com/starset2/music/album/transmissions-19455773',
+ 'info_dict': {
+ 'title': 'Transmissions',
+ 'id': '19455773',
+ },
+ 'playlist_count': 14,
+ 'skip': 'this album is only available in some countries',
+ }, {
+ 'url': 'https://myspace.com/killsorrow/music/album/the-demo-18596029',
+ 'info_dict': {
+ 'title': 'The Demo',
+ 'id': '18596029',
+ },
+ 'playlist_count': 5,
+ }]
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ playlist_id = mobj.group('id')
+ display_id = mobj.group('title') + playlist_id
+ webpage = self._download_webpage(url, display_id)
+ tracks_paths = re.findall(r'"music:song" content="(.*?)"', webpage)
+ if not tracks_paths:
+ raise ExtractorError(
+ '%s: No songs found, try using proxy' % display_id,
+ expected=True)
+ entries = [
+ self.url_result(t_path, ie=MySpaceIE.ie_key())
+ for t_path in tracks_paths]
+ return {
+ '_type': 'playlist',
+ 'id': playlist_id,
+ 'display_id': display_id,
+ 'title': self._og_search_title(webpage),
+ 'entries': entries,
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/myspass.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/myspass.py
new file mode 100644
index 0000000000..5b9b9fbcd0
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/myspass.py
@@ -0,0 +1,72 @@
+from __future__ import unicode_literals
+import os.path
+
+from .common import InfoExtractor
+from ..compat import (
+ compat_urllib_parse_urlparse,
+)
+from ..utils import (
+ ExtractorError,
+)
+
+
+class MySpassIE(InfoExtractor):
+ _VALID_URL = r'http://www\.myspass\.de/.*'
+ _TEST = {
+ 'url': 'http://www.myspass.de/myspass/shows/tvshows/absolute-mehrheit/Absolute-Mehrheit-vom-17022013-Die-Highlights-Teil-2--/11741/',
+ 'md5': '0b49f4844a068f8b33f4b7c88405862b',
+ 'info_dict': {
+ 'id': '11741',
+ 'ext': 'mp4',
+ "description": "Wer kann in die Fu\u00dfstapfen von Wolfgang Kubicki treten und die Mehrheit der Zuschauer hinter sich versammeln? Wird vielleicht sogar die Absolute Mehrheit geknackt und der Jackpot von 200.000 Euro mit nach Hause genommen?",
+ "title": "Absolute Mehrheit vom 17.02.2013 - Die Highlights, Teil 2",
+ },
+ }
+
+ def _real_extract(self, url):
+ META_DATA_URL_TEMPLATE = 'http://www.myspass.de/myspass/includes/apps/video/getvideometadataxml.php?id=%s'
+
+ # video id is the last path element of the URL
+ # usually there is a trailing slash, so also try the second but last
+ url_path = compat_urllib_parse_urlparse(url).path
+ url_parent_path, video_id = os.path.split(url_path)
+ if not video_id:
+ _, video_id = os.path.split(url_parent_path)
+
+ # get metadata
+ metadata_url = META_DATA_URL_TEMPLATE % video_id
+ metadata = self._download_xml(metadata_url, video_id)
+
+ # extract values from metadata
+ url_flv_el = metadata.find('url_flv')
+ if url_flv_el is None:
+ raise ExtractorError('Unable to extract download url')
+ video_url = url_flv_el.text
+ title_el = metadata.find('title')
+ if title_el is None:
+ raise ExtractorError('Unable to extract title')
+ title = title_el.text
+ format_id_el = metadata.find('format_id')
+ if format_id_el is None:
+ format = 'mp4'
+ else:
+ format = format_id_el.text
+ description_el = metadata.find('description')
+ if description_el is not None:
+ description = description_el.text
+ else:
+ description = None
+ imagePreview_el = metadata.find('imagePreview')
+ if imagePreview_el is not None:
+ thumbnail = imagePreview_el.text
+ else:
+ thumbnail = None
+
+ return {
+ 'id': video_id,
+ 'url': video_url,
+ 'title': title,
+ 'format': format,
+ 'thumbnail': thumbnail,
+ 'description': description,
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/myvideo.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/myvideo.py
new file mode 100644
index 0000000000..5e754fcffb
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/myvideo.py
@@ -0,0 +1,175 @@
+from __future__ import unicode_literals
+
+import binascii
+import base64
+import hashlib
+import re
+import json
+
+from .common import InfoExtractor
+from ..compat import (
+ compat_ord,
+ compat_urllib_parse,
+ compat_urllib_request,
+)
+from ..utils import (
+ ExtractorError,
+)
+
+
+class MyVideoIE(InfoExtractor):
+ _VALID_URL = r'http://(?:www\.)?myvideo\.de/(?:[^/]+/)?watch/(?P<id>[0-9]+)/[^?/]+.*'
+ IE_NAME = 'myvideo'
+ _TEST = {
+ 'url': 'http://www.myvideo.de/watch/8229274/bowling_fail_or_win',
+ 'md5': '2d2753e8130479ba2cb7e0a37002053e',
+ 'info_dict': {
+ 'id': '8229274',
+ 'ext': 'flv',
+ 'title': 'bowling-fail-or-win',
+ }
+ }
+
+ # Original Code from: https://github.com/dersphere/plugin.video.myvideo_de.git
+ # Released into the Public Domain by Tristan Fischer on 2013-05-19
+ # https://github.com/rg3/youtube-dl/pull/842
+ def __rc4crypt(self, data, key):
+ x = 0
+ box = list(range(256))
+ for i in list(range(256)):
+ x = (x + box[i] + compat_ord(key[i % len(key)])) % 256
+ box[i], box[x] = box[x], box[i]
+ x = 0
+ y = 0
+ out = ''
+ for char in data:
+ x = (x + 1) % 256
+ y = (y + box[x]) % 256
+ box[x], box[y] = box[y], box[x]
+ out += chr(compat_ord(char) ^ box[(box[x] + box[y]) % 256])
+ return out
+
+ def __md5(self, s):
+ return hashlib.md5(s).hexdigest().encode()
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+
+ GK = (
+ b'WXpnME1EZGhNRGhpTTJNM01XVmhOREU0WldNNVpHTTJOakpt'
+ b'TW1FMU5tVTBNR05pWkRaa05XRXhNVFJoWVRVd1ptSXhaVEV3'
+ b'TnpsbA0KTVRkbU1tSTRNdz09'
+ )
+
+ # Get video webpage
+ webpage_url = 'http://www.myvideo.de/watch/%s' % video_id
+ webpage = self._download_webpage(webpage_url, video_id)
+
+ mobj = re.search('source src=\'(.+?)[.]([^.]+)\'', webpage)
+ if mobj is not None:
+ self.report_extraction(video_id)
+ video_url = mobj.group(1) + '.flv'
+
+ video_title = self._html_search_regex('<title>([^<]+) ',
+ webpage, 'title')
+
+ return {
+ 'id': video_id,
+ 'url': video_url,
+ 'title': video_title,
+ }
+
+ mobj = re.search(r'data-video-service="/service/data/video/%s/config' % video_id, webpage)
+ if mobj is not None:
+ request = compat_urllib_request.Request('http://www.myvideo.de/service/data/video/%s/config' % video_id, '')
+ response = self._download_webpage(request, video_id,
+ 'Downloading video info')
+ info = json.loads(base64.b64decode(response).decode('utf-8'))
+ return {
+ 'id': video_id,
+ 'title': info['title'],
+ 'url': info['streaming_url'].replace('rtmpe', 'rtmpt'),
+ 'play_path': info['filename'],
+ 'ext': 'flv',
+ 'thumbnail': info['thumbnail'][0]['url'],
+ }
+
+ # try encxml
+ mobj = re.search('var flashvars={(.+?)}', webpage)
+ if mobj is None:
+ raise ExtractorError('Unable to extract video')
+
+ params = {}
+ encxml = ''
+ sec = mobj.group(1)
+ for (a, b) in re.findall('(.+?):\'(.+?)\',?', sec):
+ if not a == '_encxml':
+ params[a] = b
+ else:
+ encxml = compat_urllib_parse.unquote(b)
+ if not params.get('domain'):
+ params['domain'] = 'www.myvideo.de'
+ xmldata_url = '%s?%s' % (encxml, compat_urllib_parse.urlencode(params))
+ if 'flash_playertype=MTV' in xmldata_url:
+ self._downloader.report_warning('avoiding MTV player')
+ xmldata_url = (
+ 'http://www.myvideo.de/dynamic/get_player_video_xml.php'
+ '?flash_playertype=D&ID=%s&_countlimit=4&autorun=yes'
+ ) % video_id
+
+ # get enc data
+ enc_data = self._download_webpage(xmldata_url, video_id).split('=')[1]
+ enc_data_b = binascii.unhexlify(enc_data)
+ sk = self.__md5(
+ base64.b64decode(base64.b64decode(GK)) +
+ self.__md5(
+ str(video_id).encode('utf-8')
+ )
+ )
+ dec_data = self.__rc4crypt(enc_data_b, sk)
+
+ # extracting infos
+ self.report_extraction(video_id)
+
+ video_url = None
+ mobj = re.search('connectionurl=\'(.*?)\'', dec_data)
+ if mobj:
+ video_url = compat_urllib_parse.unquote(mobj.group(1))
+ if 'myvideo2flash' in video_url:
+ self.report_warning(
+ 'Rewriting URL to use unencrypted rtmp:// ...',
+ video_id)
+ video_url = video_url.replace('rtmpe://', 'rtmp://')
+
+ if not video_url:
+ # extract non rtmp videos
+ mobj = re.search('path=\'(http.*?)\' source=\'(.*?)\'', dec_data)
+ if mobj is None:
+ raise ExtractorError('unable to extract url')
+ video_url = compat_urllib_parse.unquote(mobj.group(1)) + compat_urllib_parse.unquote(mobj.group(2))
+
+ video_file = self._search_regex('source=\'(.*?)\'', dec_data, 'video file')
+ video_file = compat_urllib_parse.unquote(video_file)
+
+ if not video_file.endswith('f4m'):
+ ppath, prefix = video_file.split('.')
+ video_playpath = '%s:%s' % (prefix, ppath)
+ else:
+ video_playpath = ''
+
+ video_swfobj = self._search_regex('swfobject.embedSWF\(\'(.+?)\'', webpage, 'swfobj')
+ video_swfobj = compat_urllib_parse.unquote(video_swfobj)
+
+ video_title = self._html_search_regex("(.*?)",
+ webpage, 'title')
+
+ return {
+ 'id': video_id,
+ 'url': video_url,
+ 'tc_url': video_url,
+ 'title': video_title,
+ 'ext': 'flv',
+ 'play_path': video_playpath,
+ 'player_url': video_swfobj,
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/myvidster.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/myvidster.py
new file mode 100644
index 0000000000..a94ab8358c
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/myvidster.py
@@ -0,0 +1,29 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class MyVidsterIE(InfoExtractor):
+ _VALID_URL = r'http://(?:www\.)?myvidster\.com/video/(?P\d+)/'
+
+ _TEST = {
+ 'url': 'http://www.myvidster.com/video/32059805/Hot_chemistry_with_raw_love_making',
+ 'md5': '95296d0231c1363222c3441af62dc4ca',
+ 'info_dict': {
+ 'id': '3685814',
+ 'title': 'md5:7d8427d6d02c4fbcef50fe269980c749',
+ 'upload_date': '20141027',
+ 'uploader_id': 'utkualp',
+ 'ext': 'mp4',
+ 'age_limit': 18,
+ },
+ 'add_ie': ['XHamster'],
+ }
+
+ def _real_extract(self, url):
+ video_id = self._match_id(url)
+ webpage = self._download_webpage(url, video_id)
+
+ return self.url_result(self._html_search_regex(
+ r'rel="videolink" href="(?P.*)">',
+ webpage, 'real video url'))
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/naver.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/naver.py
new file mode 100644
index 0000000000..c10405f04d
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/naver.py
@@ -0,0 +1,83 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..compat import (
+ compat_urllib_parse,
+)
+from ..utils import (
+ ExtractorError,
+ clean_html,
+)
+
+
+class NaverIE(InfoExtractor):
+ _VALID_URL = r'https?://(?:m\.)?tvcast\.naver\.com/v/(?P\d+)'
+
+ _TEST = {
+ 'url': 'http://tvcast.naver.com/v/81652',
+ 'info_dict': {
+ 'id': '81652',
+ 'ext': 'mp4',
+ 'title': '[9Л⌡■ К╙╗Л²≤ЙЁ═Л┌╛ М∙╢Л└╓Й╟∙Л²≤][Л┬≤М∙≥_Й╧─Л┐│М²╛] Л┬≤М∙≥ AМ≤∙ 16~20К╡┬',
+ 'description': 'М∙╘Й╡╘К╤┬КЁ─Л²≤ К╡∙Л╧≥ К╘■Й╟─Л┼╓М└╟К■■ | К╘■Й╟─Л┼╓М└╟К■■ Л┬≤М∙≥ Й╧─Л┐│М²╛ Л└═Л┐²К▀≤Л²╢ 9Л⌡■ К╙╗Л²≤ЙЁ═Л┌╛ Л┬≤М∙≥AМ≤∙ 16К╡┬Л≈░Л└° 20К╡┬Й╧▄Л╖─ М∙╢Л└╓Й╟∙Л²≤К╔╪ ЙЁ╣Й╟°М∙╘К▀┬К▀╓.',
+ 'upload_date': '20130903',
+ },
+ }
+
+ def _real_extract(self, url):
+ video_id = self._match_id(url)
+ webpage = self._download_webpage(url, video_id)
+
+ m_id = re.search(r'var rmcPlayer = new nhn.rmcnmv.RMCVideoPlayer\("(.+?)", "(.+?)"',
+ webpage)
+ if m_id is None:
+ m_error = re.search(
+ r'(?s)',
+ webpage)
+ if m_error:
+ raise ExtractorError(clean_html(m_error.group('msg')), expected=True)
+ raise ExtractorError('couldn\'t extract vid and key')
+ vid = m_id.group(1)
+ key = m_id.group(2)
+ query = compat_urllib_parse.urlencode({'vid': vid, 'inKey': key, })
+ query_urls = compat_urllib_parse.urlencode({
+ 'masterVid': vid,
+ 'protocol': 'p2p',
+ 'inKey': key,
+ })
+ info = self._download_xml(
+ 'http://serviceapi.rmcnmv.naver.com/flash/videoInfo.nhn?' + query,
+ video_id, 'Downloading video info')
+ urls = self._download_xml(
+ 'http://serviceapi.rmcnmv.naver.com/flash/playableEncodingOption.nhn?' + query_urls,
+ video_id, 'Downloading video formats info')
+
+ formats = []
+ for format_el in urls.findall('EncodingOptions/EncodingOption'):
+ domain = format_el.find('Domain').text
+ f = {
+ 'url': domain + format_el.find('uri').text,
+ 'ext': 'mp4',
+ 'width': int(format_el.find('width').text),
+ 'height': int(format_el.find('height').text),
+ }
+ if domain.startswith('rtmp'):
+ f.update({
+ 'ext': 'flv',
+ 'rtmp_protocol': '1', # rtmpt
+ })
+ formats.append(f)
+ self._sort_formats(formats)
+
+ return {
+ 'id': video_id,
+ 'title': info.find('Subject').text,
+ 'formats': formats,
+ 'description': self._og_search_description(webpage),
+ 'thumbnail': self._og_search_thumbnail(webpage),
+ 'upload_date': info.find('WriteDate').text.replace('.', ''),
+ 'view_count': int(info.find('PlayCount').text),
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/nba.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/nba.py
new file mode 100644
index 0000000000..862b706bf9
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/nba.py
@@ -0,0 +1,47 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+ remove_end,
+ parse_duration,
+)
+
+
+class NBAIE(InfoExtractor):
+ _VALID_URL = r'https?://(?:watch\.|www\.)?nba\.com/(?:nba/)?video(?P/[^?]*?)/?(?:/index\.html)?(?:\?.*)?$'
+ _TESTS = [{
+ 'url': 'http://www.nba.com/video/games/nets/2012/12/04/0021200253-okc-bkn-recap.nba/index.html',
+ 'md5': 'c0edcfc37607344e2ff8f13c378c88a4',
+ 'info_dict': {
+ 'id': '0021200253-okc-bkn-recap.nba',
+ 'ext': 'mp4',
+ 'title': 'Thunder vs. Nets',
+ 'description': 'Kevin Durant scores 32 points and dishes out six assists as the Thunder beat the Nets in Brooklyn.',
+ 'duration': 181,
+ },
+ }, {
+ 'url': 'http://www.nba.com/video/games/hornets/2014/12/05/0021400276-nyk-cha-play5.nba/',
+ 'only_matching': True,
+ }]
+
+ def _real_extract(self, url):
+ video_id = self._match_id(url)
+ webpage = self._download_webpage(url, video_id)
+
+ video_url = 'http://ht-mobile.cdn.turner.com/nba/big' + video_id + '_nba_1280x720.mp4'
+
+ shortened_video_id = video_id.rpartition('/')[2]
+ title = remove_end(
+ self._og_search_title(webpage, default=shortened_video_id), ' : NBA.com')
+
+ description = self._og_search_description(webpage)
+ duration = parse_duration(
+ self._html_search_meta('duration', webpage, 'duration'))
+
+ return {
+ 'id': shortened_video_id,
+ 'url': video_url,
+ 'title': title,
+ 'description': description,
+ 'duration': duration,
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/nbc.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/nbc.py
new file mode 100644
index 0000000000..690c46b6a5
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/nbc.py
@@ -0,0 +1,134 @@
+from __future__ import unicode_literals
+
+import re
+import json
+
+from .common import InfoExtractor
+from ..compat import (
+ compat_str,
+)
+from ..utils import (
+ ExtractorError,
+ find_xpath_attr,
+)
+
+
+class NBCIE(InfoExtractor):
+ _VALID_URL = r'http://www\.nbc\.com/(?:[^/]+/)+(?Pn?\d+)'
+
+ _TESTS = [
+ {
+ 'url': 'http://www.nbc.com/chicago-fire/video/i-am-a-firefighter/2734188',
+ # md5 checksum is not stable
+ 'info_dict': {
+ 'id': 'bTmnLCvIbaaH',
+ 'ext': 'flv',
+ 'title': 'I Am a Firefighter',
+ 'description': 'An emergency puts Dawson\'sf irefighter skills to the ultimate test in this four-part digital series.',
+ },
+ },
+ {
+ 'url': 'http://www.nbc.com/the-tonight-show/episodes/176',
+ 'info_dict': {
+ 'id': 'XwU9KZkp98TH',
+ 'ext': 'flv',
+ 'title': 'Ricky Gervais, Steven Van Zandt, ILoveMakonnen',
+ 'description': 'A brand new episode of The Tonight Show welcomes Ricky Gervais, Steven Van Zandt and ILoveMakonnen.',
+ },
+ 'skip': 'Only works from US',
+ },
+ ]
+
+ def _real_extract(self, url):
+ video_id = self._match_id(url)
+ webpage = self._download_webpage(url, video_id)
+ theplatform_url = self._search_regex(
+ '(?:class="video-player video-player-full" data-mpx-url|class="player" src)="(.*?)"',
+ webpage, 'theplatform url').replace('_no_endcard', '')
+ if theplatform_url.startswith('//'):
+ theplatform_url = 'http:' + theplatform_url
+ return self.url_result(theplatform_url)
+
+
+class NBCNewsIE(InfoExtractor):
+ _VALID_URL = r'''(?x)https?://www\.nbcnews\.com/
+ ((video/.+?/(?P\d+))|
+ (feature/[^/]+/(?P.+)))
+ '''
+
+ _TESTS = [
+ {
+ 'url': 'http://www.nbcnews.com/video/nbc-news/52753292',
+ 'md5': '47abaac93c6eaf9ad37ee6c4463a5179',
+ 'info_dict': {
+ 'id': '52753292',
+ 'ext': 'flv',
+ 'title': 'Crew emerges after four-month Mars food study',
+ 'description': 'md5:24e632ffac72b35f8b67a12d1b6ddfc1',
+ },
+ },
+ {
+ 'url': 'http://www.nbcnews.com/feature/edward-snowden-interview/how-twitter-reacted-snowden-interview-n117236',
+ 'md5': 'b2421750c9f260783721d898f4c42063',
+ 'info_dict': {
+ 'id': 'I1wpAI_zmhsQ',
+ 'ext': 'mp4',
+ 'title': 'How Twitter Reacted To The Snowden Interview',
+ 'description': 'md5:65a0bd5d76fe114f3c2727aa3a81fe64',
+ },
+ 'add_ie': ['ThePlatform'],
+ },
+ ]
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+ if video_id is not None:
+ all_info = self._download_xml('http://www.nbcnews.com/id/%s/displaymode/1219' % video_id, video_id)
+ info = all_info.find('video')
+
+ return {
+ 'id': video_id,
+ 'title': info.find('headline').text,
+ 'ext': 'flv',
+ 'url': find_xpath_attr(info, 'media', 'type', 'flashVideo').text,
+ 'description': compat_str(info.find('caption').text),
+ 'thumbnail': find_xpath_attr(info, 'media', 'type', 'thumbnail').text,
+ }
+ else:
+ # "feature" pages use theplatform.com
+ title = mobj.group('title')
+ webpage = self._download_webpage(url, title)
+ bootstrap_json = self._search_regex(
+ r'var bootstrapJson = ({.+})\s*$', webpage, 'bootstrap json',
+ flags=re.MULTILINE)
+ bootstrap = json.loads(bootstrap_json)
+ info = bootstrap['results'][0]['video']
+ mpxid = info['mpxId']
+
+ base_urls = [
+ info['fallbackPlaylistUrl'],
+ info['associatedPlaylistUrl'],
+ ]
+
+ for base_url in base_urls:
+ if not base_url:
+ continue
+ playlist_url = base_url + '?form=MPXNBCNewsAPI'
+ all_videos = self._download_json(playlist_url, title)['videos']
+
+ try:
+ info = next(v for v in all_videos if v['mpxId'] == mpxid)
+ break
+ except StopIteration:
+ continue
+
+ if info is None:
+ raise ExtractorError('Could not find video in playlists')
+
+ return {
+ '_type': 'url',
+ # We get the best quality video
+ 'url': info['videoAssets'][-1]['publicUrl'],
+ 'ie_key': 'ThePlatform',
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/ndr.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/ndr.py
new file mode 100644
index 0000000000..f49c666909
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/ndr.py
@@ -0,0 +1,94 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+ ExtractorError,
+ int_or_none,
+ qualities,
+)
+
+
+class NDRIE(InfoExtractor):
+ IE_NAME = 'ndr'
+ IE_DESC = 'NDR.de - Mediathek'
+ _VALID_URL = r'https?://www\.ndr\.de/.+?(?P<id>\d+)\.html'
+
+ _TESTS = [
+ {
+ 'url': 'http://www.ndr.de/fernsehen/sendungen/nordmagazin/Kartoffeltage-in-der-Lewitz,nordmagazin25866.html',
+ 'md5': '5bc5f5b92c82c0f8b26cddca34f8bb2c',
+ 'note': 'Video file',
+ 'info_dict': {
+ 'id': '25866',
+ 'ext': 'mp4',
+ 'title': 'Kartoffeltage in der Lewitz',
+ 'description': 'md5:48c4c04dde604c8a9971b3d4e3b9eaa8',
+ 'duration': 166,
+ }
+ },
+ {
+ 'url': 'http://www.ndr.de/info/audio51535.html',
+ 'md5': 'bb3cd38e24fbcc866d13b50ca59307b8',
+ 'note': 'Audio file',
+ 'info_dict': {
+ 'id': '51535',
+ 'ext': 'mp3',
+ 'title': 'La Valette entgeht der Hinrichtung',
+ 'description': 'md5:22f9541913a40fe50091d5cdd7c9f536',
+ 'duration': 884,
+ }
+ }
+ ]
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+
+ page = self._download_webpage(url, video_id, 'Downloading page')
+
+ title = self._og_search_title(page).strip()
+ description = self._og_search_description(page)
+ if description:
+ description = description.strip()
+
+ duration = int_or_none(self._html_search_regex(r'duration: (\d+),\n', page, 'duration', fatal=False))
+
+ formats = []
+
+ mp3_url = re.search(r'''\{src:'(?P<audio>[^']+)', type:"audio/mp3"},''', page)
+ if mp3_url:
+ formats.append({
+ 'url': mp3_url.group('audio'),
+ 'format_id': 'mp3',
+ })
+
+ thumbnail = None
+
+ video_url = re.search(r'''3: \{src:'(?P<video>.+?)\.(lo|hi|hq)\.mp4', type:"video/mp4"},''', page)
+ if video_url:
+ thumbnails = re.findall(r'''\d+: \{src: "([^"]+)"(?: \|\| '[^']+')?, quality: '([^']+)'}''', page)
+ if thumbnails:
+ quality_key = qualities(['xs', 's', 'm', 'l', 'xl'])
+ largest = max(thumbnails, key=lambda thumb: quality_key(thumb[1]))
+ thumbnail = 'http://www.ndr.de' + largest[0]
+
+ for format_id in 'lo', 'hi', 'hq':
+ formats.append({
+ 'url': '%s.%s.mp4' % (video_url.group('video'), format_id),
+ 'format_id': format_id,
+ })
+
+ if not formats:
+ raise ExtractorError('No media links available for %s' % video_id)
+
+ return {
+ 'id': video_id,
+ 'title': title,
+ 'description': description,
+ 'thumbnail': thumbnail,
+ 'duration': duration,
+ 'formats': formats,
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/ndtv.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/ndtv.py
new file mode 100644
index 0000000000..95e7d63aad
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/ndtv.py
@@ -0,0 +1,74 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+ month_by_name,
+ int_or_none,
+)
+
+
+class NDTVIE(InfoExtractor):
+ _VALID_URL = r'^https?://(?:www\.)?ndtv\.com/video/player/[^/]*/[^/]*/(?P<id>[a-z0-9]+)'
+
+ _TEST = {
+ 'url': 'http://www.ndtv.com/video/player/news/ndtv-exclusive-don-t-need-character-certificate-from-rahul-gandhi-says-arvind-kejriwal/300710',
+ 'md5': '39f992dbe5fb531c395d8bbedb1e5e88',
+ 'info_dict': {
+ 'id': '300710',
+ 'ext': 'mp4',
+ 'title': "NDTV exclusive: Don't need character certificate from Rahul Gandhi, says Arvind Kejriwal",
+ 'description': 'md5:ab2d4b4a6056c5cb4caa6d729deabf02',
+ 'upload_date': '20131208',
+ 'duration': 1327,
+ 'thumbnail': 'http://i.ndtvimg.com/video/images/vod/medium/2013-12/big_300710_1386518307.jpg',
+ },
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+
+ webpage = self._download_webpage(url, video_id)
+
+ filename = self._search_regex(
+ r"__filename='([^']+)'", webpage, 'video filename')
+ video_url = ('http://bitcast-b.bitgravity.com/ndtvod/23372/ndtv/%s' %
+ filename)
+
+ duration = int_or_none(self._search_regex(
+ r"__duration='([^']+)'", webpage, 'duration', fatal=False))
+
+ date_m = re.search(r'''(?x)
+ <p\s+class="vod_dateline">\s*
+ Published\s+On:\s*
+ (?P<monthname>[A-Za-z]+)\s+(?P<day>[0-9]+),\s*(?P<year>[0-9]+)
+ ''', webpage)
+ upload_date = None
+
+ if date_m is not None:
+ month = month_by_name(date_m.group('monthname'))
+ if month is not None:
+ upload_date = '%s%02d%02d' % (
+ date_m.group('year'), month, int(date_m.group('day')))
+
+ description = self._og_search_description(webpage)
+ READ_MORE = ' (Read more)'
+ if description.endswith(READ_MORE):
+ description = description[:-len(READ_MORE)]
+
+ title = self._og_search_title(webpage)
+ TITLE_SUFFIX = ' - NDTV'
+ if title.endswith(TITLE_SUFFIX):
+ title = title[:-len(TITLE_SUFFIX)]
+
+ return {
+ 'id': video_id,
+ 'url': video_url,
+ 'title': title,
+ 'description': description,
+ 'thumbnail': self._og_search_thumbnail(webpage),
+ 'duration': duration,
+ 'upload_date': upload_date,
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/nerdcubed.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/nerdcubed.py
new file mode 100644
index 0000000000..efc903afa9
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/nerdcubed.py
@@ -0,0 +1,35 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import datetime
+
+from .common import InfoExtractor
+
+
+class NerdCubedFeedIE(InfoExtractor):
+ _VALID_URL = r'https?://(?:www\.)?nerdcubed\.co\.uk/feed\.json'
+ _TEST = {
+ 'url': 'http://www.nerdcubed.co.uk/feed.json',
+ 'info_dict': {
+ 'title': 'nerdcubed.co.uk feed',
+ },
+ 'playlist_mincount': 1300,
+ }
+
+ def _real_extract(self, url):
+ feed = self._download_json(url, url, "Downloading NerdCubed JSON feed")
+
+ entries = [{
+ '_type': 'url',
+ 'title': feed_entry['title'],
+ 'uploader': feed_entry['source']['name'] if feed_entry['source'] else None,
+ 'upload_date': datetime.datetime.strptime(feed_entry['date'], '%Y-%m-%d').strftime('%Y%m%d'),
+ 'url': "http://www.youtube.com/watch?v=" + feed_entry['youtube_id'],
+ } for feed_entry in feed]
+
+ return {
+ '_type': 'playlist',
+ 'title': 'nerdcubed.co.uk feed',
+ 'id': 'nerdcubed-feed',
+ 'entries': entries,
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/newgrounds.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/newgrounds.py
new file mode 100644
index 0000000000..cd117b04ed
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/newgrounds.py
@@ -0,0 +1,42 @@
+from __future__ import unicode_literals
+
+import json
+import re
+
+from .common import InfoExtractor
+
+
+class NewgroundsIE(InfoExtractor):
+ _VALID_URL = r'https?://(?:www\.)?newgrounds\.com/audio/listen/(?P<id>[0-9]+)'
+ _TEST = {
+ 'url': 'http://www.newgrounds.com/audio/listen/549479',
+ 'md5': 'fe6033d297591288fa1c1f780386f07a',
+ 'info_dict': {
+ 'id': '549479',
+ 'ext': 'mp3',
+ 'title': 'B7 - BusMode',
+ 'uploader': 'Burn7',
+ }
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ music_id = mobj.group('id')
+ webpage = self._download_webpage(url, music_id)
+
+ title = self._html_search_regex(
+ r',"name":"([^"]+)",', webpage, 'music title')
+ uploader = self._html_search_regex(
+ r',"artist":"([^"]+)",', webpage, 'music uploader')
+
+ music_url_json_string = self._html_search_regex(
+ r'({"url":"[^"]+"),', webpage, 'music url') + '}'
+ music_url_json = json.loads(music_url_json_string)
+ music_url = music_url_json['url']
+
+ return {
+ 'id': music_id,
+ 'title': title,
+ 'url': music_url,
+ 'uploader': uploader,
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/newstube.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/newstube.py
new file mode 100644
index 0000000000..85fcad06b5
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/newstube.py
@@ -0,0 +1,92 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import ExtractorError
+
+
+class NewstubeIE(InfoExtractor):
+ _VALID_URL = r'https?://(?:www\.)?newstube\.ru/media/(?P<id>.+)'
+ _TEST = {
+ 'url': 'http://www.newstube.ru/media/telekanal-cnn-peremestil-gorod-slavyansk-v-krym',
+ 'info_dict': {
+ 'id': '728e0ef2-e187-4012-bac0-5a081fdcb1f6',
+ 'ext': 'flv',
+ 'title': 'п╒п╣п╩п╣п╨п╟п╫п╟п╩ CNN п©п╣я─п╣п╪п╣я│я┌п╦п╩ пЁп╬я─п╬п╢ п║п╩п╟п╡я▐п╫я│п╨ п╡ п я─я▀п╪',
+ 'description': 'md5:419a8c9f03442bc0b0a794d689360335',
+ 'duration': 31.05,
+ },
+ 'params': {
+ # rtmp download
+ 'skip_download': True,
+ },
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+
+ page = self._download_webpage(url, video_id, 'Downloading page')
+
+ video_guid = self._html_search_regex(
+ r'<meta property="og:video" content="https?://(?:www\.)?newstube\.ru/freshplayer\.swf\?guid=(?P<guid>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})',
+ page, 'video GUID')
+
+ player = self._download_xml(
+ 'http://p.newstube.ru/v2/player.asmx/GetAutoPlayInfo6?state=&url=%s&sessionId=&id=%s&placement=profile&location=n2' % (url, video_guid),
+ video_guid, 'Downloading player XML')
+
+ def ns(s):
+ return s.replace('/', '/%(ns)s') % {'ns': '{http://app1.newstube.ru/N2SiteWS/player.asmx}'}
+
+ error_message = player.find(ns('./ErrorMessage'))
+ if error_message is not None:
+ raise ExtractorError('%s returned error: %s' % (self.IE_NAME, error_message.text), expected=True)
+
+ session_id = player.find(ns('./SessionId')).text
+ media_info = player.find(ns('./Medias/MediaInfo'))
+ title = media_info.find(ns('./Name')).text
+ description = self._og_search_description(page)
+ thumbnail = media_info.find(ns('./KeyFrame')).text
+ duration = int(media_info.find(ns('./Duration')).text) / 1000.0
+
+ formats = []
+
+ for stream_info in media_info.findall(ns('./Streams/StreamInfo')):
+ media_location = stream_info.find(ns('./MediaLocation'))
+ if media_location is None:
+ continue
+
+ server = media_location.find(ns('./Server')).text
+ app = media_location.find(ns('./App')).text
+ media_id = stream_info.find(ns('./Id')).text
+ quality_id = stream_info.find(ns('./QualityId')).text
+ name = stream_info.find(ns('./Name')).text
+ width = int(stream_info.find(ns('./Width')).text)
+ height = int(stream_info.find(ns('./Height')).text)
+
+ formats.append({
+ 'url': 'rtmp://%s/%s' % (server, app),
+ 'app': app,
+ 'play_path': '01/%s' % video_guid.upper(),
+ 'rtmp_conn': ['S:%s' % session_id, 'S:%s' % media_id, 'S:n2'],
+ 'page_url': url,
+ 'ext': 'flv',
+ 'format_id': quality_id,
+ 'format_note': name,
+ 'width': width,
+ 'height': height,
+ })
+
+ self._sort_formats(formats)
+
+ return {
+ 'id': video_guid,
+ 'title': title,
+ 'description': description,
+ 'thumbnail': thumbnail,
+ 'duration': duration,
+ 'formats': formats,
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/nfb.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/nfb.py
new file mode 100644
index 0000000000..ea077254b4
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/nfb.py
@@ -0,0 +1,94 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..compat import (
+ compat_urllib_request,
+ compat_urllib_parse,
+)
+
+
+class NFBIE(InfoExtractor):
+ IE_NAME = 'nfb'
+ IE_DESC = 'National Film Board of Canada'
+ _VALID_URL = r'https?://(?:www\.)?(?:nfb|onf)\.ca/film/(?P<id>[\da-z_-]+)'
+
+ _TEST = {
+ 'url': 'https://www.nfb.ca/film/qallunaat_why_white_people_are_funny',
+ 'info_dict': {
+ 'id': 'qallunaat_why_white_people_are_funny',
+ 'ext': 'mp4',
+ 'title': 'Qallunaat! Why White People Are Funny ',
+ 'description': 'md5:836d8aff55e087d04d9f6df554d4e038',
+ 'duration': 3128,
+ 'uploader': 'Mark Sandiford',
+ 'uploader_id': 'mark-sandiford',
+ },
+ 'params': {
+ # rtmp download
+ 'skip_download': True,
+ }
+ }
+
+ def _real_extract(self, url):
+ video_id = self._match_id(url)
+ page = self._download_webpage(
+ 'https://www.nfb.ca/film/%s' % video_id, video_id,
+ 'Downloading film page')
+
+ uploader_id = self._html_search_regex(r'<a class="director-link" href="/explore-all-directors/([^/]+)/"',
+ page, 'director id', fatal=False)
+ uploader = self._html_search_regex(r'<em class="director-name" itemprop="name">([^<]+)</em>',
+ page, 'director name', fatal=False)
+
+ request = compat_urllib_request.Request('https://www.nfb.ca/film/%s/player_config' % video_id,
+ compat_urllib_parse.urlencode({'getConfig': 'true'}).encode('ascii'))
+ request.add_header('Content-Type', 'application/x-www-form-urlencoded')
+ request.add_header('X-NFB-Referer', 'http://www.nfb.ca/medias/flash/NFBVideoPlayer.swf')
+
+ config = self._download_xml(request, video_id, 'Downloading player config XML')
+
+ title = None
+ description = None
+ thumbnail = None
+ duration = None
+ formats = []
+
+ def extract_thumbnail(media):
+ thumbnails = {}
+ for asset in media.findall('assets/asset'):
+ thumbnails[asset.get('quality')] = asset.find('default/url').text
+ if not thumbnails:
+ return None
+ if 'high' in thumbnails:
+ return thumbnails['high']
+ return list(thumbnails.values())[0]
+
+ for media in config.findall('./player/stream/media'):
+ if media.get('type') == 'posterImage':
+ thumbnail = extract_thumbnail(media)
+ elif media.get('type') == 'video':
+ duration = int(media.get('duration'))
+ title = media.find('title').text
+ description = media.find('description').text
+ # It seems assets always go from lower to better quality, so no need to sort
+ for asset in media.findall('assets/asset'):
+ for x in asset:
+ formats.append({
+ 'url': x.find('streamerURI').text,
+ 'app': x.find('streamerURI').text.split('/', 3)[3],
+ 'play_path': x.find('url').text,
+ 'rtmp_live': False,
+ 'ext': 'mp4',
+ 'format_id': '%s-%s' % (x.tag, asset.get('quality')),
+ })
+
+ return {
+ 'id': video_id,
+ 'title': title,
+ 'description': description,
+ 'thumbnail': thumbnail,
+ 'duration': duration,
+ 'uploader': uploader,
+ 'uploader_id': uploader_id,
+ 'formats': formats,
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/nfl.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/nfl.py
new file mode 100644
index 0000000000..606e2294ef
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/nfl.py
@@ -0,0 +1,146 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..compat import (
+ compat_urllib_parse_urlparse,
+)
+from ..utils import (
+ ExtractorError,
+ int_or_none,
+ remove_end,
+)
+
+
+class NFLIE(InfoExtractor):
+ IE_NAME = 'nfl.com'
+ _VALID_URL = r'''(?x)https?://
+ (?P<host>(?:www\.)?(?:nfl\.com|.*?\.clubs\.nfl\.com))/
+ (?:.+?/)*
+ (?P<id>(?:\d[a-z]{2}\d{13}|\w{8}\-(?:\w{4}\-){3}\w{12}))'''
+ _TESTS = [
+ {
+ 'url': 'http://www.nfl.com/videos/nfl-game-highlights/0ap3000000398478/Week-3-Redskins-vs-Eagles-highlights',
+ 'md5': '394ef771ddcd1354f665b471d78ec4c6',
+ 'info_dict': {
+ 'id': '0ap3000000398478',
+ 'ext': 'mp4',
+ 'title': 'Week 3: Redskins vs. Eagles highlights',
+ 'description': 'md5:56323bfb0ac4ee5ab24bd05fdf3bf478',
+ 'upload_date': '20140921',
+ 'timestamp': 1411337580,
+ 'thumbnail': 're:^https?://.*\.jpg$',
+ }
+ },
+ {
+ 'url': 'http://prod.www.steelers.clubs.nfl.com/video-and-audio/videos/LIVE_Post_Game_vs_Browns/9d72f26a-9e2b-4718-84d3-09fb4046c266',
+ 'md5': 'cf85bdb4bc49f6e9d3816d130c78279c',
+ 'info_dict': {
+ 'id': '9d72f26a-9e2b-4718-84d3-09fb4046c266',
+ 'ext': 'mp4',
+ 'title': 'LIVE: Post Game vs. Browns',
+ 'description': 'md5:6a97f7e5ebeb4c0e69a418a89e0636e8',
+ 'upload_date': '20131229',
+ 'timestamp': 1388354455,
+ 'thumbnail': 're:^https?://.*\.jpg$',
+ }
+ }
+ ]
+
+ @staticmethod
+ def prepend_host(host, url):
+ if not url.startswith('http'):
+ if not url.startswith('/'):
+ url = '/%s' % url
+ url = 'http://{0:}{1:}'.format(host, url)
+ return url
+
+ @staticmethod
+ def format_from_stream(stream, protocol, host, path_prefix='',
+ preference=0, note=None):
+ url = '{protocol:}://{host:}/{prefix:}{path:}'.format(
+ protocol=protocol,
+ host=host,
+ prefix=path_prefix,
+ path=stream.get('path'),
+ )
+ return {
+ 'url': url,
+ 'vbr': int_or_none(stream.get('rate', 0), 1000),
+ 'preference': preference,
+ 'format_note': note,
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id, host = mobj.group('id'), mobj.group('host')
+
+ webpage = self._download_webpage(url, video_id)
+
+ config_url = NFLIE.prepend_host(host, self._search_regex(
+ r'(?:config|configURL)\s*:\s*"([^"]+)"', webpage, 'config URL'))
+ config = self._download_json(config_url, video_id,
+ note='Downloading player config')
+ url_template = NFLIE.prepend_host(
+ host, '{contentURLTemplate:}'.format(**config))
+ video_data = self._download_json(
+ url_template.format(id=video_id), video_id)
+
+ formats = []
+ cdn_data = video_data.get('cdnData', {})
+ streams = cdn_data.get('bitrateInfo', [])
+ if cdn_data.get('format') == 'EXTERNAL_HTTP_STREAM':
+ parts = compat_urllib_parse_urlparse(cdn_data.get('uri'))
+ protocol, host = parts.scheme, parts.netloc
+ for stream in streams:
+ formats.append(
+ NFLIE.format_from_stream(stream, protocol, host))
+ else:
+ cdns = config.get('cdns')
+ if not cdns:
+ raise ExtractorError('Failed to get CDN data', expected=True)
+
+ for name, cdn in cdns.items():
+ # LimeLight streams don't seem to work
+ if cdn.get('name') == 'LIMELIGHT':
+ continue
+
+ protocol = cdn.get('protocol')
+ host = remove_end(cdn.get('host', ''), '/')
+ if not (protocol and host):
+ continue
+
+ prefix = cdn.get('pathprefix', '')
+ if prefix and not prefix.endswith('/'):
+ prefix = '%s/' % prefix
+
+ preference = 0
+ if protocol == 'rtmp':
+ preference = -2
+ elif 'prog' in name.lower():
+ preference = 1
+
+ for stream in streams:
+ formats.append(
+ NFLIE.format_from_stream(stream, protocol, host,
+ prefix, preference, name))
+
+ self._sort_formats(formats)
+
+ thumbnail = None
+ for q in ('xl', 'l', 'm', 's', 'xs'):
+ thumbnail = video_data.get('imagePaths', {}).get(q)
+ if thumbnail:
+ break
+
+ return {
+ 'id': video_id,
+ 'title': video_data.get('headline'),
+ 'formats': formats,
+ 'description': video_data.get('caption'),
+ 'duration': video_data.get('duration'),
+ 'thumbnail': thumbnail,
+ 'timestamp': int_or_none(video_data.get('posted'), 1000),
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/nhl.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/nhl.py
new file mode 100644
index 0000000000..d3a4fc5138
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/nhl.py
@@ -0,0 +1,155 @@
+from __future__ import unicode_literals
+
+import re
+import json
+import os
+
+from .common import InfoExtractor
+from ..compat import (
+ compat_urlparse,
+ compat_urllib_parse,
+ compat_urllib_parse_urlparse
+)
+from ..utils import (
+ unified_strdate,
+)
+
+
+class NHLBaseInfoExtractor(InfoExtractor):
+ @staticmethod
+ def _fix_json(json_string):
+ return json_string.replace('\\\'', '\'')
+
+ def _extract_video(self, info):
+ video_id = info['id']
+ self.report_extraction(video_id)
+
+ initial_video_url = info['publishPoint']
+ if info['formats'] == '1':
+ parsed_url = compat_urllib_parse_urlparse(initial_video_url)
+ filename, ext = os.path.splitext(parsed_url.path)
+ path = '%s_sd%s' % (filename, ext)
+ data = compat_urllib_parse.urlencode({
+ 'type': 'fvod',
+ 'path': compat_urlparse.urlunparse(parsed_url[:2] + (path,) + parsed_url[3:])
+ })
+ path_url = 'http://video.nhl.com/videocenter/servlets/encryptvideopath?' + data
+ path_doc = self._download_xml(
+ path_url, video_id, 'Downloading final video url')
+ video_url = path_doc.find('path').text
+ else:
+ video_url = initial_video_url
+
+ join = compat_urlparse.urljoin
+ return {
+ 'id': video_id,
+ 'title': info['name'],
+ 'url': video_url,
+ 'description': info['description'],
+ 'duration': int(info['duration']),
+ 'thumbnail': join(join(video_url, '/u/'), info['bigImage']),
+ 'upload_date': unified_strdate(info['releaseDate'].split('.')[0]),
+ }
+
+
+class NHLIE(NHLBaseInfoExtractor):
+ IE_NAME = 'nhl.com'
+ _VALID_URL = r'https?://video(?P<team>\.[^.]*)?\.nhl\.com/videocenter/console(?:\?(?:.*?[?&])?)id=(?P<id>[-0-9a-zA-Z]+)'
+
+ _TESTS = [{
+ 'url': 'http://video.canucks.nhl.com/videocenter/console?catid=6?id=453614',
+ 'md5': 'db704a4ea09e8d3988c85e36cc892d09',
+ 'info_dict': {
+ 'id': '453614',
+ 'ext': 'mp4',
+ 'title': 'Quick clip: Weise 4-3 goal vs Flames',
+ 'description': 'Dale Weise scores his first of the season to put the Canucks up 4-3.',
+ 'duration': 18,
+ 'upload_date': '20131006',
+ },
+ }, {
+ 'url': 'http://video.nhl.com/videocenter/console?id=2014020024-628-h',
+ 'md5': 'd22e82bc592f52d37d24b03531ee9696',
+ 'info_dict': {
+ 'id': '2014020024-628-h',
+ 'ext': 'mp4',
+ 'title': 'Alex Galchenyuk Goal on Ray Emery (14:40/3rd)',
+ 'description': 'Home broadcast - Montreal Canadiens at Philadelphia Flyers - October 11, 2014',
+ 'duration': 0,
+ 'upload_date': '20141011',
+ },
+ }, {
+ 'url': 'http://video.mapleleafs.nhl.com/videocenter/console?id=58665&catid=802',
+ 'md5': 'c78fc64ea01777e426cfc202b746c825',
+ 'info_dict': {
+ 'id': '58665',
+ 'ext': 'flv',
+ 'title': 'Classic Game In Six - April 22, 1979',
+ 'description': 'It was the last playoff game for the Leafs in the decade, and the last time the Leafs and Habs played in the playoffs. Great game, not a great ending.',
+ 'duration': 400,
+ 'upload_date': '20100129'
+ },
+ }, {
+ 'url': 'http://video.flames.nhl.com/videocenter/console?id=630616',
+ 'only_matching': True,
+ }]
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+ json_url = 'http://video.nhl.com/videocenter/servlets/playlist?ids=%s&format=json' % video_id
+ data = self._download_json(
+ json_url, video_id, transform_source=self._fix_json)
+ return self._extract_video(data[0])
+
+
+class NHLVideocenterIE(NHLBaseInfoExtractor):
+ IE_NAME = 'nhl.com:videocenter'
+ IE_DESC = 'NHL videocenter category'
+ _VALID_URL = r'https?://video\.(?P<team>[^.]*)\.nhl\.com/videocenter/(console\?[^(id=)]*catid=(?P<catid>[0-9]+)(?![&?]id=).*?)?$'
+ _TEST = {
+ 'url': 'http://video.canucks.nhl.com/videocenter/console?catid=999',
+ 'info_dict': {
+ 'id': '999',
+ 'title': 'Highlights',
+ },
+ 'playlist_count': 12,
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ team = mobj.group('team')
+ webpage = self._download_webpage(url, team)
+ cat_id = self._search_regex(
+ [r'var defaultCatId = "(.+?)";',
+ r'{statusIndex:0,index:0,.*?id:(.*?),'],
+ webpage, 'category id')
+ playlist_title = self._html_search_regex(
+ r'tab0"[^>]*?>(.*?)</td>',
+ webpage, 'playlist title', flags=re.DOTALL).lower().capitalize()
+
+ data = compat_urllib_parse.urlencode({
+ 'cid': cat_id,
+ # This is the default value
+ 'count': 12,
+ 'ptrs': 3,
+ 'format': 'json',
+ })
+ path = '/videocenter/servlets/browse?' + data
+ request_url = compat_urlparse.urljoin(url, path)
+ response = self._download_webpage(request_url, playlist_title)
+ response = self._fix_json(response)
+ if not response.strip():
+ self._downloader.report_warning('Got an empty reponse, trying '
+ 'adding the "newvideos" parameter')
+ response = self._download_webpage(request_url + '&newvideos=true',
+ playlist_title)
+ response = self._fix_json(response)
+ videos = json.loads(response)
+
+ return {
+ '_type': 'playlist',
+ 'title': playlist_title,
+ 'id': cat_id,
+ 'entries': [self._extract_video(v) for v in videos],
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/niconico.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/niconico.py
new file mode 100644
index 0000000000..4c18904169
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/niconico.py
@@ -0,0 +1,189 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+import re
+import json
+
+from .common import InfoExtractor
+from ..compat import (
+ compat_urllib_parse,
+ compat_urllib_request,
+ compat_urlparse,
+)
+from ..utils import (
+ ExtractorError,
+ int_or_none,
+ parse_duration,
+ unified_strdate,
+)
+
+
+class NiconicoIE(InfoExtractor):
+ IE_NAME = 'niconico'
+ IE_DESC = 'Ц┐▀Ц┌ЁЦ┐▀Ц┌ЁЕ▀∙Г■╩'
+
+ _TEST = {
+ 'url': 'http://www.nicovideo.jp/watch/sm22312215',
+ 'md5': 'd1a75c0823e2f629128c43e1212760f9',
+ 'info_dict': {
+ 'id': 'sm22312215',
+ 'ext': 'mp4',
+ 'title': 'Big Buck Bunny',
+ 'uploader': 'takuya0301',
+ 'uploader_id': '2698420',
+ 'upload_date': '20131123',
+ 'description': '(c) copyright 2008, Blender Foundation / www.bigbuckbunny.org',
+ 'duration': 33,
+ },
+ 'params': {
+ 'username': 'ydl.niconico@gmail.com',
+ 'password': 'youtube-dl',
+ },
+ }
+
+ _VALID_URL = r'https?://(?:www\.|secure\.)?nicovideo\.jp/watch/((?:[a-z]{2})?[0-9]+)'
+ _NETRC_MACHINE = 'niconico'
+ # Determine whether the downloader used authentication to download video
+ _AUTHENTICATED = False
+
+ def _real_initialize(self):
+ self._login()
+
+ def _login(self):
+ (username, password) = self._get_login_info()
+ # No authentication to be performed
+ if not username:
+ return True
+
+ # Log in
+ login_form_strs = {
+ 'mail': username,
+ 'password': password,
+ }
+ # Convert to UTF-8 *before* urlencode because Python 2.x's urlencode
+ # chokes on unicode
+ login_form = dict((k.encode('utf-8'), v.encode('utf-8')) for k, v in login_form_strs.items())
+ login_data = compat_urllib_parse.urlencode(login_form).encode('utf-8')
+ request = compat_urllib_request.Request(
+ 'https://secure.nicovideo.jp/secure/login', login_data)
+ login_results = self._download_webpage(
+ request, None, note='Logging in', errnote='Unable to log in')
+ if re.search(r'(?i)<h1 class="mb8p4">Log in error</h1>', login_results) is not None:
+ self._downloader.report_warning('unable to log in: bad username or password')
+ return False
+ # Successful login
+ self._AUTHENTICATED = True
+ return True
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group(1)
+
+ # Get video webpage. We are not actually interested in it, but need
+ # the cookies in order to be able to download the info webpage
+ self._download_webpage('http://www.nicovideo.jp/watch/' + video_id, video_id)
+
+ video_info = self._download_xml(
+ 'http://ext.nicovideo.jp/api/getthumbinfo/' + video_id, video_id,
+ note='Downloading video info page')
+
+ if self._AUTHENTICATED:
+ # Get flv info
+ flv_info_webpage = self._download_webpage(
+ 'http://flapi.nicovideo.jp/api/getflv?v=' + video_id,
+ video_id, 'Downloading flv info')
+ else:
+ # Get external player info
+ ext_player_info = self._download_webpage(
+ 'http://ext.nicovideo.jp/thumb_watch/' + video_id, video_id)
+ thumb_play_key = self._search_regex(
+ r'\'thumbPlayKey\'\s*:\s*\'(.*?)\'', ext_player_info, 'thumbPlayKey')
+
+ # Get flv info
+ flv_info_data = compat_urllib_parse.urlencode({
+ 'k': thumb_play_key,
+ 'v': video_id
+ })
+ flv_info_request = compat_urllib_request.Request(
+ 'http://ext.nicovideo.jp/thumb_watch', flv_info_data,
+ {'Content-Type': 'application/x-www-form-urlencoded'})
+ flv_info_webpage = self._download_webpage(
+ flv_info_request, video_id,
+ note='Downloading flv info', errnote='Unable to download flv info')
+
+ if 'deleted=' in flv_info_webpage:
+ raise ExtractorError('The video has been deleted.',
+ expected=True)
+ video_real_url = compat_urlparse.parse_qs(flv_info_webpage)['url'][0]
+
+ # Start extracting information
+ title = video_info.find('.//title').text
+ extension = video_info.find('.//movie_type').text
+ video_format = extension.upper()
+ thumbnail = video_info.find('.//thumbnail_url').text
+ description = video_info.find('.//description').text
+ upload_date = unified_strdate(video_info.find('.//first_retrieve').text.split('+')[0])
+ view_count = int_or_none(video_info.find('.//view_counter').text)
+ comment_count = int_or_none(video_info.find('.//comment_num').text)
+ duration = parse_duration(video_info.find('.//length').text)
+ webpage_url = video_info.find('.//watch_url').text
+
+ if video_info.find('.//ch_id') is not None:
+ uploader_id = video_info.find('.//ch_id').text
+ uploader = video_info.find('.//ch_name').text
+ elif video_info.find('.//user_id') is not None:
+ uploader_id = video_info.find('.//user_id').text
+ uploader = video_info.find('.//user_nickname').text
+ else:
+ uploader_id = uploader = None
+
+ return {
+ 'id': video_id,
+ 'url': video_real_url,
+ 'title': title,
+ 'ext': extension,
+ 'format': video_format,
+ 'thumbnail': thumbnail,
+ 'description': description,
+ 'uploader': uploader,
+ 'upload_date': upload_date,
+ 'uploader_id': uploader_id,
+ 'view_count': view_count,
+ 'comment_count': comment_count,
+ 'duration': duration,
+ 'webpage_url': webpage_url,
+ }
+
+
+class NiconicoPlaylistIE(InfoExtractor):
+ _VALID_URL = r'https?://www\.nicovideo\.jp/mylist/(?P<id>\d+)'
+
+ _TEST = {
+ 'url': 'http://www.nicovideo.jp/mylist/27411728',
+ 'info_dict': {
+ 'id': '27411728',
+ 'title': 'AKB48Ц│╝Ц┌╙Ц┐╪Ц┐╚Ц┐┼Ц┌╓Ц┐┬Ц┐▀Ц┐┐Ц┐²Ц┐Ё',
+ },
+ 'playlist_mincount': 225,
+ }
+
+ def _real_extract(self, url):
+ list_id = self._match_id(url)
+ webpage = self._download_webpage(url, list_id)
+
+ entries_json = self._search_regex(r'Mylist\.preload\(\d+, (\[.*\])\);',
+ webpage, 'entries')
+ entries = json.loads(entries_json)
+ entries = [{
+ '_type': 'url',
+ 'ie_key': NiconicoIE.ie_key(),
+ 'url': ('http://www.nicovideo.jp/watch/%s' %
+ entry['item_data']['video_id']),
+ } for entry in entries]
+
+ return {
+ '_type': 'playlist',
+ 'title': self._search_regex(r'\s+name: "(.*?)"', webpage, 'title'),
+ 'id': list_id,
+ 'entries': entries,
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/ninegag.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/ninegag.py
new file mode 100644
index 0000000000..7f842b5c25
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/ninegag.py
@@ -0,0 +1,73 @@
+from __future__ import unicode_literals
+
+import re
+import json
+
+from .common import InfoExtractor
+from ..utils import str_to_int
+
+
+class NineGagIE(InfoExtractor):
+ IE_NAME = '9gag'
+ _VALID_URL = r'''(?x)^https?://(?:www\.)?9gag\.tv/
+ (?:
+ v/(?P<numid>[0-9]+)|
+ p/(?P<id>[a-zA-Z0-9]+)/(?P<display_id>[^?#/]+)
+ )
+ '''
+
+ _TESTS = [{
+ "url": "http://9gag.tv/v/1912",
+ "info_dict": {
+ "id": "1912",
+ "ext": "mp4",
+ "description": "This 3-minute video will make you smile and then make you feel untalented and insignificant. Anyway, you should share this awesomeness. (Thanks, Dino!)",
+ "title": "\"People Are Awesome 2013\" Is Absolutely Awesome",
+ 'uploader_id': 'UCdEH6EjDKwtTe-sO2f0_1XA',
+ 'uploader': 'CompilationChannel',
+ 'upload_date': '20131110',
+ "view_count": int,
+ "thumbnail": "re:^https?://",
+ },
+ 'add_ie': ['Youtube']
+ }, {
+ 'url': 'http://9gag.tv/p/KklwM/alternate-banned-opening-scene-of-gravity?ref=fsidebar',
+ 'info_dict': {
+ 'id': 'KklwM',
+ 'ext': 'mp4',
+ 'display_id': 'alternate-banned-opening-scene-of-gravity',
+ "description": "While Gravity was a pretty awesome movie already, YouTuber Krishna Shenoi came up with a way to improve upon it, introducing a much better solution to Sandra Bullock's seemingly endless tumble in space. The ending is priceless.",
+ 'title': "Banned Opening Scene Of \"Gravity\" That Changes The Whole Movie",
+ 'uploader': 'Krishna Shenoi',
+ 'upload_date': '20140401',
+ 'uploader_id': 'krishnashenoi93',
+ },
+ }]
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('numid') or mobj.group('id')
+ display_id = mobj.group('display_id') or video_id
+
+ webpage = self._download_webpage(url, display_id)
+
+ post_view = json.loads(self._html_search_regex(
+ r'var postView = new app\.PostView\({\s*post:\s*({.+?}),\s*posts:\s*prefetchedCurrentPost', webpage, 'post view'))
+
+ youtube_id = post_view['videoExternalId']
+ title = post_view['title']
+ description = post_view['description']
+ view_count = str_to_int(post_view['externalView'])
+ thumbnail = post_view.get('thumbnail_700w') or post_view.get('ogImageUrl') or post_view.get('thumbnail_300w')
+
+ return {
+ '_type': 'url_transparent',
+ 'url': youtube_id,
+ 'ie_key': 'Youtube',
+ 'id': video_id,
+ 'display_id': display_id,
+ 'title': title,
+ 'description': description,
+ 'view_count': view_count,
+ 'thumbnail': thumbnail,
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/noco.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/noco.py
new file mode 100644
index 0000000000..251e6da074
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/noco.py
@@ -0,0 +1,174 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+import re
+import time
+import hashlib
+
+from .common import InfoExtractor
+from ..compat import (
+ compat_str,
+ compat_urllib_parse,
+ compat_urllib_request,
+)
+from ..utils import (
+ clean_html,
+ ExtractorError,
+ unified_strdate,
+)
+
+
+class NocoIE(InfoExtractor):
+ _VALID_URL = r'http://(?:(?:www\.)?noco\.tv/emission/|player\.noco\.tv/\?idvideo=)(?P<id>\d+)'
+ _LOGIN_URL = 'http://noco.tv/do.php'
+ _API_URL_TEMPLATE = 'https://api.noco.tv/1.1/%s?ts=%s&tk=%s'
+ _SUB_LANG_TEMPLATE = '&sub_lang=%s'
+ _NETRC_MACHINE = 'noco'
+
+ _TEST = {
+ 'url': 'http://noco.tv/emission/11538/nolife/ami-ami-idol-hello-france/',
+ 'md5': '0a993f0058ddbcd902630b2047ef710e',
+ 'info_dict': {
+ 'id': '11538',
+ 'ext': 'mp4',
+ 'title': 'Ami Ami Idol - Hello! France',
+ 'description': 'md5:4eaab46ab68fa4197a317a88a53d3b86',
+ 'upload_date': '20140412',
+ 'uploader': 'Nolife',
+ 'uploader_id': 'NOL',
+ 'duration': 2851.2,
+ },
+ 'skip': 'Requires noco account',
+ }
+
+ def _real_initialize(self):
+ self._login()
+
+ def _login(self):
+ (username, password) = self._get_login_info()
+ if username is None:
+ return
+
+ login_form = {
+ 'a': 'login',
+ 'cookie': '1',
+ 'username': username,
+ 'password': password,
+ }
+ request = compat_urllib_request.Request(self._LOGIN_URL, compat_urllib_parse.urlencode(login_form))
+ request.add_header('Content-Type', 'application/x-www-form-urlencoded; charset=UTF-8')
+
+ login = self._download_json(request, None, 'Logging in as %s' % username)
+
+ if 'erreur' in login:
+ raise ExtractorError('Unable to login: %s' % clean_html(login['erreur']), expected=True)
+
+ def _call_api(self, path, video_id, note, sub_lang=None):
+ ts = compat_str(int(time.time() * 1000))
+ tk = hashlib.md5((hashlib.md5(ts.encode('ascii')).hexdigest() + '#8S?uCraTedap6a').encode('ascii')).hexdigest()
+ url = self._API_URL_TEMPLATE % (path, ts, tk)
+ if sub_lang:
+ url += self._SUB_LANG_TEMPLATE % sub_lang
+
+ resp = self._download_json(url, video_id, note)
+
+ if isinstance(resp, dict) and resp.get('error'):
+ self._raise_error(resp['error'], resp['description'])
+
+ return resp
+
+ def _raise_error(self, error, description):
+ raise ExtractorError(
+ '%s returned error: %s - %s' % (self.IE_NAME, error, description),
+ expected=True)
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+
+ medias = self._call_api(
+ 'shows/%s/medias' % video_id,
+ video_id, 'Downloading video JSON')
+
+ qualities = self._call_api(
+ 'qualities',
+ video_id, 'Downloading qualities JSON')
+
+ formats = []
+
+ for lang, lang_dict in medias['fr']['video_list'].items():
+ for format_id, fmt in lang_dict['quality_list'].items():
+ format_id_extended = '%s-%s' % (lang, format_id) if lang != 'none' else format_id
+
+ video = self._call_api(
+ 'shows/%s/video/%s/fr' % (video_id, format_id.lower()),
+ video_id, 'Downloading %s video JSON' % format_id_extended,
+ lang if lang != 'none' else None)
+
+ file_url = video['file']
+ if not file_url:
+ continue
+
+ if file_url in ['forbidden', 'not found']:
+ popmessage = video['popmessage']
+ self._raise_error(popmessage['title'], popmessage['message'])
+
+ formats.append({
+ 'url': file_url,
+ 'format_id': format_id_extended,
+ 'width': fmt['res_width'],
+ 'height': fmt['res_lines'],
+ 'abr': fmt['audiobitrate'],
+ 'vbr': fmt['videobitrate'],
+ 'filesize': fmt['filesize'],
+ 'format_note': qualities[format_id]['quality_name'],
+ 'preference': qualities[format_id]['priority'],
+ })
+
+ self._sort_formats(formats)
+
+ show = self._call_api(
+ 'shows/by_id/%s' % video_id,
+ video_id, 'Downloading show JSON')[0]
+
+ upload_date = unified_strdate(show['online_date_start_utc'])
+ uploader = show['partner_name']
+ uploader_id = show['partner_key']
+ duration = show['duration_ms'] / 1000.0
+
+ thumbnails = []
+ for thumbnail_key, thumbnail_url in show.items():
+ m = re.search(r'^screenshot_(?P<width>\d+)x(?P<height>\d+)$', thumbnail_key)
+ if not m:
+ continue
+ thumbnails.append({
+ 'url': thumbnail_url,
+ 'width': int(m.group('width')),
+ 'height': int(m.group('height')),
+ })
+
+ episode = show.get('show_TT') or show.get('show_OT')
+ family = show.get('family_TT') or show.get('family_OT')
+ episode_number = show.get('episode_number')
+
+ title = ''
+ if family:
+ title += family
+ if episode_number:
+ title += ' #' + compat_str(episode_number)
+ if episode:
+ title += ' - ' + episode
+
+ description = show.get('show_resume') or show.get('family_resume')
+
+ return {
+ 'id': video_id,
+ 'title': title,
+ 'description': description,
+ 'thumbnails': thumbnails,
+ 'upload_date': upload_date,
+ 'uploader': uploader,
+ 'uploader_id': uploader_id,
+ 'duration': duration,
+ 'formats': formats,
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/normalboots.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/normalboots.py
new file mode 100644
index 0000000000..3d35b11ac8
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/normalboots.py
@@ -0,0 +1,51 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+
+from ..utils import (
+ unified_strdate,
+)
+
+
+class NormalbootsIE(InfoExtractor):
+ _VALID_URL = r'http://(?:www\.)?normalboots\.com/video/(?P<videoid>[0-9a-z-]*)/?$'
+ _TEST = {
+ 'url': 'http://normalboots.com/video/home-alone-games-jontron/',
+ 'md5': '8bf6de238915dd501105b44ef5f1e0f6',
+ 'info_dict': {
+ 'id': 'home-alone-games-jontron',
+ 'ext': 'mp4',
+ 'title': 'Home Alone Games - JonTron - NormalBoots',
+ 'description': 'Jon is late for Christmas. Typical. Thanks to: Paul Ritchey for Co-Writing/Filming: http://www.youtube.com/user/ContinueShow Michael Azzi for Christmas Intro Animation: http://michafrar.tumblr.com/ Jerrod Waters for Christmas Intro Music: http://www.youtube.com/user/xXJerryTerryXx Casey Ormond for Б─≤Tense Battle ThemeБ─≥:\xa0http://www.youtube.com/Kiamet/',
+ 'uploader': 'JonTron',
+ 'upload_date': '20140125',
+ }
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('videoid')
+
+ webpage = self._download_webpage(url, video_id)
+ video_uploader = self._html_search_regex(r'Posted\sby\s<a\shref="[A-Za-z0-9/]*">(?P<uploader>[A-Za-z]*)\s</a>',
+ webpage, 'uploader')
+ raw_upload_date = self._html_search_regex('<span style="text-transform:uppercase; font-size:inherit;">[A-Za-z]+, (?P<date>.*)</span>',
+ webpage, 'date')
+ video_upload_date = unified_strdate(raw_upload_date)
+
+ player_url = self._html_search_regex(r'<iframe\swidth="[0-9]+"\sheight="[0-9]+"\ssrc="(?P<url>[\S]+)"', webpage, 'url')
+ player_page = self._download_webpage(player_url, video_id)
+ video_url = self._html_search_regex(r"file:\s'(?P<file>[^']+\.mp4)'", player_page, 'file')
+
+ return {
+ 'id': video_id,
+ 'url': video_url,
+ 'title': self._og_search_title(webpage),
+ 'description': self._og_search_description(webpage),
+ 'thumbnail': self._og_search_thumbnail(webpage),
+ 'uploader': video_uploader,
+ 'upload_date': video_upload_date,
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/nosvideo.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/nosvideo.py
new file mode 100644
index 0000000000..f5ef856db0
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/nosvideo.py
@@ -0,0 +1,77 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..compat import (
+ compat_urllib_request,
+)
+from ..utils import (
+ ExtractorError,
+ urlencode_postdata,
+ xpath_text,
+ xpath_with_ns,
+)
+
+_x = lambda p: xpath_with_ns(p, {'xspf': 'http://xspf.org/ns/0/'})
+
+
+class NosVideoIE(InfoExtractor):
+ _VALID_URL = r'https?://(?:www\.)?nosvideo\.com/' + \
+ '(?:embed/|\?v=)(?P<id>[A-Za-z0-9]{12})/?'
+ _PLAYLIST_URL = 'http://nosvideo.com/xml/{xml_id:s}.xml'
+ _FILE_DELETED_REGEX = r'<b>File Not Found</b>'
+ _TEST = {
+ 'url': 'http://nosvideo.com/?v=mu8fle7g7rpq',
+ 'md5': '6124ed47130d8be3eacae635b071e6b6',
+ 'info_dict': {
+ 'id': 'mu8fle7g7rpq',
+ 'ext': 'mp4',
+ 'title': 'big_buck_bunny_480p_surround-fix.avi.mp4',
+ 'thumbnail': 're:^https?://.*\.jpg$',
+ }
+ }
+
+ def _real_extract(self, url):
+ video_id = self._match_id(url)
+
+ fields = {
+ 'id': video_id,
+ 'op': 'download1',
+ 'method_free': 'Continue to Video',
+ }
+ req = compat_urllib_request.Request(url, urlencode_postdata(fields))
+ req.add_header('Content-type', 'application/x-www-form-urlencoded')
+ webpage = self._download_webpage(req, video_id,
+ 'Downloading download page')
+ if re.search(self._FILE_DELETED_REGEX, webpage) is not None:
+ raise ExtractorError('Video %s does not exist' % video_id,
+ expected=True)
+
+ xml_id = self._search_regex(r'php\|([^\|]+)\|', webpage, 'XML ID')
+ playlist_url = self._PLAYLIST_URL.format(xml_id=xml_id)
+ playlist = self._download_xml(playlist_url, video_id)
+
+ track = playlist.find(_x('.//xspf:track'))
+ if track is None:
+ raise ExtractorError(
+ 'XML playlist is missing the \'track\' element',
+ expected=True)
+ title = xpath_text(track, _x('./xspf:title'), 'title')
+ url = xpath_text(track, _x('./xspf:file'), 'URL', fatal=True)
+ thumbnail = xpath_text(track, _x('./xspf:image'), 'thumbnail')
+ if title is not None:
+ title = title.strip()
+
+ formats = [{
+ 'format_id': 'sd',
+ 'url': url,
+ }]
+
+ return {
+ 'id': video_id,
+ 'title': title,
+ 'thumbnail': thumbnail,
+ 'formats': formats,
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/novamov.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/novamov.py
new file mode 100644
index 0000000000..04d779890a
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/novamov.py
@@ -0,0 +1,71 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..compat import (
+ compat_urlparse,
+)
+from ..utils import (
+ ExtractorError,
+)
+
+
+class NovaMovIE(InfoExtractor):
+ IE_NAME = 'novamov'
+ IE_DESC = 'NovaMov'
+
+ _VALID_URL_TEMPLATE = r'http://(?:(?:www\.)?%(host)s/(?:file|video)/|(?:(?:embed|www)\.)%(host)s/embed\.php\?(?:.*?&)?v=)(?P<id>[a-z\d]{13})'
+ _VALID_URL = _VALID_URL_TEMPLATE % {'host': 'novamov\.com'}
+
+ _HOST = 'www.novamov.com'
+
+ _FILE_DELETED_REGEX = r'This file no longer exists on our servers!</h2>'
+ _FILEKEY_REGEX = r'flashvars\.filekey="(?P<filekey>[^"]+)";'
+ _TITLE_REGEX = r'(?s)<div class="v_tab blockborder rounded5" id="v_tab1">\s*<h3>([^<]+)</h3>'
+ _DESCRIPTION_REGEX = r'(?s)<div class="v_tab blockborder rounded5" id="v_tab1">\s*<h3>[^<]+</h3><p>([^<]+)</p>'
+
+ _TEST = {
+ 'url': 'http://www.novamov.com/video/4rurhn9x446jj',
+ 'md5': '7205f346a52bbeba427603ba10d4b935',
+ 'info_dict': {
+ 'id': '4rurhn9x446jj',
+ 'ext': 'flv',
+ 'title': 'search engine optimization',
+ 'description': 'search engine optimization is used to rank the web page in the google search engine'
+ },
+ 'skip': '"Invalid token" errors abound (in web interface as well as youtube-dl, there is nothing we can do about it.)'
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+
+ page = self._download_webpage(
+ 'http://%s/video/%s' % (self._HOST, video_id), video_id, 'Downloading video page')
+
+ if re.search(self._FILE_DELETED_REGEX, page) is not None:
+ raise ExtractorError('Video %s does not exist' % video_id, expected=True)
+
+ filekey = self._search_regex(self._FILEKEY_REGEX, page, 'filekey')
+
+ title = self._html_search_regex(self._TITLE_REGEX, page, 'title', fatal=False)
+ description = self._html_search_regex(self._DESCRIPTION_REGEX, page, 'description', default='', fatal=False)
+
+ api_response = self._download_webpage(
+ 'http://%s/api/player.api.php?key=%s&file=%s' % (self._HOST, filekey, video_id), video_id,
+ 'Downloading video api response')
+
+ response = compat_urlparse.parse_qs(api_response)
+
+ if 'error_msg' in response:
+ raise ExtractorError('%s returned error: %s' % (self.IE_NAME, response['error_msg'][0]), expected=True)
+
+ video_url = response['url'][0]
+
+ return {
+ 'id': video_id,
+ 'url': video_url,
+ 'title': title,
+ 'description': description
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/nowness.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/nowness.py
new file mode 100644
index 0000000000..6b2f3f55a6
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/nowness.py
@@ -0,0 +1,64 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .brightcove import BrightcoveIE
+from .common import InfoExtractor
+from ..utils import ExtractorError
+
+
+class NownessIE(InfoExtractor):
+ _VALID_URL = r'https?://(?:(?:www|cn)\.)?nowness\.com/[^?#]*?/(?P<id>[0-9]+)/(?P<slug>[^/]+?)(?:$|[?#])'
+
+ _TESTS = [
+ {
+ 'url': 'http://www.nowness.com/day/2013/6/27/3131/candor--the-art-of-gesticulation',
+ 'md5': '068bc0202558c2e391924cb8cc470676',
+ 'info_dict': {
+ 'id': '2520295746001',
+ 'ext': 'mp4',
+ 'title': 'Candor: The Art of Gesticulation',
+ 'description': 'Candor: The Art of Gesticulation',
+ 'thumbnail': 're:^https?://.*\.jpg',
+ 'uploader': 'Nowness',
+ }
+ },
+ {
+ 'url': 'http://cn.nowness.com/day/2014/8/7/4069/kasper-bj-rke-ft-jaakko-eino-kalevi--tnr',
+ 'md5': 'e79cf125e387216f86b2e0a5b5c63aa3',
+ 'info_dict': {
+ 'id': '3716354522001',
+ 'ext': 'mp4',
+ 'title': 'Kasper Bjц╦rke ft. Jaakko Eino Kalevi: TNR',
+ 'description': 'Kasper Bjц╦rke ft. Jaakko Eino Kalevi: TNR',
+ 'thumbnail': 're:^https?://.*\.jpg',
+ 'uploader': 'Nowness',
+ }
+ },
+ ]
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('slug')
+
+ webpage = self._download_webpage(url, video_id)
+ player_url = self._search_regex(
+ r'"([^"]+/content/issue-[0-9.]+.js)"', webpage, 'player URL')
+ real_id = self._search_regex(
+ r'\sdata-videoId="([0-9]+)"', webpage, 'internal video ID')
+
+ player_code = self._download_webpage(
+ player_url, video_id,
+ note='Downloading player JavaScript',
+ errnote='Player download failed')
+ player_code = player_code.replace("'+d+'", real_id)
+
+ bc_url = BrightcoveIE._extract_brightcove_url(player_code)
+ if bc_url is None:
+ raise ExtractorError('Could not find player definition')
+ return {
+ '_type': 'url',
+ 'url': bc_url,
+ 'ie_key': 'Brightcove',
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/nowvideo.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/nowvideo.py
new file mode 100644
index 0000000000..dec09cdfef
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/nowvideo.py
@@ -0,0 +1,28 @@
+from __future__ import unicode_literals
+
+from .novamov import NovaMovIE
+
+
+class NowVideoIE(NovaMovIE):
+ IE_NAME = 'nowvideo'
+ IE_DESC = 'NowVideo'
+
+ _VALID_URL = NovaMovIE._VALID_URL_TEMPLATE % {'host': 'nowvideo\.(?:ch|sx|eu|at|ag|co|li)'}
+
+ _HOST = 'www.nowvideo.ch'
+
+ _FILE_DELETED_REGEX = r'>This file no longer exists on our servers.<'
+ _FILEKEY_REGEX = r'var fkzd="([^"]+)";'
+ _TITLE_REGEX = r'<h4>([^<]+)</h4>'
+ _DESCRIPTION_REGEX = r'</h4>\s*<p>([^<]+)</p>'
+
+ _TEST = {
+ 'url': 'http://www.nowvideo.ch/video/0mw0yow7b6dxa',
+ 'md5': 'f8fbbc8add72bd95b7850c6a02fc8817',
+ 'info_dict': {
+ 'id': '0mw0yow7b6dxa',
+ 'ext': 'flv',
+ 'title': 'youtubedl test video _BaW_jenozKc.mp4',
+ 'description': 'Description',
+ }
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/npo.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/npo.py
new file mode 100644
index 0000000000..8da76ae45a
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/npo.py
@@ -0,0 +1,181 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+ unified_strdate,
+ parse_duration,
+ qualities,
+ strip_jsonp,
+ url_basename,
+ fix_xml_ampersands,
+)
+
+
+class NPOIE(InfoExtractor):
+ IE_NAME = 'npo.nl'
+ _VALID_URL = r'https?://www\.npo\.nl/[^/]+/[^/]+/(?P<id>[^/?]+)'
+
+ _TESTS = [
+ {
+ 'url': 'http://www.npo.nl/nieuwsuur/22-06-2014/VPWON_1220719',
+ 'md5': '4b3f9c429157ec4775f2c9cb7b911016',
+ 'info_dict': {
+ 'id': 'VPWON_1220719',
+ 'ext': 'm4v',
+ 'title': 'Nieuwsuur',
+ 'description': 'Dagelijks tussen tien en elf: nieuws, sport en achtergronden.',
+ 'upload_date': '20140622',
+ },
+ },
+ {
+ 'url': 'http://www.npo.nl/de-mega-mike-mega-thomas-show/27-02-2009/VARA_101191800',
+ 'md5': 'da50a5787dbfc1603c4ad80f31c5120b',
+ 'info_dict': {
+ 'id': 'VARA_101191800',
+ 'ext': 'm4v',
+ 'title': 'De Mega Mike & Mega Thomas show',
+ 'description': 'md5:3b74c97fc9d6901d5a665aac0e5400f4',
+ 'upload_date': '20090227',
+ 'duration': 2400,
+ },
+ },
+ {
+ 'url': 'http://www.npo.nl/tegenlicht/25-02-2013/VPWON_1169289',
+ 'md5': 'f8065e4e5a7824068ed3c7e783178f2c',
+ 'info_dict': {
+ 'id': 'VPWON_1169289',
+ 'ext': 'm4v',
+ 'title': 'Tegenlicht',
+ 'description': 'md5:d6476bceb17a8c103c76c3b708f05dd1',
+ 'upload_date': '20130225',
+ 'duration': 3000,
+ },
+ },
+ {
+ 'url': 'http://www.npo.nl/de-nieuwe-mens-deel-1/21-07-2010/WO_VPRO_043706',
+ 'info_dict': {
+ 'id': 'WO_VPRO_043706',
+ 'ext': 'wmv',
+ 'title': 'De nieuwe mens - Deel 1',
+ 'description': 'md5:518ae51ba1293ffb80d8d8ce90b74e4b',
+ 'duration': 4680,
+ },
+ 'params': {
+ # mplayer mms download
+ 'skip_download': True,
+ }
+ },
+ ]
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+ return self._get_info(video_id)
+
+ def _get_info(self, video_id):
+ metadata = self._download_json(
+ 'http://e.omroep.nl/metadata/aflevering/%s' % video_id,
+ video_id,
+ # We have to remove the javascript callback
+ transform_source=strip_jsonp,
+ )
+ token_page = self._download_webpage(
+ 'http://ida.omroep.nl/npoplayer/i.js',
+ video_id,
+ note='Downloading token'
+ )
+ token = self._search_regex(r'npoplayer\.token = "(.+?)"', token_page, 'token')
+
+ formats = []
+
+ pubopties = metadata.get('pubopties')
+ if pubopties:
+ quality = qualities(['adaptive', 'wmv_sb', 'h264_sb', 'wmv_bb', 'h264_bb', 'wvc1_std', 'h264_std'])
+ for format_id in pubopties:
+ format_info = self._download_json(
+ 'http://ida.omroep.nl/odi/?prid=%s&puboptions=%s&adaptive=yes&token=%s'
+ % (video_id, format_id, token),
+ video_id, 'Downloading %s JSON' % format_id)
+ if format_info.get('error_code', 0) or format_info.get('errorcode', 0):
+ continue
+ streams = format_info.get('streams')
+ if streams:
+ video_info = self._download_json(
+ streams[0] + '&type=json',
+ video_id, 'Downloading %s stream JSON' % format_id)
+ else:
+ video_info = format_info
+ video_url = video_info.get('url')
+ if not video_url:
+ continue
+ if format_id == 'adaptive':
+ formats.extend(self._extract_m3u8_formats(video_url, video_id))
+ else:
+ formats.append({
+ 'url': video_url,
+ 'format_id': format_id,
+ 'quality': quality(format_id),
+ })
+
+ streams = metadata.get('streams')
+ if streams:
+ for i, stream in enumerate(streams):
+ stream_url = stream.get('url')
+ if not stream_url:
+ continue
+ asx = self._download_xml(
+ stream_url, video_id,
+ 'Downloading stream %d ASX playlist' % i,
+ transform_source=fix_xml_ampersands)
+ ref = asx.find('./ENTRY/Ref')
+ if ref is None:
+ continue
+ video_url = ref.get('href')
+ if not video_url:
+ continue
+ formats.append({
+ 'url': video_url,
+ 'ext': stream.get('formaat', 'asf'),
+ 'quality': stream.get('kwaliteit'),
+ })
+
+ self._sort_formats(formats)
+
+ return {
+ 'id': video_id,
+ 'title': metadata['titel'],
+ 'description': metadata['info'],
+ 'thumbnail': metadata.get('images', [{'url': None}])[-1]['url'],
+ 'upload_date': unified_strdate(metadata.get('gidsdatum')),
+ 'duration': parse_duration(metadata.get('tijdsduur')),
+ 'formats': formats,
+ }
+
+
+class TegenlichtVproIE(NPOIE):
+ IE_NAME = 'tegenlicht.vpro.nl'
+ _VALID_URL = r'https?://tegenlicht\.vpro\.nl/afleveringen/.*?'
+
+ _TESTS = [
+ {
+ 'url': 'http://tegenlicht.vpro.nl/afleveringen/2012-2013/de-toekomst-komt-uit-afrika.html',
+ 'md5': 'f8065e4e5a7824068ed3c7e783178f2c',
+ 'info_dict': {
+ 'id': 'VPWON_1169289',
+ 'ext': 'm4v',
+ 'title': 'Tegenlicht',
+ 'description': 'md5:d6476bceb17a8c103c76c3b708f05dd1',
+ 'upload_date': '20130225',
+ },
+ },
+ ]
+
+ def _real_extract(self, url):
+ name = url_basename(url)
+ webpage = self._download_webpage(url, name)
+ urn = self._html_search_meta('mediaurn', webpage)
+ info_page = self._download_json(
+ 'http://rs.vpro.nl/v2/api/media/%s.json' % urn, name)
+ return self._get_info(info_page['mid'])
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/nrk.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/nrk.py
new file mode 100644
index 0000000000..96f0ae1ebd
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/nrk.py
@@ -0,0 +1,144 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+ ExtractorError,
+ float_or_none,
+ unified_strdate,
+)
+
+
+class NRKIE(InfoExtractor):
+ _VALID_URL = r'http://(?:www\.)?nrk\.no/(?:video|lyd)/[^/]+/(?P<id>[\dA-F]{16})'
+
+ _TESTS = [
+ {
+ 'url': 'http://www.nrk.no/video/dompap_og_andre_fugler_i_piip_show/D0FA54B5C8B6CE59/emne/piipshow/',
+ 'md5': 'a6eac35052f3b242bb6bb7f43aed5886',
+ 'info_dict': {
+ 'id': '150533',
+ 'ext': 'flv',
+ 'title': 'Dompap og andre fugler i Piip-Show',
+ 'description': 'md5:d9261ba34c43b61c812cb6b0269a5c8f'
+ }
+ },
+ {
+ 'url': 'http://www.nrk.no/lyd/lyd_av_oppleser_for_blinde/AEFDDD5473BA0198/',
+ 'md5': '3471f2a51718195164e88f46bf427668',
+ 'info_dict': {
+ 'id': '154915',
+ 'ext': 'flv',
+ 'title': 'Slik hц╦res internett ut nц╔r du er blind',
+ 'description': 'md5:a621f5cc1bd75c8d5104cb048c6b8568',
+ }
+ },
+ ]
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+
+ page = self._download_webpage(url, video_id)
+
+ video_id = self._html_search_regex(r'<div class="nrk-video" data-nrk-id="(\d+)">', page, 'video id')
+
+ data = self._download_json(
+ 'http://v7.psapi.nrk.no/mediaelement/%s' % video_id, video_id, 'Downloading media JSON')
+
+ if data['usageRights']['isGeoBlocked']:
+ raise ExtractorError('NRK har ikke rettig-heter til ц╔ vise dette programmet utenfor Norge', expected=True)
+
+ video_url = data['mediaUrl'] + '?hdcore=3.1.1&plugin=aasp-3.1.1.69.124'
+
+ images = data.get('images')
+ if images:
+ thumbnails = images['webImages']
+ thumbnails.sort(key=lambda image: image['pixelWidth'])
+ thumbnail = thumbnails[-1]['imageUrl']
+ else:
+ thumbnail = None
+
+ return {
+ 'id': video_id,
+ 'url': video_url,
+ 'ext': 'flv',
+ 'title': data['title'],
+ 'description': data['description'],
+ 'thumbnail': thumbnail,
+ }
+
+
+class NRKTVIE(InfoExtractor):
+ _VALID_URL = r'http://tv\.nrk(?:super)?\.no/(?:serie/[^/]+|program)/(?P<id>[a-zA-Z]{4}\d{8})'
+
+ _TESTS = [
+ {
+ 'url': 'http://tv.nrk.no/serie/20-spoersmaal-tv/MUHH48000314/23-05-2014',
+ 'md5': '7b96112fbae1faf09a6f9ae1aff6cb84',
+ 'info_dict': {
+ 'id': 'MUHH48000314',
+ 'ext': 'flv',
+ 'title': '20 spц╦rsmц╔l',
+ 'description': 'md5:bdea103bc35494c143c6a9acdd84887a',
+ 'upload_date': '20140523',
+ 'duration': 1741.52,
+ }
+ },
+ {
+ 'url': 'http://tv.nrk.no/program/mdfp15000514',
+ 'md5': 'af01795a31f1cf7265c8657534d8077b',
+ 'info_dict': {
+ 'id': 'mdfp15000514',
+ 'ext': 'flv',
+ 'title': 'Kunnskapskanalen: Grunnlovsjubilц╘et - Stor stц╔hei for ingenting',
+ 'description': 'md5:654c12511f035aed1e42bdf5db3b206a',
+ 'upload_date': '20140524',
+ 'duration': 4605.0,
+ }
+ },
+ ]
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+
+ page = self._download_webpage(url, video_id)
+
+ title = self._html_search_meta('title', page, 'title')
+ description = self._html_search_meta('description', page, 'description')
+ thumbnail = self._html_search_regex(r'data-posterimage="([^"]+)"', page, 'thumbnail', fatal=False)
+ upload_date = unified_strdate(self._html_search_meta('rightsfrom', page, 'upload date', fatal=False))
+ duration = float_or_none(
+ self._html_search_regex(r'data-duration="([^"]+)"', page, 'duration', fatal=False))
+
+ formats = []
+
+ f4m_url = re.search(r'data-media="([^"]+)"', page)
+ if f4m_url:
+ formats.append({
+ 'url': f4m_url.group(1) + '?hdcore=3.1.1&plugin=aasp-3.1.1.69.124',
+ 'format_id': 'f4m',
+ 'ext': 'flv',
+ })
+
+ m3u8_url = re.search(r'data-hls-media="([^"]+)"', page)
+ if m3u8_url:
+ formats.append({
+ 'url': m3u8_url.group(1),
+ 'format_id': 'm3u8',
+ })
+
+ self._sort_formats(formats)
+
+ return {
+ 'id': video_id,
+ 'title': title,
+ 'description': description,
+ 'thumbnail': thumbnail,
+ 'upload_date': upload_date,
+ 'duration': duration,
+ 'formats': formats,
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/ntv.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/ntv.py
new file mode 100644
index 0000000000..ee740cd9c0
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/ntv.py
@@ -0,0 +1,148 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+ unescapeHTML
+)
+
+
+class NTVIE(InfoExtractor):
+ _VALID_URL = r'http://(?:www\.)?ntv\.ru/(?P<id>.+)'
+
+ _TESTS = [
+ {
+ 'url': 'http://www.ntv.ru/novosti/863142/',
+ 'info_dict': {
+ 'id': '746000',
+ 'ext': 'flv',
+ 'title': 'п п╬п╪п╟п╫п╢я┐я▌я┴п╦п╧ п╖п╣я─п╫п╬п╪п╬я─я│п╨п╦п╪ я└п╩п╬я┌п╬п╪ п©я─п╬п╡п╣п╩ п©п╣я─п╣пЁп╬п╡п╬я─я▀ п╡ я┬я┌п╟п╠п╣ п▓п°п║ пёп╨я─п╟п╦п╫я▀',
+ 'description': 'п п╬п╪п╟п╫п╢я┐я▌я┴п╦п╧ п╖п╣я─п╫п╬п╪п╬я─я│п╨п╦п╪ я└п╩п╬я┌п╬п╪ п©я─п╬п╡п╣п╩ п©п╣я─п╣пЁп╬п╡п╬я─я▀ п╡ я┬я┌п╟п╠п╣ п▓п°п║ пёп╨я─п╟п╦п╫я▀',
+ 'duration': 136,
+ },
+ 'params': {
+ # rtmp download
+ 'skip_download': True,
+ },
+ },
+ {
+ 'url': 'http://www.ntv.ru/video/novosti/750370/',
+ 'info_dict': {
+ 'id': '750370',
+ 'ext': 'flv',
+ 'title': 'п═п╬п╢п╫я▀п╣ п©п╟я│я│п╟п╤п╦я─п╬п╡ п©я─п╬п©п╟п╡я┬п╣пЁп╬ Boeing п╫п╣ п╡п╣я─я▐я┌ п╡ я┌я─п╟пЁп╦я┤п╣я│п╨п╦п╧ п╦я│я┘п╬п╢',
+ 'description': 'п═п╬п╢п╫я▀п╣ п©п╟я│я│п╟п╤п╦я─п╬п╡ п©я─п╬п©п╟п╡я┬п╣пЁп╬ Boeing п╫п╣ п╡п╣я─я▐я┌ п╡ я┌я─п╟пЁп╦я┤п╣я│п╨п╦п╧ п╦я│я┘п╬п╢',
+ 'duration': 172,
+ },
+ 'params': {
+ # rtmp download
+ 'skip_download': True,
+ },
+ },
+ {
+ 'url': 'http://www.ntv.ru/peredacha/segodnya/m23700/o232416',
+ 'info_dict': {
+ 'id': '747480',
+ 'ext': 'flv',
+ 'title': 'б╚п║п╣пЁп╬п╢п╫я▐б╩. 21б═п╪п╟я─я┌п╟ 2014б═пЁп╬п╢п╟. 16:00 ',
+ 'description': 'б╚п║п╣пЁп╬п╢п╫я▐б╩. 21б═п╪п╟я─я┌п╟ 2014б═пЁп╬п╢п╟. 16:00 ',
+ 'duration': 1496,
+ },
+ 'params': {
+ # rtmp download
+ 'skip_download': True,
+ },
+ },
+ {
+ 'url': 'http://www.ntv.ru/kino/Koma_film',
+ 'info_dict': {
+ 'id': '758100',
+ 'ext': 'flv',
+ 'title': 'п·я│я┌я─п╬я│я▌п╤п╣я┌п╫я▀п╧ я└п╦п╩я▄п╪ б╚п п╬п╪п╟б╩',
+ 'description': 'п·я│я┌я─п╬я│я▌п╤п╣я┌п╫я▀п╧ я└п╦п╩я▄п╪ б╚п п╬п╪п╟б╩',
+ 'duration': 5592,
+ },
+ 'params': {
+ # rtmp download
+ 'skip_download': True,
+ },
+ },
+ {
+ 'url': 'http://www.ntv.ru/serial/Delo_vrachey/m31760/o233916/',
+ 'info_dict': {
+ 'id': '751482',
+ 'ext': 'flv',
+ 'title': 'б╚п■п╣п╩п╬ п╡я─п╟я┤п╣п╧б╩: б╚п■п╣я─п╣п╡я├п╣ п╤п╦п╥п╫п╦б╩',
+ 'description': 'б╚п■п╣п╩п╬ п╡я─п╟я┤п╣п╧б╩: б╚п■п╣я─п╣п╡я├п╣ п╤п╦п╥п╫п╦б╩',
+ 'duration': 2590,
+ },
+ 'params': {
+ # rtmp download
+ 'skip_download': True,
+ },
+ },
+ ]
+
+ _VIDEO_ID_REGEXES = [
+ r'<meta property="og:url" content="http://www\.ntv\.ru/video/(\d+)',
+ r'<video embed=[^>]+><id>(\d+)</id>',
+ r'<video restriction[^>]+><key>(\d+)</key>',
+ ]
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+
+ page = self._download_webpage(url, video_id)
+
+ video_id = self._html_search_regex(self._VIDEO_ID_REGEXES, page, 'video id')
+
+ player = self._download_xml('http://www.ntv.ru/vi%s/' % video_id, video_id, 'Downloading video XML')
+ title = unescapeHTML(player.find('./data/title').text)
+ description = unescapeHTML(player.find('./data/description').text)
+
+ video = player.find('./data/video')
+ video_id = video.find('./id').text
+ thumbnail = video.find('./splash').text
+ duration = int(video.find('./totaltime').text)
+ view_count = int(video.find('./views').text)
+ puid22 = video.find('./puid22').text
+
+ apps = {
+ '4': 'video1',
+ '7': 'video2',
+ }
+
+ app = apps.get(puid22, apps['4'])
+
+ formats = []
+ for format_id in ['', 'hi', 'webm']:
+ file = video.find('./%sfile' % format_id)
+ if file is None:
+ continue
+ size = video.find('./%ssize' % format_id)
+ formats.append({
+ 'url': 'rtmp://media.ntv.ru/%s' % app,
+ 'app': app,
+ 'play_path': file.text,
+ 'rtmp_conn': 'B:1',
+ 'player_url': 'http://www.ntv.ru/swf/vps1.swf?update=20131128',
+ 'page_url': 'http://www.ntv.ru',
+ 'flash_version': 'LNX 11,2,202,341',
+ 'rtmp_live': True,
+ 'ext': 'flv',
+ 'filesize': int(size.text),
+ })
+ self._sort_formats(formats)
+
+ return {
+ 'id': video_id,
+ 'title': title,
+ 'description': description,
+ 'thumbnail': thumbnail,
+ 'duration': duration,
+ 'view_count': view_count,
+ 'formats': formats,
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/nuvid.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/nuvid.py
new file mode 100644
index 0000000000..57928f2aed
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/nuvid.py
@@ -0,0 +1,75 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..compat import (
+ compat_urllib_request,
+)
+from ..utils import (
+ parse_duration,
+ unified_strdate,
+)
+
+
+class NuvidIE(InfoExtractor):
+ _VALID_URL = r'https?://(?:www|m)\.nuvid\.com/video/(?P<id>[0-9]+)'
+ _TEST = {
+ 'url': 'http://m.nuvid.com/video/1310741/',
+ 'md5': 'eab207b7ac4fccfb4e23c86201f11277',
+ 'info_dict': {
+ 'id': '1310741',
+ 'ext': 'mp4',
+ 'title': 'Horny babes show their awesome bodeis and',
+ 'duration': 129,
+ 'upload_date': '20140508',
+ 'age_limit': 18,
+ }
+ }
+
+ def _real_extract(self, url):
+ video_id = self._match_id(url)
+
+ formats = []
+
+ for dwnld_speed, format_id in [(0, '3gp'), (5, 'mp4')]:
+ request = compat_urllib_request.Request(
+ 'http://m.nuvid.com/play/%s' % video_id)
+ request.add_header('Cookie', 'skip_download_page=1; dwnld_speed=%d; adv_show=1' % dwnld_speed)
+ webpage = self._download_webpage(
+ request, video_id, 'Downloading %s page' % format_id)
+ video_url = self._html_search_regex(
+ r'<a\s+href="([^"]+)"\s+class="b_link">', webpage, '%s video URL' % format_id, fatal=False)
+ if not video_url:
+ continue
+ formats.append({
+ 'url': video_url,
+ 'format_id': format_id,
+ })
+
+ webpage = self._download_webpage(
+ 'http://m.nuvid.com/video/%s' % video_id, video_id, 'Downloading video page')
+ title = self._html_search_regex(
+ [r'<span title="([^"]+)">',
+ r'<div class="thumb-holder video">\s*<h5[^>]*>([^<]+)</h5>'], webpage, 'title').strip()
+ thumbnails = [
+ {
+ 'url': thumb_url,
+ } for thumb_url in re.findall(r'<img src="([^"]+)" alt="" />', webpage)
+ ]
+ thumbnail = thumbnails[0]['url'] if thumbnails else None
+ duration = parse_duration(self._html_search_regex(
+ r'<i class="fa fa-clock-o"></i>\s*(\d{2}:\d{2})', webpage, 'duration', fatal=False))
+ upload_date = unified_strdate(self._html_search_regex(
+ r'<i class="fa fa-user"></i>\s*(\d{4}-\d{2}-\d{2})', webpage, 'upload date', fatal=False))
+
+ return {
+ 'id': video_id,
+ 'title': title,
+ 'thumbnails': thumbnails,
+ 'thumbnail': thumbnail,
+ 'duration': duration,
+ 'upload_date': upload_date,
+ 'age_limit': 18,
+ 'formats': formats,
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/nytimes.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/nytimes.py
new file mode 100644
index 0000000000..56e1cad3b0
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/nytimes.py
@@ -0,0 +1,77 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import parse_iso8601
+
+
+class NYTimesIE(InfoExtractor):
+ _VALID_URL = r'https?://(?:www\.)?nytimes\.com/video/(?:[^/]+/)+(?P<id>\d+)'
+
+ _TEST = {
+ 'url': 'http://www.nytimes.com/video/opinion/100000002847155/verbatim-what-is-a-photocopier.html?playlistId=100000001150263',
+ 'md5': '18a525a510f942ada2720db5f31644c0',
+ 'info_dict': {
+ 'id': '100000002847155',
+ 'ext': 'mov',
+ 'title': 'Verbatim: What Is a Photocopier?',
+ 'description': 'md5:93603dada88ddbda9395632fdc5da260',
+ 'timestamp': 1398631707,
+ 'upload_date': '20140427',
+ 'uploader': 'Brett Weiner',
+ 'duration': 419,
+ }
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+
+ video_data = self._download_json(
+ 'http://www.nytimes.com/svc/video/api/v2/video/%s' % video_id, video_id, 'Downloading video JSON')
+
+ title = video_data['headline']
+ description = video_data['summary']
+ duration = video_data['duration'] / 1000.0
+
+ uploader = video_data['byline']
+ timestamp = parse_iso8601(video_data['publication_date'][:-8])
+
+ def get_file_size(file_size):
+ if isinstance(file_size, int):
+ return file_size
+ elif isinstance(file_size, dict):
+ return int(file_size.get('value', 0))
+ else:
+ return 0
+
+ formats = [
+ {
+ 'url': video['url'],
+ 'format_id': video['type'],
+ 'vcodec': video['video_codec'],
+ 'width': video['width'],
+ 'height': video['height'],
+ 'filesize': get_file_size(video['fileSize']),
+ } for video in video_data['renditions']
+ ]
+ self._sort_formats(formats)
+
+ thumbnails = [
+ {
+ 'url': 'http://www.nytimes.com/%s' % image['url'],
+ 'resolution': '%dx%d' % (image['width'], image['height']),
+ } for image in video_data['images']
+ ]
+
+ return {
+ 'id': video_id,
+ 'title': title,
+ 'description': description,
+ 'timestamp': timestamp,
+ 'uploader': uploader,
+ 'duration': duration,
+ 'formats': formats,
+ 'thumbnails': thumbnails,
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/oe1.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/oe1.py
new file mode 100644
index 0000000000..38971ab4de
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/oe1.py
@@ -0,0 +1,40 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import calendar
+import datetime
+import re
+
+from .common import InfoExtractor
+
+# audios on oe1.orf.at are only available for 7 days, so we can't
+# add tests.
+
+
+class OE1IE(InfoExtractor):
+ IE_DESC = 'oe1.orf.at'
+ _VALID_URL = r'http://oe1\.orf\.at/programm/(?P<id>[0-9]+)'
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ show_id = mobj.group('id')
+
+ data = self._download_json(
+ 'http://oe1.orf.at/programm/%s/konsole' % show_id,
+ show_id
+ )
+
+ timestamp = datetime.datetime.strptime('%s %s' % (
+ data['item']['day_label'],
+ data['item']['time']
+ ), '%d.%m.%Y %H:%M')
+ unix_timestamp = calendar.timegm(timestamp.utctimetuple())
+
+ return {
+ 'id': show_id,
+ 'title': data['item']['title'],
+ 'url': data['item']['url_stream'],
+ 'ext': 'mp3',
+ 'description': data['item'].get('info'),
+ 'timestamp': unix_timestamp
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/oktoberfesttv.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/oktoberfesttv.py
new file mode 100644
index 0000000000..4a41c05421
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/oktoberfesttv.py
@@ -0,0 +1,47 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class OktoberfestTVIE(InfoExtractor):
+ _VALID_URL = r'https?://www\.oktoberfest-tv\.de/[^/]+/[^/]+/video/(?P<id>[^/?#]+)'
+
+ _TEST = {
+ 'url': 'http://www.oktoberfest-tv.de/de/kameras/video/hb-zelt',
+ 'info_dict': {
+ 'id': 'hb-zelt',
+ 'ext': 'mp4',
+ 'title': 're:^Live-Kamera: Hofbrц╓uzelt [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
+ 'thumbnail': 're:^https?://.*\.jpg$',
+ 'is_live': True,
+ },
+ 'params': {
+ 'skip_download': True,
+ }
+ }
+
+ def _real_extract(self, url):
+ video_id = self._match_id(url)
+ webpage = self._download_webpage(url, video_id)
+
+ title = self._live_title(self._html_search_regex(
+ r'<h1><strong>.*?</strong>(.*?)</h1>', webpage, 'title'))
+
+ clip = self._search_regex(
+ r"clip:\s*\{\s*url:\s*'([^']+)'", webpage, 'clip')
+ ncurl = self._search_regex(
+ r"netConnectionUrl:\s*'([^']+)'", webpage, 'rtmp base')
+ video_url = ncurl + clip
+ thumbnail = self._search_regex(
+ r"canvas:\s*\{\s*backgroundImage:\s*'url\(([^)]+)\)'", webpage,
+ 'thumbnail', fatal=False)
+
+ return {
+ 'id': video_id,
+ 'title': title,
+ 'url': video_url,
+ 'ext': 'mp4',
+ 'is_live': True,
+ 'thumbnail': thumbnail,
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/ooyala.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/ooyala.py
new file mode 100644
index 0000000000..d5b05c18fe
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/ooyala.py
@@ -0,0 +1,97 @@
+from __future__ import unicode_literals
+import re
+import json
+
+from .common import InfoExtractor
+from ..utils import (
+ unescapeHTML,
+ ExtractorError,
+)
+
+
+class OoyalaIE(InfoExtractor):
+ _VALID_URL = r'(?:ooyala:|https?://.+?\.ooyala\.com/.*?(?:embedCode|ec)=)(?P<id>.+?)(&|$)'
+
+ _TESTS = [
+ {
+ # From http://it.slashdot.org/story/13/04/25/178216/recovering-data-from-broken-hard-drives-and-ssds-video
+ 'url': 'http://player.ooyala.com/player.js?embedCode=pxczE2YjpfHfn1f3M-ykG_AmJRRn0PD8',
+ 'info_dict': {
+ 'id': 'pxczE2YjpfHfn1f3M-ykG_AmJRRn0PD8',
+ 'ext': 'mp4',
+ 'title': 'Explaining Data Recovery from Hard Drives and SSDs',
+ 'description': 'How badly damaged does a drive have to be to defeat Russell and his crew? Apparently, smashed to bits.',
+ },
+ }, {
+ # Only available for ipad
+ 'url': 'http://player.ooyala.com/player.js?embedCode=x1b3lqZDq9y_7kMyC2Op5qo-p077tXD0',
+ 'info_dict': {
+ 'id': 'x1b3lqZDq9y_7kMyC2Op5qo-p077tXD0',
+ 'ext': 'mp4',
+ 'title': 'Simulation Overview - Levels of Simulation',
+ 'description': '',
+ },
+ },
+ ]
+
+ @staticmethod
+ def _url_for_embed_code(embed_code):
+ return 'http://player.ooyala.com/player.js?embedCode=%s' % embed_code
+
+ @classmethod
+ def _build_url_result(cls, embed_code):
+ return cls.url_result(cls._url_for_embed_code(embed_code),
+ ie=cls.ie_key())
+
+ def _extract_result(self, info, more_info):
+ return {
+ 'id': info['embedCode'],
+ 'ext': 'mp4',
+ 'title': unescapeHTML(info['title']),
+ 'url': info.get('ipad_url') or info['url'],
+ 'description': unescapeHTML(more_info['description']),
+ 'thumbnail': more_info['promo'],
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ embedCode = mobj.group('id')
+ player_url = 'http://player.ooyala.com/player.js?embedCode=%s' % embedCode
+ player = self._download_webpage(player_url, embedCode)
+ mobile_url = self._search_regex(r'mobile_player_url="(.+?)&device="',
+ player, 'mobile player url')
+ # Looks like some videos are only available for particular devices
+ # (e.g. http://player.ooyala.com/player.js?embedCode=x1b3lqZDq9y_7kMyC2Op5qo-p077tXD0
+ # is only available for ipad)
+ # Working around with fetching URLs for all the devices found starting with 'unknown'
+ # until we succeed or eventually fail for each device.
+ devices = re.findall(r'device\s*=\s*"([^"]+)";', player)
+ devices.remove('unknown')
+ devices.insert(0, 'unknown')
+ for device in devices:
+ mobile_player = self._download_webpage(
+ '%s&device=%s' % (mobile_url, device), embedCode,
+ 'Downloading mobile player JS for %s device' % device)
+ videos_info = self._search_regex(
+ r'var streams=window.oo_testEnv\?\[\]:eval\("\((\[{.*?}\])\)"\);',
+ mobile_player, 'info', fatal=False, default=None)
+ if videos_info:
+ break
+ if not videos_info:
+ raise ExtractorError('Unable to extract info')
+ videos_info = videos_info.replace('\\"', '"')
+ videos_more_info = self._search_regex(
+ r'eval\("\(({.*?\\"promo\\".*?})\)"', mobile_player, 'more info').replace('\\"', '"')
+ videos_info = json.loads(videos_info)
+ videos_more_info = json.loads(videos_more_info)
+
+ if videos_more_info.get('lineup'):
+ videos = [self._extract_result(info, more_info) for (info, more_info) in zip(videos_info, videos_more_info['lineup'])]
+ return {
+ '_type': 'playlist',
+ 'id': embedCode,
+ 'title': unescapeHTML(videos_more_info['title']),
+ 'entries': videos,
+ }
+ else:
+ return self._extract_result(videos_info[0], videos_more_info)
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/orf.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/orf.py
new file mode 100644
index 0000000000..4fed83bd61
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/orf.py
@@ -0,0 +1,196 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import json
+import re
+import calendar
+import datetime
+
+from .common import InfoExtractor
+from ..utils import (
+ HEADRequest,
+ unified_strdate,
+ ExtractorError,
+)
+
+
+class ORFTVthekIE(InfoExtractor):
+ IE_NAME = 'orf:tvthek'
+ IE_DESC = 'ORF TVthek'
+ _VALID_URL = r'https?://tvthek\.orf\.at/(?:programs/.+?/episodes|topics?/.+?|program/[^/]+)/(?P<id>\d+)'
+
+ _TESTS = [{
+ 'url': 'http://tvthek.orf.at/program/Aufgetischt/2745173/Aufgetischt-Mit-der-Steirischen-Tafelrunde/8891389',
+ 'playlist': [{
+ 'md5': '2942210346ed779588f428a92db88712',
+ 'info_dict': {
+ 'id': '8896777',
+ 'ext': 'mp4',
+ 'title': 'Aufgetischt: Mit der Steirischen Tafelrunde',
+ 'description': 'md5:c1272f0245537812d4e36419c207b67d',
+ 'duration': 2668,
+ 'upload_date': '20141208',
+ },
+ }],
+ 'skip': 'Blocked outside of Austria / Germany',
+ }, {
+ 'url': 'http://tvthek.orf.at/topic/Im-Wandel-der-Zeit/8002126/Best-of-Ingrid-Thurnher/7982256',
+ 'playlist': [{
+ 'md5': '68f543909aea49d621dfc7703a11cfaf',
+ 'info_dict': {
+ 'id': '7982259',
+ 'ext': 'mp4',
+ 'title': 'Best of Ingrid Thurnher',
+ 'upload_date': '20140527',
+ 'description': 'Viele Jahre war Ingrid Thurnher das "Gesicht" der ZIB 2. Vor ihrem Wechsel zur ZIB 2 im jahr 1995 moderierte sie unter anderem "Land und Leute", "ц√sterreich-Bild" und "Niederц╤sterreich heute".',
+ }
+ }],
+ '_skip': 'Blocked outside of Austria / Germany',
+ }]
+
+ def _real_extract(self, url):
+ playlist_id = self._match_id(url)
+ webpage = self._download_webpage(url, playlist_id)
+
+ data_json = self._search_regex(
+ r'initializeAdworx\((.+?)\);\n', webpage, 'video info')
+ all_data = json.loads(data_json)
+
+ def get_segments(all_data):
+ for data in all_data:
+ if data['name'] in (
+ 'Tracker::EPISODE_DETAIL_PAGE_OVER_PROGRAM',
+ 'Tracker::EPISODE_DETAIL_PAGE_OVER_TOPIC'):
+ return data['values']['segments']
+
+ sdata = get_segments(all_data)
+ if not sdata:
+ raise ExtractorError('Unable to extract segments')
+
+ def quality_to_int(s):
+ m = re.search('([0-9]+)', s)
+ if m is None:
+ return -1
+ return int(m.group(1))
+
+ entries = []
+ for sd in sdata:
+ video_id = sd['id']
+ formats = [{
+ 'preference': -10 if fd['delivery'] == 'hls' else None,
+ 'format_id': '%s-%s-%s' % (
+ fd['delivery'], fd['quality'], fd['quality_string']),
+ 'url': fd['src'],
+ 'protocol': fd['protocol'],
+ 'quality': quality_to_int(fd['quality']),
+ } for fd in sd['playlist_item_array']['sources']]
+
+ # Check for geoblocking.
+ # There is a property is_geoprotection, but that's always false
+ geo_str = sd.get('geoprotection_string')
+ if geo_str:
+ try:
+ http_url = next(
+ f['url']
+ for f in formats
+ if re.match(r'^https?://.*\.mp4$', f['url']))
+ except StopIteration:
+ pass
+ else:
+ req = HEADRequest(http_url)
+ self._request_webpage(
+ req, video_id,
+ note='Testing for geoblocking',
+ errnote=((
+ 'This video seems to be blocked outside of %s. '
+ 'You may want to try the streaming-* formats.')
+ % geo_str),
+ fatal=False)
+
+ self._sort_formats(formats)
+
+ upload_date = unified_strdate(sd['created_date'])
+ entries.append({
+ '_type': 'video',
+ 'id': video_id,
+ 'title': sd['header'],
+ 'formats': formats,
+ 'description': sd.get('description'),
+ 'duration': int(sd['duration_in_seconds']),
+ 'upload_date': upload_date,
+ 'thumbnail': sd.get('image_full_url'),
+ })
+
+ return {
+ '_type': 'playlist',
+ 'entries': entries,
+ 'id': playlist_id,
+ }
+
+
+# Audios on ORF radio are only available for 7 days, so we can't add tests.
+
+
+class ORFOE1IE(InfoExtractor):
+ IE_NAME = 'orf:oe1'
+ IE_DESC = 'Radio ц√sterreich 1'
+ _VALID_URL = r'http://oe1\.orf\.at/programm/(?P<id>[0-9]+)'
+
+ def _real_extract(self, url):
+ show_id = self._match_id(url)
+ data = self._download_json(
+ 'http://oe1.orf.at/programm/%s/konsole' % show_id,
+ show_id
+ )
+
+ timestamp = datetime.datetime.strptime('%s %s' % (
+ data['item']['day_label'],
+ data['item']['time']
+ ), '%d.%m.%Y %H:%M')
+ unix_timestamp = calendar.timegm(timestamp.utctimetuple())
+
+ return {
+ 'id': show_id,
+ 'title': data['item']['title'],
+ 'url': data['item']['url_stream'],
+ 'ext': 'mp3',
+ 'description': data['item'].get('info'),
+ 'timestamp': unix_timestamp
+ }
+
+
+class ORFFM4IE(InfoExtractor):
+ IE_DESC = 'orf:fm4'
+ IE_DESC = 'radio FM4'
+ _VALID_URL = r'http://fm4\.orf\.at/7tage/?#(?P<date>[0-9]+)/(?P<show>\w+)'
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ show_date = mobj.group('date')
+ show_id = mobj.group('show')
+
+ data = self._download_json(
+ 'http://audioapi.orf.at/fm4/json/2.0/broadcasts/%s/4%s' % (show_date, show_id),
+ show_id
+ )
+
+ def extract_entry_dict(info, title, subtitle):
+ return {
+ 'id': info['loopStreamId'].replace('.mp3', ''),
+ 'url': 'http://loopstream01.apa.at/?channel=fm4&id=%s' % info['loopStreamId'],
+ 'title': title,
+ 'description': subtitle,
+ 'duration': (info['end'] - info['start']) / 1000,
+ 'timestamp': info['start'] / 1000,
+ 'ext': 'mp3'
+ }
+
+ entries = [extract_entry_dict(t, data['title'], data['subtitle']) for t in data['streams']]
+
+ return {
+ '_type': 'playlist',
+ 'id': show_id,
+ 'title': data['title'],
+ 'description': data['subtitle'],
+ 'entries': entries
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/parliamentliveuk.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/parliamentliveuk.py
new file mode 100644
index 0000000000..0a423a08f0
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/parliamentliveuk.py
@@ -0,0 +1,53 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+
+
+class ParliamentLiveUKIE(InfoExtractor):
+ IE_NAME = 'parliamentlive.tv'
+ IE_DESC = 'UK parliament videos'
+ _VALID_URL = r'https?://www\.parliamentlive\.tv/Main/Player\.aspx\?(?:[^&]+&)*?meetingId=(?P<id>[0-9]+)'
+
+ _TEST = {
+ 'url': 'http://www.parliamentlive.tv/Main/Player.aspx?meetingId=15121&player=windowsmedia',
+ 'info_dict': {
+ 'id': '15121',
+ 'ext': 'asf',
+ 'title': 'hoc home affairs committee, 18 mar 2014.pm',
+ 'description': 'md5:033b3acdf83304cd43946b2d5e5798d1',
+ },
+ 'params': {
+ 'skip_download': True, # Requires mplayer (mms)
+ }
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+ webpage = self._download_webpage(url, video_id)
+
+ asx_url = self._html_search_regex(
+ r'embed.*?src="([^"]+)" name="MediaPlayer"', webpage,
+ 'metadata URL')
+ asx = self._download_xml(asx_url, video_id, 'Downloading ASX metadata')
+ video_url = asx.find('.//REF').attrib['HREF']
+
+ title = self._search_regex(
+ r'''(?x)player\.setClipDetails\(
+ (?:(?:[0-9]+|"[^"]+"),\s*){2}
+ "([^"]+",\s*"[^"]+)"
+ ''',
+ webpage, 'title').replace('", "', ', ')
+ description = self._html_search_regex(
+ r'(?s)<span id="MainContentPlaceHolder_CaptionsBlock_WitnessInfo">(.*?)</span>',
+ webpage, 'description')
+
+ return {
+ 'id': video_id,
+ 'ext': 'asf',
+ 'url': video_url,
+ 'title': title,
+ 'description': description,
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/patreon.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/patreon.py
new file mode 100644
index 0000000000..5429592a75
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/patreon.py
@@ -0,0 +1,100 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+import json
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+ js_to_json,
+)
+
+
+class PatreonIE(InfoExtractor):
+ _VALID_URL = r'https?://(?:www\.)?patreon\.com/creation\?hid=(.+)'
+ _TESTS = [
+ {
+ 'url': 'http://www.patreon.com/creation?hid=743933',
+ 'md5': 'e25505eec1053a6e6813b8ed369875cc',
+ 'info_dict': {
+ 'id': '743933',
+ 'ext': 'mp3',
+ 'title': 'Episode 166: David Smalley of Dogma Debate',
+ 'uploader': 'Cognitive Dissonance Podcast',
+ 'thumbnail': 're:^https?://.*$',
+ },
+ },
+ {
+ 'url': 'http://www.patreon.com/creation?hid=754133',
+ 'md5': '3eb09345bf44bf60451b8b0b81759d0a',
+ 'info_dict': {
+ 'id': '754133',
+ 'ext': 'mp3',
+ 'title': 'CD 167 Extra',
+ 'uploader': 'Cognitive Dissonance Podcast',
+ 'thumbnail': 're:^https?://.*$',
+ },
+ },
+ ]
+
+ # Currently Patreon exposes download URL via hidden CSS, so login is not
+ # needed. Keeping this commented for when this inevitably changes.
+ '''
+ def _login(self):
+ (username, password) = self._get_login_info()
+ if username is None:
+ return
+
+ login_form = {
+ 'redirectUrl': 'http://www.patreon.com/',
+ 'email': username,
+ 'password': password,
+ }
+
+ request = compat_urllib_request.Request(
+ 'https://www.patreon.com/processLogin',
+ compat_urllib_parse.urlencode(login_form).encode('utf-8')
+ )
+ login_page = self._download_webpage(request, None, note='Logging in as %s' % username)
+
+ if re.search(r'onLoginFailed', login_page):
+ raise ExtractorError('Unable to login, incorrect username and/or password', expected=True)
+
+ def _real_initialize(self):
+ self._login()
+ '''
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group(1)
+
+ webpage = self._download_webpage(url, video_id)
+ title = self._og_search_title(webpage).strip()
+
+ attach_fn = self._html_search_regex(
+ r'<div class="attach"><a target="_blank" href="([^"]+)">',
+ webpage, 'attachment URL', default=None)
+ if attach_fn is not None:
+ video_url = 'http://www.patreon.com' + attach_fn
+ thumbnail = self._og_search_thumbnail(webpage)
+ uploader = self._html_search_regex(
+ r'<strong>(.*?)</strong> is creating', webpage, 'uploader')
+ else:
+ playlist_js = self._search_regex(
+ r'(?s)new\s+jPlayerPlaylist\(\s*\{\s*[^}]*},\s*(\[.*?,?\s*\])',
+ webpage, 'playlist JSON')
+ playlist_json = js_to_json(playlist_js)
+ playlist = json.loads(playlist_json)
+ data = playlist[0]
+ video_url = self._proto_relative_url(data['mp3'])
+ thumbnail = self._proto_relative_url(data.get('cover'))
+ uploader = data.get('artist')
+
+ return {
+ 'id': video_id,
+ 'url': video_url,
+ 'ext': 'mp3',
+ 'title': title,
+ 'uploader': uploader,
+ 'thumbnail': thumbnail,
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/pbs.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/pbs.py
new file mode 100644
index 0000000000..6118ed5c20
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/pbs.py
@@ -0,0 +1,170 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+ unified_strdate,
+ US_RATINGS,
+)
+
+
+class PBSIE(InfoExtractor):
+ _VALID_URL = r'''(?x)https?://
+ (?:
+ # Direct video URL
+ video\.pbs\.org/(?:viralplayer|video)/(?P<id>[0-9]+)/? |
+ # Article with embedded player (or direct video)
+ (?:www\.)?pbs\.org/(?:[^/]+/){2,5}(?P<presumptive_id>[^/]+?)(?:\.html)?/?(?:$|[?\#]) |
+ # Player
+ video\.pbs\.org/(?:widget/)?partnerplayer/(?P<player_id>[^/]+)/
+ )
+ '''
+
+ _TESTS = [
+ {
+ 'url': 'http://www.pbs.org/tpt/constitution-usa-peter-sagal/watch/a-more-perfect-union/',
+ 'md5': 'ce1888486f0908d555a8093cac9a7362',
+ 'info_dict': {
+ 'id': '2365006249',
+ 'ext': 'mp4',
+ 'title': 'A More Perfect Union',
+ 'description': 'md5:ba0c207295339c8d6eced00b7c363c6a',
+ 'duration': 3190,
+ },
+ },
+ {
+ 'url': 'http://www.pbs.org/wgbh/pages/frontline/losing-iraq/',
+ 'md5': '143c98aa54a346738a3d78f54c925321',
+ 'info_dict': {
+ 'id': '2365297690',
+ 'ext': 'mp4',
+ 'title': 'Losing Iraq',
+ 'description': 'md5:f5bfbefadf421e8bb8647602011caf8e',
+ 'duration': 5050,
+ },
+ },
+ {
+ 'url': 'http://www.pbs.org/newshour/bb/education-jan-june12-cyberschools_02-23/',
+ 'md5': 'b19856d7f5351b17a5ab1dc6a64be633',
+ 'info_dict': {
+ 'id': '2201174722',
+ 'ext': 'mp4',
+ 'title': 'Cyber Schools Gain Popularity, but Quality Questions Persist',
+ 'description': 'md5:5871c15cba347c1b3d28ac47a73c7c28',
+ 'duration': 801,
+ },
+ },
+ {
+ 'url': 'http://www.pbs.org/wnet/gperf/dudamel-conducts-verdi-requiem-hollywood-bowl-full-episode/3374/',
+ 'md5': 'c62859342be2a0358d6c9eb306595978',
+ 'info_dict': {
+ 'id': '2365297708',
+ 'ext': 'mp4',
+ 'description': 'md5:68d87ef760660eb564455eb30ca464fe',
+ 'title': 'Dudamel Conducts Verdi Requiem at the Hollywood Bowl - Full',
+ 'duration': 6559,
+ 'thumbnail': 're:^https?://.*\.jpg$',
+ }
+ },
+ {
+ 'url': 'http://www.pbs.org/wgbh/nova/earth/killer-typhoon.html',
+ 'md5': '908f3e5473a693b266b84e25e1cf9703',
+ 'info_dict': {
+ 'id': '2365160389',
+ 'display_id': 'killer-typhoon',
+ 'ext': 'mp4',
+ 'description': 'md5:c741d14e979fc53228c575894094f157',
+ 'title': 'Killer Typhoon',
+ 'duration': 3172,
+ 'thumbnail': 're:^https?://.*\.jpg$',
+ 'upload_date': '20140122',
+ }
+ },
+ {
+ 'url': 'http://www.pbs.org/wgbh/pages/frontline/united-states-of-secrets/',
+ 'info_dict': {
+ 'id': 'united-states-of-secrets',
+ },
+ 'playlist_count': 2,
+ }
+ ]
+
+ def _extract_webpage(self, url):
+ mobj = re.match(self._VALID_URL, url)
+
+ presumptive_id = mobj.group('presumptive_id')
+ display_id = presumptive_id
+ if presumptive_id:
+ webpage = self._download_webpage(url, display_id)
+
+ upload_date = unified_strdate(self._search_regex(
+ r'<input type="hidden" id="air_date_[0-9]+" value="([^"]+)"',
+ webpage, 'upload date', default=None))
+
+ # tabbed frontline videos
+ tabbed_videos = re.findall(
+ r'<div[^>]+class="videotab[^"]*"[^>]+vid="(\d+)"', webpage)
+ if tabbed_videos:
+ return tabbed_videos, presumptive_id, upload_date
+
+ MEDIA_ID_REGEXES = [
+ r"div\s*:\s*'videoembed'\s*,\s*mediaid\s*:\s*'(\d+)'", # frontline video embed
+ r'class="coveplayerid">([^<]+)<', # coveplayer
+ r'<input type="hidden" id="pbs_video_id_[0-9]+" value="([0-9]+)"/>', # jwplayer
+ ]
+
+ media_id = self._search_regex(
+ MEDIA_ID_REGEXES, webpage, 'media ID', fatal=False, default=None)
+ if media_id:
+ return media_id, presumptive_id, upload_date
+
+ url = self._search_regex(
+ r'<iframe\s+(?:class|id)=["\']partnerPlayer["\'].*?\s+src=["\'](.*?)["\']>',
+ webpage, 'player URL')
+ mobj = re.match(self._VALID_URL, url)
+
+ player_id = mobj.group('player_id')
+ if not display_id:
+ display_id = player_id
+ if player_id:
+ player_page = self._download_webpage(
+ url, display_id, note='Downloading player page',
+ errnote='Could not download player page')
+ video_id = self._search_regex(
+ r'<div\s+id="video_([0-9]+)"', player_page, 'video ID')
+ else:
+ video_id = mobj.group('id')
+ display_id = video_id
+
+ return video_id, display_id, None
+
+ def _real_extract(self, url):
+ video_id, display_id, upload_date = self._extract_webpage(url)
+
+ if isinstance(video_id, list):
+ entries = [self.url_result(
+ 'http://video.pbs.org/video/%s' % vid_id, 'PBS', vid_id)
+ for vid_id in video_id]
+ return self.playlist_result(entries, display_id)
+
+ info_url = 'http://video.pbs.org/videoInfo/%s?format=json' % video_id
+ info = self._download_json(info_url, display_id)
+
+ rating_str = info.get('rating')
+ if rating_str is not None:
+ rating_str = rating_str.rpartition('-')[2]
+ age_limit = US_RATINGS.get(rating_str)
+
+ return {
+ 'id': video_id,
+ 'display_id': display_id,
+ 'title': info['title'],
+ 'url': info['alternate_encoding']['url'],
+ 'ext': 'mp4',
+ 'description': info['program'].get('description'),
+ 'thumbnail': info.get('image_url'),
+ 'duration': info.get('duration'),
+ 'age_limit': age_limit,
+ 'upload_date': upload_date,
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/phoenix.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/phoenix.py
new file mode 100644
index 0000000000..a20672c0cc
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/phoenix.py
@@ -0,0 +1,31 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from .zdf import extract_from_xml_url
+
+
+class PhoenixIE(InfoExtractor):
+ _VALID_URL = r'https?://(?:www\.)?phoenix\.de/content/(?P<id>[0-9]+)'
+ _TEST = {
+ 'url': 'http://www.phoenix.de/content/884301',
+ 'md5': 'ed249f045256150c92e72dbb70eadec6',
+ 'info_dict': {
+ 'id': '884301',
+ 'ext': 'mp4',
+ 'title': 'Michael Krons mit Hans-Werner Sinn',
+ 'description': 'Im Dialog - Sa. 25.10.14, 00.00 - 00.35 Uhr',
+ 'upload_date': '20141025',
+ 'uploader': 'Im Dialog',
+ }
+ }
+
+ def _real_extract(self, url):
+ video_id = self._match_id(url)
+ webpage = self._download_webpage(url, video_id)
+
+ internal_id = self._search_regex(
+ r'<div class="phx_vod" id="phx_vod_([0-9]+)"',
+ webpage, 'internal video ID')
+
+ api_url = 'http://www.phoenix.de/php/zdfplayer-v1.3/data/beitragsDetails.php?ak=web&id=%s' % internal_id
+ return extract_from_xml_url(self, video_id, api_url)
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/photobucket.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/photobucket.py
new file mode 100644
index 0000000000..c66db3cdc8
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/photobucket.py
@@ -0,0 +1,46 @@
+from __future__ import unicode_literals
+
+import json
+import re
+
+from .common import InfoExtractor
+from ..compat import compat_urllib_parse
+
+
+class PhotobucketIE(InfoExtractor):
+ _VALID_URL = r'http://(?:[a-z0-9]+\.)?photobucket\.com/.*(([\?\&]current=)|_)(?P<id>.*)\.(?P<ext>(flv)|(mp4))'
+ _TEST = {
+ 'url': 'http://media.photobucket.com/user/rachaneronas/media/TiredofLinkBuildingTryBacklinkMyDomaincom_zpsc0c3b9fa.mp4.html?filters[term]=search&filters[primary]=videos&filters[secondary]=images&sort=1&o=0',
+ 'md5': '7dabfb92b0a31f6c16cebc0f8e60ff99',
+ 'info_dict': {
+ 'id': 'zpsc0c3b9fa',
+ 'ext': 'mp4',
+ 'timestamp': 1367669341,
+ 'upload_date': '20130504',
+ 'uploader': 'rachaneronas',
+ 'title': 'Tired of Link Building? Try BacklinkMyDomain.com!',
+ }
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+ video_extension = mobj.group('ext')
+
+ webpage = self._download_webpage(url, video_id)
+
+ # Extract URL, uploader, and title from webpage
+ self.report_extraction(video_id)
+ info_json = self._search_regex(r'Pb\.Data\.Shared\.put\(Pb\.Data\.Shared\.MEDIA, (.*?)\);',
+ webpage, 'info json')
+ info = json.loads(info_json)
+ url = compat_urllib_parse.unquote(self._html_search_regex(r'file=(.+\.mp4)', info['linkcodes']['html'], 'url'))
+ return {
+ 'id': video_id,
+ 'url': url,
+ 'uploader': info['username'],
+ 'timestamp': info['creationDate'],
+ 'title': info['title'],
+ 'ext': video_extension,
+ 'thumbnail': info['thumbUrl'],
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/planetaplay.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/planetaplay.py
new file mode 100644
index 0000000000..596c621d75
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/planetaplay.py
@@ -0,0 +1,60 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import ExtractorError
+
+
+class PlanetaPlayIE(InfoExtractor):
+ _VALID_URL = r'https?://(?:www\.)?planetaplay\.com/\?sng=(?P<id>[0-9]+)'
+ _API_URL = 'http://planetaplay.com/action/playlist/?sng={0:}'
+ _THUMBNAIL_URL = 'http://planetaplay.com/img/thumb/{thumb:}'
+ _TEST = {
+ 'url': 'http://planetaplay.com/?sng=3586',
+ 'md5': '9d569dceb7251a4e01355d5aea60f9db',
+ 'info_dict': {
+ 'id': '3586',
+ 'ext': 'flv',
+ 'title': 'md5:e829428ee28b1deed00de90de49d1da1',
+ }
+ }
+
+ _SONG_FORMATS = {
+ 'lq': (0, 'http://www.planetaplay.com/videoplayback/{med_hash:}'),
+ 'hq': (1, 'http://www.planetaplay.com/videoplayback/hi/{med_hash:}'),
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+
+ response = self._download_json(
+ self._API_URL.format(video_id), video_id)['response']
+ try:
+ data = response.get('data')[0]
+ except IndexError:
+ raise ExtractorError(
+ '%s: failed to get the playlist' % self.IE_NAME, expected=True)
+
+ title = '{song_artists:} - {sng_name:}'.format(**data)
+ thumbnail = self._THUMBNAIL_URL.format(**data)
+
+ formats = []
+ for format_id, (quality, url_template) in self._SONG_FORMATS.items():
+ formats.append({
+ 'format_id': format_id,
+ 'url': url_template.format(**data),
+ 'quality': quality,
+ 'ext': 'flv',
+ })
+
+ self._sort_formats(formats)
+
+ return {
+ 'id': video_id,
+ 'title': title,
+ 'formats': formats,
+ 'thumbnail': thumbnail,
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/played.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/played.py
new file mode 100644
index 0000000000..449d4836c3
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/played.py
@@ -0,0 +1,63 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+import os.path
+
+from .common import InfoExtractor
+from ..compat import (
+ compat_urllib_parse,
+ compat_urllib_request,
+)
+from ..utils import (
+ ExtractorError,
+)
+
+
+class PlayedIE(InfoExtractor):
+ IE_NAME = 'played.to'
+ _VALID_URL = r'https?://(?:www\.)?played\.to/(?P<id>[a-zA-Z0-9_-]+)'
+
+ _TEST = {
+ 'url': 'http://played.to/j2f2sfiiukgt',
+ 'md5': 'c2bd75a368e82980e7257bf500c00637',
+ 'info_dict': {
+ 'id': 'j2f2sfiiukgt',
+ 'ext': 'flv',
+ 'title': 'youtube-dl_test_video.mp4',
+ },
+ }
+
+ def _real_extract(self, url):
+ video_id = self._match_id(url)
+ orig_webpage = self._download_webpage(url, video_id)
+
+ m_error = re.search(
+ r'(?s)Reason for deletion:.*?<b class="err"[^>]*>(?P<msg>[^<]+)</b>', orig_webpage)
+ if m_error:
+ raise ExtractorError(m_error.group('msg'), expected=True)
+
+ fields = re.findall(
+ r'type="hidden" name="([^"]+)"\s+value="([^"]+)">', orig_webpage)
+ data = dict(fields)
+
+ self._sleep(2, video_id)
+
+ post = compat_urllib_parse.urlencode(data)
+ headers = {
+ b'Content-Type': b'application/x-www-form-urlencoded',
+ }
+ req = compat_urllib_request.Request(url, post, headers)
+ webpage = self._download_webpage(
+ req, video_id, note='Downloading video page ...')
+
+ title = os.path.splitext(data['fname'])[0]
+
+ video_url = self._search_regex(
+ r'file: "?(.+?)",', webpage, 'video URL')
+
+ return {
+ 'id': video_id,
+ 'title': title,
+ 'url': video_url,
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/playfm.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/playfm.py
new file mode 100644
index 0000000000..9576aed0e6
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/playfm.py
@@ -0,0 +1,88 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..compat import (
+ compat_urllib_parse,
+ compat_urllib_request,
+)
+from ..utils import (
+ ExtractorError,
+ float_or_none,
+ int_or_none,
+ str_to_int,
+)
+
+
+class PlayFMIE(InfoExtractor):
+ IE_NAME = 'play.fm'
+ _VALID_URL = r'https?://(?:www\.)?play\.fm/[^?#]*(?P<upload_date>[0-9]{8})(?P<id>[0-9]{6})(?:$|[?#])'
+
+ _TEST = {
+ 'url': 'http://www.play.fm/recording/leipzigelectronicmusicbatofarparis_fr20140712137220',
+ 'md5': 'c505f8307825a245d0c7ad1850001f22',
+ 'info_dict': {
+ 'id': '137220',
+ 'ext': 'mp3',
+ 'title': 'LEIPZIG ELECTRONIC MUSIC @ Batofar (Paris,FR) - 2014-07-12',
+ 'uploader': 'Sven Tasnadi',
+ 'uploader_id': 'sventasnadi',
+ 'duration': 5627.428,
+ 'upload_date': '20140712',
+ 'view_count': int,
+ 'comment_count': int,
+ 'thumbnail': 're:^https?://.*\.jpg$',
+ },
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+ upload_date = mobj.group('upload_date')
+
+ rec_data = compat_urllib_parse.urlencode({'rec_id': video_id})
+ req = compat_urllib_request.Request(
+ 'http://www.play.fm/flexRead/recording', data=rec_data)
+ req.add_header('Content-Type', 'application/x-www-form-urlencoded')
+ rec_doc = self._download_xml(req, video_id)
+
+ error_node = rec_doc.find('./error')
+ if error_node is not None:
+ raise ExtractorError('An error occured: %s (code %s)' % (
+ error_node.text, rec_doc.find('./status').text))
+
+ recording = rec_doc.find('./recording')
+ title = recording.find('./title').text
+ view_count = str_to_int(recording.find('./stats/playcount').text)
+ comment_count = str_to_int(recording.find('./stats/comments').text)
+ duration = float_or_none(recording.find('./duration').text, scale=1000)
+ thumbnail = recording.find('./image').text
+
+ artist = recording.find('./artists/artist')
+ uploader = artist.find('./name').text
+ uploader_id = artist.find('./slug').text
+
+ video_url = '%s//%s/%s/%s/offset/0/sh/%s/rec/%s/jingle/%s/loc/%s' % (
+ 'http:', recording.find('./url').text,
+ recording.find('./_class').text, recording.find('./file_id').text,
+ rec_doc.find('./uuid').text, video_id,
+ rec_doc.find('./jingle/file_id').text,
+ 'http%3A%2F%2Fwww.play.fm%2Fplayer',
+ )
+
+ return {
+ 'id': video_id,
+ 'url': video_url,
+ 'ext': 'mp3',
+ 'filesize': int_or_none(recording.find('./size').text),
+ 'title': title,
+ 'upload_date': upload_date,
+ 'view_count': view_count,
+ 'comment_count': comment_count,
+ 'duration': duration,
+ 'thumbnail': thumbnail,
+ 'uploader': uploader,
+ 'uploader_id': uploader_id,
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/playvid.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/playvid.py
new file mode 100644
index 0000000000..c3e667e9e7
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/playvid.py
@@ -0,0 +1,87 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..compat import (
+ compat_urllib_parse,
+)
+from ..utils import (
+ clean_html,
+ ExtractorError,
+)
+
+
+class PlayvidIE(InfoExtractor):
+ _VALID_URL = r'https?://www\.playvid\.com/watch(\?v=|/)(?P<id>.+?)(?:#|$)'
+ _TEST = {
+ 'url': 'http://www.playvid.com/watch/RnmBNgtrrJu',
+ 'md5': 'ffa2f6b2119af359f544388d8c01eb6c',
+ 'info_dict': {
+ 'id': 'RnmBNgtrrJu',
+ 'ext': 'mp4',
+ 'title': 'md5:9256d01c6317e3f703848b5906880dc8',
+ 'duration': 82,
+ 'age_limit': 18,
+ }
+ }
+
+ def _real_extract(self, url):
+ video_id = self._match_id(url)
+ webpage = self._download_webpage(url, video_id)
+
+ m_error = re.search(
+ r'<div class="block-error">\s*<div class="heading">\s*<div>(?P<msg>.+?)</div>\s*</div>', webpage)
+ if m_error:
+ raise ExtractorError(clean_html(m_error.group('msg')), expected=True)
+
+ video_title = None
+ duration = None
+ video_thumbnail = None
+ formats = []
+
+ # most of the information is stored in the flashvars
+ flashvars = self._html_search_regex(
+ r'flashvars="(.+?)"', webpage, 'flashvars')
+
+ infos = compat_urllib_parse.unquote(flashvars).split(r'&')
+ for info in infos:
+ videovars_match = re.match(r'^video_vars\[(.+?)\]=(.+?)$', info)
+ if videovars_match:
+ key = videovars_match.group(1)
+ val = videovars_match.group(2)
+
+ if key == 'title':
+ video_title = compat_urllib_parse.unquote_plus(val)
+ if key == 'duration':
+ try:
+ duration = int(val)
+ except ValueError:
+ pass
+ if key == 'big_thumb':
+ video_thumbnail = val
+
+ videourl_match = re.match(
+ r'^video_urls\]\[(?P<resolution>[0-9]+)p', key)
+ if videourl_match:
+ height = int(videourl_match.group('resolution'))
+ formats.append({
+ 'height': height,
+ 'url': val,
+ })
+ self._sort_formats(formats)
+
+ # Extract title - should be in the flashvars; if not, look elsewhere
+ if video_title is None:
+ video_title = self._html_search_regex(
+ r'<title>(.*?)</title', webpage, 'title')
+
+ return {
+ 'id': video_id,
+ 'formats': formats,
+ 'title': video_title,
+ 'thumbnail': video_thumbnail,
+ 'duration': duration,
+ 'description': None,
+ 'age_limit': 18
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/podomatic.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/podomatic.py
new file mode 100644
index 0000000000..f20946a2bd
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/podomatic.py
@@ -0,0 +1,69 @@
+from __future__ import unicode_literals
+
+import json
+import re
+
+from .common import InfoExtractor
+from ..utils import int_or_none
+
+
+class PodomaticIE(InfoExtractor):
+ IE_NAME = 'podomatic'
+ _VALID_URL = r'^(?P<proto>https?)://(?P<channel>[^.]+)\.podomatic\.com/entry/(?P<id>[^?]+)'
+
+ _TESTS = [
+ {
+ 'url': 'http://scienceteachingtips.podomatic.com/entry/2009-01-02T16_03_35-08_00',
+ 'md5': '84bb855fcf3429e6bf72460e1eed782d',
+ 'info_dict': {
+ 'id': '2009-01-02T16_03_35-08_00',
+ 'ext': 'mp3',
+ 'uploader': 'Science Teaching Tips',
+ 'uploader_id': 'scienceteachingtips',
+ 'title': '64. When the Moon Hits Your Eye',
+ 'duration': 446,
+ }
+ },
+ {
+ 'url': 'http://ostbahnhof.podomatic.com/entry/2013-11-15T16_31_21-08_00',
+ 'md5': 'd2cf443931b6148e27638650e2638297',
+ 'info_dict': {
+ 'id': '2013-11-15T16_31_21-08_00',
+ 'ext': 'mp3',
+ 'uploader': 'Ostbahnhof / Techno Mix',
+ 'uploader_id': 'ostbahnhof',
+ 'title': 'Einunddreizig',
+ 'duration': 3799,
+ }
+ },
+ ]
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+ channel = mobj.group('channel')
+
+ json_url = (('%s://%s.podomatic.com/entry/embed_params/%s' +
+ '?permalink=true&rtmp=0') %
+ (mobj.group('proto'), channel, video_id))
+ data_json = self._download_webpage(
+ json_url, video_id, 'Downloading video info')
+ data = json.loads(data_json)
+
+ video_url = data['downloadLink']
+ if not video_url:
+ video_url = '%s/%s' % (data['streamer'].replace('rtmp', 'http'), data['mediaLocation'])
+ uploader = data['podcast']
+ title = data['title']
+ thumbnail = data['imageLocation']
+ duration = int_or_none(data.get('length'), 1000)
+
+ return {
+ 'id': video_id,
+ 'url': video_url,
+ 'title': title,
+ 'uploader': uploader,
+ 'uploader_id': channel,
+ 'thumbnail': thumbnail,
+ 'duration': duration,
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/pornhd.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/pornhd.py
new file mode 100644
index 0000000000..954dfccb75
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/pornhd.py
@@ -0,0 +1,70 @@
+from __future__ import unicode_literals
+
+import re
+import json
+
+from .common import InfoExtractor
+from ..utils import (
+ int_or_none,
+ js_to_json,
+ qualities,
+)
+
+
+class PornHdIE(InfoExtractor):
+ _VALID_URL = r'http://(?:www\.)?pornhd\.com/(?:[a-z]{2,4}/)?videos/(?P<id>\d+)(?:/(?P<display_id>.+))?'
+ _TEST = {
+ 'url': 'http://www.pornhd.com/videos/1962/sierra-day-gets-his-cum-all-over-herself-hd-porn-video',
+ 'md5': '956b8ca569f7f4d8ec563e2c41598441',
+ 'info_dict': {
+ 'id': '1962',
+ 'display_id': 'sierra-day-gets-his-cum-all-over-herself-hd-porn-video',
+ 'ext': 'mp4',
+ 'title': 'Sierra loves doing laundry',
+ 'description': 'md5:8ff0523848ac2b8f9b065ba781ccf294',
+ 'thumbnail': 're:^https?://.*\.jpg',
+ 'view_count': int,
+ 'age_limit': 18,
+ }
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+ display_id = mobj.group('display_id')
+
+ webpage = self._download_webpage(url, display_id or video_id)
+
+ title = self._html_search_regex(
+ r'<title>(.+) porn HD.+? ', webpage, 'title')
+ description = self._html_search_regex(
+ r'([^<]+)
', webpage, 'description', fatal=False)
+ view_count = int_or_none(self._html_search_regex(
+ r'(\d+) views\s*', webpage, 'view count', fatal=False))
+ thumbnail = self._search_regex(
+ r"'poster'\s*:\s*'([^']+)'", webpage, 'thumbnail', fatal=False)
+
+ quality = qualities(['sd', 'hd'])
+ sources = json.loads(js_to_json(self._search_regex(
+ r"(?s)'sources'\s*:\s*(\{.+?\})\s*\}\);", webpage, 'sources')))
+ formats = []
+ for container, s in sources.items():
+ for qname, video_url in s.items():
+ formats.append({
+ 'url': video_url,
+ 'container': container,
+ 'format_id': '%s-%s' % (container, qname),
+ 'quality': quality(qname),
+ })
+ self._sort_formats(formats)
+
+ return {
+ 'id': video_id,
+ 'display_id': display_id,
+ 'title': title,
+ 'description': description,
+ 'thumbnail': thumbnail,
+ 'view_count': view_count,
+ 'formats': formats,
+ 'age_limit': 18,
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/pornhub.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/pornhub.py
new file mode 100644
index 0000000000..634142d0d2
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/pornhub.py
@@ -0,0 +1,102 @@
+from __future__ import unicode_literals
+
+import os
+import re
+
+from .common import InfoExtractor
+from ..compat import (
+ compat_urllib_parse,
+ compat_urllib_parse_urlparse,
+ compat_urllib_request,
+)
+from ..utils import (
+ str_to_int,
+)
+from ..aes import (
+ aes_decrypt_text
+)
+
+
+class PornHubIE(InfoExtractor):
+ _VALID_URL = r'https?://(?:www\.)?pornhub\.com/view_video\.php\?viewkey=(?P[0-9a-f]+)'
+ _TEST = {
+ 'url': 'http://www.pornhub.com/view_video.php?viewkey=648719015',
+ 'md5': '882f488fa1f0026f023f33576004a2ed',
+ 'info_dict': {
+ 'id': '648719015',
+ 'ext': 'mp4',
+ "uploader": "Babes",
+ "title": "Seductive Indian beauty strips down and fingers her pink pussy",
+ "age_limit": 18
+ }
+ }
+
+ def _extract_count(self, pattern, webpage, name):
+ count = self._html_search_regex(pattern, webpage, '%s count' % name, fatal=False)
+ if count:
+ count = str_to_int(count)
+ return count
+
+ def _real_extract(self, url):
+ video_id = self._match_id(url)
+
+ req = compat_urllib_request.Request(url)
+ req.add_header('Cookie', 'age_verified=1')
+ webpage = self._download_webpage(req, video_id)
+
+ video_title = self._html_search_regex(r']+>([^<]+)', webpage, 'title')
+ video_uploader = self._html_search_regex(
+ r'(?s)From: .+?<(?:a href="/users/|a href="/channels/|([\d,\.]+) views', webpage, 'view')
+ like_count = self._extract_count(r'([\d,\.]+) ', webpage, 'like')
+ dislike_count = self._extract_count(r'([\d,\.]+) ', webpage, 'dislike')
+ comment_count = self._extract_count(
+ r'All comments \(', webpage, 'comment')
+
+ video_urls = list(map(compat_urllib_parse.unquote, re.findall(r'"quality_[0-9]{3}p":"([^"]+)', webpage)))
+ if webpage.find('"encrypted":true') != -1:
+ password = compat_urllib_parse.unquote_plus(self._html_search_regex(r'"video_title":"([^"]+)', webpage, 'password'))
+ video_urls = list(map(lambda s: aes_decrypt_text(s, password, 32).decode('utf-8'), video_urls))
+
+ formats = []
+ for video_url in video_urls:
+ path = compat_urllib_parse_urlparse(video_url).path
+ extension = os.path.splitext(path)[1][1:]
+ format = path.split('/')[5].split('_')[:2]
+ format = "-".join(format)
+
+ m = re.match(r'^(?P[0-9]+)P-(?P[0-9]+)K$', format)
+ if m is None:
+ height = None
+ tbr = None
+ else:
+ height = int(m.group('height'))
+ tbr = int(m.group('tbr'))
+
+ formats.append({
+ 'url': video_url,
+ 'ext': extension,
+ 'format': format,
+ 'format_id': format,
+ 'tbr': tbr,
+ 'height': height,
+ })
+ self._sort_formats(formats)
+
+ return {
+ 'id': video_id,
+ 'uploader': video_uploader,
+ 'title': video_title,
+ 'thumbnail': thumbnail,
+ 'view_count': view_count,
+ 'like_count': like_count,
+ 'dislike_count': dislike_count,
+ 'comment_count': comment_count,
+ 'formats': formats,
+ 'age_limit': 18,
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/pornotube.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/pornotube.py
new file mode 100644
index 0000000000..34735c51e1
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/pornotube.py
@@ -0,0 +1,94 @@
+from __future__ import unicode_literals
+
+import json
+
+from .common import InfoExtractor
+from ..compat import (
+ compat_urllib_request,
+)
+from ..utils import (
+ int_or_none,
+)
+
+
+class PornotubeIE(InfoExtractor):
+ _VALID_URL = r'https?://(?:\w+\.)?pornotube\.com/(?:[^?#]*?)/video/(?P[0-9]+)'
+ _TEST = {
+ 'url': 'http://www.pornotube.com/orientation/straight/video/4964/title/weird-hot-and-wet-science',
+ 'md5': '60fc5a4f0d93a97968fc7999d98260c9',
+ 'info_dict': {
+ 'id': '4964',
+ 'ext': 'mp4',
+ 'upload_date': '20141203',
+ 'title': 'Weird Hot and Wet Science',
+ 'description': 'md5:a8304bef7ef06cb4ab476ca6029b01b0',
+ 'categories': ['Adult Humor', 'Blondes'],
+ 'uploader': 'Alpha Blue Archives',
+ 'thumbnail': 're:^https?://.*\\.jpg$',
+ 'timestamp': 1417582800,
+ 'age_limit': 18,
+ }
+ }
+
+ def _real_extract(self, url):
+ video_id = self._match_id(url)
+
+ # Fetch origin token
+ js_config = self._download_webpage(
+ 'http://www.pornotube.com/assets/src/app/config.js', video_id,
+ note='Download JS config')
+ originAuthenticationSpaceKey = self._search_regex(
+ r"constant\('originAuthenticationSpaceKey',\s*'([^']+)'",
+ js_config, 'originAuthenticationSpaceKey')
+
+ # Fetch actual token
+ token_req_data = {
+ 'authenticationSpaceKey': originAuthenticationSpaceKey,
+ 'credentials': 'Clip Application',
+ }
+ token_req = compat_urllib_request.Request(
+ 'https://api.aebn.net/auth/v1/token/primal',
+ data=json.dumps(token_req_data).encode('utf-8'))
+ token_req.add_header('Content-Type', 'application/json')
+ token_req.add_header('Origin', 'http://www.pornotube.com')
+ token_answer = self._download_json(
+ token_req, video_id, note='Requesting primal token')
+ token = token_answer['tokenKey']
+
+ # Get video URL
+ delivery_req = compat_urllib_request.Request(
+ 'https://api.aebn.net/delivery/v1/clips/%s/MP4' % video_id)
+ delivery_req.add_header('Authorization', token)
+ delivery_info = self._download_json(
+ delivery_req, video_id, note='Downloading delivery information')
+ video_url = delivery_info['mediaUrl']
+
+ # Get additional info (title etc.)
+ info_req = compat_urllib_request.Request(
+ 'https://api.aebn.net/content/v1/clips/%s?expand='
+ 'title,description,primaryImageNumber,startSecond,endSecond,'
+ 'movie.title,movie.MovieId,movie.boxCoverFront,movie.stars,'
+ 'movie.studios,stars.name,studios.name,categories.name,'
+ 'clipActive,movieActive,publishDate,orientations' % video_id)
+ info_req.add_header('Authorization', token)
+ info = self._download_json(
+ info_req, video_id, note='Downloading metadata')
+
+ timestamp = int_or_none(info.get('publishDate'), scale=1000)
+ uploader = info.get('studios', [{}])[0].get('name')
+ movie_id = info['movie']['movieId']
+ thumbnail = 'http://pic.aebn.net/dis/t/%s/%s_%08d.jpg' % (
+ movie_id, movie_id, info['primaryImageNumber'])
+ categories = [c['name'] for c in info.get('categories')]
+
+ return {
+ 'id': video_id,
+ 'url': video_url,
+ 'title': info['title'],
+ 'description': info.get('description'),
+ 'timestamp': timestamp,
+ 'uploader': uploader,
+ 'thumbnail': thumbnail,
+ 'categories': categories,
+ 'age_limit': 18,
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/pornoxo.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/pornoxo.py
new file mode 100644
index 0000000000..202f58673a
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/pornoxo.py
@@ -0,0 +1,65 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+ str_to_int,
+)
+
+
+class PornoXOIE(InfoExtractor):
+ _VALID_URL = r'https?://(?:www\.)?pornoxo\.com/videos/(?P\d+)/(?P[^/]+)\.html'
+ _TEST = {
+ 'url': 'http://www.pornoxo.com/videos/7564/striptease-from-sexy-secretary.html',
+ 'md5': '582f28ecbaa9e6e24cb90f50f524ce87',
+ 'info_dict': {
+ 'id': '7564',
+ 'ext': 'flv',
+ 'title': 'Striptease From Sexy Secretary!',
+ 'description': 'Striptease From Sexy Secretary!',
+ 'categories': list, # NSFW
+ 'thumbnail': 're:https?://.*\.jpg$',
+ 'age_limit': 18,
+ }
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+
+ webpage = self._download_webpage(url, video_id)
+
+ video_url = self._html_search_regex(
+ r'\'file\'\s*:\s*"([^"]+)"', webpage, 'video_url')
+
+ title = self._html_search_regex(
+ r'([^<]+)\s*-\s*PornoXO', webpage, 'title')
+
+ description = self._html_search_regex(
+ r'<meta name="description" content="([^"]+)\s*featuring',
+ webpage, 'description', fatal=False)
+
+ thumbnail = self._html_search_regex(
+ r'\'image\'\s*:\s*"([^"]+)"', webpage, 'thumbnail', fatal=False)
+
+ view_count = str_to_int(self._html_search_regex(
+ r'[vV]iews:\s*([0-9,]+)', webpage, 'view count', fatal=False))
+
+ categories_str = self._html_search_regex(
+ r'<meta name="description" content=".*featuring\s*([^"]+)"',
+ webpage, 'categories', fatal=False)
+ categories = (
+ None if categories_str is None
+ else categories_str.split(','))
+
+ return {
+ 'id': video_id,
+ 'url': video_url,
+ 'title': title,
+ 'description': description,
+ 'thumbnail': thumbnail,
+ 'categories': categories,
+ 'view_count': view_count,
+ 'age_limit': 18,
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/promptfile.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/promptfile.py
new file mode 100644
index 0000000000..f536e6e6cd
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/promptfile.py
@@ -0,0 +1,67 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..compat import (
+ compat_urllib_parse,
+ compat_urllib_request,
+)
+from ..utils import (
+ determine_ext,
+ ExtractorError,
+)
+
+
+class PromptFileIE(InfoExtractor):
+ _VALID_URL = r'https?://(?:www\.)?promptfile\.com/l/(?P<id>[0-9A-Z\-]+)'
+ _TEST = {
+ 'url': 'http://www.promptfile.com/l/D21B4746E9-F01462F0FF',
+ 'md5': 'd1451b6302da7215485837aaea882c4c',
+ 'info_dict': {
+ 'id': 'D21B4746E9-F01462F0FF',
+ 'ext': 'mp4',
+ 'title': 'Birds.mp4',
+ 'thumbnail': 're:^https?://.*\.jpg$',
+ }
+ }
+
+ def _real_extract(self, url):
+ video_id = self._match_id(url)
+ webpage = self._download_webpage(url, video_id)
+
+ if re.search(r'<div.+id="not_found_msg".+>(?!We are).+</div>[^-]', webpage) is not None:
+ raise ExtractorError('Video %s does not exist' % video_id,
+ expected=True)
+
+ fields = dict(re.findall(r'''(?x)type="hidden"\s+
+ name="(.+?)"\s+
+ value="(.*?)"
+ ''', webpage))
+ post = compat_urllib_parse.urlencode(fields)
+ req = compat_urllib_request.Request(url, post)
+ req.add_header('Content-type', 'application/x-www-form-urlencoded')
+ webpage = self._download_webpage(
+ req, video_id, 'Downloading video page')
+
+ url = self._html_search_regex(r'url:\s*\'([^\']+)\'', webpage, 'URL')
+ title = self._html_search_regex(
+ r'<span.+title="([^"]+)">', webpage, 'title')
+ thumbnail = self._html_search_regex(
+ r'<div id="player_overlay">.*button>.*?<img src="([^"]+)"',
+ webpage, 'thumbnail', fatal=False, flags=re.DOTALL)
+
+ formats = [{
+ 'format_id': 'sd',
+ 'url': url,
+ 'ext': determine_ext(title),
+ }]
+ self._sort_formats(formats)
+
+ return {
+ 'id': video_id,
+ 'title': title,
+ 'thumbnail': thumbnail,
+ 'formats': formats,
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/prosiebensat1.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/prosiebensat1.py
new file mode 100644
index 0000000000..385681d06e
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/prosiebensat1.py
@@ -0,0 +1,331 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from hashlib import sha1
+from .common import InfoExtractor
+from ..compat import (
+ compat_urllib_parse,
+)
+from ..utils import (
+ unified_strdate,
+)
+
+
+class ProSiebenSat1IE(InfoExtractor):
+ IE_NAME = 'prosiebensat1'
+ IE_DESC = 'ProSiebenSat.1 Digital'
+ _VALID_URL = r'https?://(?:www\.)?(?:(?:prosieben|prosiebenmaxx|sixx|sat1|kabeleins|ran|the-voice-of-germany)\.de|fem\.com)/(?P<id>.+)'
+
+ _TESTS = [
+ {
+ 'url': 'http://www.prosieben.de/tv/circus-halligalli/videos/218-staffel-2-episode-18-jahresrueckblick-ganze-folge',
+ 'info_dict': {
+ 'id': '2104602',
+ 'ext': 'mp4',
+ 'title': 'Staffel 2, Episode 18 - Jahresrц╪ckblick',
+ 'description': 'md5:8733c81b702ea472e069bc48bb658fc1',
+ 'upload_date': '20131231',
+ 'duration': 5845.04,
+ },
+ 'params': {
+ # rtmp download
+ 'skip_download': True,
+ },
+ },
+ {
+ 'url': 'http://www.prosieben.de/videokatalog/Gesellschaft/Leben/Trends/video-Lady-Umstyling-f%C3%BCr-Audrina-Rebekka-Audrina-Fergen-billig-aussehen-Battal-Modica-700544.html',
+ 'info_dict': {
+ 'id': '2570327',
+ 'ext': 'mp4',
+ 'title': 'Lady-Umstyling fц╪r Audrina',
+ 'description': 'md5:4c16d0c17a3461a0d43ea4084e96319d',
+ 'upload_date': '20131014',
+ 'duration': 606.76,
+ },
+ 'params': {
+ # rtmp download
+ 'skip_download': True,
+ },
+ 'skip': 'Seems to be broken',
+ },
+ {
+ 'url': 'http://www.prosiebenmaxx.de/tv/experience/video/144-countdown-fuer-die-autowerkstatt-ganze-folge',
+ 'info_dict': {
+ 'id': '2429369',
+ 'ext': 'mp4',
+ 'title': 'Countdown fц╪r die Autowerkstatt',
+ 'description': 'md5:809fc051a457b5d8666013bc40698817',
+ 'upload_date': '20140223',
+ 'duration': 2595.04,
+ },
+ 'params': {
+ # rtmp download
+ 'skip_download': True,
+ },
+ },
+ {
+ 'url': 'http://www.sixx.de/stars-style/video/sexy-laufen-in-ugg-boots-clip',
+ 'info_dict': {
+ 'id': '2904997',
+ 'ext': 'mp4',
+ 'title': 'Sexy laufen in Ugg Boots',
+ 'description': 'md5:edf42b8bd5bc4e5da4db4222c5acb7d6',
+ 'upload_date': '20140122',
+ 'duration': 245.32,
+ },
+ 'params': {
+ # rtmp download
+ 'skip_download': True,
+ },
+ },
+ {
+ 'url': 'http://www.sat1.de/film/der-ruecktritt/video/im-interview-kai-wiesinger-clip',
+ 'info_dict': {
+ 'id': '2906572',
+ 'ext': 'mp4',
+ 'title': 'Im Interview: Kai Wiesinger',
+ 'description': 'md5:e4e5370652ec63b95023e914190b4eb9',
+ 'upload_date': '20140203',
+ 'duration': 522.56,
+ },
+ 'params': {
+ # rtmp download
+ 'skip_download': True,
+ },
+ },
+ {
+ 'url': 'http://www.kabeleins.de/tv/rosins-restaurants/videos/jagd-auf-fertigkost-im-elsthal-teil-2-ganze-folge',
+ 'info_dict': {
+ 'id': '2992323',
+ 'ext': 'mp4',
+ 'title': 'Jagd auf Fertigkost im Elsthal - Teil 2',
+ 'description': 'md5:2669cde3febe9bce13904f701e774eb6',
+ 'upload_date': '20141014',
+ 'duration': 2410.44,
+ },
+ 'params': {
+ # rtmp download
+ 'skip_download': True,
+ },
+ },
+ {
+ 'url': 'http://www.ran.de/fussball/bundesliga/video/schalke-toennies-moechte-raul-zurueck-ganze-folge',
+ 'info_dict': {
+ 'id': '3004256',
+ 'ext': 'mp4',
+ 'title': 'Schalke: Tц╤nnies mц╤chte Raul zurц╪ck',
+ 'description': 'md5:4b5b271d9bcde223b54390754c8ece3f',
+ 'upload_date': '20140226',
+ 'duration': 228.96,
+ },
+ 'params': {
+ # rtmp download
+ 'skip_download': True,
+ },
+ },
+ {
+ 'url': 'http://www.the-voice-of-germany.de/video/31-andreas-kuemmert-rocket-man-clip',
+ 'info_dict': {
+ 'id': '2572814',
+ 'ext': 'mp4',
+ 'title': 'Andreas Kц╪mmert: Rocket Man',
+ 'description': 'md5:6ddb02b0781c6adf778afea606652e38',
+ 'upload_date': '20131017',
+ 'duration': 469.88,
+ },
+ 'params': {
+ # rtmp download
+ 'skip_download': True,
+ },
+ },
+ {
+ 'url': 'http://www.fem.com/wellness/videos/wellness-video-clip-kurztripps-zum-valentinstag.html',
+ 'info_dict': {
+ 'id': '2156342',
+ 'ext': 'mp4',
+ 'title': 'Kurztrips zum Valentinstag',
+ 'description': 'Romantischer Kurztrip zum Valentinstag? Wir verraten, was sich hier wirklich lohnt.',
+ 'duration': 307.24,
+ },
+ 'params': {
+ # rtmp download
+ 'skip_download': True,
+ },
+ },
+ {
+ 'url': 'http://www.prosieben.de/tv/joko-gegen-klaas/videos/playlists/episode-8-ganze-folge-playlist',
+ 'info_dict': {
+ 'id': '439664',
+ 'title': 'Episode 8 - Ganze Folge - Playlist',
+ 'description': 'md5:63b8963e71f481782aeea877658dec84',
+ },
+ 'playlist_count': 2,
+ },
+ ]
+
+ _CLIPID_REGEXES = [
+ r'"clip_id"\s*:\s+"(\d+)"',
+ r'clipid: "(\d+)"',
+ r'clip[iI]d=(\d+)',
+ r"'itemImageUrl'\s*:\s*'/dynamic/thumbnails/full/\d+/(\d+)",
+ ]
+ _TITLE_REGEXES = [
+ r'<h2 class="subtitle" itemprop="name">\s*(.+?)</h2>',
+ r'<header class="clearfix">\s*<h3>(.+?)</h3>',
+ r'<!-- start video -->\s*<h1>(.+?)</h1>',
+ r'<h1 class="att-name">\s*(.+?)</h1>',
+ ]
+ _DESCRIPTION_REGEXES = [
+ r'<p itemprop="description">\s*(.+?)</p>',
+ r'<div class="videoDecription">\s*<p><strong>Beschreibung</strong>: (.+?)</p>',
+ r'<div class="g-plusone" data-size="medium"></div>\s*</div>\s*</header>\s*(.+?)\s*<footer>',
+ r'<p class="att-description">\s*(.+?)\s*</p>',
+ ]
+ _UPLOAD_DATE_REGEXES = [
+ r'<meta property="og:published_time" content="(.+?)">',
+ r'<span>\s*(\d{2}\.\d{2}\.\d{4} \d{2}:\d{2}) \|\s*<span itemprop="duration"',
+ r'<footer>\s*(\d{2}\.\d{2}\.\d{4}) \d{2}:\d{2} Uhr',
+ r'<span style="padding-left: 4px;line-height:20px; color:#404040">(\d{2}\.\d{2}\.\d{4})</span>',
+ r'(\d{2}\.\d{2}\.\d{4}) \| \d{2}:\d{2} Min<br/>',
+ ]
+ _PAGE_TYPE_REGEXES = [
+ r'<meta name="page_type" content="([^"]+)">',
+ r"'itemType'\s*:\s*'([^']*)'",
+ ]
+ _PLAYLIST_ID_REGEXES = [
+ r'content[iI]d=(\d+)',
+ r"'itemId'\s*:\s*'([^']*)'",
+ ]
+ _PLAYLIST_CLIP_REGEXES = [
+ r'(?s)data-qvt=.+?<a href="([^"]+)"',
+ ]
+
+ def _extract_clip(self, url, webpage):
+ clip_id = self._html_search_regex(self._CLIPID_REGEXES, webpage, 'clip id')
+
+ access_token = 'testclient'
+ client_name = 'kolibri-1.2.5'
+ client_location = url
+
+ videos_api_url = 'http://vas.sim-technik.de/vas/live/v2/videos?%s' % compat_urllib_parse.urlencode({
+ 'access_token': access_token,
+ 'client_location': client_location,
+ 'client_name': client_name,
+ 'ids': clip_id,
+ })
+
+ videos = self._download_json(videos_api_url, clip_id, 'Downloading videos JSON')
+
+ duration = float(videos[0]['duration'])
+ source_ids = [source['id'] for source in videos[0]['sources']]
+ source_ids_str = ','.join(map(str, source_ids))
+
+ g = '01!8d8F_)r9]4s[qeuXfP%'
+
+ client_id = g[:2] + sha1(''.join([clip_id, g, access_token, client_location, g, client_name])
+ .encode('utf-8')).hexdigest()
+
+ sources_api_url = 'http://vas.sim-technik.de/vas/live/v2/videos/%s/sources?%s' % (clip_id, compat_urllib_parse.urlencode({
+ 'access_token': access_token,
+ 'client_id': client_id,
+ 'client_location': client_location,
+ 'client_name': client_name,
+ }))
+
+ sources = self._download_json(sources_api_url, clip_id, 'Downloading sources JSON')
+ server_id = sources['server_id']
+
+ client_id = g[:2] + sha1(''.join([g, clip_id, access_token, server_id,
+ client_location, source_ids_str, g, client_name])
+ .encode('utf-8')).hexdigest()
+
+ url_api_url = 'http://vas.sim-technik.de/vas/live/v2/videos/%s/sources/url?%s' % (clip_id, compat_urllib_parse.urlencode({
+ 'access_token': access_token,
+ 'client_id': client_id,
+ 'client_location': client_location,
+ 'client_name': client_name,
+ 'server_id': server_id,
+ 'source_ids': source_ids_str,
+ }))
+
+ urls = self._download_json(url_api_url, clip_id, 'Downloading urls JSON')
+
+ title = self._html_search_regex(self._TITLE_REGEXES, webpage, 'title')
+ description = self._html_search_regex(self._DESCRIPTION_REGEXES, webpage, 'description', fatal=False)
+ thumbnail = self._og_search_thumbnail(webpage)
+
+ upload_date = unified_strdate(self._html_search_regex(
+ self._UPLOAD_DATE_REGEXES, webpage, 'upload date', default=None))
+
+ formats = []
+
+ urls_sources = urls['sources']
+ if isinstance(urls_sources, dict):
+ urls_sources = urls_sources.values()
+
+ def fix_bitrate(bitrate):
+ return (bitrate // 1000) if bitrate % 1000 == 0 else bitrate
+
+ for source in urls_sources:
+ protocol = source['protocol']
+ if protocol == 'rtmp' or protocol == 'rtmpe':
+ mobj = re.search(r'^(?P<url>rtmpe?://[^/]+/(?P<app>[^/]+))/(?P<playpath>.+)$', source['url'])
+ if not mobj:
+ continue
+ formats.append({
+ 'url': mobj.group('url'),
+ 'app': mobj.group('app'),
+ 'play_path': mobj.group('playpath'),
+ 'player_url': 'http://livepassdl.conviva.com/hf/ver/2.79.0.17083/LivePassModuleMain.swf',
+ 'page_url': 'http://www.prosieben.de',
+ 'vbr': fix_bitrate(source['bitrate']),
+ 'ext': 'mp4',
+ 'format_id': '%s_%s' % (source['cdn'], source['bitrate']),
+ })
+ else:
+ formats.append({
+ 'url': source['url'],
+ 'vbr': fix_bitrate(source['bitrate']),
+ })
+
+ self._sort_formats(formats)
+
+ return {
+ 'id': clip_id,
+ 'title': title,
+ 'description': description,
+ 'thumbnail': thumbnail,
+ 'upload_date': upload_date,
+ 'duration': duration,
+ 'formats': formats,
+ }
+
+ def _extract_playlist(self, url, webpage):
+ playlist_id = self._html_search_regex(
+ self._PLAYLIST_ID_REGEXES, webpage, 'playlist id')
+ for regex in self._PLAYLIST_CLIP_REGEXES:
+ playlist_clips = re.findall(regex, webpage)
+ if playlist_clips:
+ title = self._html_search_regex(
+ self._TITLE_REGEXES, webpage, 'title')
+ description = self._html_search_regex(
+ self._DESCRIPTION_REGEXES, webpage, 'description', fatal=False)
+ entries = [
+ self.url_result(
+ re.match('(.+?//.+?)/', url).group(1) + clip_path,
+ 'ProSiebenSat1')
+ for clip_path in playlist_clips]
+ return self.playlist_result(entries, playlist_id, title, description)
+
+ def _real_extract(self, url):
+ video_id = self._match_id(url)
+ webpage = self._download_webpage(url, video_id)
+ page_type = self._search_regex(
+ self._PAGE_TYPE_REGEXES, webpage,
+ 'page type', default='clip').lower()
+ if page_type == 'clip':
+ return self._extract_clip(url, webpage)
+ elif page_type == 'playlist':
+ return self._extract_playlist(url, webpage)
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/pyvideo.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/pyvideo.py
new file mode 100644
index 0000000000..6d5732d45c
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/pyvideo.py
@@ -0,0 +1,59 @@
+from __future__ import unicode_literals
+
+import re
+import os
+
+from .common import InfoExtractor
+
+
+class PyvideoIE(InfoExtractor):
+ _VALID_URL = r'http://(?:www\.)?pyvideo\.org/video/(?P<id>\d+)/(.*)'
+
+ _TESTS = [
+ {
+ 'url': 'http://pyvideo.org/video/1737/become-a-logging-expert-in-30-minutes',
+ 'md5': 'de317418c8bc76b1fd8633e4f32acbc6',
+ 'info_dict': {
+ 'id': '24_4WWkSmNo',
+ 'ext': 'mp4',
+ 'title': 'Become a logging expert in 30 minutes',
+ 'description': 'md5:9665350d466c67fb5b1598de379021f7',
+ 'upload_date': '20130320',
+ 'uploader': 'NextDayVideo',
+ 'uploader_id': 'NextDayVideo',
+ },
+ 'add_ie': ['Youtube'],
+ },
+ {
+ 'url': 'http://pyvideo.org/video/2542/gloriajw-spotifywitherikbernhardsson182m4v',
+ 'md5': '5fe1c7e0a8aa5570330784c847ff6d12',
+ 'info_dict': {
+ 'id': '2542',
+ 'ext': 'm4v',
+ 'title': 'Gloriajw-SpotifyWithErikBernhardsson182',
+ },
+ },
+ ]
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+
+ webpage = self._download_webpage(url, video_id)
+
+ m_youtube = re.search(r'(https?://www\.youtube\.com/watch\?v=.*)', webpage)
+ if m_youtube is not None:
+ return self.url_result(m_youtube.group(1), 'Youtube')
+
+ title = self._html_search_regex(
+ r'<div class="section">\s*<h3(?:\s+class="[^"]*"[^>]*)?>([^>]+?)</h3>',
+ webpage, 'title', flags=re.DOTALL)
+ video_url = self._search_regex(
+ [r'<source src="(.*?)"', r'<dt>Download</dt>.*?<a href="(.+?)"'],
+ webpage, 'video url', flags=re.DOTALL)
+
+ return {
+ 'id': video_id,
+ 'title': os.path.splitext(title)[0],
+ 'url': video_url,
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/quickvid.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/quickvid.py
new file mode 100644
index 0000000000..af7d76cf47
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/quickvid.py
@@ -0,0 +1,53 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..compat import (
+ compat_urlparse,
+)
+from ..utils import (
+ determine_ext,
+ int_or_none,
+)
+
+
+class QuickVidIE(InfoExtractor):
+ _VALID_URL = r'https?://(www\.)?quickvid\.org/watch\.php\?v=(?P<id>[a-zA-Z_0-9-]+)'
+ _TEST = {
+ 'url': 'http://quickvid.org/watch.php?v=sUQT3RCG8dx',
+ 'md5': 'c0c72dd473f260c06c808a05d19acdc5',
+ 'info_dict': {
+ 'id': 'sUQT3RCG8dx',
+ 'ext': 'mp4',
+ 'title': 'Nick Offerman\'s Summer Reading Recap',
+ 'thumbnail': 're:^https?://.*\.(?:png|jpg|gif)$',
+ 'view_count': int,
+ },
+ }
+
+ def _real_extract(self, url):
+ video_id = self._match_id(url)
+ webpage = self._download_webpage(url, video_id)
+
+ title = self._html_search_regex(r'<h2>(.*?)</h2>', webpage, 'title')
+ view_count = int_or_none(self._html_search_regex(
+ r'(?s)<div id="views">(.*?)</div>',
+ webpage, 'view count', fatal=False))
+ video_code = self._search_regex(
+ r'(?s)<video id="video"[^>]*>(.*?)</video>', webpage, 'video code')
+ formats = [
+ {
+ 'url': compat_urlparse.urljoin(url, src),
+ 'format_id': determine_ext(src, None),
+ } for src in re.findall('<source\s+src="([^"]+)"', video_code)
+ ]
+ self._sort_formats(formats)
+
+ return {
+ 'id': video_id,
+ 'title': title,
+ 'formats': formats,
+ 'thumbnail': self._og_search_thumbnail(webpage),
+ 'view_count': view_count,
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/radiode.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/radiode.py
new file mode 100644
index 0000000000..f95bc94543
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/radiode.py
@@ -0,0 +1,55 @@
+from __future__ import unicode_literals
+
+import json
+
+from .common import InfoExtractor
+
+
+class RadioDeIE(InfoExtractor):
+ IE_NAME = 'radio.de'
+ _VALID_URL = r'https?://(?P<id>.+?)\.(?:radio\.(?:de|at|fr|pt|es|pl|it)|rad\.io)'
+ _TEST = {
+ 'url': 'http://ndr2.radio.de/',
+ 'md5': '3b4cdd011bc59174596b6145cda474a4',
+ 'info_dict': {
+ 'id': 'ndr2',
+ 'ext': 'mp3',
+ 'title': 're:^NDR 2 [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
+ 'description': 'md5:591c49c702db1a33751625ebfb67f273',
+ 'thumbnail': 're:^https?://.*\.png',
+ },
+ 'params': {
+ 'skip_download': True,
+ }
+ }
+
+ def _real_extract(self, url):
+ radio_id = self._match_id(url)
+
+ webpage = self._download_webpage(url, radio_id)
+
+ broadcast = json.loads(self._search_regex(
+ r'_getBroadcast\s*=\s*function\(\s*\)\s*{\s*return\s+({.+?})\s*;\s*}',
+ webpage, 'broadcast'))
+
+ title = self._live_title(broadcast['name'])
+ description = broadcast.get('description') or broadcast.get('shortDescription')
+ thumbnail = broadcast.get('picture4Url') or broadcast.get('picture4TransUrl')
+
+ formats = [{
+ 'url': stream['streamUrl'],
+ 'ext': stream['streamContentFormat'].lower(),
+ 'acodec': stream['streamContentFormat'],
+ 'abr': stream['bitRate'],
+ 'asr': stream['sampleRate']
+ } for stream in broadcast['streamUrls']]
+ self._sort_formats(formats)
+
+ return {
+ 'id': radio_id,
+ 'title': title,
+ 'description': description,
+ 'thumbnail': thumbnail,
+ 'is_live': True,
+ 'formats': formats,
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/radiofrance.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/radiofrance.py
new file mode 100644
index 0000000000..09352ed825
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/radiofrance.py
@@ -0,0 +1,59 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+
+
+class RadioFranceIE(InfoExtractor):
+ _VALID_URL = r'^https?://maison\.radiofrance\.fr/radiovisions/(?P<id>[^?#]+)'
+ IE_NAME = 'radiofrance'
+
+ _TEST = {
+ 'url': 'http://maison.radiofrance.fr/radiovisions/one-one',
+ 'md5': 'bdbb28ace95ed0e04faab32ba3160daf',
+ 'info_dict': {
+ 'id': 'one-one',
+ 'ext': 'ogg',
+ "title": "One to one",
+ "description": "Plutц╢t que d'imaginer la radio de demain comme technologie ou comme crц╘ation de contenu, je veux montrer que quelles que soient ses ц╘volutions, j'ai l'intime conviction que la radio continuera d'ц╙tre un grand mц╘dia de proximitц╘ pour les auditeurs.",
+ "uploader": "Thomas Hercouц╚t",
+ },
+ }
+
+ def _real_extract(self, url):
+ m = re.match(self._VALID_URL, url)
+ video_id = m.group('id')
+
+ webpage = self._download_webpage(url, video_id)
+ title = self._html_search_regex(r'<h1>(.*?)</h1>', webpage, 'title')
+ description = self._html_search_regex(
+ r'<div class="bloc_page_wrapper"><div class="text">(.*?)</div>',
+ webpage, 'description', fatal=False)
+ uploader = self._html_search_regex(
+ r'<div class="credit"> © (.*?)</div>',
+ webpage, 'uploader', fatal=False)
+
+ formats_str = self._html_search_regex(
+ r'class="jp-jplayer[^"]*" data-source="([^"]+)">',
+ webpage, 'audio URLs')
+ formats = [
+ {
+ 'format_id': fm[0],
+ 'url': fm[1],
+ 'vcodec': 'none',
+ 'preference': i,
+ }
+ for i, fm in
+ enumerate(re.findall(r"([a-z0-9]+)\s*:\s*'([^']+)'", formats_str))
+ ]
+ self._sort_formats(formats)
+
+ return {
+ 'id': video_id,
+ 'title': title,
+ 'formats': formats,
+ 'description': description,
+ 'uploader': uploader,
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/rai.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/rai.py
new file mode 100644
index 0000000000..aa26b7e0bb
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/rai.py
@@ -0,0 +1,124 @@
+from __future__ import unicode_literals
+
+import re
+
+from .subtitles import SubtitlesInfoExtractor
+from ..compat import (
+ compat_urllib_parse,
+)
+from ..utils import (
+ parse_duration,
+ unified_strdate,
+)
+
+
+class RaiIE(SubtitlesInfoExtractor):
+ _VALID_URL = r'(?P<url>http://(?:.+?\.)?(?:rai\.it|rai\.tv|rainews\.it)/dl/.+?-(?P<id>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})(?:-.+?)?\.html)'
+ _TESTS = [
+ {
+ 'url': 'http://www.rai.tv/dl/RaiTV/programmi/media/ContentItem-cb27157f-9dd0-4aee-b788-b1f67643a391.html',
+ 'md5': 'c064c0b2d09c278fb293116ef5d0a32d',
+ 'info_dict': {
+ 'id': 'cb27157f-9dd0-4aee-b788-b1f67643a391',
+ 'ext': 'mp4',
+ 'title': 'Report del 07/04/2014',
+ 'description': 'md5:f27c544694cacb46a078db84ec35d2d9',
+ 'upload_date': '20140407',
+ 'duration': 6160,
+ }
+ },
+ {
+ 'url': 'http://www.raisport.rai.it/dl/raiSport/media/rassegna-stampa-04a9f4bd-b563-40cf-82a6-aad3529cb4a9.html',
+ 'md5': '8bb9c151924ce241b74dd52ef29ceafa',
+ 'info_dict': {
+ 'id': '04a9f4bd-b563-40cf-82a6-aad3529cb4a9',
+ 'ext': 'mp4',
+ 'title': 'TG PRIMO TEMPO',
+ 'description': '',
+ 'upload_date': '20140612',
+ 'duration': 1758,
+ },
+ 'skip': 'Error 404',
+ },
+ {
+ 'url': 'http://www.rainews.it/dl/rainews/media/state-of-the-net-Antonella-La-Carpia-regole-virali-7aafdea9-0e5d-49d5-88a6-7e65da67ae13.html',
+ 'md5': '35cf7c229f22eeef43e48b5cf923bef0',
+ 'info_dict': {
+ 'id': '7aafdea9-0e5d-49d5-88a6-7e65da67ae13',
+ 'ext': 'mp4',
+ 'title': 'State of the Net, Antonella La Carpia: regole virali',
+ 'description': 'md5:b0ba04a324126903e3da7763272ae63c',
+ 'upload_date': '20140613',
+ },
+ 'skip': 'Error 404',
+ },
+ {
+ 'url': 'http://www.rai.tv/dl/RaiTV/programmi/media/ContentItem-b4a49761-e0cc-4b14-8736-2729f6f73132-tg2.html',
+ 'md5': '35694f062977fe6619943f08ed935730',
+ 'info_dict': {
+ 'id': 'b4a49761-e0cc-4b14-8736-2729f6f73132',
+ 'ext': 'mp4',
+ 'title': 'Alluvione in Sardegna e dissesto idrogeologico',
+ 'description': 'Edizione delle ore 20:30 ',
+ }
+ },
+ ]
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+
+ media = self._download_json('%s?json' % mobj.group('url'), video_id, 'Downloading video JSON')
+
+ title = media.get('name')
+ description = media.get('desc')
+ thumbnail = media.get('image_300') or media.get('image_medium') or media.get('image')
+ duration = parse_duration(media.get('length'))
+ uploader = media.get('author')
+ upload_date = unified_strdate(media.get('date'))
+
+ formats = []
+
+ for format_id in ['wmv', 'm3u8', 'mediaUri', 'h264']:
+ media_url = media.get(format_id)
+ if not media_url:
+ continue
+ formats.append({
+ 'url': media_url,
+ 'format_id': format_id,
+ 'ext': 'mp4',
+ })
+
+ if self._downloader.params.get('listsubtitles', False):
+ page = self._download_webpage(url, video_id)
+ self._list_available_subtitles(video_id, page)
+ return
+
+ subtitles = {}
+ if self._have_to_download_any_subtitles:
+ page = self._download_webpage(url, video_id)
+ subtitles = self.extract_subtitles(video_id, page)
+
+ return {
+ 'id': video_id,
+ 'title': title,
+ 'description': description,
+ 'thumbnail': thumbnail,
+ 'uploader': uploader,
+ 'upload_date': upload_date,
+ 'duration': duration,
+ 'formats': formats,
+ 'subtitles': subtitles,
+ }
+
+ def _get_available_subtitles(self, video_id, webpage):
+ subtitles = {}
+ m = re.search(r'<meta name="closedcaption" content="(?P<captions>[^"]+)"', webpage)
+ if m:
+ captions = m.group('captions')
+ STL_EXT = '.stl'
+ SRT_EXT = '.srt'
+ if captions.endswith(STL_EXT):
+ captions = captions[:-len(STL_EXT)] + SRT_EXT
+ subtitles['it'] = 'http://www.rai.tv%s' % compat_urllib_parse.quote(captions)
+ return subtitles
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/rbmaradio.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/rbmaradio.py
new file mode 100644
index 0000000000..0f8f3ebde0
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/rbmaradio.py
@@ -0,0 +1,55 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+import json
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+ ExtractorError,
+)
+
+
+class RBMARadioIE(InfoExtractor):
+ _VALID_URL = r'https?://(?:www\.)?rbmaradio\.com/shows/(?P<videoID>[^/]+)$'
+ _TEST = {
+ 'url': 'http://www.rbmaradio.com/shows/ford-lopatin-live-at-primavera-sound-2011',
+ 'md5': '6bc6f9bcb18994b4c983bc3bf4384d95',
+ 'info_dict': {
+ 'id': 'ford-lopatin-live-at-primavera-sound-2011',
+ 'ext': 'mp3',
+ "uploader_id": "ford-lopatin",
+ "location": "Spain",
+ "description": "Joel Ford and Daniel Б─≥Oneohtrix Point NeverБ─≥ Lopatin fly their midified pop extravaganza to Spain. Live at Primavera Sound 2011.",
+ "uploader": "Ford & Lopatin",
+ "title": "Live at Primavera Sound 2011",
+ },
+ }
+
+ def _real_extract(self, url):
+ m = re.match(self._VALID_URL, url)
+ video_id = m.group('videoID')
+
+ webpage = self._download_webpage(url, video_id)
+
+ json_data = self._search_regex(r'window\.gon.*?gon\.show=(.+?);$',
+ webpage, 'json data', flags=re.MULTILINE)
+
+ try:
+ data = json.loads(json_data)
+ except ValueError as e:
+ raise ExtractorError('Invalid JSON: ' + str(e))
+
+ video_url = data['akamai_url'] + '&cbr=256'
+
+ return {
+ 'id': video_id,
+ 'url': video_url,
+ 'title': data['title'],
+ 'description': data.get('teaser_text'),
+ 'location': data.get('country_of_origin'),
+ 'uploader': data.get('host', {}).get('name'),
+ 'uploader_id': data.get('host', {}).get('slug'),
+ 'thumbnail': data.get('image', {}).get('large_url_2x'),
+ 'duration': data.get('duration'),
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/redtube.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/redtube.py
new file mode 100644
index 0000000000..846b76c815
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/redtube.py
@@ -0,0 +1,40 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class RedTubeIE(InfoExtractor):
+ _VALID_URL = r'http://(?:www\.)?redtube\.com/(?P<id>[0-9]+)'
+ _TEST = {
+ 'url': 'http://www.redtube.com/66418',
+ 'info_dict': {
+ 'id': '66418',
+ 'ext': 'mp4',
+ "title": "Sucked on a toilet",
+ "age_limit": 18,
+ }
+ }
+
+ def _real_extract(self, url):
+ video_id = self._match_id(url)
+ webpage = self._download_webpage(url, video_id)
+
+ video_url = self._html_search_regex(
+ r'<source src="(.+?)" type="video/mp4">', webpage, 'video URL')
+ video_title = self._html_search_regex(
+ r'<h1 class="videoTitle[^"]*">(.+?)</h1>',
+ webpage, 'title')
+ video_thumbnail = self._og_search_thumbnail(webpage)
+
+ # No self-labeling, but they describe themselves as
+ # "Home of Videos Porno"
+ age_limit = 18
+
+ return {
+ 'id': video_id,
+ 'url': video_url,
+ 'ext': 'mp4',
+ 'title': video_title,
+ 'thumbnail': video_thumbnail,
+ 'age_limit': age_limit,
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/restudy.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/restudy.py
new file mode 100644
index 0000000000..b17c2bfc06
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/restudy.py
@@ -0,0 +1,40 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class RestudyIE(InfoExtractor):
+ _VALID_URL = r'https?://(?:www\.)?restudy\.dk/video/play/id/(?P<id>[0-9]+)'
+ _TEST = {
+ 'url': 'https://www.restudy.dk/video/play/id/1637',
+ 'info_dict': {
+ 'id': '1637',
+ 'ext': 'flv',
+ 'title': 'Leiden-frosteffekt',
+ 'description': 'Denne video er et eksperiment med flydende kvц╕lstof.',
+ },
+ 'params': {
+ # rtmp download
+ 'skip_download': True,
+ }
+ }
+
+ def _real_extract(self, url):
+ video_id = self._match_id(url)
+
+ webpage = self._download_webpage(url, video_id)
+
+ title = self._og_search_title(webpage).strip()
+ description = self._og_search_description(webpage).strip()
+
+ formats = self._extract_smil_formats(
+ 'https://www.restudy.dk/awsmedia/SmilDirectory/video_%s.xml' % video_id,
+ video_id)
+
+ return {
+ 'id': video_id,
+ 'title': title,
+ 'description': description,
+ 'formats': formats,
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/reverbnation.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/reverbnation.py
new file mode 100644
index 0000000000..ec7e7df7bc
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/reverbnation.py
@@ -0,0 +1,44 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import str_or_none
+
+
+class ReverbNationIE(InfoExtractor):
+ _VALID_URL = r'^https?://(?:www\.)?reverbnation\.com/.*?/song/(?P<id>\d+).*?$'
+ _TESTS = [{
+ 'url': 'http://www.reverbnation.com/alkilados/song/16965047-mona-lisa',
+ 'md5': '3da12ebca28c67c111a7f8b262d3f7a7',
+ 'info_dict': {
+ "id": "16965047",
+ "ext": "mp3",
+ "title": "MONA LISA",
+ "uploader": "ALKILADOS",
+ "uploader_id": "216429",
+ "thumbnail": "re:^https://gp1\.wac\.edgecastcdn\.net/.*?\.jpg$"
+ },
+ }]
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ song_id = mobj.group('id')
+
+ api_res = self._download_json(
+ 'https://api.reverbnation.com/song/%s' % song_id,
+ song_id,
+ note='Downloading information of song %s' % song_id
+ )
+
+ return {
+ 'id': song_id,
+ 'title': api_res.get('name'),
+ 'url': api_res.get('url'),
+ 'uploader': api_res.get('artist', {}).get('name'),
+ 'uploader_id': str_or_none(api_res.get('artist', {}).get('id')),
+ 'thumbnail': self._proto_relative_url(
+ api_res.get('image', api_res.get('thumbnail'))),
+ 'ext': 'mp3',
+ 'vcodec': 'none',
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/ringtv.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/ringtv.py
new file mode 100644
index 0000000000..59dc137cc2
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/ringtv.py
@@ -0,0 +1,43 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+
+
+class RingTVIE(InfoExtractor):
+ _VALID_URL = r'(?:http://)?(?:www\.)?ringtv\.craveonline\.com/(?P<type>news|videos/video)/(?P<id>[^/?#]+)'
+ _TEST = {
+ "url": "http://ringtv.craveonline.com/news/310833-luis-collazo-says-victor-ortiz-better-not-quit-on-jan-30",
+ "file": "857645.mp4",
+ "md5": "d25945f5df41cdca2d2587165ac28720",
+ "info_dict": {
+ "title": 'Video: Luis Collazo says Victor Ortiz "better not quit on Jan. 30" - Ring TV',
+ "description": 'Luis Collazo is excited about his Jan. 30 showdown with fellow former welterweight titleholder Victor Ortiz at Barclays Center in his hometown of Brooklyn. The SuperBowl week fight headlines a Golden Boy Live! card on Fox Sports 1.',
+ }
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id').split('-')[0]
+ webpage = self._download_webpage(url, video_id)
+
+ if mobj.group('type') == 'news':
+ video_id = self._search_regex(
+ r'''(?x)<iframe[^>]+src="http://cms\.springboardplatform\.com/
+ embed_iframe/[0-9]+/video/([0-9]+)/''',
+ webpage, 'real video ID')
+ title = self._og_search_title(webpage)
+ description = self._html_search_regex(
+ r'addthis:description="([^"]+)"',
+ webpage, 'description', fatal=False)
+ final_url = "http://ringtv.craveonline.springboardplatform.com/storage/ringtv.craveonline.com/conversion/%s.mp4" % video_id
+ thumbnail_url = "http://ringtv.craveonline.springboardplatform.com/storage/ringtv.craveonline.com/snapshots/%s.jpg" % video_id
+
+ return {
+ 'id': video_id,
+ 'url': final_url,
+ 'title': title,
+ 'thumbnail': thumbnail_url,
+ 'description': description,
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/ro220.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/ro220.py
new file mode 100644
index 0000000000..962b524e94
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/ro220.py
@@ -0,0 +1,43 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..compat import compat_urllib_parse_unquote
+
+
+class Ro220IE(InfoExtractor):
+ IE_NAME = '220.ro'
+ _VALID_URL = r'(?x)(?:https?://)?(?:www\.)?220\.ro/(?P<category>[^/]+)/(?P<shorttitle>[^/]+)/(?P<id>[^/]+)'
+ _TEST = {
+ 'url': 'http://www.220.ro/sport/Luati-Le-Banii-Sez-4-Ep-1/LYV6doKo7f/',
+ 'md5': '03af18b73a07b4088753930db7a34add',
+ 'info_dict': {
+ 'id': 'LYV6doKo7f',
+ 'ext': 'mp4',
+ 'title': 'Luati-le Banii sez 4 ep 1',
+ 'description': 're:^Iata-ne reveniti dupa o binemeritata vacanta\. +Va astept si pe Facebook cu pareri si comentarii.$',
+ }
+ }
+
+ def _real_extract(self, url):
+ video_id = self._match_id(url)
+
+ webpage = self._download_webpage(url, video_id)
+ url = compat_urllib_parse_unquote(self._search_regex(
+ r'(?s)clip\s*:\s*{.*?url\s*:\s*\'([^\']+)\'', webpage, 'url'))
+ title = self._og_search_title(webpage)
+ description = self._og_search_description(webpage)
+ thumbnail = self._og_search_thumbnail(webpage)
+
+ formats = [{
+ 'format_id': 'sd',
+ 'url': url,
+ 'ext': 'mp4',
+ }]
+
+ return {
+ 'id': video_id,
+ 'formats': formats,
+ 'title': title,
+ 'description': description,
+ 'thumbnail': thumbnail,
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/rottentomatoes.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/rottentomatoes.py
new file mode 100644
index 0000000000..c1500b82fe
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/rottentomatoes.py
@@ -0,0 +1,18 @@
+from __future__ import unicode_literals
+
+from .videodetective import VideoDetectiveIE
+
+
+# It just uses the same method as videodetective.com,
+# the internetvideoarchive.com is extracted from the og:video property
+class RottenTomatoesIE(VideoDetectiveIE):
+ _VALID_URL = r'https?://www\.rottentomatoes\.com/m/[^/]+/trailers/(?P<id>\d+)'
+
+ _TEST = {
+ 'url': 'http://www.rottentomatoes.com/m/toy_story_3/trailers/11028566/',
+ 'file': '613340.mp4',
+ 'info_dict': {
+ 'title': 'TOY STORY 3',
+ 'description': 'From the creators of the beloved TOY STORY films, comes a story that will reunite the gang in a whole new way.',
+ },
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/roxwel.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/roxwel.py
new file mode 100644
index 0000000000..41638c1d01
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/roxwel.py
@@ -0,0 +1,53 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import unified_strdate, determine_ext
+
+
+class RoxwelIE(InfoExtractor):
+ _VALID_URL = r'https?://www\.roxwel\.com/player/(?P<filename>.+?)(\.|\?|$)'
+
+ _TEST = {
+ 'url': 'http://www.roxwel.com/player/passionpittakeawalklive.html',
+ 'info_dict': {
+ 'id': 'passionpittakeawalklive',
+ 'ext': 'flv',
+ 'title': 'Take A Walk (live)',
+ 'uploader': 'Passion Pit',
+ 'uploader_id': 'passionpit',
+ 'upload_date': '20120928',
+ 'description': 'Passion Pit performs "Take A Walk\" live at The Backyard in Austin, Texas. ',
+ },
+ 'params': {
+ # rtmp download
+ 'skip_download': True,
+ }
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ filename = mobj.group('filename')
+ info_url = 'http://www.roxwel.com/api/videos/%s' % filename
+ info = self._download_json(info_url, filename)
+
+ rtmp_rates = sorted([int(r.replace('flv_', '')) for r in info['media_rates'] if r.startswith('flv_')])
+ best_rate = rtmp_rates[-1]
+ url_page_url = 'http://roxwel.com/pl_one_time.php?filename=%s&quality=%s' % (filename, best_rate)
+ rtmp_url = self._download_webpage(url_page_url, filename, 'Downloading video url')
+ ext = determine_ext(rtmp_url)
+ if ext == 'f4v':
+ rtmp_url = rtmp_url.replace(filename, 'mp4:%s' % filename)
+
+ return {
+ 'id': filename,
+ 'title': info['title'],
+ 'url': rtmp_url,
+ 'ext': 'flv',
+ 'description': info['description'],
+ 'thumbnail': info.get('player_image_url') or info.get('image_url_large'),
+ 'uploader': info['artist'],
+ 'uploader_id': info['artistname'],
+ 'upload_date': unified_strdate(info['dbdate']),
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/rtbf.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/rtbf.py
new file mode 100644
index 0000000000..dce64e1517
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/rtbf.py
@@ -0,0 +1,49 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+import json
+
+from .common import InfoExtractor
+
+
+class RTBFIE(InfoExtractor):
+ _VALID_URL = r'https?://www.rtbf.be/video/[^\?]+\?id=(?P<id>\d+)'
+ _TEST = {
+ 'url': 'https://www.rtbf.be/video/detail_les-diables-au-coeur-episode-2?id=1921274',
+ 'md5': '799f334ddf2c0a582ba80c44655be570',
+ 'info_dict': {
+ 'id': '1921274',
+ 'ext': 'mp4',
+ 'title': 'Les Diables au coeur (ц╘pisode 2)',
+ 'description': 'Football - Diables Rouges',
+ 'duration': 3099,
+ 'timestamp': 1398456336,
+ 'upload_date': '20140425',
+ }
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+
+ page = self._download_webpage('https://www.rtbf.be/video/embed?id=%s' % video_id, video_id)
+
+ data = json.loads(self._html_search_regex(
+ r'<div class="js-player-embed(?: player-embed)?" data-video="([^"]+)"', page, 'data video'))['data']
+
+ video_url = data.get('downloadUrl') or data.get('url')
+
+ if data['provider'].lower() == 'youtube':
+ return self.url_result(video_url, 'Youtube')
+
+ return {
+ 'id': video_id,
+ 'url': video_url,
+ 'title': data['title'],
+ 'description': data.get('description') or data.get('subtitle'),
+ 'thumbnail': data['thumbnail']['large'],
+ 'duration': data.get('duration') or data.get('realDuration'),
+ 'timestamp': data['created'],
+ 'view_count': data['viewCount'],
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/rtlnl.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/rtlnl.py
new file mode 100644
index 0000000000..d029b0ec52
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/rtlnl.py
@@ -0,0 +1,71 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import parse_duration
+
+
+class RtlXlIE(InfoExtractor):
+ IE_NAME = 'rtlxl.nl'
+ _VALID_URL = r'https?://www\.rtlxl\.nl/#!/[^/]+/(?P<uuid>[^/?]+)'
+
+ _TEST = {
+ 'url': 'http://www.rtlxl.nl/#!/rtl-nieuws-132237/6e4203a6-0a5e-3596-8424-c599a59e0677',
+ 'md5': 'cc16baa36a6c169391f0764fa6b16654',
+ 'info_dict': {
+ 'id': '6e4203a6-0a5e-3596-8424-c599a59e0677',
+ 'ext': 'mp4',
+ 'title': 'RTL Nieuws - Laat',
+ 'description': 'md5:6b61f66510c8889923b11f2778c72dc5',
+ 'timestamp': 1408051800,
+ 'upload_date': '20140814',
+ 'duration': 576.880,
+ },
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ uuid = mobj.group('uuid')
+
+ info = self._download_json(
+ 'http://www.rtl.nl/system/s4m/vfd/version=2/uuid=%s/fmt=flash/' % uuid,
+ uuid)
+
+ material = info['material'][0]
+ episode_info = info['episodes'][0]
+
+ progname = info['abstracts'][0]['name']
+ subtitle = material['title'] or info['episodes'][0]['name']
+
+ # Use unencrypted m3u8 streams (See https://github.com/rg3/youtube-dl/issues/4118)
+ videopath = material['videopath'].replace('.f4m', '.m3u8')
+ m3u8_url = 'http://manifest.us.rtl.nl' + videopath
+
+ formats = self._extract_m3u8_formats(m3u8_url, uuid, ext='mp4')
+
+ video_urlpart = videopath.split('/flash/')[1][:-5]
+ PG_URL_TEMPLATE = 'http://pg.us.rtl.nl/rtlxl/network/%s/progressive/%s.mp4'
+
+ formats.extend([
+ {
+ 'url': PG_URL_TEMPLATE % ('a2m', video_urlpart),
+ 'format_id': 'pg-sd',
+ },
+ {
+ 'url': PG_URL_TEMPLATE % ('a3m', video_urlpart),
+ 'format_id': 'pg-hd',
+ 'quality': 0,
+ }
+ ])
+
+ self._sort_formats(formats)
+
+ return {
+ 'id': uuid,
+ 'title': '%s - %s' % (progname, subtitle),
+ 'formats': formats,
+ 'timestamp': material['original_date'],
+ 'description': episode_info['synopsis'],
+ 'duration': parse_duration(material.get('duration')),
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/rtlnow.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/rtlnow.py
new file mode 100644
index 0000000000..285c3c4beb
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/rtlnow.py
@@ -0,0 +1,156 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+ ExtractorError,
+ clean_html,
+ unified_strdate,
+ int_or_none,
+)
+
+
+class RTLnowIE(InfoExtractor):
+ """Information Extractor for RTL NOW, RTL2 NOW, RTL NITRO, SUPER RTL NOW, VOX NOW and n-tv NOW"""
+ _VALID_URL = r'''(?x)
+ (?:https?://)?
+ (?P<url>
+ (?P<domain>
+ rtl-now\.rtl\.de|
+ rtl2now\.rtl2\.de|
+ (?:www\.)?voxnow\.de|
+ (?:www\.)?rtlnitronow\.de|
+ (?:www\.)?superrtlnow\.de|
+ (?:www\.)?n-tvnow\.de)
+ /+[a-zA-Z0-9-]+/[a-zA-Z0-9-]+\.php\?
+ (?:container_id|film_id)=(?P<video_id>[0-9]+)&
+ player=1(?:&season=[0-9]+)?(?:&.*)?
+ )'''
+
+ _TESTS = [
+ {
+ 'url': 'http://rtl-now.rtl.de/ahornallee/folge-1.php?film_id=90419&player=1&season=1',
+ 'info_dict': {
+ 'id': '90419',
+ 'ext': 'flv',
+ 'title': 'Ahornallee - Folge 1 - Der Einzug',
+ 'description': 'md5:ce843b6b5901d9a7f7d04d1bbcdb12de',
+ 'upload_date': '20070416',
+ 'duration': 1685,
+ },
+ 'params': {
+ 'skip_download': True,
+ },
+ 'skip': 'Only works from Germany',
+ },
+ {
+ 'url': 'http://rtl2now.rtl2.de/aerger-im-revier/episode-15-teil-1.php?film_id=69756&player=1&season=2&index=5',
+ 'info_dict': {
+ 'id': '69756',
+ 'ext': 'flv',
+ 'title': 'ц└rger im Revier - Ein junger Ladendieb, ein handfester Streit u.a.',
+ 'description': 'md5:3fb247005ed21a935ffc82b7dfa70cf0',
+ 'thumbnail': 'http://autoimg.static-fra.de/rtl2now/219850/1500x1500/image2.jpg',
+ 'upload_date': '20120519',
+ 'duration': 1245,
+ },
+ 'params': {
+ 'skip_download': True,
+ },
+ 'skip': 'Only works from Germany',
+ },
+ {
+ 'url': 'http://www.voxnow.de/voxtours/suedafrika-reporter-ii.php?film_id=13883&player=1&season=17',
+ 'info_dict': {
+ 'id': '13883',
+ 'ext': 'flv',
+ 'title': 'Voxtours - Sц╪dafrika-Reporter II',
+ 'description': 'md5:de7f8d56be6fd4fed10f10f57786db00',
+ 'upload_date': '20090627',
+ 'duration': 1800,
+ },
+ 'params': {
+ 'skip_download': True,
+ },
+ },
+ {
+ 'url': 'http://superrtlnow.de/medicopter-117/angst.php?film_id=99205&player=1',
+ 'info_dict': {
+ 'id': '99205',
+ 'ext': 'flv',
+ 'title': 'Medicopter 117 - Angst!',
+ 'description': 're:^Im Therapiezentrum \'Sonnalm\' kommen durch eine Unachtsamkeit die fц╪r die B.handlung mit Phobikern gehaltenen Voglespinnen frei\. Eine Ausreiц÷erin',
+ 'thumbnail': 'http://autoimg.static-fra.de/superrtlnow/287529/1500x1500/image2.jpg',
+ 'upload_date': '20080928',
+ 'duration': 2691,
+ },
+ 'params': {
+ 'skip_download': True,
+ },
+ },
+ {
+ 'url': 'http://www.n-tvnow.de/deluxe-alles-was-spass-macht/thema-ua-luxushotel-fuer-vierbeiner.php?container_id=153819&player=1&season=0',
+ 'only_matching': True,
+ },
+ ]
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_page_url = 'http://%s/' % mobj.group('domain')
+ video_id = mobj.group('video_id')
+
+ webpage = self._download_webpage('http://' + mobj.group('url'), video_id)
+
+ mobj = re.search(r'(?s)<div style="margin-left: 20px; font-size: 13px;">(.*?)<div id="playerteaser">', webpage)
+ if mobj:
+ raise ExtractorError(clean_html(mobj.group(1)), expected=True)
+
+ title = self._og_search_title(webpage)
+ description = self._og_search_description(webpage)
+ thumbnail = self._og_search_thumbnail(webpage, default=None)
+
+ upload_date = unified_strdate(self._html_search_meta('uploadDate', webpage, 'upload date'))
+
+ mobj = re.search(r'<meta itemprop="duration" content="PT(?P<seconds>\d+)S" />', webpage)
+ duration = int(mobj.group('seconds')) if mobj else None
+
+ playerdata_url = self._html_search_regex(
+ r"'playerdata': '(?P<playerdata_url>[^']+)'", webpage, 'playerdata_url')
+
+ playerdata = self._download_xml(playerdata_url, video_id, 'Downloading player data XML')
+
+ videoinfo = playerdata.find('./playlist/videoinfo')
+
+ formats = []
+ for filename in videoinfo.findall('filename'):
+ mobj = re.search(r'(?P<url>rtmpe://(?:[^/]+/){2})(?P<play_path>.+)', filename.text)
+ if mobj:
+ fmt = {
+ 'url': mobj.group('url'),
+ 'play_path': 'mp4:' + mobj.group('play_path'),
+ 'page_url': video_page_url,
+ 'player_url': video_page_url + 'includes/vodplayer.swf',
+ }
+ else:
+ fmt = {
+ 'url': filename.text,
+ }
+ fmt.update({
+ 'width': int_or_none(filename.get('width')),
+ 'height': int_or_none(filename.get('height')),
+ 'vbr': int_or_none(filename.get('bitrate')),
+ 'ext': 'flv',
+ })
+ formats.append(fmt)
+
+ return {
+ 'id': video_id,
+ 'title': title,
+ 'description': description,
+ 'thumbnail': thumbnail,
+ 'upload_date': upload_date,
+ 'duration': duration,
+ 'formats': formats,
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/rtp.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/rtp.py
new file mode 100644
index 0000000000..7736cabbac
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/rtp.py
@@ -0,0 +1,60 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import json
+
+from .common import InfoExtractor
+from ..utils import js_to_json
+
+
+class RTPIE(InfoExtractor):
+ _VALID_URL = r'https?://(?:www\.)?rtp\.pt/play/p(?P<program_id>[0-9]+)/(?P<id>[^/?#]+)/?'
+ _TESTS = [{
+ 'url': 'http://www.rtp.pt/play/p405/e174042/paixoes-cruzadas',
+ 'info_dict': {
+ 'id': 'e174042',
+ 'ext': 'mp3',
+ 'title': 'Paixц╣es Cruzadas',
+ 'description': 'As paixц╣es musicais de AntцЁnio Cartaxo e AntцЁnio Macedo',
+ 'thumbnail': 're:^https?://.*\.jpg',
+ },
+ 'params': {
+ 'skip_download': True, # RTMP download
+ },
+ }, {
+ 'url': 'http://www.rtp.pt/play/p831/a-quimica-das-coisas',
+ 'only_matching': True,
+ }]
+
+ def _real_extract(self, url):
+ video_id = self._match_id(url)
+
+ webpage = self._download_webpage(url, video_id)
+ title = self._html_search_meta(
+ 'twitter:title', webpage, display_name='title', fatal=True)
+ description = self._html_search_meta('description', webpage)
+ thumbnail = self._og_search_thumbnail(webpage)
+
+ player_config = self._search_regex(
+ r'(?s)RTPPLAY\.player\.newPlayer\(\s*(\{.*?\})\s*\)', webpage, 'player config')
+ config = json.loads(js_to_json(player_config))
+
+ path, ext = config.get('file').rsplit('.', 1)
+ formats = [{
+ 'app': config.get('application'),
+ 'play_path': '{ext:s}:{path:s}'.format(ext=ext, path=path),
+ 'page_url': url,
+ 'url': 'rtmp://{streamer:s}/{application:s}'.format(**config),
+ 'rtmp_live': config.get('live', False),
+ 'ext': ext,
+ 'vcodec': config.get('type') == 'audio' and 'none' or None,
+ 'player_url': 'http://programas.rtp.pt/play/player.swf?v3',
+ }]
+
+ return {
+ 'id': video_id,
+ 'title': title,
+ 'formats': formats,
+ 'description': description,
+ 'thumbnail': thumbnail,
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/rts.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/rts.py
new file mode 100644
index 0000000000..5e84c10980
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/rts.py
@@ -0,0 +1,188 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..compat import (
+ compat_str,
+)
+from ..utils import (
+ int_or_none,
+ parse_duration,
+ parse_iso8601,
+ unescapeHTML,
+)
+
+
+class RTSIE(InfoExtractor):
+ IE_DESC = 'RTS.ch'
+ _VALID_URL = r'https?://(?:www\.)?rts\.ch/(?:(?:[^/]+/){2,}(?P<id>[0-9]+)-(?P<display_id>.+?)\.html|play/tv/[^/]+/video/(?P<display_id_new>.+?)\?id=(?P<id_new>[0-9]+))'
+
+ _TESTS = [
+ {
+ 'url': 'http://www.rts.ch/archives/tv/divers/3449373-les-enfants-terribles.html',
+ 'md5': '753b877968ad8afaeddccc374d4256a5',
+ 'info_dict': {
+ 'id': '3449373',
+ 'display_id': 'les-enfants-terribles',
+ 'ext': 'mp4',
+ 'duration': 1488,
+ 'title': 'Les Enfants Terribles',
+ 'description': 'France Pommier et sa soeur Luce Feral, les deux filles de ce groupe de 5.',
+ 'uploader': 'Divers',
+ 'upload_date': '19680921',
+ 'timestamp': -40280400,
+ 'thumbnail': 're:^https?://.*\.image',
+ 'view_count': int,
+ },
+ },
+ {
+ 'url': 'http://www.rts.ch/emissions/passe-moi-les-jumelles/5624067-entre-ciel-et-mer.html',
+ 'md5': 'c148457a27bdc9e5b1ffe081a7a8337b',
+ 'info_dict': {
+ 'id': '5624067',
+ 'display_id': 'entre-ciel-et-mer',
+ 'ext': 'mp4',
+ 'duration': 3720,
+ 'title': 'Les yeux dans les cieux - Mon homard au Canada',
+ 'description': 'md5:d22ee46f5cc5bac0912e5a0c6d44a9f7',
+ 'uploader': 'Passe-moi les jumelles',
+ 'upload_date': '20140404',
+ 'timestamp': 1396635300,
+ 'thumbnail': 're:^https?://.*\.image',
+ 'view_count': int,
+ },
+ },
+ {
+ 'url': 'http://www.rts.ch/video/sport/hockey/5745975-1-2-kloten-fribourg-5-2-second-but-pour-gotteron-par-kwiatowski.html',
+ 'md5': 'b4326fecd3eb64a458ba73c73e91299d',
+ 'info_dict': {
+ 'id': '5745975',
+ 'display_id': '1-2-kloten-fribourg-5-2-second-but-pour-gotteron-par-kwiatowski',
+ 'ext': 'mp4',
+ 'duration': 48,
+ 'title': '1/2, Kloten - Fribourg (5-2): second but pour Gottц╘ron par Kwiatowski',
+ 'description': 'Hockey - Playoff',
+ 'uploader': 'Hockey',
+ 'upload_date': '20140403',
+ 'timestamp': 1396556882,
+ 'thumbnail': 're:^https?://.*\.image',
+ 'view_count': int,
+ },
+ 'skip': 'Blocked outside Switzerland',
+ },
+ {
+ 'url': 'http://www.rts.ch/video/info/journal-continu/5745356-londres-cachee-par-un-epais-smog.html',
+ 'md5': '9bb06503773c07ce83d3cbd793cebb91',
+ 'info_dict': {
+ 'id': '5745356',
+ 'display_id': 'londres-cachee-par-un-epais-smog',
+ 'ext': 'mp4',
+ 'duration': 33,
+ 'title': 'Londres cachц╘e par un ц╘pais smog',
+ 'description': 'Un important voile de smog recouvre Londres depuis mercredi, provoquц╘ par la pollution et du sable du Sahara.',
+ 'uploader': 'Le Journal en continu',
+ 'upload_date': '20140403',
+ 'timestamp': 1396537322,
+ 'thumbnail': 're:^https?://.*\.image',
+ 'view_count': int,
+ },
+ },
+ {
+ 'url': 'http://www.rts.ch/audio/couleur3/programmes/la-belle-video-de-stephane-laurenceau/5706148-urban-hippie-de-damien-krisl-03-04-2014.html',
+ 'md5': 'dd8ef6a22dff163d063e2a52bc8adcae',
+ 'info_dict': {
+ 'id': '5706148',
+ 'display_id': 'urban-hippie-de-damien-krisl-03-04-2014',
+ 'ext': 'mp3',
+ 'duration': 123,
+ 'title': '"Urban Hippie", de Damien Krisl',
+ 'description': 'Des Hippies super glam.',
+ 'upload_date': '20140403',
+ 'timestamp': 1396551600,
+ },
+ },
+ {
+ 'url': 'http://www.rts.ch/play/tv/-/video/le-19h30?id=6348260',
+ 'md5': '968777c8779e5aa2434be96c54e19743',
+ 'info_dict': {
+ 'id': '6348260',
+ 'display_id': 'le-19h30',
+ 'ext': 'mp4',
+ 'duration': 1796,
+ 'title': 'Le 19h30',
+ 'description': '',
+ 'uploader': 'Le 19h30',
+ 'upload_date': '20141201',
+ 'timestamp': 1417458600,
+ 'thumbnail': 're:^https?://.*\.image',
+ 'view_count': int,
+ },
+ },
+ {
+ 'url': 'http://www.rts.ch/play/tv/le-19h30/video/le-chantier-du-nouveau-parlement-vaudois-a-permis-une-trouvaille-historique?id=6348280',
+ 'only_matching': True,
+ }
+ ]
+
+ def _real_extract(self, url):
+ m = re.match(self._VALID_URL, url)
+ video_id = m.group('id') or m.group('id_new')
+ display_id = m.group('display_id') or m.group('display_id_new')
+
+ def download_json(internal_id):
+ return self._download_json(
+ 'http://www.rts.ch/a/%s.html?f=json/article' % internal_id,
+ display_id)
+
+ all_info = download_json(video_id)
+
+ # video_id extracted out of URL is not always a real id
+ if 'video' not in all_info and 'audio' not in all_info:
+ page = self._download_webpage(url, display_id)
+ internal_id = self._html_search_regex(
+ r'<(?:video|audio) data-id="([0-9]+)"', page,
+ 'internal video id')
+ all_info = download_json(internal_id)
+
+ info = all_info['video']['JSONinfo'] if 'video' in all_info else all_info['audio']
+
+ upload_timestamp = parse_iso8601(info.get('broadcast_date'))
+ duration = info.get('duration') or info.get('cutout') or info.get('cutduration')
+ if isinstance(duration, compat_str):
+ duration = parse_duration(duration)
+ view_count = info.get('plays')
+ thumbnail = unescapeHTML(info.get('preview_image_url'))
+
+ def extract_bitrate(url):
+ return int_or_none(self._search_regex(
+ r'-([0-9]+)k\.', url, 'bitrate', default=None))
+
+ formats = [{
+ 'format_id': fid,
+ 'url': furl,
+ 'tbr': extract_bitrate(furl),
+ } for fid, furl in info['streams'].items()]
+
+ if 'media' in info:
+ formats.extend([{
+ 'format_id': '%s-%sk' % (media['ext'], media['rate']),
+ 'url': 'http://download-video.rts.ch/%s' % media['url'],
+ 'tbr': media['rate'] or extract_bitrate(media['url']),
+ } for media in info['media'] if media.get('rate')])
+
+ self._sort_formats(formats)
+
+ return {
+ 'id': video_id,
+ 'display_id': display_id,
+ 'formats': formats,
+ 'title': info['title'],
+ 'description': info.get('intro'),
+ 'duration': duration,
+ 'view_count': view_count,
+ 'uploader': info.get('programName'),
+ 'timestamp': upload_timestamp,
+ 'thumbnail': thumbnail,
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/rtve.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/rtve.py
new file mode 100644
index 0000000000..0ce22d60c7
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/rtve.py
@@ -0,0 +1,141 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+import base64
+import re
+import time
+
+from .common import InfoExtractor
+from ..utils import (
+ struct_unpack,
+ remove_end,
+)
+
+
+def _decrypt_url(png):
+ encrypted_data = base64.b64decode(png)
+ text_index = encrypted_data.find(b'tEXt')
+ text_chunk = encrypted_data[text_index - 4:]
+ length = struct_unpack('!I', text_chunk[:4])[0]
+ # Use bytearray to get integers when iterating in both python 2.x and 3.x
+ data = bytearray(text_chunk[8:8 + length])
+ data = [chr(b) for b in data if b != 0]
+ hash_index = data.index('#')
+ alphabet_data = data[:hash_index]
+ url_data = data[hash_index + 1:]
+
+ alphabet = []
+ e = 0
+ d = 0
+ for l in alphabet_data:
+ if d == 0:
+ alphabet.append(l)
+ d = e = (e + 1) % 4
+ else:
+ d -= 1
+ url = ''
+ f = 0
+ e = 3
+ b = 1
+ for letter in url_data:
+ if f == 0:
+ l = int(letter) * 10
+ f = 1
+ else:
+ if e == 0:
+ l += int(letter)
+ url += alphabet[l]
+ e = (b + 3) % 4
+ f = 0
+ b += 1
+ else:
+ e -= 1
+
+ return url
+
+
+class RTVEALaCartaIE(InfoExtractor):
+ IE_NAME = 'rtve.es:alacarta'
+ IE_DESC = 'RTVE a la carta'
+ _VALID_URL = r'http://www\.rtve\.es/alacarta/videos/[^/]+/[^/]+/(?P<id>\d+)'
+
+ _TESTS = [{
+ 'url': 'http://www.rtve.es/alacarta/videos/balonmano/o-swiss-cup-masculina-final-espana-suecia/2491869/',
+ 'md5': '1d49b7e1ca7a7502c56a4bf1b60f1b43',
+ 'info_dict': {
+ 'id': '2491869',
+ 'ext': 'mp4',
+ 'title': 'Balonmano - Swiss Cup masculina. Final: Espaц╠a-Suecia',
+ },
+ }, {
+ 'note': 'Live stream',
+ 'url': 'http://www.rtve.es/alacarta/videos/television/24h-live/1694255/',
+ 'info_dict': {
+ 'id': '1694255',
+ 'ext': 'flv',
+ 'title': 'TODO',
+ }
+ }]
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+ info = self._download_json(
+ 'http://www.rtve.es/api/videos/%s/config/alacarta_videos.json' % video_id,
+ video_id)['page']['items'][0]
+ png_url = 'http://www.rtve.es/ztnr/movil/thumbnail/default/videos/%s.png' % video_id
+ png = self._download_webpage(png_url, video_id, 'Downloading url information')
+ video_url = _decrypt_url(png)
+
+ return {
+ 'id': video_id,
+ 'title': info['title'],
+ 'url': video_url,
+ 'thumbnail': info.get('image'),
+ 'page_url': url,
+ }
+
+
+class RTVELiveIE(InfoExtractor):
+ IE_NAME = 'rtve.es:live'
+ IE_DESC = 'RTVE.es live streams'
+ _VALID_URL = r'http://www\.rtve\.es/(?:deportes/directo|noticias|television)/(?P<id>[a-zA-Z0-9-]+)'
+
+ _TESTS = [{
+ 'url': 'http://www.rtve.es/noticias/directo-la-1/',
+ 'info_dict': {
+ 'id': 'directo-la-1',
+ 'ext': 'flv',
+ 'title': 're:^La 1 de TVE [0-9]{4}-[0-9]{2}-[0-9]{2}Z[0-9]{6}$',
+ },
+ 'params': {
+ 'skip_download': 'live stream',
+ }
+ }]
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ start_time = time.gmtime()
+ video_id = mobj.group('id')
+
+ webpage = self._download_webpage(url, video_id)
+ player_url = self._search_regex(
+ r'<param name="movie" value="([^"]+)"/>', webpage, 'player URL')
+ title = remove_end(self._og_search_title(webpage), ' en directo')
+ title += ' ' + time.strftime('%Y-%m-%dZ%H%M%S', start_time)
+
+ vidplayer_id = self._search_regex(
+ r' id="vidplayer([0-9]+)"', webpage, 'internal video ID')
+ png_url = 'http://www.rtve.es/ztnr/movil/thumbnail/default/videos/%s.png' % vidplayer_id
+ png = self._download_webpage(png_url, video_id, 'Downloading url information')
+ video_url = _decrypt_url(png)
+
+ return {
+ 'id': video_id,
+ 'ext': 'flv',
+ 'title': title,
+ 'url': video_url,
+ 'app': 'rtve-live-live?ovpfv=2.1.2',
+ 'player_url': player_url,
+ 'rtmp_live': True,
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/ruhd.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/ruhd.py
new file mode 100644
index 0000000000..0e470e73f5
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/ruhd.py
@@ -0,0 +1,45 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class RUHDIE(InfoExtractor):
+ _VALID_URL = r'http://(?:www\.)?ruhd\.ru/play\.php\?vid=(?P<id>\d+)'
+ _TEST = {
+ 'url': 'http://www.ruhd.ru/play.php?vid=207',
+ 'md5': 'd1a9ec4edf8598e3fbd92bb16072ba83',
+ 'info_dict': {
+ 'id': '207',
+ 'ext': 'divx',
+ 'title': 'п п·п╒ п╠п╟п╟п╟п╟п╟п╪',
+ 'description': 'п╨п╩п╟я│я│п╫я▀п╧ п╨п╬я┌)',
+ 'thumbnail': 're:^http://.*\.jpg$',
+ }
+ }
+
+ def _real_extract(self, url):
+ video_id = self._match_id(url)
+ webpage = self._download_webpage(url, video_id)
+
+ video_url = self._html_search_regex(
+ r'<param name="src" value="([^"]+)"', webpage, 'video url')
+ title = self._html_search_regex(
+ r'<title>([^<]+) RUHD.ru - п▓п╦п╢п╣п╬ п▓я▀я│п╬п╨п╬пЁп╬ п╨п╟я┤п╣я│я┌п╡п╟ Б└√1 п╡ п═п╬я│я│п╦п╦! ',
+ webpage, 'title')
+ description = self._html_search_regex(
+ r'(?s)(.+?)
',
+ webpage, 'description', fatal=False)
+ thumbnail = self._html_search_regex(
+ r'\d+)'
+ _TESTS = [{
+ 'url': 'http://rutube.ru/tags/video/1800/',
+ 'info_dict': {
+ 'id': '1800',
+ },
+ 'playlist_mincount': 68,
+ }]
+
+ _PAGE_TEMPLATE = 'http://rutube.ru/api/tags/video/%s/?page=%s&format=json'
+
+ def _extract_videos(self, channel_id, channel_title=None):
+ entries = []
+ for pagenum in itertools.count(1):
+ page = self._download_json(
+ self._PAGE_TEMPLATE % (channel_id, pagenum),
+ channel_id, 'Downloading page %s' % pagenum)
+ results = page['results']
+ if not results:
+ break
+ entries.extend(self.url_result(result['video_url'], 'Rutube') for result in results)
+ if not page['has_next']:
+ break
+ return self.playlist_result(entries, channel_id, channel_title)
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ channel_id = mobj.group('id')
+ return self._extract_videos(channel_id)
+
+
+class RutubeMovieIE(RutubeChannelIE):
+ IE_NAME = 'rutube:movie'
+ IE_DESC = 'Rutube movies'
+ _VALID_URL = r'http://rutube\.ru/metainfo/tv/(?P\d+)'
+ _TESTS = []
+
+ _MOVIE_TEMPLATE = 'http://rutube.ru/api/metainfo/tv/%s/?format=json'
+ _PAGE_TEMPLATE = 'http://rutube.ru/api/metainfo/tv/%s/video?page=%s&format=json'
+
+ def _real_extract(self, url):
+ movie_id = self._match_id(url)
+ movie = self._download_json(
+ self._MOVIE_TEMPLATE % movie_id, movie_id,
+ 'Downloading movie JSON')
+ movie_name = movie['name']
+ return self._extract_videos(movie_id, movie_name)
+
+
+class RutubePersonIE(RutubeChannelIE):
+ IE_NAME = 'rutube:person'
+ IE_DESC = 'Rutube person videos'
+ _VALID_URL = r'http://rutube\.ru/video/person/(?P\d+)'
+ _TESTS = [{
+ 'url': 'http://rutube.ru/video/person/313878/',
+ 'info_dict': {
+ 'id': '313878',
+ },
+ 'playlist_mincount': 37,
+ }]
+
+ _PAGE_TEMPLATE = 'http://rutube.ru/api/video/person/%s/?page=%s&format=json'
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/rutv.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/rutv.py
new file mode 100644
index 0000000000..a73e6f331f
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/rutv.py
@@ -0,0 +1,194 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+ ExtractorError,
+ int_or_none
+)
+
+
+class RUTVIE(InfoExtractor):
+ IE_DESC = 'RUTV.RU'
+ _VALID_URL = r'''(?x)
+ https?://player\.(?:rutv\.ru|vgtrk\.com)/
+ (?Pflash2v/container\.swf\?id=
+ |iframe/(?Pswf|video|live)/id/
+ |index/iframe/cast_id/)
+ (?P\d+)'''
+
+ _TESTS = [
+ {
+ 'url': 'http://player.rutv.ru/flash2v/container.swf?id=774471&sid=kultura&fbv=true&isPlay=true&ssl=false&i=560&acc_video_id=episode_id/972347/video_id/978186/brand_id/31724',
+ 'info_dict': {
+ 'id': '774471',
+ 'ext': 'mp4',
+ 'title': 'п°п╬п╫п╬п╩п╬пЁп╦ п╫п╟ п╡я│п╣ п╡я─п╣п╪п╣п╫п╟',
+ 'description': 'md5:18d8b5e6a41fb1faa53819471852d5d5',
+ 'duration': 2906,
+ },
+ 'params': {
+ # m3u8 download
+ 'skip_download': True,
+ },
+ },
+ {
+ 'url': 'https://player.vgtrk.com/flash2v/container.swf?id=774016&sid=russiatv&fbv=true&isPlay=true&ssl=false&i=560&acc_video_id=episode_id/972098/video_id/977760/brand_id/57638',
+ 'info_dict': {
+ 'id': '774016',
+ 'ext': 'mp4',
+ 'title': 'п╖я┐п╤п╬п╧ п╡ я│п╣п╪я▄п╣ п║я┌п╟п╩п╦п╫п╟',
+ 'description': '',
+ 'duration': 2539,
+ },
+ 'params': {
+ # m3u8 download
+ 'skip_download': True,
+ },
+ },
+ {
+ 'url': 'http://player.rutv.ru/iframe/swf/id/766888/sid/hitech/?acc_video_id=4000',
+ 'info_dict': {
+ 'id': '766888',
+ 'ext': 'mp4',
+ 'title': 'п▓п╣я│я┌п╦.net: п╦п╫я┌п╣я─п╫п╣я┌-пЁп╦пЁп╟п╫я┌я▀ п╫п╟я┤п╟п╩п╦ п©п╣я─п╣я┌я▐пЁп╦п╡п╟п╫п╦п╣ п©я─п╬пЁя─п╟п╪п╪п╫я▀я┘ "п╬п╢п╣я▐п╩"',
+ 'description': 'md5:65ddd47f9830c4f42ed6475f8730c995',
+ 'duration': 279,
+ },
+ 'params': {
+ # m3u8 download
+ 'skip_download': True,
+ },
+ },
+ {
+ 'url': 'http://player.rutv.ru/iframe/video/id/771852/start_zoom/true/showZoomBtn/false/sid/russiatv/?acc_video_id=episode_id/970443/video_id/975648/brand_id/5169',
+ 'info_dict': {
+ 'id': '771852',
+ 'ext': 'mp4',
+ 'title': 'п÷я─я▐п╪п╬п╧ я█я└п╦я─. п√п╣я─я┌п╡я▀ п╥п╟пЁп╟п╢п╬я┤п╫п╬п╧ п╠п╬п╩п╣п╥п╫п╦: я│п╪п╣я─я┌я▄ п╬я┌ я│я┌п╟я─п╬я│я┌п╦ п╡ 17 п╩п╣я┌',
+ 'description': 'md5:b81c8c55247a4bd996b43ce17395b2d8',
+ 'duration': 3096,
+ },
+ 'params': {
+ # m3u8 download
+ 'skip_download': True,
+ },
+ },
+ {
+ 'url': 'http://player.rutv.ru/iframe/live/id/51499/showZoomBtn/false/isPlay/true/sid/sochi2014',
+ 'info_dict': {
+ 'id': '51499',
+ 'ext': 'flv',
+ 'title': 'п║п╬я┤п╦-2014. п▒п╦п╟я┌п╩п╬п╫. п≤п╫п╢п╦п╡п╦п╢я┐п╟п╩я▄п╫п╟я▐ пЁп╬п╫п╨п╟. п°я┐п╤я┤п╦п╫я▀ ',
+ 'description': 'md5:9e0ed5c9d2fa1efbfdfed90c9a6d179c',
+ },
+ 'params': {
+ # rtmp download
+ 'skip_download': True,
+ },
+ 'skip': 'Translation has finished',
+ },
+ ]
+
+ @classmethod
+ def _extract_url(cls, webpage):
+ mobj = re.search(
+ r']+?src=(["\'])(?Phttps?://player\.rutv\.ru/(?:iframe/(?:swf|video|live)/id|index/iframe/cast_id)/.+?)\1', webpage)
+ if mobj:
+ return mobj.group('url')
+
+ mobj = re.search(
+ r']+?property=(["\'])og:video\1[^>]+?content=(["\'])(?Phttps?://player\.(?:rutv\.ru|vgtrk\.com)/flash2v/container\.swf\?id=.+?\2)',
+ webpage)
+ if mobj:
+ return mobj.group('url')
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+ video_path = mobj.group('path')
+
+ if video_path.startswith('flash2v'):
+ video_type = 'video'
+ elif video_path.startswith('iframe'):
+ video_type = mobj.group('type')
+ if video_type == 'swf':
+ video_type = 'video'
+ elif video_path.startswith('index/iframe/cast_id'):
+ video_type = 'live'
+
+ json_data = self._download_json(
+ 'http://player.rutv.ru/iframe/%splay/id/%s' % ('live-' if video_type == 'live' else '', video_id),
+ video_id, 'Downloading JSON')
+
+ if json_data['errors']:
+ raise ExtractorError('%s said: %s' % (self.IE_NAME, json_data['errors']), expected=True)
+
+ playlist = json_data['data']['playlist']
+ medialist = playlist['medialist']
+ media = medialist[0]
+
+ if media['errors']:
+ raise ExtractorError('%s said: %s' % (self.IE_NAME, media['errors']), expected=True)
+
+ view_count = playlist.get('count_views')
+ priority_transport = playlist['priority_transport']
+
+ thumbnail = media['picture']
+ width = int_or_none(media['width'])
+ height = int_or_none(media['height'])
+ description = media['anons']
+ title = media['title']
+ duration = int_or_none(media.get('duration'))
+
+ formats = []
+
+ for transport, links in media['sources'].items():
+ for quality, url in links.items():
+ if transport == 'rtmp':
+ mobj = re.search(r'^(?Prtmp://[^/]+/(?P.+))/(?P.+)$', url)
+ if not mobj:
+ continue
+ fmt = {
+ 'url': mobj.group('url'),
+ 'play_path': mobj.group('playpath'),
+ 'app': mobj.group('app'),
+ 'page_url': 'http://player.rutv.ru',
+ 'player_url': 'http://player.rutv.ru/flash2v/osmf.swf?i=22',
+ 'rtmp_live': True,
+ 'ext': 'flv',
+ 'vbr': int(quality),
+ }
+ elif transport == 'm3u8':
+ fmt = {
+ 'url': url,
+ 'ext': 'mp4',
+ }
+ else:
+ fmt = {
+ 'url': url
+ }
+ fmt.update({
+ 'width': width,
+ 'height': height,
+ 'format_id': '%s-%s' % (transport, quality),
+ 'preference': -1 if priority_transport == transport else -2,
+ })
+ formats.append(fmt)
+
+ if not formats:
+ raise ExtractorError('No media links available for %s' % video_id)
+
+ self._sort_formats(formats)
+
+ return {
+ 'id': video_id,
+ 'title': title,
+ 'description': description,
+ 'thumbnail': thumbnail,
+ 'view_count': view_count,
+ 'duration': duration,
+ 'formats': formats,
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/sapo.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/sapo.py
new file mode 100644
index 0000000000..172cc12752
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/sapo.py
@@ -0,0 +1,119 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+ parse_duration,
+ unified_strdate,
+)
+
+
+class SapoIE(InfoExtractor):
+ IE_DESC = 'SAPO Vц╜deos'
+ _VALID_URL = r'https?://(?:(?:v2|www)\.)?videos\.sapo\.(?:pt|cv|ao|mz|tl)/(?P[\da-zA-Z]{20})'
+
+ _TESTS = [
+ {
+ 'url': 'http://videos.sapo.pt/UBz95kOtiWYUMTA5Ghfi',
+ 'md5': '79ee523f6ecb9233ac25075dee0eda83',
+ 'note': 'SD video',
+ 'info_dict': {
+ 'id': 'UBz95kOtiWYUMTA5Ghfi',
+ 'ext': 'mp4',
+ 'title': 'Benfica - Marcas na HitцЁria',
+ 'description': 'md5:c9082000a128c3fd57bf0299e1367f22',
+ 'duration': 264,
+ 'uploader': 'tiago_1988',
+ 'upload_date': '20080229',
+ 'categories': ['benfica', 'cabral', 'desporto', 'futebol', 'geovanni', 'hooijdonk', 'joao', 'karel', 'lisboa', 'miccoli'],
+ },
+ },
+ {
+ 'url': 'http://videos.sapo.pt/IyusNAZ791ZdoCY5H5IF',
+ 'md5': '90a2f283cfb49193fe06e861613a72aa',
+ 'note': 'HD video',
+ 'info_dict': {
+ 'id': 'IyusNAZ791ZdoCY5H5IF',
+ 'ext': 'mp4',
+ 'title': 'Codebits VII - Report',
+ 'description': 'md5:6448d6fd81ce86feac05321f354dbdc8',
+ 'duration': 144,
+ 'uploader': 'codebits',
+ 'upload_date': '20140427',
+ 'categories': ['codebits', 'codebits2014'],
+ },
+ },
+ {
+ 'url': 'http://v2.videos.sapo.pt/yLqjzPtbTimsn2wWBKHz',
+ 'md5': 'e5aa7cc0bdc6db9b33df1a48e49a15ac',
+ 'note': 'v2 video',
+ 'info_dict': {
+ 'id': 'yLqjzPtbTimsn2wWBKHz',
+ 'ext': 'mp4',
+ 'title': 'Hipnose Condicionativa 4',
+ 'description': 'md5:ef0481abf8fb4ae6f525088a6dadbc40',
+ 'duration': 692,
+ 'uploader': 'sapozen',
+ 'upload_date': '20090609',
+ 'categories': ['condicionativa', 'heloisa', 'hipnose', 'miranda', 'sapo', 'zen'],
+ },
+ },
+ ]
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+
+ item = self._download_xml(
+ 'http://rd3.videos.sapo.pt/%s/rss2' % video_id, video_id).find('./channel/item')
+
+ title = item.find('./title').text
+ description = item.find('./{http://videos.sapo.pt/mrss/}synopse').text
+ thumbnail = item.find('./{http://search.yahoo.com/mrss/}content').get('url')
+ duration = parse_duration(item.find('./{http://videos.sapo.pt/mrss/}time').text)
+ uploader = item.find('./{http://videos.sapo.pt/mrss/}author').text
+ upload_date = unified_strdate(item.find('./pubDate').text)
+ view_count = int(item.find('./{http://videos.sapo.pt/mrss/}views').text)
+ comment_count = int(item.find('./{http://videos.sapo.pt/mrss/}comment_count').text)
+ tags = item.find('./{http://videos.sapo.pt/mrss/}tags').text
+ categories = tags.split() if tags else []
+ age_limit = 18 if item.find('./{http://videos.sapo.pt/mrss/}m18').text == 'true' else 0
+
+ video_url = item.find('./{http://videos.sapo.pt/mrss/}videoFile').text
+ video_size = item.find('./{http://videos.sapo.pt/mrss/}videoSize').text.split('x')
+
+ formats = [{
+ 'url': video_url,
+ 'ext': 'mp4',
+ 'format_id': 'sd',
+ 'width': int(video_size[0]),
+ 'height': int(video_size[1]),
+ }]
+
+ if item.find('./{http://videos.sapo.pt/mrss/}HD').text == 'true':
+ formats.append({
+ 'url': re.sub(r'/mov/1$', '/mov/39', video_url),
+ 'ext': 'mp4',
+ 'format_id': 'hd',
+ 'width': 1280,
+ 'height': 720,
+ })
+
+ self._sort_formats(formats)
+
+ return {
+ 'id': video_id,
+ 'title': title,
+ 'description': description,
+ 'thumbnail': thumbnail,
+ 'duration': duration,
+ 'uploader': uploader,
+ 'upload_date': upload_date,
+ 'view_count': view_count,
+ 'comment_count': comment_count,
+ 'categories': categories,
+ 'age_limit': age_limit,
+ 'formats': formats,
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/savefrom.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/savefrom.py
new file mode 100644
index 0000000000..5b7367b941
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/savefrom.py
@@ -0,0 +1,37 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import os.path
+import re
+
+from .common import InfoExtractor
+
+
+class SaveFromIE(InfoExtractor):
+ IE_NAME = 'savefrom.net'
+ _VALID_URL = r'https?://[^.]+\.savefrom\.net/\#url=(?P.*)$'
+
+ _TEST = {
+ 'url': 'http://en.savefrom.net/#url=http://youtube.com/watch?v=UlVRAPW2WJY&utm_source=youtube.com&utm_medium=short_domains&utm_campaign=ssyoutube.com',
+ 'info_dict': {
+ 'id': 'UlVRAPW2WJY',
+ 'ext': 'mp4',
+ 'title': 'About Team Radical MMA | MMA Fighting',
+ 'upload_date': '20120816',
+ 'uploader': 'Howcast',
+ 'uploader_id': 'Howcast',
+ 'description': 're:(?s).* Hi, my name is Rene Dreifuss\. And I\'m here to show you some MMA.*',
+ },
+ 'params': {
+ 'skip_download': True
+ }
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = os.path.splitext(url.split('/')[-1])[0]
+ return {
+ '_type': 'url',
+ 'id': video_id,
+ 'url': mobj.group('url'),
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/sbs.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/sbs.py
new file mode 100644
index 0000000000..b8775c2f99
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/sbs.py
@@ -0,0 +1,59 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+import json
+import re
+from .common import InfoExtractor
+from ..utils import (
+ js_to_json,
+ remove_end,
+)
+
+
+class SBSIE(InfoExtractor):
+ IE_DESC = 'sbs.com.au'
+ _VALID_URL = r'https?://(?:www\.)?sbs\.com\.au/ondemand/video/(?:single/)?(?P[0-9]+)'
+
+ _TESTS = [{
+ # Original URL is handled by the generic IE which finds the iframe:
+ # http://www.sbs.com.au/thefeed/blog/2014/08/21/dingo-conservation
+ 'url': 'http://www.sbs.com.au/ondemand/video/single/320403011771/?source=drupal&vertical=thefeed',
+ 'md5': '3150cf278965eeabb5b4cea1c963fe0a',
+ 'info_dict': {
+ 'id': '320403011771',
+ 'ext': 'mp4',
+ 'title': 'Dingo Conservation',
+ 'description': 'Dingoes are on the brink of extinction; most of the animals we think are dingoes are in fact crossbred with wild dogs. This family run a dingo conservation park to prevent their extinction',
+ 'thumbnail': 're:http://.*\.jpg',
+ },
+ 'add_ies': ['generic'],
+ }, {
+ 'url': 'http://www.sbs.com.au/ondemand/video/320403011771/Dingo-Conservation-The-Feed',
+ 'only_matching': True,
+ }]
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+ webpage = self._download_webpage(url, video_id)
+
+ release_urls_json = js_to_json(self._search_regex(
+ r'(?s)playerParams\.releaseUrls\s*=\s*(\{.*?\n\});\n',
+ webpage, ''))
+ release_urls = json.loads(release_urls_json)
+ theplatform_url = (
+ release_urls.get('progressive') or release_urls.get('standard'))
+
+ title = remove_end(self._og_search_title(webpage), ' (The Feed)')
+ description = self._html_search_meta('description', webpage)
+ thumbnail = self._og_search_thumbnail(webpage)
+
+ return {
+ '_type': 'url_transparent',
+ 'id': video_id,
+ 'url': theplatform_url,
+
+ 'title': title,
+ 'description': description,
+ 'thumbnail': thumbnail,
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/scivee.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/scivee.py
new file mode 100644
index 0000000000..3bf93c870b
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/scivee.py
@@ -0,0 +1,56 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import int_or_none
+
+
+class SciVeeIE(InfoExtractor):
+ _VALID_URL = r'https?://(?:www\.)?scivee\.tv/node/(?P\d+)'
+
+ _TEST = {
+ 'url': 'http://www.scivee.tv/node/62352',
+ 'md5': 'b16699b74c9e6a120f6772a44960304f',
+ 'info_dict': {
+ 'id': '62352',
+ 'ext': 'mp4',
+ 'title': 'Adam Arkin at the 2014 DOE JGI Genomics of Energy & Environment Meeting',
+ 'description': 'md5:81f1710638e11a481358fab1b11059d7',
+ },
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+
+ # annotations XML is malformed
+ annotations = self._download_webpage(
+ 'http://www.scivee.tv/assets/annotations/%s' % video_id, video_id, 'Downloading annotations')
+
+ title = self._html_search_regex(r'([^<]+) ', annotations, 'title')
+ description = self._html_search_regex(r'([^<]+) ', annotations, 'abstract', fatal=False)
+ filesize = int_or_none(self._html_search_regex(
+ r'([^<]+) ', annotations, 'filesize', fatal=False))
+
+ formats = [
+ {
+ 'url': 'http://www.scivee.tv/assets/audio/%s' % video_id,
+ 'ext': 'mp3',
+ 'format_id': 'audio',
+ },
+ {
+ 'url': 'http://www.scivee.tv/assets/video/%s' % video_id,
+ 'ext': 'mp4',
+ 'format_id': 'video',
+ 'filesize': filesize,
+ },
+ ]
+
+ return {
+ 'id': video_id,
+ 'title': title,
+ 'description': description,
+ 'thumbnail': 'http://www.scivee.tv/assets/videothumb/%s' % video_id,
+ 'formats': formats,
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/screencast.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/screencast.py
new file mode 100644
index 0000000000..dfd897ba3a
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/screencast.py
@@ -0,0 +1,111 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..compat import (
+ compat_parse_qs,
+ compat_urllib_request,
+)
+from ..utils import (
+ ExtractorError,
+)
+
+
+class ScreencastIE(InfoExtractor):
+ _VALID_URL = r'https?://www\.screencast\.com/t/(?P[a-zA-Z0-9]+)'
+ _TESTS = [{
+ 'url': 'http://www.screencast.com/t/3ZEjQXlT',
+ 'md5': '917df1c13798a3e96211dd1561fded83',
+ 'info_dict': {
+ 'id': '3ZEjQXlT',
+ 'ext': 'm4v',
+ 'title': 'Color Measurement with Ocean Optics Spectrometers',
+ 'description': 'md5:240369cde69d8bed61349a199c5fb153',
+ 'thumbnail': 're:^https?://.*\.(?:gif|jpg)$',
+ }
+ }, {
+ 'url': 'http://www.screencast.com/t/V2uXehPJa1ZI',
+ 'md5': 'e8e4b375a7660a9e7e35c33973410d34',
+ 'info_dict': {
+ 'id': 'V2uXehPJa1ZI',
+ 'ext': 'mov',
+ 'title': 'The Amadeus Spectrometer',
+ 'description': 're:^In this video, our friends at.*To learn more about Amadeus, visit',
+ 'thumbnail': 're:^https?://.*\.(?:gif|jpg)$',
+ }
+ }, {
+ 'url': 'http://www.screencast.com/t/aAB3iowa',
+ 'md5': 'dedb2734ed00c9755761ccaee88527cd',
+ 'info_dict': {
+ 'id': 'aAB3iowa',
+ 'ext': 'mp4',
+ 'title': 'Google Earth Export',
+ 'description': 'Provides a demo of a CommunityViz export to Google Earth, one of the 3D viewing options.',
+ 'thumbnail': 're:^https?://.*\.(?:gif|jpg)$',
+ }
+ }, {
+ 'url': 'http://www.screencast.com/t/X3ddTrYh',
+ 'md5': '669ee55ff9c51988b4ebc0877cc8b159',
+ 'info_dict': {
+ 'id': 'X3ddTrYh',
+ 'ext': 'wmv',
+ 'title': 'Toolkit 6 User Group Webinar (2014-03-04) - Default Judgment and First Impression',
+ 'description': 'md5:7b9f393bc92af02326a5c5889639eab0',
+ 'thumbnail': 're:^https?://.*\.(?:gif|jpg)$',
+ }
+ },
+ ]
+
+ def _real_extract(self, url):
+ video_id = self._match_id(url)
+ webpage = self._download_webpage(url, video_id)
+
+ video_url = self._html_search_regex(
+ r'Title: ([^<]*) (.*?)<'],
+ webpage, 'title')
+ thumbnail = self._og_search_thumbnail(webpage)
+ description = self._og_search_description(webpage, default=None)
+ if description is None:
+ description = self._html_search_meta('description', webpage)
+
+ return {
+ 'id': video_id,
+ 'url': video_url,
+ 'title': title,
+ 'description': description,
+ 'thumbnail': thumbnail,
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/screencastomatic.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/screencastomatic.py
new file mode 100644
index 0000000000..05337421ca
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/screencastomatic.py
@@ -0,0 +1,49 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..compat import compat_urlparse
+from ..utils import (
+ ExtractorError,
+ js_to_json,
+)
+
+
+class ScreencastOMaticIE(InfoExtractor):
+ _VALID_URL = r'https?://screencast-o-matic\.com/watch/(?P[0-9a-zA-Z]+)'
+ _TEST = {
+ 'url': 'http://screencast-o-matic.com/watch/c2lD3BeOPl',
+ 'md5': '483583cb80d92588f15ccbedd90f0c18',
+ 'info_dict': {
+ 'id': 'c2lD3BeOPl',
+ 'ext': 'mp4',
+ 'title': 'Welcome to 3-4 Philosophy @ DECV!',
+ 'thumbnail': 're:^https?://.*\.jpg$',
+ 'description': 'as the title says! also: some general info re 1) VCE philosophy and 2) distance learning.',
+ }
+ }
+
+ def _real_extract(self, url):
+ video_id = self._match_id(url)
+ webpage = self._download_webpage(url, video_id)
+
+ setup_js = self._search_regex(
+ r"(?s)jwplayer\('mp4Player'\).setup\((\{.*?\})\);",
+ webpage, 'setup code')
+ data = self._parse_json(setup_js, video_id, transform_source=js_to_json)
+ try:
+ video_data = next(
+ m for m in data['modes'] if m.get('type') == 'html5')
+ except StopIteration:
+ raise ExtractorError('Could not find any video entries!')
+ video_url = compat_urlparse.urljoin(url, video_data['config']['file'])
+ thumbnail = data.get('image')
+
+ return {
+ 'id': video_id,
+ 'title': self._og_search_title(webpage),
+ 'description': self._og_search_description(webpage),
+ 'url': video_url,
+ 'ext': 'mp4',
+ 'thumbnail': thumbnail,
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/screenwavemedia.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/screenwavemedia.py
new file mode 100644
index 0000000000..6c9fdb7c1a
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/screenwavemedia.py
@@ -0,0 +1,178 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+ int_or_none,
+ unified_strdate,
+)
+
+
+class ScreenwaveMediaIE(InfoExtractor):
+ _VALID_URL = r'http://player\.screenwavemedia\.com/play/[a-zA-Z]+\.php\?[^"]*\bid=(?P.+)'
+
+ _TESTS = [{
+ 'url': 'http://player.screenwavemedia.com/play/play.php?playerdiv=videoarea&companiondiv=squareAd&id=Cinemassacre-19911',
+ 'only_matching': True,
+ }]
+
+ def _real_extract(self, url):
+ video_id = self._match_id(url)
+ playerdata = self._download_webpage(url, video_id, 'Downloading player webpage')
+
+ vidtitle = self._search_regex(
+ r'\'vidtitle\'\s*:\s*"([^"]+)"', playerdata, 'vidtitle').replace('\\/', '/')
+ vidurl = self._search_regex(
+ r'\'vidurl\'\s*:\s*"([^"]+)"', playerdata, 'vidurl').replace('\\/', '/')
+
+ videolist_url = None
+
+ mobj = re.search(r"'videoserver'\s*:\s*'(?P[^']+)'", playerdata)
+ if mobj:
+ videoserver = mobj.group('videoserver')
+ mobj = re.search(r'\'vidid\'\s*:\s*"(?P[^\']+)"', playerdata)
+ vidid = mobj.group('vidid') if mobj else video_id
+ videolist_url = 'http://%s/vod/smil:%s.smil/jwplayer.smil' % (videoserver, vidid)
+ else:
+ mobj = re.search(r"file\s*:\s*'(?Phttp.+?/jwplayer\.smil)'", playerdata)
+ if mobj:
+ videolist_url = mobj.group('smil')
+
+ if videolist_url:
+ videolist = self._download_xml(videolist_url, video_id, 'Downloading videolist XML')
+ formats = []
+ baseurl = vidurl[:vidurl.rfind('/') + 1]
+ for video in videolist.findall('.//video'):
+ src = video.get('src')
+ if not src:
+ continue
+ file_ = src.partition(':')[-1]
+ width = int_or_none(video.get('width'))
+ height = int_or_none(video.get('height'))
+ bitrate = int_or_none(video.get('system-bitrate'), scale=1000)
+ format = {
+ 'url': baseurl + file_,
+ 'format_id': src.rpartition('.')[0].rpartition('_')[-1],
+ }
+ if width or height:
+ format.update({
+ 'tbr': bitrate,
+ 'width': width,
+ 'height': height,
+ })
+ else:
+ format.update({
+ 'abr': bitrate,
+ 'vcodec': 'none',
+ })
+ formats.append(format)
+ else:
+ formats = [{
+ 'url': vidurl,
+ }]
+ self._sort_formats(formats)
+
+ return {
+ 'id': video_id,
+ 'title': vidtitle,
+ 'formats': formats,
+ }
+
+
+class CinemassacreIE(InfoExtractor):
+ _VALID_URL = 'https?://(?:www\.)?cinemassacre\.com/(?P[0-9]{4})/(?P[0-9]{2})/(?P[0-9]{2})/(?P[^?#/]+)'
+ _TESTS = [
+ {
+ 'url': 'http://cinemassacre.com/2012/11/10/avgn-the-movie-trailer/',
+ 'md5': 'fde81fbafaee331785f58cd6c0d46190',
+ 'info_dict': {
+ 'id': 'Cinemassacre-19911',
+ 'ext': 'mp4',
+ 'upload_date': '20121110',
+ 'title': 'Б─°Angry Video Game Nerd: The MovieБ─² Б─⌠ Trailer',
+ 'description': 'md5:fb87405fcb42a331742a0dce2708560b',
+ },
+ },
+ {
+ 'url': 'http://cinemassacre.com/2013/10/02/the-mummys-hand-1940',
+ 'md5': 'd72f10cd39eac4215048f62ab477a511',
+ 'info_dict': {
+ 'id': 'Cinemassacre-521be8ef82b16',
+ 'ext': 'mp4',
+ 'upload_date': '20131002',
+ 'title': 'The MummyБ─≥s Hand (1940)',
+ },
+ }
+ ]
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ display_id = mobj.group('display_id')
+ video_date = mobj.group('date_y') + mobj.group('date_m') + mobj.group('date_d')
+
+ webpage = self._download_webpage(url, display_id)
+
+ playerdata_url = self._search_regex(
+ r'src="(http://player\.screenwavemedia\.com/play/[a-zA-Z]+\.php\?[^"]*\bid=.+?)"',
+ webpage, 'player data URL')
+ video_title = self._html_search_regex(
+ r'(?P<title>.+?)\|', webpage, 'title')
+ video_description = self._html_search_regex(
+ r'<div class="entry-content">(?P<description>.+?)</div>',
+ webpage, 'description', flags=re.DOTALL, fatal=False)
+ video_thumbnail = self._og_search_thumbnail(webpage)
+
+ return {
+ '_type': 'url_transparent',
+ 'display_id': display_id,
+ 'title': video_title,
+ 'description': video_description,
+ 'upload_date': video_date,
+ 'thumbnail': video_thumbnail,
+ 'url': playerdata_url,
+ }
+
+
+class TeamFourIE(InfoExtractor):
+ _VALID_URL = r'https?://(?:www\.)?teamfourstar\.com/video/(?P<id>[a-z0-9\-]+)/?'
+ _TEST = {
+ 'url': 'http://teamfourstar.com/video/a-moment-with-tfs-episode-4/',
+ 'info_dict': {
+ 'id': 'TeamFourStar-5292a02f20bfa',
+ 'ext': 'mp4',
+ 'upload_date': '20130401',
+ 'description': 'Check out this and more on our website: http://teamfourstar.com\nTFS Store: http://sharkrobot.com/team-four-star\nFollow on Twitter: http://twitter.com/teamfourstar\nLike on FB: http://facebook.com/teamfourstar',
+ 'title': 'A Moment With TFS Episode 4',
+ }
+ }
+
+ def _real_extract(self, url):
+ display_id = self._match_id(url)
+ webpage = self._download_webpage(url, display_id)
+
+ playerdata_url = self._search_regex(
+ r'src="(http://player\.screenwavemedia\.com/play/[a-zA-Z]+\.php\?[^"]*\bid=.+?)"',
+ webpage, 'player data URL')
+
+ video_title = self._html_search_regex(
+ r'<div class="heroheadingtitle">(?P<title>.+?)</div>',
+ webpage, 'title')
+ video_date = unified_strdate(self._html_search_regex(
+ r'<div class="heroheadingdate">(?P<date>.+?)</div>',
+ webpage, 'date', fatal=False))
+ video_description = self._html_search_regex(
+ r'(?s)<div class="postcontent">(?P<description>.+?)</div>',
+ webpage, 'description', fatal=False)
+ video_thumbnail = self._og_search_thumbnail(webpage)
+
+ return {
+ '_type': 'url_transparent',
+ 'display_id': display_id,
+ 'title': video_title,
+ 'description': video_description,
+ 'upload_date': video_date,
+ 'thumbnail': video_thumbnail,
+ 'url': playerdata_url,
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/servingsys.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/servingsys.py
new file mode 100644
index 0000000000..16dc3736b4
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/servingsys.py
@@ -0,0 +1,69 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+ int_or_none,
+)
+
+
+class ServingSysIE(InfoExtractor):
+ _VALID_URL = r'https?://(?:[^.]+\.)?serving-sys\.com/BurstingPipe/adServer\.bs\?.*?&pli=(?P<id>[0-9]+)'
+
+ _TEST = {
+ 'url': 'http://bs.serving-sys.com/BurstingPipe/adServer.bs?cn=is&c=23&pl=VAST&pli=5349193&PluID=0&pos=7135&ord=[timestamp]&cim=1?',
+ 'playlist': [{
+ 'file': '29955898.flv',
+ 'md5': 'baed851342df6846eb8677a60a011a0f',
+ 'info_dict': {
+ 'title': 'AdAPPter_Hyundai_demo (1)',
+ 'duration': 74,
+ 'tbr': 1378,
+ 'width': 640,
+ 'height': 400,
+ },
+ }, {
+ 'file': '29907998.flv',
+ 'md5': '979b4da2655c4bc2d81aeb915a8c5014',
+ 'info_dict': {
+ 'title': 'AdAPPter_Hyundai_demo (2)',
+ 'duration': 34,
+ 'width': 854,
+ 'height': 480,
+ 'tbr': 516,
+ },
+ }],
+ 'params': {
+ 'playlistend': 2,
+ },
+ 'skip': 'Blocked in the US [sic]',
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ pl_id = mobj.group('id')
+
+ vast_doc = self._download_xml(url, pl_id)
+ title = vast_doc.find('.//AdTitle').text
+ media = vast_doc.find('.//MediaFile').text
+ info_url = self._search_regex(r'&adData=([^&]+)&', media, 'info URL')
+
+ doc = self._download_xml(info_url, pl_id, 'Downloading video info')
+ entries = [{
+ '_type': 'video',
+ 'id': a.attrib['id'],
+ 'title': '%s (%s)' % (title, a.attrib['assetID']),
+ 'url': a.attrib['URL'],
+ 'duration': int_or_none(a.attrib.get('length')),
+ 'tbr': int_or_none(a.attrib.get('bitrate')),
+ 'height': int_or_none(a.attrib.get('height')),
+ 'width': int_or_none(a.attrib.get('width')),
+ } for a in doc.findall('.//AdditionalAssets/asset')]
+
+ return {
+ '_type': 'playlist',
+ 'id': pl_id,
+ 'title': title,
+ 'entries': entries,
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/sexu.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/sexu.py
new file mode 100644
index 0000000000..6365a8779d
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/sexu.py
@@ -0,0 +1,61 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+
+
+class SexuIE(InfoExtractor):
+ _VALID_URL = r'https?://(?:www\.)?sexu\.com/(?P<id>\d+)'
+ _TEST = {
+ 'url': 'http://sexu.com/961791/',
+ 'md5': 'ff615aca9691053c94f8f10d96cd7884',
+ 'info_dict': {
+ 'id': '961791',
+ 'ext': 'mp4',
+ 'title': 'md5:4d05a19a5fc049a63dbbaf05fb71d91b',
+ 'description': 'md5:c5ed8625eb386855d5a7967bd7b77a54',
+ 'categories': list, # NSFW
+ 'thumbnail': 're:https?://.*\.jpg$',
+ 'age_limit': 18,
+ }
+ }
+
+ def _real_extract(self, url):
+ video_id = self._match_id(url)
+ webpage = self._download_webpage(url, video_id)
+
+ quality_arr = self._search_regex(
+ r'sources:\s*\[([^\]]+)\]', webpage, 'forrmat string')
+ formats = [{
+ 'url': fmt[0].replace('\\', ''),
+ 'format_id': fmt[1],
+ 'height': int(fmt[1][:3]),
+ } for fmt in re.findall(r'"file":"([^"]+)","label":"([^"]+)"', quality_arr)]
+ self._sort_formats(formats)
+
+ title = self._html_search_regex(
+ r'<title>([^<]+)\s*-\s*Sexu\.Com ', webpage, 'title')
+
+ description = self._html_search_meta(
+ 'description', webpage, 'description')
+
+ thumbnail = self._html_search_regex(
+ r'image:\s*"([^"]+)"',
+ webpage, 'thumbnail', fatal=False)
+
+ categories_str = self._html_search_meta(
+ 'keywords', webpage, 'categories')
+ categories = (
+ None if categories_str is None
+ else categories_str.split(','))
+
+ return {
+ 'id': video_id,
+ 'title': title,
+ 'description': description,
+ 'thumbnail': thumbnail,
+ 'categories': categories,
+ 'formats': formats,
+ 'age_limit': 18,
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/sexykarma.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/sexykarma.py
new file mode 100644
index 0000000000..c833fc8ee8
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/sexykarma.py
@@ -0,0 +1,117 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+ unified_strdate,
+ parse_duration,
+ int_or_none,
+)
+
+
+class SexyKarmaIE(InfoExtractor):
+ IE_DESC = 'Sexy Karma and Watch Indian Porn'
+ _VALID_URL = r'https?://(?:www\.)?(?:sexykarma\.com|watchindianporn\.net)/(?:[^/]+/)*video/(?P[^/]+)-(?P[a-zA-Z0-9]+)\.html'
+ _TESTS = [{
+ 'url': 'http://www.sexykarma.com/gonewild/video/taking-a-quick-pee-yHI70cOyIHt.html',
+ 'md5': 'b9798e7d1ef1765116a8f516c8091dbd',
+ 'info_dict': {
+ 'id': 'yHI70cOyIHt',
+ 'display_id': 'taking-a-quick-pee',
+ 'ext': 'mp4',
+ 'title': 'Taking a quick pee.',
+ 'thumbnail': 're:^https?://.*\.jpg$',
+ 'uploader': 'wildginger7',
+ 'upload_date': '20141007',
+ 'duration': 22,
+ 'view_count': int,
+ 'comment_count': int,
+ 'categories': list,
+ }
+ }, {
+ 'url': 'http://www.sexykarma.com/gonewild/video/pot-pixie-tribute-8Id6EZPbuHf.html',
+ 'md5': 'dd216c68d29b49b12842b9babe762a5d',
+ 'info_dict': {
+ 'id': '8Id6EZPbuHf',
+ 'display_id': 'pot-pixie-tribute',
+ 'ext': 'mp4',
+ 'title': 'pot_pixie tribute',
+ 'thumbnail': 're:^https?://.*\.jpg$',
+ 'uploader': 'banffite',
+ 'upload_date': '20141013',
+ 'duration': 16,
+ 'view_count': int,
+ 'comment_count': int,
+ 'categories': list,
+ }
+ }, {
+ 'url': 'http://www.watchindianporn.net/video/desi-dancer-namrata-stripping-completely-nude-and-dancing-on-a-hot-number-dW2mtctxJfs.html',
+ 'md5': '9afb80675550406ed9a63ac2819ef69d',
+ 'info_dict': {
+ 'id': 'dW2mtctxJfs',
+ 'display_id': 'desi-dancer-namrata-stripping-completely-nude-and-dancing-on-a-hot-number',
+ 'ext': 'mp4',
+ 'title': 'Desi dancer namrata stripping completely nude and dancing on a hot number',
+ 'thumbnail': 're:^https?://.*\.jpg$',
+ 'uploader': 'Don',
+ 'upload_date': '20140213',
+ 'duration': 83,
+ 'view_count': int,
+ 'comment_count': int,
+ 'categories': list,
+ }
+ }]
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+ display_id = mobj.group('display_id')
+
+ webpage = self._download_webpage(url, display_id)
+
+ video_url = self._html_search_regex(
+ r"url: escape\('([^']+)'\)", webpage, 'url')
+
+ title = self._html_search_regex(
+ r'(.*?) ',
+ webpage, 'title')
+ thumbnail = self._html_search_regex(
+ r'\s*(.*?)',
+ webpage, 'uploader')
+ upload_date = unified_strdate(self._html_search_regex(
+ r'Added: (.+?) ', webpage, 'upload date', fatal=False))
+
+ duration = parse_duration(self._search_regex(
+ r'Time:\s* \s*\s*(.+?)\s* ',
+ webpage, 'duration', fatal=False))
+
+ view_count = int_or_none(self._search_regex(
+ r'Views:\s* \s*\s*(\d+)\s* ',
+ webpage, 'view count', fatal=False))
+ comment_count = int_or_none(self._search_regex(
+ r'Comments:\s* \s*\s*(\d+)\s* ',
+ webpage, 'comment count', fatal=False))
+
+ categories = re.findall(
+ r'([^<]+) [\da-z]{10})'
+
+ _TEST = {
+ 'url': 'http://shared.sx/0060718775',
+ 'md5': '106fefed92a8a2adb8c98e6a0652f49b',
+ 'info_dict': {
+ 'id': '0060718775',
+ 'ext': 'mp4',
+ 'title': 'Bmp4',
+ },
+ }
+
+ def _real_extract(self, url):
+ video_id = self._match_id(url)
+ webpage = self._download_webpage(url, video_id)
+
+ if '>File does not exist<' in webpage:
+ raise ExtractorError(
+ 'Video %s does not exist' % video_id, expected=True)
+
+ download_form = dict(re.findall(
+ r'Filename: .+?(.+?) ', webpage, 'title')
+ duration = parse_duration(
+ self._search_regex(
+ r'(?s)Length: .+?(.+?) ',
+ webpage,
+ 'duration',
+ fatal=False
+ )
+ )
+
+ m = re.search(
+ r'''(?xs)Width\sx\sHeight .+?
+ (?P\d+)\sx\s(?P\d+) ''',
+ webpage
+ )
+ width = height = None
+ if m:
+ width, height = int(m.group('width')), int(m.group('height'))
+
+ formats = [{
+ 'format_id': 'sd',
+ 'url': video_url,
+ 'width': width,
+ 'height': height,
+ }]
+
+ return {
+ 'id': video_id,
+ 'title': title,
+ 'duration': duration,
+ 'formats': formats,
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/sina.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/sina.py
new file mode 100644
index 0000000000..a63d126d45
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/sina.py
@@ -0,0 +1,76 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..compat import (
+ compat_urllib_request,
+ compat_urllib_parse,
+)
+
+
+class SinaIE(InfoExtractor):
+ _VALID_URL = r'''https?://(.*?\.)?video\.sina\.com\.cn/
+ (
+ (.+?/(((?P\d+).html)|(.*?(\#|(vid=)|b/)(?P\d+?)($|&|\-))))
+ |
+ # This is used by external sites like Weibo
+ (api/sinawebApi/outplay.php/(?P.+?)\.swf)
+ )
+ '''
+
+ _TESTS = [
+ {
+ 'url': 'http://video.sina.com.cn/news/vlist/zt/chczlj2013/?opsubject_id=top12#110028898',
+ 'file': '110028898.flv',
+ 'md5': 'd65dd22ddcf44e38ce2bf58a10c3e71f',
+ 'info_dict': {
+ 'title': 'Ц─┼Д╦╜Е⌡╫Ф√╟И≈╩Ц─▀ Ф°²И╡°Х╕│Ф╠┌Е╥╢Ф▀©И╘╛Г╚▀Е█ЁИ┤┼Ф■╬Х╒╚Ф┴ёХ┬╧Е▒≤',
+ }
+ },
+ {
+ 'url': 'http://video.sina.com.cn/v/b/101314253-1290078633.html',
+ 'info_dict': {
+ 'id': '101314253',
+ 'ext': 'flv',
+ 'title': 'Е├⌡Ф√╧Ф▐░И╚≤Е╞╧Ф°²Ф┐┘Ф┼╔Г⌡▒Х╖├Г╨╖Е┬╚',
+ },
+ },
+ ]
+
+ @classmethod
+ def suitable(cls, url):
+ return re.match(cls._VALID_URL, url, flags=re.VERBOSE) is not None
+
+ def _extract_video(self, video_id):
+ data = compat_urllib_parse.urlencode({'vid': video_id})
+ url_doc = self._download_xml('http://v.iask.com/v_play.php?%s' % data,
+ video_id, 'Downloading video url')
+ image_page = self._download_webpage(
+ 'http://interface.video.sina.com.cn/interface/common/getVideoImage.php?%s' % data,
+ video_id, 'Downloading thumbnail info')
+
+ return {'id': video_id,
+ 'url': url_doc.find('./durl/url').text,
+ 'ext': 'flv',
+ 'title': url_doc.find('./vname').text,
+ 'thumbnail': image_page.split('=')[1],
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url, flags=re.VERBOSE)
+ video_id = mobj.group('id')
+ if mobj.group('token') is not None:
+ # The video id is in the redirected url
+ self.to_screen('Getting video id')
+ request = compat_urllib_request.Request(url)
+ request.get_method = lambda: 'HEAD'
+ (_, urlh) = self._download_webpage_handle(request, 'NA', False)
+ return self._real_extract(urlh.geturl())
+ elif video_id is None:
+ pseudo_id = mobj.group('pseudo_id')
+ webpage = self._download_webpage(url, pseudo_id)
+ video_id = self._search_regex(r'vid:\'(\d+?)\'', webpage, 'video id')
+
+ return self._extract_video(video_id)
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/slideshare.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/slideshare.py
new file mode 100644
index 0000000000..e7d776e7bd
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/slideshare.py
@@ -0,0 +1,55 @@
+from __future__ import unicode_literals
+
+import re
+import json
+
+from .common import InfoExtractor
+from ..compat import (
+ compat_urlparse,
+)
+from ..utils import (
+ ExtractorError,
+)
+
+
+class SlideshareIE(InfoExtractor):
+ _VALID_URL = r'https?://www\.slideshare\.net/[^/]+?/(?P.+?)($|\?)'
+
+ _TEST = {
+ 'url': 'http://www.slideshare.net/Dataversity/keynote-presentation-managing-scale-and-complexity',
+ 'info_dict': {
+ 'id': '25665706',
+ 'ext': 'mp4',
+ 'title': 'Managing Scale and Complexity',
+ 'description': 'This was a keynote presentation at the NoSQL Now! 2013 Conference & Expo (http://www.nosqlnow.com). This presentation was given by Adrian Cockcroft from Netflix.',
+ },
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ page_title = mobj.group('title')
+ webpage = self._download_webpage(url, page_title)
+ slideshare_obj = self._search_regex(
+ r'var slideshare_object = ({.*?}); var user_info =',
+ webpage, 'slideshare object')
+ info = json.loads(slideshare_obj)
+ if info['slideshow']['type'] != 'video':
+ raise ExtractorError('Webpage type is "%s": only video extraction is supported for Slideshare' % info['slideshow']['type'], expected=True)
+
+ doc = info['doc']
+ bucket = info['jsplayer']['video_bucket']
+ ext = info['jsplayer']['video_extension']
+ video_url = compat_urlparse.urljoin(bucket, doc + '-SD.' + ext)
+ description = self._html_search_regex(
+ r'<p\s+(?:style="[^"]*"\s+)?class=".*?description.*?"[^>]*>(.*?)</p>', webpage,
+ 'description', fatal=False)
+
+ return {
+ '_type': 'video',
+ 'id': info['slideshow']['id'],
+ 'title': info['slideshow']['title'],
+ 'ext': ext,
+ 'url': video_url,
+ 'thumbnail': info['slideshow']['pin_image_url'],
+ 'description': description,
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/slutload.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/slutload.py
new file mode 100644
index 0000000000..3df71304da
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/slutload.py
@@ -0,0 +1,44 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+
+
+class SlutloadIE(InfoExtractor):
+ _VALID_URL = r'^https?://(?:\w+\.)?slutload\.com/video/[^/]+/(?P<id>[^/]+)/?$'
+ _TEST = {
+ 'url': 'http://www.slutload.com/video/virginie-baisee-en-cam/TD73btpBqSxc/',
+ 'md5': '0cf531ae8006b530bd9df947a6a0df77',
+ 'info_dict': {
+ 'id': 'TD73btpBqSxc',
+ 'ext': 'mp4',
+ "title": "virginie baisee en cam",
+ "age_limit": 18,
+ 'thumbnail': 're:https?://.*?\.jpg'
+ }
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+
+ webpage = self._download_webpage(url, video_id)
+
+ video_title = self._html_search_regex(r'<h1><strong>([^<]+)</strong>',
+ webpage, 'title').strip()
+
+ video_url = self._html_search_regex(
+ r'(?s)<div id="vidPlayer"\s+data-url="([^"]+)"',
+ webpage, 'video URL')
+ thumbnail = self._html_search_regex(
+ r'(?s)<div id="vidPlayer"\s+.*?previewer-file="([^"]+)"',
+ webpage, 'thumbnail', fatal=False)
+
+ return {
+ 'id': video_id,
+ 'url': video_url,
+ 'title': video_title,
+ 'thumbnail': thumbnail,
+ 'age_limit': 18
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/smotri.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/smotri.py
new file mode 100644
index 0000000000..baef3daa04
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/smotri.py
@@ -0,0 +1,374 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+import re
+import json
+import hashlib
+import uuid
+
+from .common import InfoExtractor
+from ..compat import (
+ compat_urllib_parse,
+ compat_urllib_request,
+)
+from ..utils import (
+ ExtractorError,
+ int_or_none,
+ unified_strdate,
+)
+
+
+class SmotriIE(InfoExtractor):
+ IE_DESC = 'Smotri.com'
+ IE_NAME = 'smotri'
+ _VALID_URL = r'^https?://(?:www\.)?(?:smotri\.com/video/view/\?id=|pics\.smotri\.com/(?:player|scrubber_custom8)\.swf\?file=)(?P<id>v(?P<realvideoid>[0-9]+)[a-z0-9]{4})'
+ _NETRC_MACHINE = 'smotri'
+
+ _TESTS = [
+ # real video id 2610366
+ {
+ 'url': 'http://smotri.com/video/view/?id=v261036632ab',
+ 'md5': '2a7b08249e6f5636557579c368040eb9',
+ 'info_dict': {
+ 'id': 'v261036632ab',
+ 'ext': 'mp4',
+ 'title': 'п╨п╟я┌п╟я│я┌я─п╬я└п╟ я│ п╨п╟п╪п╣я─ п╡п╦п╢п╣п╬п╫п╟п╠п╩я▌п╢п╣п╫п╦я▐',
+ 'uploader': 'rbc2008',
+ 'uploader_id': 'rbc08',
+ 'upload_date': '20131118',
+ 'thumbnail': 'http://frame6.loadup.ru/8b/a9/2610366.3.3.jpg',
+ },
+ },
+ # real video id 57591
+ {
+ 'url': 'http://smotri.com/video/view/?id=v57591cb20',
+ 'md5': '830266dfc21f077eac5afd1883091bcd',
+ 'info_dict': {
+ 'id': 'v57591cb20',
+ 'ext': 'flv',
+ 'title': 'test',
+ 'uploader': 'Support Photofile@photofile',
+ 'uploader_id': 'support-photofile',
+ 'upload_date': '20070704',
+ 'thumbnail': 'http://frame4.loadup.ru/03/ed/57591.2.3.jpg',
+ },
+ },
+ # video-password
+ {
+ 'url': 'http://smotri.com/video/view/?id=v1390466a13c',
+ 'md5': 'f6331cef33cad65a0815ee482a54440b',
+ 'info_dict': {
+ 'id': 'v1390466a13c',
+ 'ext': 'mp4',
+ 'title': 'TOCCA_A_NOI_-_LE_COSE_NON_VANNO_CAMBIAMOLE_ORA-1',
+ 'uploader': 'timoxa40',
+ 'uploader_id': 'timoxa40',
+ 'upload_date': '20100404',
+ 'thumbnail': 'http://frame7.loadup.ru/af/3f/1390466.3.3.jpg',
+ },
+ 'params': {
+ 'videopassword': 'qwerty',
+ },
+ 'skip': 'Video is not approved by moderator',
+ },
+ # age limit + video-password
+ {
+ 'url': 'http://smotri.com/video/view/?id=v15408898bcf',
+ 'md5': '91e909c9f0521adf5ee86fbe073aad70',
+ 'info_dict': {
+ 'id': 'v15408898bcf',
+ 'ext': 'flv',
+ 'title': 'я█я┌п╬я┌ я─п╬п╩п╦п╨ п╫п╣ п©п╬п╨п╟п╤я┐я┌ п©п╬ п╒п▓',
+ 'uploader': 'zzxxx',
+ 'uploader_id': 'ueggb',
+ 'upload_date': '20101001',
+ 'thumbnail': 'http://frame3.loadup.ru/75/75/1540889.1.3.jpg',
+ 'age_limit': 18,
+ },
+ 'params': {
+ 'videopassword': '333'
+ },
+ 'skip': 'Video is not approved by moderator',
+ },
+ # swf player
+ {
+ 'url': 'http://pics.smotri.com/scrubber_custom8.swf?file=v9188090500',
+ 'md5': '4d47034979d9390d14acdf59c4935bc2',
+ 'info_dict': {
+ 'id': 'v9188090500',
+ 'ext': 'mp4',
+ 'title': 'Shakira - Don\'t Bother',
+ 'uploader': 'HannahL',
+ 'uploader_id': 'lisaha95',
+ 'upload_date': '20090331',
+ 'thumbnail': 'http://frame8.loadup.ru/44/0b/918809.7.3.jpg',
+ },
+ },
+ ]
+
+ @classmethod
+ def _extract_url(cls, webpage):
+ mobj = re.search(
+ r'<embed[^>]src=(["\'])(?P<url>http://pics\.smotri\.com/(?:player|scrubber_custom8)\.swf\?file=v.+?\1)',
+ webpage)
+ if mobj is not None:
+ return mobj.group('url')
+
+ mobj = re.search(
+ r'''(?x)<div\s+class="video_file">http://smotri\.com/video/download/file/[^<]+</div>\s*
+ <div\s+class="video_image">[^<]+</div>\s*
+ <div\s+class="video_id">(?P<id>[^<]+)</div>''', webpage)
+ if mobj is not None:
+ return 'http://smotri.com/video/view/?id=%s' % mobj.group('id')
+
+ def _search_meta(self, name, html, display_name=None):
+ if display_name is None:
+ display_name = name
+ return self._html_search_regex(
+ r'<meta itemprop="%s" content="([^"]+)" />' % re.escape(name),
+ html, display_name, fatal=False)
+ return self._html_search_meta(name, html, display_name)
+
+ def _real_extract(self, url):
+ video_id = self._match_id(url)
+
+ video_form = {
+ 'ticket': video_id,
+ 'video_url': '1',
+ 'frame_url': '1',
+ 'devid': 'LoadupFlashPlayer',
+ 'getvideoinfo': '1',
+ }
+
+ request = compat_urllib_request.Request(
+ 'http://smotri.com/video/view/url/bot/', compat_urllib_parse.urlencode(video_form))
+ request.add_header('Content-Type', 'application/x-www-form-urlencoded')
+
+ video = self._download_json(request, video_id, 'Downloading video JSON')
+
+ if video.get('_moderate_no') or not video.get('moderated'):
+ raise ExtractorError('Video %s has not been approved by moderator' % video_id, expected=True)
+
+ if video.get('error'):
+ raise ExtractorError('Video %s does not exist' % video_id, expected=True)
+
+ video_url = video.get('_vidURL') or video.get('_vidURL_mp4')
+ title = video['title']
+ thumbnail = video['_imgURL']
+ upload_date = unified_strdate(video['added'])
+ uploader = video['userNick']
+ uploader_id = video['userLogin']
+ duration = int_or_none(video['duration'])
+
+ # Video JSON does not provide enough meta data
+ # We will extract some from the video web page instead
+ webpage_url = 'http://smotri.com/video/view/?id=%s' % video_id
+ webpage = self._download_webpage(webpage_url, video_id, 'Downloading video page')
+
+ # Warning if video is unavailable
+ warning = self._html_search_regex(
+ r'<div class="videoUnModer">(.*?)</div>', webpage,
+ 'warning message', default=None)
+ if warning is not None:
+ self._downloader.report_warning(
+ 'Video %s may not be available; smotri said: %s ' %
+ (video_id, warning))
+
+ # Adult content
+ if re.search('EroConfirmText">', webpage) is not None:
+ self.report_age_confirmation()
+ confirm_string = self._html_search_regex(
+ r'<a href="/video/view/\?id=%s&confirm=([^"]+)" title="[^"]+">' % video_id,
+ webpage, 'confirm string')
+ confirm_url = webpage_url + '&confirm=%s' % confirm_string
+ webpage = self._download_webpage(confirm_url, video_id, 'Downloading video page (age confirmed)')
+ adult_content = True
+ else:
+ adult_content = False
+
+ view_count = self._html_search_regex(
+ 'п·п╠я┴п╣п╣ п╨п╬п╩п╦я┤п╣я│я┌п╡п╬ п©я─п╬я│п╪п╬я┌я─п╬п╡.*?<span class="Number">(\\d+)</span>',
+ webpage, 'view count', fatal=False, flags=re.MULTILINE | re.DOTALL)
+
+ return {
+ 'id': video_id,
+ 'url': video_url,
+ 'title': title,
+ 'thumbnail': thumbnail,
+ 'uploader': uploader,
+ 'upload_date': upload_date,
+ 'uploader_id': uploader_id,
+ 'duration': duration,
+ 'view_count': int_or_none(view_count),
+ 'age_limit': 18 if adult_content else 0,
+ }
+
+
+class SmotriCommunityIE(InfoExtractor):
+ IE_DESC = 'Smotri.com community videos'
+ IE_NAME = 'smotri:community'
+ _VALID_URL = r'^https?://(?:www\.)?smotri\.com/community/video/(?P<communityid>[0-9A-Za-z_\'-]+)'
+ _TEST = {
+ 'url': 'http://smotri.com/community/video/kommuna',
+ 'info_dict': {
+ 'id': 'kommuna',
+ 'title': 'п п÷п═п╓',
+ },
+ 'playlist_mincount': 4,
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ community_id = mobj.group('communityid')
+
+ url = 'http://smotri.com/export/rss/video/by/community/-/%s/video.xml' % community_id
+ rss = self._download_xml(url, community_id, 'Downloading community RSS')
+
+ entries = [self.url_result(video_url.text, 'Smotri')
+ for video_url in rss.findall('./channel/item/link')]
+
+ description_text = rss.find('./channel/description').text
+ community_title = self._html_search_regex(
+ '^п▓п╦п╢п╣п╬ я│п╬п╬п╠я┴п╣я│я┌п╡п╟ "([^"]+)"$', description_text, 'community title')
+
+ return self.playlist_result(entries, community_id, community_title)
+
+
+class SmotriUserIE(InfoExtractor):
+ IE_DESC = 'Smotri.com user videos'
+ IE_NAME = 'smotri:user'
+ _VALID_URL = r'^https?://(?:www\.)?smotri\.com/user/(?P<userid>[0-9A-Za-z_\'-]+)'
+ _TESTS = [{
+ 'url': 'http://smotri.com/user/inspector',
+ 'info_dict': {
+ 'id': 'inspector',
+ 'title': 'Inspector',
+ },
+ 'playlist_mincount': 9,
+ }]
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ user_id = mobj.group('userid')
+
+ url = 'http://smotri.com/export/rss/user/video/-/%s/video.xml' % user_id
+ rss = self._download_xml(url, user_id, 'Downloading user RSS')
+
+ entries = [self.url_result(video_url.text, 'Smotri')
+ for video_url in rss.findall('./channel/item/link')]
+
+ description_text = rss.find('./channel/description').text
+ user_nickname = self._html_search_regex(
+ '^п▓п╦п╢п╣п╬ я─п╣п╤п╦я│я│п╣я─п╟ (.*)$', description_text,
+ 'user nickname')
+
+ return self.playlist_result(entries, user_id, user_nickname)
+
+
+class SmotriBroadcastIE(InfoExtractor):
+ IE_DESC = 'Smotri.com broadcasts'
+ IE_NAME = 'smotri:broadcast'
+ _VALID_URL = r'^https?://(?:www\.)?(?P<url>smotri\.com/live/(?P<broadcastid>[^/]+))/?.*'
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ broadcast_id = mobj.group('broadcastid')
+
+ broadcast_url = 'http://' + mobj.group('url')
+ broadcast_page = self._download_webpage(broadcast_url, broadcast_id, 'Downloading broadcast page')
+
+ if re.search('>п═п╣п╤п╦я│я│п╣я─ я│ п╩п╬пЁп╦п╫п╬п╪ <br/>"%s"<br/> <span>п╫п╣ я│я┐я┴п╣я│я┌п╡я┐п╣я┌<' % broadcast_id, broadcast_page) is not None:
+ raise ExtractorError(
+ 'Broadcast %s does not exist' % broadcast_id, expected=True)
+
+ # Adult content
+ if re.search('EroConfirmText">', broadcast_page) is not None:
+
+ (username, password) = self._get_login_info()
+ if username is None:
+ raise ExtractorError(
+ 'Erotic broadcasts allowed only for registered users, '
+ 'use --username and --password options to provide account credentials.',
+ expected=True)
+
+ login_form = {
+ 'login-hint53': '1',
+ 'confirm_erotic': '1',
+ 'login': username,
+ 'password': password,
+ }
+
+ request = compat_urllib_request.Request(
+ broadcast_url + '/?no_redirect=1', compat_urllib_parse.urlencode(login_form))
+ request.add_header('Content-Type', 'application/x-www-form-urlencoded')
+ broadcast_page = self._download_webpage(
+ request, broadcast_id, 'Logging in and confirming age')
+
+ if re.search('>п²п╣п╡п╣я─п╫я▀п╧ п╩п╬пЁп╦п╫ п╦п╩п╦ п©п╟я─п╬п╩я▄<', broadcast_page) is not None:
+ raise ExtractorError('Unable to log in: bad username or password', expected=True)
+
+ adult_content = True
+ else:
+ adult_content = False
+
+ ticket = self._html_search_regex(
+ r"window\.broadcast_control\.addFlashVar\('file'\s*,\s*'([^']+)'\)",
+ broadcast_page, 'broadcast ticket')
+
+ url = 'http://smotri.com/broadcast/view/url/?ticket=%s' % ticket
+
+ broadcast_password = self._downloader.params.get('videopassword', None)
+ if broadcast_password:
+ url += '&pass=%s' % hashlib.md5(broadcast_password.encode('utf-8')).hexdigest()
+
+ broadcast_json_page = self._download_webpage(
+ url, broadcast_id, 'Downloading broadcast JSON')
+
+ try:
+ broadcast_json = json.loads(broadcast_json_page)
+
+ protected_broadcast = broadcast_json['_pass_protected'] == 1
+ if protected_broadcast and not broadcast_password:
+ raise ExtractorError(
+ 'This broadcast is protected by a password, use the --video-password option',
+ expected=True)
+
+ broadcast_offline = broadcast_json['is_play'] == 0
+ if broadcast_offline:
+ raise ExtractorError('Broadcast %s is offline' % broadcast_id, expected=True)
+
+ rtmp_url = broadcast_json['_server']
+ mobj = re.search(r'^rtmp://[^/]+/(?P<app>.+)/?$', rtmp_url)
+ if not mobj:
+ raise ExtractorError('Unexpected broadcast rtmp URL')
+
+ broadcast_playpath = broadcast_json['_streamName']
+ broadcast_app = '%s/%s' % (mobj.group('app'), broadcast_json['_vidURL'])
+ broadcast_thumbnail = broadcast_json['_imgURL']
+ broadcast_title = self._live_title(broadcast_json['title'])
+ broadcast_description = broadcast_json['description']
+ broadcaster_nick = broadcast_json['nick']
+ broadcaster_login = broadcast_json['login']
+ rtmp_conn = 'S:%s' % uuid.uuid4().hex
+ except KeyError:
+ if protected_broadcast:
+ raise ExtractorError('Bad broadcast password', expected=True)
+ raise ExtractorError('Unexpected broadcast JSON')
+
+ return {
+ 'id': broadcast_id,
+ 'url': rtmp_url,
+ 'title': broadcast_title,
+ 'thumbnail': broadcast_thumbnail,
+ 'description': broadcast_description,
+ 'uploader': broadcaster_nick,
+ 'uploader_id': broadcaster_login,
+ 'age_limit': 18 if adult_content else 0,
+ 'ext': 'flv',
+ 'play_path': broadcast_playpath,
+ 'player_url': 'http://pics.smotri.com/broadcast_play.swf',
+ 'app': broadcast_app,
+ 'rtmp_live': True,
+ 'rtmp_conn': rtmp_conn,
+ 'is_live': True,
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/snotr.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/snotr.py
new file mode 100644
index 0000000000..da3b05a8dc
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/snotr.py
@@ -0,0 +1,68 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+ float_or_none,
+ str_to_int,
+ parse_duration,
+)
+
+
+class SnotrIE(InfoExtractor):
+ _VALID_URL = r'http?://(?:www\.)?snotr\.com/video/(?P<id>\d+)/([\w]+)'
+ _TESTS = [{
+ 'url': 'http://www.snotr.com/video/13708/Drone_flying_through_fireworks',
+ 'info_dict': {
+ 'id': '13708',
+ 'ext': 'flv',
+ 'title': 'Drone flying through fireworks!',
+ 'duration': 247,
+ 'filesize_approx': 98566144,
+ 'description': 'A drone flying through Fourth of July Fireworks',
+ }
+ }, {
+ 'url': 'http://www.snotr.com/video/530/David_Letteman_-_George_W_Bush_Top_10',
+ 'info_dict': {
+ 'id': '530',
+ 'ext': 'flv',
+ 'title': 'David Letteman - George W. Bush Top 10',
+ 'duration': 126,
+ 'filesize_approx': 8912896,
+ 'description': 'The top 10 George W. Bush moments, brought to you by David Letterman!',
+ }
+ }]
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+
+ webpage = self._download_webpage(url, video_id)
+ title = self._og_search_title(webpage)
+
+ description = self._og_search_description(webpage)
+ video_url = "http://cdn.videos.snotr.com/%s.flv" % video_id
+
+ view_count = str_to_int(self._html_search_regex(
+ r'<p>\n<strong>Views:</strong>\n([\d,\.]+)</p>',
+ webpage, 'view count', fatal=False))
+
+ duration = parse_duration(self._html_search_regex(
+ r'<p>\n<strong>Length:</strong>\n\s*([0-9:]+).*?</p>',
+ webpage, 'duration', fatal=False))
+
+ filesize_approx = float_or_none(self._html_search_regex(
+ r'<p>\n<strong>Filesize:</strong>\n\s*([0-9.]+)\s*megabyte</p>',
+ webpage, 'filesize', fatal=False), invscale=1024 * 1024)
+
+ return {
+ 'id': video_id,
+ 'description': description,
+ 'title': title,
+ 'url': video_url,
+ 'view_count': view_count,
+ 'duration': duration,
+ 'filesize_approx': filesize_approx,
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/sockshare.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/sockshare.py
new file mode 100644
index 0000000000..7d3c0e9378
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/sockshare.py
@@ -0,0 +1,84 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from ..compat import (
+ compat_urllib_parse,
+ compat_urllib_request,
+)
+from ..utils import (
+ determine_ext,
+ ExtractorError,
+)
+
+from .common import InfoExtractor
+
+
+class SockshareIE(InfoExtractor):
+ _VALID_URL = r'https?://(?:www\.)?sockshare\.com/file/(?P<id>[0-9A-Za-z]+)'
+ _FILE_DELETED_REGEX = r'This file doesn\'t exist, or has been removed\.</div>'
+ _TEST = {
+ 'url': 'http://www.sockshare.com/file/437BE28B89D799D7',
+ 'md5': '9d0bf1cfb6dbeaa8d562f6c97506c5bd',
+ 'info_dict': {
+ 'id': '437BE28B89D799D7',
+ 'title': 'big_buck_bunny_720p_surround.avi',
+ 'ext': 'avi',
+ 'thumbnail': 're:^http://.*\.jpg$',
+ }
+ }
+
+ def _real_extract(self, url):
+ video_id = self._match_id(url)
+ url = 'http://sockshare.com/file/%s' % video_id
+ webpage = self._download_webpage(url, video_id)
+
+ if re.search(self._FILE_DELETED_REGEX, webpage) is not None:
+ raise ExtractorError('Video %s does not exist' % video_id,
+ expected=True)
+
+ confirm_hash = self._html_search_regex(r'''(?x)<input\s+
+ type="hidden"\s+
+ value="([^"]*)"\s+
+ name="hash"
+ ''', webpage, 'hash')
+
+ fields = {
+ "hash": confirm_hash,
+ "confirm": "Continue as Free User"
+ }
+
+ post = compat_urllib_parse.urlencode(fields)
+ req = compat_urllib_request.Request(url, post)
+ # Apparently, this header is required for confirmation to work.
+ req.add_header('Host', 'www.sockshare.com')
+ req.add_header('Content-type', 'application/x-www-form-urlencoded')
+
+ webpage = self._download_webpage(
+ req, video_id, 'Downloading video page')
+
+ video_url = self._html_search_regex(
+ r'<a href="([^"]*)".+class="download_file_link"',
+ webpage, 'file url')
+ video_url = "http://www.sockshare.com" + video_url
+ title = self._html_search_regex((
+ r'<h1>(.+)<strong>',
+ r'var name = "([^"]+)";'),
+ webpage, 'title', default=None)
+ thumbnail = self._html_search_regex(
+ r'<img\s+src="([^"]*)".+?name="bg"',
+ webpage, 'thumbnail')
+
+ formats = [{
+ 'format_id': 'sd',
+ 'url': video_url,
+ 'ext': determine_ext(title),
+ }]
+
+ return {
+ 'id': video_id,
+ 'title': title,
+ 'thumbnail': thumbnail,
+ 'formats': formats,
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/sohu.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/sohu.py
new file mode 100644
index 0000000000..07f514a462
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/sohu.py
@@ -0,0 +1,97 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+import json
+import re
+
+from .common import InfoExtractor
+from ..utils import ExtractorError
+
+
+class SohuIE(InfoExtractor):
+ _VALID_URL = r'https?://(?P<mytv>my\.)?tv\.sohu\.com/.+?/(?(mytv)|n)(?P<id>\d+)\.shtml.*?'
+
+ _TEST = {
+ 'url': 'http://tv.sohu.com/20130724/n382479172.shtml#super',
+ 'md5': 'bde8d9a6ffd82c63a1eefaef4eeefec7',
+ 'info_dict': {
+ 'id': '382479172',
+ 'ext': 'mp4',
+ 'title': 'MVО╪ Far East MovementЦ─┼The IllestЦ─▀',
+ },
+ 'skip': 'Only available from China',
+ }
+
+ def _real_extract(self, url):
+
+ def _fetch_data(vid_id, mytv=False):
+ if mytv:
+ base_data_url = 'http://my.tv.sohu.com/play/videonew.do?vid='
+ else:
+ base_data_url = 'http://hot.vrs.sohu.com/vrs_flash.action?vid='
+ data_url = base_data_url + str(vid_id)
+ data_json = self._download_webpage(
+ data_url, video_id,
+ note='Downloading JSON data for ' + str(vid_id))
+ return json.loads(data_json)
+
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+ mytv = mobj.group('mytv') is not None
+
+ webpage = self._download_webpage(url, video_id)
+ raw_title = self._html_search_regex(r'(?s)<title>(.+?) ',
+ webpage, 'video title')
+ title = raw_title.partition('-')[0].strip()
+
+ vid = self._html_search_regex(r'var vid ?= ?["\'](\d+)["\']', webpage,
+ 'video path')
+ data = _fetch_data(vid, mytv)
+
+ QUALITIES = ('ori', 'super', 'high', 'nor')
+ vid_ids = [data['data'][q + 'Vid']
+ for q in QUALITIES
+ if data['data'][q + 'Vid'] != 0]
+ if not vid_ids:
+ raise ExtractorError('No formats available for this video')
+
+ # For now, we just pick the highest available quality
+ vid_id = vid_ids[-1]
+
+ format_data = data if vid == vid_id else _fetch_data(vid_id, mytv)
+ part_count = format_data['data']['totalBlocks']
+ allot = format_data['allot']
+ prot = format_data['prot']
+ clipsURL = format_data['data']['clipsURL']
+ su = format_data['data']['su']
+
+ playlist = []
+ for i in range(part_count):
+ part_url = ('http://%s/?prot=%s&file=%s&new=%s' %
+ (allot, prot, clipsURL[i], su[i]))
+ part_str = self._download_webpage(
+ part_url, video_id,
+ note='Downloading part %d of %d' % (i + 1, part_count))
+
+ part_info = part_str.split('|')
+ video_url = '%s%s?key=%s' % (part_info[0], su[i], part_info[3])
+
+ video_info = {
+ 'id': '%s_part%02d' % (video_id, i + 1),
+ 'title': title,
+ 'url': video_url,
+ 'ext': 'mp4',
+ }
+ playlist.append(video_info)
+
+ if len(playlist) == 1:
+ info = playlist[0]
+ info['id'] = video_id
+ else:
+ info = {
+ '_type': 'playlist',
+ 'entries': playlist,
+ 'id': video_id,
+ }
+
+ return info
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/soundcloud.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/soundcloud.py
new file mode 100644
index 0000000000..5d60c49395
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/soundcloud.py
@@ -0,0 +1,383 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+import re
+import itertools
+
+from .common import InfoExtractor
+from ..compat import (
+ compat_str,
+ compat_urlparse,
+ compat_urllib_parse,
+)
+from ..utils import (
+ ExtractorError,
+ int_or_none,
+ unified_strdate,
+)
+
+
+class SoundcloudIE(InfoExtractor):
+ """Information extractor for soundcloud.com
+ To access the media, the uid of the song and a stream token
+ must be extracted from the page source and the script must make
+ a request to media.soundcloud.com/crossdomain.xml. Then
+ the media can be grabbed by requesting from an url composed
+ of the stream token and uid
+ """
+
+ _VALID_URL = r'''(?x)^(?:https?://)?
+ (?:(?:(?:www\.|m\.)?soundcloud\.com/
+ (?P[\w\d-]+)/
+ (?!sets/|likes/?(?:$|[?#]))
+ (?P[\w\d-]+)/?
+ (?P<token>[^?]+?)?(?:[?].*)?$)
+ |(?:api\.soundcloud\.com/tracks/(?P<track_id>\d+)
+ (?:/?\?secret_token=(?P<secret_token>[^&]+))?)
+ |(?P<player>(?:w|player|p.)\.soundcloud\.com/player/?.*?url=.*)
+ )
+ '''
+ IE_NAME = 'soundcloud'
+ _TESTS = [
+ {
+ 'url': 'http://soundcloud.com/ethmusic/lostin-powers-she-so-heavy',
+ 'md5': 'ebef0a451b909710ed1d7787dddbf0d7',
+ 'info_dict': {
+ 'id': '62986583',
+ 'ext': 'mp3',
+ 'upload_date': '20121011',
+ 'description': 'No Downloads untill we record the finished version this weekend, i was too pumped n i had to post it , earl is prolly gonna b hella p.o\'d',
+ 'uploader': 'E.T. ExTerrestrial Music',
+ 'title': 'Lostin Powers - She so Heavy (SneakPreview) Adrian Ackers Blueprint 1',
+ 'duration': 143,
+ }
+ },
+ # not streamable song
+ {
+ 'url': 'https://soundcloud.com/the-concept-band/goldrushed-mastered?in=the-concept-band/sets/the-royal-concept-ep',
+ 'info_dict': {
+ 'id': '47127627',
+ 'ext': 'mp3',
+ 'title': 'Goldrushed',
+ 'description': 'From Stockholm Sweden\r\nPovel / Magnus / Filip / David\r\nwww.theroyalconcept.com',
+ 'uploader': 'The Royal Concept',
+ 'upload_date': '20120521',
+ 'duration': 227,
+ },
+ 'params': {
+ # rtmp
+ 'skip_download': True,
+ },
+ },
+ # private link
+ {
+ 'url': 'https://soundcloud.com/jaimemf/youtube-dl-test-video-a-y-baw/s-8Pjrp',
+ 'md5': 'aa0dd32bfea9b0c5ef4f02aacd080604',
+ 'info_dict': {
+ 'id': '123998367',
+ 'ext': 'mp3',
+ 'title': 'Youtube - Dl Test Video \'\' Aл┬Б├╜',
+ 'uploader': 'jaimeMF',
+ 'description': 'test chars: \"\'/\\ц╓Б├╜',
+ 'upload_date': '20131209',
+ 'duration': 9,
+ },
+ },
+ # private link (alt format)
+ {
+ 'url': 'https://api.soundcloud.com/tracks/123998367?secret_token=s-8Pjrp',
+ 'md5': 'aa0dd32bfea9b0c5ef4f02aacd080604',
+ 'info_dict': {
+ 'id': '123998367',
+ 'ext': 'mp3',
+ 'title': 'Youtube - Dl Test Video \'\' Aл┬Б├╜',
+ 'uploader': 'jaimeMF',
+ 'description': 'test chars: \"\'/\\ц╓Б├╜',
+ 'upload_date': '20131209',
+ 'duration': 9,
+ },
+ },
+ # downloadable song
+ {
+ 'url': 'https://soundcloud.com/oddsamples/bus-brakes',
+ 'md5': '7624f2351f8a3b2e7cd51522496e7631',
+ 'info_dict': {
+ 'id': '128590877',
+ 'ext': 'mp3',
+ 'title': 'Bus Brakes',
+ 'description': 'md5:0053ca6396e8d2fd7b7e1595ef12ab66',
+ 'uploader': 'oddsamples',
+ 'upload_date': '20140109',
+ 'duration': 17,
+ },
+ },
+ ]
+
+ _CLIENT_ID = 'b45b1aa10f1ac2941910a7f0d10f8e28'
+ _IPHONE_CLIENT_ID = '376f225bf427445fc4bfb6b99b72e0bf'
+
+ def report_resolve(self, video_id):
+ """Report information extraction."""
+ self.to_screen('%s: Resolving id' % video_id)
+
+ @classmethod
+ def _resolv_url(cls, url):
+ return 'http://api.soundcloud.com/resolve.json?url=' + url + '&client_id=' + cls._CLIENT_ID
+
+ def _extract_info_dict(self, info, full_title=None, quiet=False, secret_token=None):
+ track_id = compat_str(info['id'])
+ name = full_title or track_id
+ if quiet:
+ self.report_extraction(name)
+
+ thumbnail = info['artwork_url']
+ if thumbnail is not None:
+ thumbnail = thumbnail.replace('-large', '-t500x500')
+ ext = 'mp3'
+ result = {
+ 'id': track_id,
+ 'uploader': info['user']['username'],
+ 'upload_date': unified_strdate(info['created_at']),
+ 'title': info['title'],
+ 'description': info['description'],
+ 'thumbnail': thumbnail,
+ 'duration': int_or_none(info.get('duration'), 1000),
+ 'webpage_url': info.get('permalink_url'),
+ }
+ formats = []
+ if info.get('downloadable', False):
+ # We can build a direct link to the song
+ format_url = (
+ 'https://api.soundcloud.com/tracks/{0}/download?client_id={1}'.format(
+ track_id, self._CLIENT_ID))
+ formats.append({
+ 'format_id': 'download',
+ 'ext': info.get('original_format', 'mp3'),
+ 'url': format_url,
+ 'vcodec': 'none',
+ 'preference': 10,
+ })
+
+ # We have to retrieve the url
+ streams_url = ('http://api.soundcloud.com/i1/tracks/{0}/streams?'
+ 'client_id={1}&secret_token={2}'.format(track_id, self._IPHONE_CLIENT_ID, secret_token))
+ format_dict = self._download_json(
+ streams_url,
+ track_id, 'Downloading track url')
+
+ for key, stream_url in format_dict.items():
+ if key.startswith('http'):
+ formats.append({
+ 'format_id': key,
+ 'ext': ext,
+ 'url': stream_url,
+ 'vcodec': 'none',
+ })
+ elif key.startswith('rtmp'):
+ # The url doesn't have an rtmp app, we have to extract the playpath
+ url, path = stream_url.split('mp3:', 1)
+ formats.append({
+ 'format_id': key,
+ 'url': url,
+ 'play_path': 'mp3:' + path,
+ 'ext': ext,
+ 'vcodec': 'none',
+ })
+
+ if not formats:
+ # We fallback to the stream_url in the original info, this
+ # cannot be always used, sometimes it can give an HTTP 404 error
+ formats.append({
+ 'format_id': 'fallback',
+ 'url': info['stream_url'] + '?client_id=' + self._CLIENT_ID,
+ 'ext': ext,
+ 'vcodec': 'none',
+ })
+
+ for f in formats:
+ if f['format_id'].startswith('http'):
+ f['protocol'] = 'http'
+ if f['format_id'].startswith('rtmp'):
+ f['protocol'] = 'rtmp'
+
+ self._sort_formats(formats)
+ result['formats'] = formats
+
+ return result
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url, flags=re.VERBOSE)
+ if mobj is None:
+ raise ExtractorError('Invalid URL: %s' % url)
+
+ track_id = mobj.group('track_id')
+ token = None
+ if track_id is not None:
+ info_json_url = 'http://api.soundcloud.com/tracks/' + track_id + '.json?client_id=' + self._CLIENT_ID
+ full_title = track_id
+ token = mobj.group('secret_token')
+ if token:
+ info_json_url += "&secret_token=" + token
+ elif mobj.group('player'):
+ query = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)
+ return self.url_result(query['url'][0])
+ else:
+ # extract uploader (which is in the url)
+ uploader = mobj.group('uploader')
+ # extract simple title (uploader + slug of song title)
+ slug_title = mobj.group('title')
+ token = mobj.group('token')
+ full_title = resolve_title = '%s/%s' % (uploader, slug_title)
+ if token:
+ resolve_title += '/%s' % token
+
+ self.report_resolve(full_title)
+
+ url = 'http://soundcloud.com/%s' % resolve_title
+ info_json_url = self._resolv_url(url)
+ info = self._download_json(info_json_url, full_title, 'Downloading info JSON')
+
+ return self._extract_info_dict(info, full_title, secret_token=token)
+
+
+class SoundcloudSetIE(SoundcloudIE):
+ _VALID_URL = r'https?://(?:www\.)?soundcloud\.com/(?P<uploader>[\w\d-]+)/sets/(?P<slug_title>[\w\d-]+)(?:/(?P<token>[^?/]+))?'
+ IE_NAME = 'soundcloud:set'
+ _TESTS = [{
+ 'url': 'https://soundcloud.com/the-concept-band/sets/the-royal-concept-ep',
+ 'info_dict': {
+ 'title': 'The Royal Concept EP',
+ },
+ 'playlist_mincount': 6,
+ }]
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+
+ # extract uploader (which is in the url)
+ uploader = mobj.group('uploader')
+ # extract simple title (uploader + slug of song title)
+ slug_title = mobj.group('slug_title')
+ full_title = '%s/sets/%s' % (uploader, slug_title)
+ url = 'http://soundcloud.com/%s/sets/%s' % (uploader, slug_title)
+
+ token = mobj.group('token')
+ if token:
+ full_title += '/' + token
+ url += '/' + token
+
+ self.report_resolve(full_title)
+
+ resolv_url = self._resolv_url(url)
+ info = self._download_json(resolv_url, full_title)
+
+ if 'errors' in info:
+ for err in info['errors']:
+ self._downloader.report_error('unable to download video webpage: %s' % compat_str(err['error_message']))
+ return
+
+ return {
+ '_type': 'playlist',
+ 'entries': [self._extract_info_dict(track, secret_token=token) for track in info['tracks']],
+ 'id': info['id'],
+ 'title': info['title'],
+ }
+
+
+class SoundcloudUserIE(SoundcloudIE):
+ _VALID_URL = r'https?://(www\.)?soundcloud\.com/(?P<user>[^/]+)/?((?P<rsrc>tracks|likes)/?)?(\?.*)?$'
+ IE_NAME = 'soundcloud:user'
+ _TESTS = [{
+ 'url': 'https://soundcloud.com/the-concept-band',
+ 'info_dict': {
+ 'id': '9615865',
+ 'title': 'The Royal Concept',
+ },
+ 'playlist_mincount': 12
+ }, {
+ 'url': 'https://soundcloud.com/the-concept-band/likes',
+ 'info_dict': {
+ 'id': '9615865',
+ 'title': 'The Royal Concept',
+ },
+ 'playlist_mincount': 1,
+ }]
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ uploader = mobj.group('user')
+ resource = mobj.group('rsrc')
+ if resource is None:
+ resource = 'tracks'
+ elif resource == 'likes':
+ resource = 'favorites'
+
+ url = 'http://soundcloud.com/%s/' % uploader
+ resolv_url = self._resolv_url(url)
+ user = self._download_json(
+ resolv_url, uploader, 'Downloading user info')
+ base_url = 'http://api.soundcloud.com/users/%s/%s.json?' % (uploader, resource)
+
+ entries = []
+ for i in itertools.count():
+ data = compat_urllib_parse.urlencode({
+ 'offset': i * 50,
+ 'limit': 50,
+ 'client_id': self._CLIENT_ID,
+ })
+ new_entries = self._download_json(
+ base_url + data, uploader, 'Downloading track page %s' % (i + 1))
+ if len(new_entries) == 0:
+ self.to_screen('%s: End page received' % uploader)
+ break
+ entries.extend(self._extract_info_dict(e, quiet=True) for e in new_entries)
+
+ return {
+ '_type': 'playlist',
+ 'id': compat_str(user['id']),
+ 'title': user['username'],
+ 'entries': entries,
+ }
+
+
+class SoundcloudPlaylistIE(SoundcloudIE):
+ _VALID_URL = r'https?://api\.soundcloud\.com/playlists/(?P<id>[0-9]+)(?:/?\?secret_token=(?P<token>[^&]+?))?$'
+ IE_NAME = 'soundcloud:playlist'
+ _TESTS = [{
+ 'url': 'http://api.soundcloud.com/playlists/4110309',
+ 'info_dict': {
+ 'id': '4110309',
+ 'title': 'TILT Brass - Bowery Poetry Club, August \'03 [Non-Site SCR 02]',
+ 'description': 're:.*?TILT Brass - Bowery Poetry Club',
+ },
+ 'playlist_count': 6,
+ }]
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ playlist_id = mobj.group('id')
+ base_url = '%s//api.soundcloud.com/playlists/%s.json?' % (self.http_scheme(), playlist_id)
+
+ data_dict = {
+ 'client_id': self._CLIENT_ID,
+ }
+ token = mobj.group('token')
+
+ if token:
+ data_dict['secret_token'] = token
+
+ data = compat_urllib_parse.urlencode(data_dict)
+ data = self._download_json(
+ base_url + data, playlist_id, 'Downloading playlist')
+
+ entries = [
+ self._extract_info_dict(t, quiet=True, secret_token=token)
+ for t in data['tracks']]
+
+ return {
+ '_type': 'playlist',
+ 'id': playlist_id,
+ 'title': data.get('title'),
+ 'description': data.get('description'),
+ 'entries': entries,
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/soundgasm.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/soundgasm.py
new file mode 100644
index 0000000000..a4f8ce6c3c
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/soundgasm.py
@@ -0,0 +1,40 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+
+
+class SoundgasmIE(InfoExtractor):
+ _VALID_URL = r'https?://(?:www\.)?soundgasm\.net/u/(?P<user>[0-9a-zA-Z_\-]+)/(?P<title>[0-9a-zA-Z_\-]+)'
+ _TEST = {
+ 'url': 'http://soundgasm.net/u/ytdl/Piano-sample',
+ 'md5': '010082a2c802c5275bb00030743e75ad',
+ 'info_dict': {
+ 'id': '88abd86ea000cafe98f96321b23cc1206cbcbcc9',
+ 'ext': 'm4a',
+ 'title': 'ytdl_Piano-sample',
+ 'description': 'Royalty Free Sample Music'
+ }
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ display_id = mobj.group('title')
+ audio_title = mobj.group('user') + '_' + mobj.group('title')
+ webpage = self._download_webpage(url, display_id)
+ audio_url = self._html_search_regex(
+ r'(?s)m4a\:\s"([^"]+)"', webpage, 'audio URL')
+ audio_id = re.split('\/|\.', audio_url)[-2]
+ description = self._html_search_regex(
+ r'(?s)<li>Description:\s(.*?)<\/li>', webpage, 'description',
+ fatal=False)
+
+ return {
+ 'id': audio_id,
+ 'display_id': display_id,
+ 'url': audio_url,
+ 'title': audio_title,
+ 'description': description
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/southpark.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/southpark.py
new file mode 100644
index 0000000000..c20397b3d1
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/southpark.py
@@ -0,0 +1,36 @@
+from __future__ import unicode_literals
+
+from .mtv import MTVServicesInfoExtractor
+
+
+class SouthParkIE(MTVServicesInfoExtractor):
+ IE_NAME = 'southpark.cc.com'
+ _VALID_URL = r'https?://(www\.)?(?P<url>southpark\.cc\.com/(clips|full-episodes)/(?P<id>.+?)(\?|#|$))'
+
+ _FEED_URL = 'http://www.southparkstudios.com/feeds/video-player/mrss'
+
+ _TESTS = [{
+ 'url': 'http://southpark.cc.com/clips/104437/bat-daded#tab=featured',
+ 'info_dict': {
+ 'id': 'a7bff6c2-ed00-11e0-aca6-0026b9414f30',
+ 'ext': 'mp4',
+ 'title': 'South Park|Bat Daded',
+ 'description': 'Randy disqualifies South Park by getting into a fight with Bat Dad.',
+ },
+ }]
+
+
+class SouthparkDeIE(SouthParkIE):
+ IE_NAME = 'southpark.de'
+ _VALID_URL = r'https?://(www\.)?(?P<url>southpark\.de/(clips|alle-episoden)/(?P<id>.+?)(\?|#|$))'
+ _FEED_URL = 'http://www.southpark.de/feeds/video-player/mrss/'
+
+ _TESTS = [{
+ 'url': 'http://www.southpark.de/clips/uygssh/the-government-wont-respect-my-privacy#tab=featured',
+ 'info_dict': {
+ 'id': '85487c96-b3b9-4e39-9127-ad88583d9bf2',
+ 'ext': 'mp4',
+ 'title': 'The Government Won\'t Respect My Privacy',
+ 'description': 'Cartman explains the benefits of "Shitter" to Stan, Kyle and Craig.',
+ },
+ }]
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/southparkstudios.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/southparkstudios.py
new file mode 100644
index 0000000000..aea8e64393
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/southparkstudios.py
@@ -0,0 +1,36 @@
+from __future__ import unicode_literals
+
+from .mtv import MTVServicesInfoExtractor
+
+
+class SouthParkStudiosIE(MTVServicesInfoExtractor):
+ IE_NAME = 'southparkstudios.com'
+ _VALID_URL = r'https?://(www\.)?(?P<url>southparkstudios\.com/(clips|full-episodes)/(?P<id>.+?)(\?|#|$))'
+
+ _FEED_URL = 'http://www.southparkstudios.com/feeds/video-player/mrss'
+
+ _TESTS = [{
+ 'url': 'http://www.southparkstudios.com/clips/104437/bat-daded#tab=featured',
+ 'info_dict': {
+ 'id': 'a7bff6c2-ed00-11e0-aca6-0026b9414f30',
+ 'ext': 'mp4',
+ 'title': 'Bat Daded',
+ 'description': 'Randy disqualifies South Park by getting into a fight with Bat Dad.',
+ },
+ }]
+
+
+class SouthparkDeIE(SouthParkStudiosIE):
+ IE_NAME = 'southpark.de'
+ _VALID_URL = r'https?://(www\.)?(?P<url>southpark\.de/(clips|alle-episoden)/(?P<id>.+?)(\?|#|$))'
+ _FEED_URL = 'http://www.southpark.de/feeds/video-player/mrss/'
+
+ _TESTS = [{
+ 'url': 'http://www.southpark.de/clips/uygssh/the-government-wont-respect-my-privacy#tab=featured',
+ 'info_dict': {
+ 'id': '85487c96-b3b9-4e39-9127-ad88583d9bf2',
+ 'ext': 'mp4',
+ 'title': 'The Government Won\'t Respect My Privacy',
+ 'description': 'Cartman explains the benefits of "Shitter" to Stan, Kyle and Craig.',
+ },
+ }]
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/space.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/space.py
new file mode 100644
index 0000000000..c2d0d36a69
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/space.py
@@ -0,0 +1,38 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from .brightcove import BrightcoveIE
+from ..utils import RegexNotFoundError, ExtractorError
+
+
+class SpaceIE(InfoExtractor):
+ _VALID_URL = r'https?://(?:(?:www|m)\.)?space\.com/\d+-(?P<title>[^/\.\?]*?)-video\.html'
+ _TEST = {
+ 'add_ie': ['Brightcove'],
+ 'url': 'http://www.space.com/23373-huge-martian-landforms-detail-revealed-by-european-probe-video.html',
+ 'info_dict': {
+ 'id': '2780937028001',
+ 'ext': 'mp4',
+ 'title': 'Huge Martian Landforms\' Detail Revealed By European Probe | Video',
+ 'description': 'md5:db81cf7f3122f95ed234b631a6ea1e61',
+ 'uploader': 'TechMedia Networks',
+ },
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ title = mobj.group('title')
+ webpage = self._download_webpage(url, title)
+ try:
+ # Some videos require the playerKey field, which isn't define in
+ # the BrightcoveExperience object
+ brightcove_url = self._og_search_video_url(webpage)
+ except RegexNotFoundError:
+ # Other videos works fine with the info from the object
+ brightcove_url = BrightcoveIE._extract_brightcove_url(webpage)
+ if brightcove_url is None:
+ raise ExtractorError(
+ 'The webpage does not contain a video', expected=True)
+ return self.url_result(brightcove_url, BrightcoveIE.ie_key())
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/spankwire.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/spankwire.py
new file mode 100644
index 0000000000..b936202f6f
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/spankwire.py
@@ -0,0 +1,113 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..compat import (
+ compat_urllib_parse,
+ compat_urllib_parse_urlparse,
+ compat_urllib_request,
+)
+from ..utils import (
+ str_to_int,
+ unified_strdate,
+)
+from ..aes import aes_decrypt_text
+
+
+class SpankwireIE(InfoExtractor):
+ _VALID_URL = r'https?://(?:www\.)?(?P<url>spankwire\.com/[^/]*/video(?P<videoid>[0-9]+)/?)'
+ _TEST = {
+ 'url': 'http://www.spankwire.com/Buckcherry-s-X-Rated-Music-Video-Crazy-Bitch/video103545/',
+ 'md5': '8bbfde12b101204b39e4b9fe7eb67095',
+ 'info_dict': {
+ 'id': '103545',
+ 'ext': 'mp4',
+ 'title': 'Buckcherry`s X Rated Music Video Crazy Bitch',
+ 'description': 'Crazy Bitch X rated music video.',
+ 'uploader': 'oreusz',
+ 'uploader_id': '124697',
+ 'upload_date': '20070508',
+ 'age_limit': 18,
+ }
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('videoid')
+ url = 'http://www.' + mobj.group('url')
+
+ req = compat_urllib_request.Request(url)
+ req.add_header('Cookie', 'age_verified=1')
+ webpage = self._download_webpage(req, video_id)
+
+ title = self._html_search_regex(
+ r'<h1>([^<]+)', webpage, 'title')
+ description = self._html_search_regex(
+ r'<div\s+id="descriptionContent">([^<]+)<',
+ webpage, 'description', fatal=False)
+ thumbnail = self._html_search_regex(
+ r'playerData\.screenShot\s*=\s*["\']([^"\']+)["\']',
+ webpage, 'thumbnail', fatal=False)
+
+ uploader = self._html_search_regex(
+ r'by:\s*<a [^>]*>(.+?)</a>',
+ webpage, 'uploader', fatal=False)
+ uploader_id = self._html_search_regex(
+ r'by:\s*<a href="/Profile\.aspx\?.*?UserId=(\d+).*?"',
+ webpage, 'uploader id', fatal=False)
+ upload_date = unified_strdate(self._html_search_regex(
+ r'</a> on (.+?) at \d+:\d+',
+ webpage, 'upload date', fatal=False))
+
+ view_count = str_to_int(self._html_search_regex(
+ r'<div id="viewsCounter"><span>([\d,\.]+)</span> views</div>',
+ webpage, 'view count', fatal=False))
+ comment_count = str_to_int(self._html_search_regex(
+ r'Comments<span[^>]+>\s*\(([\d,\.]+)\)</span>',
+ webpage, 'comment count', fatal=False))
+
+ video_urls = list(map(
+ compat_urllib_parse.unquote,
+ re.findall(r'playerData\.cdnPath[0-9]{3,}\s*=\s*["\']([^"\']+)["\']', webpage)))
+ if webpage.find('flashvars\.encrypted = "true"') != -1:
+ password = self._html_search_regex(
+ r'flashvars\.video_title = "([^"]+)',
+ webpage, 'password').replace('+', ' ')
+ video_urls = list(map(
+ lambda s: aes_decrypt_text(s, password, 32).decode('utf-8'),
+ video_urls))
+
+ formats = []
+ for video_url in video_urls:
+ path = compat_urllib_parse_urlparse(video_url).path
+ format = path.split('/')[4].split('_')[:2]
+ resolution, bitrate_str = format
+ format = "-".join(format)
+ height = int(resolution.rstrip('Pp'))
+ tbr = int(bitrate_str.rstrip('Kk'))
+ formats.append({
+ 'url': video_url,
+ 'resolution': resolution,
+ 'format': format,
+ 'tbr': tbr,
+ 'height': height,
+ 'format_id': format,
+ })
+ self._sort_formats(formats)
+
+ age_limit = self._rta_search(webpage)
+
+ return {
+ 'id': video_id,
+ 'title': title,
+ 'description': description,
+ 'thumbnail': thumbnail,
+ 'uploader': uploader,
+ 'uploader_id': uploader_id,
+ 'upload_date': upload_date,
+ 'view_count': view_count,
+ 'comment_count': comment_count,
+ 'formats': formats,
+ 'age_limit': age_limit,
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/spiegel.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/spiegel.py
new file mode 100644
index 0000000000..1e55a9ffb5
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/spiegel.py
@@ -0,0 +1,133 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..compat import compat_urlparse
+from .spiegeltv import SpiegeltvIE
+
+
+class SpiegelIE(InfoExtractor):
+ _VALID_URL = r'https?://(?:www\.)?spiegel\.de/video/[^/]*-(?P<id>[0-9]+)(?:-embed)?(?:\.html)?(?:#.*)?$'
+ _TESTS = [{
+ 'url': 'http://www.spiegel.de/video/vulkan-tungurahua-in-ecuador-ist-wieder-aktiv-video-1259285.html',
+ 'md5': '2c2754212136f35fb4b19767d242f66e',
+ 'info_dict': {
+ 'id': '1259285',
+ 'ext': 'mp4',
+ 'title': 'Vulkanausbruch in Ecuador: Der "Feuerschlund" ist wieder aktiv',
+ 'description': 'md5:8029d8310232196eb235d27575a8b9f4',
+ 'duration': 49,
+ },
+ }, {
+ 'url': 'http://www.spiegel.de/video/schach-wm-videoanalyse-des-fuenften-spiels-video-1309159.html',
+ 'md5': 'f2cdf638d7aa47654e251e1aee360af1',
+ 'info_dict': {
+ 'id': '1309159',
+ 'ext': 'mp4',
+ 'title': 'Schach-WM in der Videoanalyse: Carlsen nutzt die Fehlgriffe des Titelverteidigers',
+ 'description': 'md5:c2322b65e58f385a820c10fa03b2d088',
+ 'duration': 983,
+ },
+ }, {
+ 'url': 'http://www.spiegel.de/video/astronaut-alexander-gerst-von-der-iss-station-beantwortet-fragen-video-1519126-embed.html',
+ 'md5': 'd8eeca6bfc8f1cd6f490eb1f44695d51',
+ 'info_dict': {
+ 'id': '1519126',
+ 'ext': 'mp4',
+ 'description': 'SPIEGEL ONLINE-Nutzer durften den deutschen Astronauten Alexander Gerst ц╪ber sein Leben auf der ISS-Station befragen. Hier kommen seine Antworten auf die besten sechs Fragen.',
+ 'title': 'Fragen an Astronaut Alexander Gerst: "Bekommen Sie die Tageszeiten mit?"',
+ }
+ }]
+
+ def _real_extract(self, url):
+ video_id = self._match_id(url)
+ webpage, handle = self._download_webpage_handle(url, video_id)
+
+ # 302 to spiegel.tv, like http://www.spiegel.de/video/der-film-zum-wochenende-die-wahrheit-ueber-maenner-video-99003272.html
+ if SpiegeltvIE.suitable(handle.geturl()):
+ return self.url_result(handle.geturl(), 'Spiegeltv')
+
+ title = re.sub(r'\s+', ' ', self._html_search_regex(
+ r'(?s)<(?:h1|div) class="module-title"[^>]*>(.*?)</(?:h1|div)>',
+ webpage, 'title'))
+ description = self._html_search_meta('description', webpage, 'description')
+
+ base_url = self._search_regex(
+ r'var\s+server\s*=\s*"([^"]+)\"', webpage, 'server URL')
+
+ xml_url = base_url + video_id + '.xml'
+ idoc = self._download_xml(xml_url, video_id)
+
+ formats = [
+ {
+ 'format_id': n.tag.rpartition('type')[2],
+ 'url': base_url + n.find('./filename').text,
+ 'width': int(n.find('./width').text),
+ 'height': int(n.find('./height').text),
+ 'abr': int(n.find('./audiobitrate').text),
+ 'vbr': int(n.find('./videobitrate').text),
+ 'vcodec': n.find('./codec').text,
+ 'acodec': 'MP4A',
+ }
+ for n in list(idoc)
+ # Blacklist type 6, it's extremely LQ and not available on the same server
+ if n.tag.startswith('type') and n.tag != 'type6'
+ ]
+ duration = float(idoc[0].findall('./duration')[0].text)
+
+ self._sort_formats(formats)
+
+ return {
+ 'id': video_id,
+ 'title': title,
+ 'description': description,
+ 'duration': duration,
+ 'formats': formats,
+ }
+
+
+class SpiegelArticleIE(InfoExtractor):
+ _VALID_URL = 'https?://www\.spiegel\.de/(?!video/)[^?#]*?-(?P<id>[0-9]+)\.html'
+ IE_NAME = 'Spiegel:Article'
+ IE_DESC = 'Articles on spiegel.de'
+ _TESTS = [{
+ 'url': 'http://www.spiegel.de/sport/sonst/badminton-wm-die-randsportart-soll-populaerer-werden-a-987092.html',
+ 'info_dict': {
+ 'id': '1516455',
+ 'ext': 'mp4',
+ 'title': 'Faszination Badminton: Nennt es bloц÷ nicht Federball',
+ 'description': 're:^Patrick Kц╓mnitz gehц╤rt.{100,}',
+ },
+ }, {
+ 'url': 'http://www.spiegel.de/wissenschaft/weltall/astronaut-alexander-gerst-antwortet-spiegel-online-lesern-a-989876.html',
+ 'info_dict': {
+
+ },
+ 'playlist_count': 6,
+ }]
+
+ def _real_extract(self, url):
+ video_id = self._match_id(url)
+ webpage = self._download_webpage(url, video_id)
+
+ # Single video on top of the page
+ video_link = self._search_regex(
+ r'<a href="([^"]+)" onclick="return spOpenVideo\(this,', webpage,
+ 'video page URL', default=None)
+ if video_link:
+ video_url = compat_urlparse.urljoin(
+ self.http_scheme() + '//spiegel.de/', video_link)
+ return self.url_result(video_url)
+
+ # Multiple embedded videos
+ embeds = re.findall(
+ r'<div class="vid_holder[0-9]+.*?</div>\s*.*?url\s*=\s*"([^"]+)"',
+ webpage)
+ entries = [
+ self.url_result(compat_urlparse.urljoin(
+ self.http_scheme() + '//spiegel.de/', embed_path))
+ for embed_path in embeds
+ ]
+ return self.playlist_result(entries)
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/spiegeltv.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/spiegeltv.py
new file mode 100644
index 0000000000..98cf92d89a
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/spiegeltv.py
@@ -0,0 +1,80 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import float_or_none
+
+
+class SpiegeltvIE(InfoExtractor):
+ _VALID_URL = r'https?://(?:www\.)?spiegel\.tv/(?:#/)?filme/(?P<id>[\-a-z0-9]+)'
+ _TESTS = [{
+ 'url': 'http://www.spiegel.tv/filme/flug-mh370/',
+ 'info_dict': {
+ 'id': 'flug-mh370',
+ 'ext': 'm4v',
+ 'title': 'Flug MH370',
+ 'description': 'Das Rц╓tsel um die Boeing 777 der Malaysia-Airlines',
+ 'thumbnail': 're:http://.*\.jpg$',
+ },
+ 'params': {
+ # rtmp download
+ 'skip_download': True,
+ }
+ }, {
+ 'url': 'http://www.spiegel.tv/#/filme/alleskino-die-wahrheit-ueber-maenner/',
+ 'only_matching': True,
+ }]
+
+ def _real_extract(self, url):
+ if '/#/' in url:
+ url = url.replace('/#/', '/')
+ video_id = self._match_id(url)
+ webpage = self._download_webpage(url, video_id)
+ title = self._html_search_regex(r'<h1.*?>(.*?)</h1>', webpage, 'title')
+
+ apihost = 'http://spiegeltv-ivms2-restapi.s3.amazonaws.com'
+ version_json = self._download_json(
+ '%s/version.json' % apihost, video_id,
+ note='Downloading version information')
+ version_name = version_json['version_name']
+
+ slug_json = self._download_json(
+ '%s/%s/restapi/slugs/%s.json' % (apihost, version_name, video_id),
+ video_id,
+ note='Downloading object information')
+ oid = slug_json['object_id']
+
+ media_json = self._download_json(
+ '%s/%s/restapi/media/%s.json' % (apihost, version_name, oid),
+ video_id, note='Downloading media information')
+ uuid = media_json['uuid']
+ is_wide = media_json['is_wide']
+
+ server_json = self._download_json(
+ 'http://www.spiegel.tv/streaming_servers/', video_id,
+ note='Downloading server information')
+ server = server_json[0]['endpoint']
+
+ thumbnails = []
+ for image in media_json['images']:
+ thumbnails.append({
+ 'url': image['url'],
+ 'width': image['width'],
+ 'height': image['height'],
+ })
+
+ description = media_json['subtitle']
+ duration = float_or_none(media_json.get('duration_in_ms'), scale=1000)
+ format = '16x9' if is_wide else '4x3'
+
+ url = server + 'mp4:' + uuid + '_spiegeltv_0500_' + format + '.m4v'
+
+ return {
+ 'id': video_id,
+ 'title': title,
+ 'url': url,
+ 'ext': 'm4v',
+ 'description': description,
+ 'duration': duration,
+ 'thumbnails': thumbnails
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/spike.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/spike.py
new file mode 100644
index 0000000000..a3adf54e30
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/spike.py
@@ -0,0 +1,32 @@
+from __future__ import unicode_literals
+
+import re
+
+from .mtv import MTVServicesInfoExtractor
+
+
+class SpikeIE(MTVServicesInfoExtractor):
+ _VALID_URL = r'''(?x)https?://
+ (www\.spike\.com/(video-clips|episodes)/.+|
+ m\.spike\.com/videos/video.rbml\?id=(?P<mobile_id>[^&]+))
+ '''
+ _TEST = {
+ 'url': 'http://www.spike.com/video-clips/lhtu8m/auction-hunters-can-allen-ride-a-hundred-year-old-motorcycle',
+ 'md5': '1a9265f32b0c375793d6c4ce45255256',
+ 'info_dict': {
+ 'id': 'b9c8221a-4e50-479a-b86d-3333323e38ba',
+ 'ext': 'mp4',
+ 'title': 'Auction Hunters|Can Allen Ride A Hundred Year-Old Motorcycle?',
+ 'description': 'md5:fbed7e82ed5fad493615b3094a9499cb',
+ },
+ }
+
+ _FEED_URL = 'http://www.spike.com/feeds/mrss/'
+ _MOBILE_TEMPLATE = 'http://m.spike.com/videos/video.rbml?id=%s'
+
+ def _real_extract(self, url):
+ mobj = re.search(self._VALID_URL, url)
+ mobile_id = mobj.group('mobile_id')
+ if mobile_id is not None:
+ url = 'http://www.spike.com/video-clips/%s' % mobile_id
+ return super(SpikeIE, self)._real_extract(url)
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/sport5.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/sport5.py
new file mode 100644
index 0000000000..dfe50ed458
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/sport5.py
@@ -0,0 +1,92 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import ExtractorError
+
+
+class Sport5IE(InfoExtractor):
+ _VALID_URL = r'http://(?:www|vod)?\.sport5\.co\.il/.*\b(?:Vi|docID)=(?P<id>\d+)'
+ _TESTS = [
+ {
+ 'url': 'http://vod.sport5.co.il/?Vc=147&Vi=176331&Page=1',
+ 'info_dict': {
+ 'id': 's5-Y59xx1-GUh2',
+ 'ext': 'mp4',
+ 'title': 'в∙в°в═в║в≥в■-в╖в∙в╗в⌠в∙в▒в■ 0:3',
+ 'description': 'в░в°в╖в░в║в╗, в▓в░в≥в≥в■ в∙в╓в▓в∙в°в≥ в║в≥в⌠в╗в∙ в°в╖в▒в∙в╕в■ в╘в° в═в∙в═в∙ в═в≥в╕в≈в∙в÷ в╒в° в╖в∙в╗в⌠в∙в▒в■ в∙в░в╙ в■в·в╖в∙в² в■в╗в░в╘в∙в÷ в▒в°в≥в▓в■',
+ 'duration': 228,
+ 'categories': list,
+ },
+ 'skip': 'Blocked outside of Israel',
+ }, {
+ 'url': 'http://www.sport5.co.il/articles.aspx?FolderID=3075&docID=176372&lang=HE',
+ 'info_dict': {
+ 'id': 's5-SiXxx1-hKh2',
+ 'ext': 'mp4',
+ 'title': 'GOALS_CELTIC_270914.mp4',
+ 'description': '',
+ 'duration': 87,
+ 'categories': list,
+ },
+ 'skip': 'Blocked outside of Israel',
+ }
+ ]
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ media_id = mobj.group('id')
+
+ webpage = self._download_webpage(url, media_id)
+
+ video_id = self._html_search_regex('clipId=([\w-]+)', webpage, 'video id')
+
+ metadata = self._download_xml(
+ 'http://sport5-metadata-rr-d.nsacdn.com/vod/vod/%s/HDS/metadata.xml' % video_id,
+ video_id)
+
+ error = metadata.find('./Error')
+ if error is not None:
+ raise ExtractorError(
+ '%s returned error: %s - %s' % (
+ self.IE_NAME,
+ error.find('./Name').text,
+ error.find('./Description').text),
+ expected=True)
+
+ title = metadata.find('./Title').text
+ description = metadata.find('./Description').text
+ duration = int(metadata.find('./Duration').text)
+
+ posters_el = metadata.find('./PosterLinks')
+ thumbnails = [{
+ 'url': thumbnail.text,
+ 'width': int(thumbnail.get('width')),
+ 'height': int(thumbnail.get('height')),
+ } for thumbnail in posters_el.findall('./PosterIMG')] if posters_el is not None else []
+
+ categories_el = metadata.find('./Categories')
+ categories = [
+ cat.get('name') for cat in categories_el.findall('./Category')
+ ] if categories_el is not None else []
+
+ formats = [{
+ 'url': fmt.text,
+ 'ext': 'mp4',
+ 'vbr': int(fmt.get('bitrate')),
+ 'width': int(fmt.get('width')),
+ 'height': int(fmt.get('height')),
+ } for fmt in metadata.findall('./PlaybackLinks/FileURL')]
+ self._sort_formats(formats)
+
+ return {
+ 'id': video_id,
+ 'title': title,
+ 'description': description,
+ 'thumbnails': thumbnails,
+ 'duration': duration,
+ 'categories': categories,
+ 'formats': formats,
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/sportbox.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/sportbox.py
new file mode 100644
index 0000000000..becdf658f6
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/sportbox.py
@@ -0,0 +1,76 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+ parse_duration,
+ parse_iso8601,
+)
+
+
+class SportBoxIE(InfoExtractor):
+ _VALID_URL = r'https?://news\.sportbox\.ru/Vidy_sporta/(?:[^/]+/)+spbvideo_NI\d+_(?P<display_id>.+)'
+ _TESTS = [
+ {
+ 'url': 'http://news.sportbox.ru/Vidy_sporta/Avtosport/Rossijskij/spbvideo_NI483529_Gonka-2-zaezd-Obyedinenniy-2000-klassi-Turing-i-S',
+ 'md5': 'ff56a598c2cf411a9a38a69709e97079',
+ 'info_dict': {
+ 'id': '80822',
+ 'ext': 'mp4',
+ 'title': 'п⌠п╬п╫п╨п╟ 2 п╥п╟п╣п╥п╢ б╚б╚п·п╠я┼п╣п╢п╦п╫п╣п╫п╫я▀п╧ 2000б╩: п╨п╩п╟я│я│я▀ п╒я┐я─п╦п╫пЁ п╦ п║я┐п©п╣я─-п©я─п╬п╢п╟п╨я┬п╫',
+ 'description': 'md5:81715fa9c4ea3d9e7915dc8180c778ed',
+ 'thumbnail': 're:^https?://.*\.jpg$',
+ 'timestamp': 1411896237,
+ 'upload_date': '20140928',
+ 'duration': 4846,
+ },
+ 'params': {
+ # m3u8 download
+ 'skip_download': True,
+ },
+ }, {
+ 'url': 'http://news.sportbox.ru/Vidy_sporta/billiard/spbvideo_NI486287_CHempionat-mira-po-dinamichnoy-piramide-4',
+ 'only_matching': True,
+ }
+ ]
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ display_id = mobj.group('display_id')
+
+ webpage = self._download_webpage(url, display_id)
+
+ video_id = self._search_regex(
+ r'src="/vdl/player/media/(\d+)"', webpage, 'video id')
+
+ player = self._download_webpage(
+ 'http://news.sportbox.ru/vdl/player/media/%s' % video_id,
+ display_id, 'Downloading player webpage')
+
+ hls = self._search_regex(
+ r"var\s+original_hls_file\s*=\s*'([^']+)'", player, 'hls file')
+
+ formats = self._extract_m3u8_formats(hls, display_id, 'mp4')
+
+ title = self._html_search_regex(
+ r'<h1 itemprop="name">([^<]+)</h1>', webpage, 'title')
+ description = self._html_search_regex(
+ r'(?s)<div itemprop="description">(.+?)</div>', webpage, 'description', fatal=False)
+ thumbnail = self._og_search_thumbnail(webpage)
+ timestamp = parse_iso8601(self._search_regex(
+ r'<span itemprop="uploadDate">([^<]+)</span>', webpage, 'timestamp', fatal=False))
+ duration = parse_duration(self._html_search_regex(
+ r'<meta itemprop="duration" content="PT([^"]+)">', webpage, 'duration', fatal=False))
+
+ return {
+ 'id': video_id,
+ 'display_id': display_id,
+ 'title': title,
+ 'description': description,
+ 'thumbnail': thumbnail,
+ 'timestamp': timestamp,
+ 'duration': duration,
+ 'formats': formats,
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/sportdeutschland.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/sportdeutschland.py
new file mode 100644
index 0000000000..1a57aebf16
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/sportdeutschland.py
@@ -0,0 +1,98 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..compat import (
+ compat_urllib_request,
+)
+from ..utils import (
+ parse_iso8601,
+)
+
+
+class SportDeutschlandIE(InfoExtractor):
+ _VALID_URL = r'https?://sportdeutschland\.tv/(?P<sport>[^/?#]+)/(?P<id>[^?#/]+)(?:$|[?#])'
+ _TESTS = [{
+ 'url': 'http://sportdeutschland.tv/badminton/live-li-ning-badminton-weltmeisterschaft-2014-kopenhagen',
+ 'info_dict': {
+ 'id': 'live-li-ning-badminton-weltmeisterschaft-2014-kopenhagen',
+ 'ext': 'mp4',
+ 'title': 're:Li-Ning Badminton Weltmeisterschaft 2014 Kopenhagen',
+ 'categories': ['Badminton'],
+ 'view_count': int,
+ 'thumbnail': 're:^https?://.*\.jpg$',
+ 'description': 're:Die Badminton-WM 2014 aus Kopenhagen bei Sportdeutschland\.TV',
+ 'timestamp': int,
+ 'upload_date': 're:^201408[23][0-9]$',
+ },
+ 'params': {
+ 'skip_download': 'Live stream',
+ },
+ }, {
+ 'url': 'http://sportdeutschland.tv/li-ning-badminton-wm-2014/lee-li-ning-badminton-weltmeisterschaft-2014-kopenhagen-herren-einzel-wei-vs',
+ 'info_dict': {
+ 'id': 'lee-li-ning-badminton-weltmeisterschaft-2014-kopenhagen-herren-einzel-wei-vs',
+ 'ext': 'mp4',
+ 'upload_date': '20140825',
+ 'description': 'md5:60a20536b57cee7d9a4ec005e8687504',
+ 'timestamp': 1408976060,
+ 'title': 'Li-Ning Badminton Weltmeisterschaft 2014 Kopenhagen: Herren Einzel, Wei Lee vs. Keun Lee',
+ 'thumbnail': 're:^https?://.*\.jpg$',
+ 'view_count': int,
+ 'categories': ['Li-Ning Badminton WM 2014'],
+ }
+ }]
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+ sport_id = mobj.group('sport')
+
+ api_url = 'http://splink.tv/api/permalinks/%s/%s' % (
+ sport_id, video_id)
+ req = compat_urllib_request.Request(api_url, headers={
+ 'Accept': 'application/vnd.vidibus.v2.html+json',
+ 'Referer': url,
+ })
+ data = self._download_json(req, video_id)
+
+ categories = list(data.get('section', {}).get('tags', {}).values())
+ asset = data['asset']
+ assets_info = self._download_json(asset['url'], video_id)
+
+ formats = []
+ smil_url = assets_info['video']
+ if '.smil' in smil_url:
+ m3u8_url = smil_url.replace('.smil', '.m3u8')
+ formats.extend(
+ self._extract_m3u8_formats(m3u8_url, video_id, ext='mp4'))
+
+ smil_doc = self._download_xml(
+ smil_url, video_id, note='Downloading SMIL metadata')
+ base_url = smil_doc.find('./head/meta').attrib['base']
+ formats.extend([{
+ 'format_id': 'rmtp',
+ 'url': base_url,
+ 'play_path': n.attrib['src'],
+ 'ext': 'flv',
+ 'preference': -100,
+ 'format_note': 'Seems to fail at example stream',
+ } for n in smil_doc.findall('./body/video')])
+ else:
+ formats.append({'url': smil_url})
+
+ self._sort_formats(formats)
+
+ return {
+ 'id': video_id,
+ 'formats': formats,
+ 'title': asset['title'],
+ 'thumbnail': asset.get('image'),
+ 'description': asset.get('teaser'),
+ 'categories': categories,
+ 'view_count': asset.get('views'),
+ 'rtmp_live': asset.get('live'),
+ 'timestamp': parse_iso8601(asset.get('date')),
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/srmediathek.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/srmediathek.py
new file mode 100644
index 0000000000..666a7dcc8a
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/srmediathek.py
@@ -0,0 +1,43 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+import json
+
+from .common import InfoExtractor
+from ..utils import js_to_json
+
+
+class SRMediathekIE(InfoExtractor):
+ IE_DESC = 'Sц╪ddeutscher Rundfunk'
+ _VALID_URL = r'https?://sr-mediathek\.sr-online\.de/index\.php\?.*?&id=(?P<id>[0-9]+)'
+
+ _TEST = {
+ 'url': 'http://sr-mediathek.sr-online.de/index.php?seite=7&id=28455',
+ 'info_dict': {
+ 'id': '28455',
+ 'ext': 'mp4',
+ 'title': 'sportarena (26.10.2014)',
+ 'description': 'Ringen: KSV Kц╤llerbach gegen Aachen-Walheim; Frauen-Fuц÷ball: 1. FC Saarbrц╪cken gegen Sindelfingen; Motorsport: Rallye in Losheim; dazu: Interview mit Timo Bernhard; Turnen: TG Saar; Reitsport: Deutscher Voltigier-Pokal; Badminton: Interview mit Michael Fuchs ',
+ 'thumbnail': 're:^https?://.*\.jpg$',
+ },
+ }
+
+ def _real_extract(self, url):
+ video_id = self._match_id(url)
+ webpage = self._download_webpage(url, video_id)
+
+ murls = json.loads(js_to_json(self._search_regex(
+ r'var mediaURLs\s*=\s*(.*?);\n', webpage, 'video URLs')))
+ formats = [{'url': murl} for murl in murls]
+ self._sort_formats(formats)
+
+ title = json.loads(js_to_json(self._search_regex(
+ r'var mediaTitles\s*=\s*(.*?);\n', webpage, 'title')))[0]
+
+ return {
+ 'id': video_id,
+ 'title': title,
+ 'formats': formats,
+ 'description': self._og_search_description(webpage),
+ 'thumbnail': self._og_search_thumbnail(webpage),
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/stanfordoc.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/stanfordoc.py
new file mode 100644
index 0000000000..4a3d8bb8f2
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/stanfordoc.py
@@ -0,0 +1,91 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+ ExtractorError,
+ orderedSet,
+ unescapeHTML,
+)
+
+
+class StanfordOpenClassroomIE(InfoExtractor):
+ IE_NAME = 'stanfordoc'
+ IE_DESC = 'Stanford Open ClassRoom'
+ _VALID_URL = r'https?://openclassroom\.stanford\.edu(?P<path>/?|(/MainFolder/(?:HomePage|CoursePage|VideoPage)\.php([?]course=(?P<course>[^&]+)(&video=(?P<video>[^&]+))?(&.*)?)?))$'
+ _TEST = {
+ 'url': 'http://openclassroom.stanford.edu/MainFolder/VideoPage.php?course=PracticalUnix&video=intro-environment&speed=100',
+ 'md5': '544a9468546059d4e80d76265b0443b8',
+ 'info_dict': {
+ 'id': 'PracticalUnix_intro-environment',
+ 'ext': 'mp4',
+ 'title': 'Intro Environment',
+ }
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+
+ if mobj.group('course') and mobj.group('video'): # A specific video
+ course = mobj.group('course')
+ video = mobj.group('video')
+ info = {
+ 'id': course + '_' + video,
+ 'uploader': None,
+ 'upload_date': None,
+ }
+
+ baseUrl = 'http://openclassroom.stanford.edu/MainFolder/courses/' + course + '/videos/'
+ xmlUrl = baseUrl + video + '.xml'
+ mdoc = self._download_xml(xmlUrl, info['id'])
+ try:
+ info['title'] = mdoc.findall('./title')[0].text
+ info['url'] = baseUrl + mdoc.findall('./videoFile')[0].text
+ except IndexError:
+ raise ExtractorError('Invalid metadata XML file')
+ return info
+ elif mobj.group('course'): # A course page
+ course = mobj.group('course')
+ info = {
+ 'id': course,
+ '_type': 'playlist',
+ 'uploader': None,
+ 'upload_date': None,
+ }
+
+ coursepage = self._download_webpage(
+ url, info['id'],
+ note='Downloading course info page',
+ errnote='Unable to download course info page')
+
+ info['title'] = self._html_search_regex(
+ r'<h1>([^<]+)</h1>', coursepage, 'title', default=info['id'])
+
+ info['description'] = self._html_search_regex(
+ r'(?s)<description>([^<]+)</description>',
+ coursepage, 'description', fatal=False)
+
+ links = orderedSet(re.findall('<a href="(VideoPage.php\?[^"]+)">', coursepage))
+ info['entries'] = [self.url_result(
+ 'http://openclassroom.stanford.edu/MainFolder/%s' % unescapeHTML(l)
+ ) for l in links]
+ return info
+ else: # Root page
+ info = {
+ 'id': 'Stanford OpenClassroom',
+ '_type': 'playlist',
+ 'uploader': None,
+ 'upload_date': None,
+ }
+ info['title'] = info['id']
+
+ rootURL = 'http://openclassroom.stanford.edu/MainFolder/HomePage.php'
+ rootpage = self._download_webpage(rootURL, info['id'],
+ errnote='Unable to download course info page')
+
+ links = orderedSet(re.findall('<a href="(CoursePage.php\?[^"]+)">', rootpage))
+ info['entries'] = [self.url_result(
+ 'http://openclassroom.stanford.edu/MainFolder/%s' % unescapeHTML(l)
+ ) for l in links]
+ return info
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/statigram.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/statigram.py
new file mode 100644
index 0000000000..d602e817a0
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/statigram.py
@@ -0,0 +1,38 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+
+
+class StatigramIE(InfoExtractor):
+ _VALID_URL = r'https?://(www\.)?statigr\.am/p/(?P<id>[^/]+)'
+ _TEST = {
+ 'url': 'http://statigr.am/p/522207370455279102_24101272',
+ 'md5': '6eb93b882a3ded7c378ee1d6884b1814',
+ 'info_dict': {
+ 'id': '522207370455279102_24101272',
+ 'ext': 'mp4',
+ 'uploader_id': 'aguynamedpatrick',
+ 'title': 'Instagram photo by @aguynamedpatrick (Patrick Janelle)',
+ },
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+ webpage = self._download_webpage(url, video_id)
+ html_title = self._html_search_regex(
+ r'<title>(.+?) ',
+ webpage, 'title')
+ title = re.sub(r'(?: *\(Videos?\))? \| Statigram$', '', html_title)
+ uploader_id = self._html_search_regex(
+ r'@([^ ]+)', title, 'uploader name', fatal=False)
+
+ return {
+ 'id': video_id,
+ 'url': self._og_search_video_url(webpage),
+ 'title': title,
+ 'thumbnail': self._og_search_thumbnail(webpage),
+ 'uploader_id': uploader_id
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/steam.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/steam.py
new file mode 100644
index 0000000000..183dcb03cc
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/steam.py
@@ -0,0 +1,123 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+ ExtractorError,
+ unescapeHTML,
+)
+
+
+class SteamIE(InfoExtractor):
+ _VALID_URL = r"""(?x)
+ https?://store\.steampowered\.com/
+ (agecheck/)?
+ (?Pvideo|app)/ #If the page is only for videos or for a game
+ (?P\d+)/?
+ (?P\d*)(?P\??) # For urltype == video we sometimes get the videoID
+ |
+ https?://(?:www\.)?steamcommunity\.com/sharedfiles/filedetails/\?id=(?P[0-9]+)
+ """
+ _VIDEO_PAGE_TEMPLATE = 'http://store.steampowered.com/video/%s/'
+ _AGECHECK_TEMPLATE = 'http://store.steampowered.com/agecheck/video/%s/?snr=1_agecheck_agecheck__age-gate&ageDay=1&ageMonth=January&ageYear=1970'
+ _TESTS = [{
+ "url": "http://store.steampowered.com/video/105600/",
+ "playlist": [
+ {
+ "md5": "f870007cee7065d7c76b88f0a45ecc07",
+ "info_dict": {
+ 'id': '81300',
+ 'ext': 'flv',
+ "title": "Terraria 1.1 Trailer",
+ 'playlist_index': 1,
+ }
+ },
+ {
+ "md5": "61aaf31a5c5c3041afb58fb83cbb5751",
+ "info_dict": {
+ 'id': '80859',
+ 'ext': 'flv',
+ "title": "Terraria Trailer",
+ 'playlist_index': 2,
+ }
+ }
+ ],
+ 'params': {
+ 'playlistend': 2,
+ }
+ }, {
+ 'url': 'http://steamcommunity.com/sharedfiles/filedetails/?id=242472205',
+ 'info_dict': {
+ 'id': 'WB5DvDOOvAY',
+ 'ext': 'mp4',
+ 'upload_date': '20140329',
+ 'title': 'FRONTIERS - Final Greenlight Trailer',
+ 'description': 'md5:dc96a773669d0ca1b36c13c1f30250d9',
+ 'uploader': 'AAD Productions',
+ 'uploader_id': 'AtomicAgeDogGames',
+ }
+ }]
+
+ def _real_extract(self, url):
+ m = re.match(self._VALID_URL, url)
+ fileID = m.group('fileID')
+ if fileID:
+ videourl = url
+ playlist_id = fileID
+ else:
+ gameID = m.group('gameID')
+ playlist_id = gameID
+ videourl = self._VIDEO_PAGE_TEMPLATE % playlist_id
+ webpage = self._download_webpage(videourl, playlist_id)
+
+ if re.search('Please enter your birth date to continue: ', webpage) is not None:
+ videourl = self._AGECHECK_TEMPLATE % playlist_id
+ self.report_age_confirmation()
+ webpage = self._download_webpage(videourl, playlist_id)
+
+ if fileID:
+ playlist_title = self._html_search_regex(
+ r'(.+)
', webpage, 'title')
+ mweb = re.finditer(r'''(?x)
+ 'movie_(?P[0-9]+)':\s*\{\s*
+ YOUTUBE_VIDEO_ID:\s*"(?P[^"]+)",
+ ''', webpage)
+ videos = [{
+ '_type': 'url',
+ 'url': vid.group('youtube_id'),
+ 'ie_key': 'Youtube',
+ } for vid in mweb]
+ else:
+ playlist_title = self._html_search_regex(
+ r'', webpage, 'game title')
+
+ mweb = re.finditer(r'''(?x)
+ 'movie_(?P[0-9]+)':\s*\{\s*
+ FILENAME:\s*"(?P[\w:/\.\?=]+)"
+ (,\s*MOVIE_NAME:\s*\"(?P[\w:/\.\?=\+-]+)\")?\s*\},
+ ''', webpage)
+ titles = re.finditer(
+ r'(?P.+?) ', webpage)
+ thumbs = re.finditer(
+ r'[a-zA-Z0-9_-]+)(?:/(?P[^#?]*)\.html)?'
+
+ _TEST = {
+ 'url': 'http://streamcloud.eu/skp9j99s4bpz/youtube-dl_test_video_____________-BaW_jenozKc.mp4.html',
+ 'md5': '6bea4c7fa5daaacc2a946b7146286686',
+ 'info_dict': {
+ 'id': 'skp9j99s4bpz',
+ 'ext': 'mp4',
+ 'title': 'youtube-dl test video \'/\\ ц╓ Б├╜',
+ },
+ 'skip': 'Only available from the EU'
+ }
+
+ def _real_extract(self, url):
+ video_id = self._match_id(url)
+ url = 'http://streamcloud.eu/%s' % video_id
+
+ orig_webpage = self._download_webpage(url, video_id)
+
+ fields = re.findall(r'''(?x)]*>([^<]+)<', webpage, 'title')
+ video_url = self._search_regex(
+ r'file:\s*"([^"]+)"', webpage, 'video URL')
+ thumbnail = self._search_regex(
+ r'image:\s*"([^"]+)"', webpage, 'thumbnail URL', fatal=False)
+
+ return {
+ 'id': video_id,
+ 'title': title,
+ 'url': video_url,
+ 'thumbnail': thumbnail,
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/streamcz.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/streamcz.py
new file mode 100644
index 0000000000..c3ceb5f76d
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/streamcz.py
@@ -0,0 +1,79 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+ int_or_none,
+)
+
+
+class StreamCZIE(InfoExtractor):
+ _VALID_URL = r'https?://(?:www\.)?stream\.cz/.+/(?P[0-9]+)'
+
+ _TESTS = [{
+ 'url': 'http://www.stream.cz/peklonataliri/765767-ecka-pro-deti',
+ 'md5': '6d3ca61a8d0633c9c542b92fcb936b0c',
+ 'info_dict': {
+ 'id': '765767',
+ 'ext': 'mp4',
+ 'title': 'Peklo na talц╜е≥i: ц┴д█ka pro dд⌡ti',
+ 'description': 'Taе║ka s grцЁnskou pomazц║nkou a dalе║ц╜ pekelnosti ZDE',
+ 'thumbnail': 're:^http://im.stream.cz/episode/52961d7e19d423f8f06f0100',
+ 'duration': 256,
+ },
+ }, {
+ 'url': 'http://www.stream.cz/blanik/10002447-tri-roky-pro-mazanka',
+ 'md5': 'e54a254fb8b871968fd8403255f28589',
+ 'info_dict': {
+ 'id': '10002447',
+ 'ext': 'mp4',
+ 'title': 'Kancelц║е≥ Blanц╜k: Tе≥i roky pro Mazц║nka',
+ 'description': 'md5:3862a00ba7bf0b3e44806b544032c859',
+ 'thumbnail': 're:^http://im.stream.cz/episode/537f838c50c11f8d21320000',
+ 'duration': 368,
+ },
+ }]
+
+ def _real_extract(self, url):
+ video_id = self._match_id(url)
+ data = self._download_json(
+ 'http://www.stream.cz/API/episode/%s' % video_id, video_id)
+
+ formats = []
+ for quality, video in enumerate(data['video_qualities']):
+ for f in video['formats']:
+ typ = f['type'].partition('/')[2]
+ qlabel = video.get('quality_label')
+ formats.append({
+ 'format_note': '%s-%s' % (qlabel, typ) if qlabel else typ,
+ 'format_id': '%s-%s' % (typ, f['quality']),
+ 'url': f['source'],
+ 'height': int_or_none(f['quality'].rstrip('p')),
+ 'quality': quality,
+ })
+ self._sort_formats(formats)
+
+ image = data.get('image')
+ if image:
+ thumbnail = self._proto_relative_url(
+ image.replace('{width}', '1240').replace('{height}', '697'),
+ scheme='http:',
+ )
+ else:
+ thumbnail = None
+
+ stream = data.get('_embedded', {}).get('stream:show', {}).get('name')
+ if stream:
+ title = '%s: %s' % (stream, data['name'])
+ else:
+ title = data['name']
+
+ return {
+ 'id': video_id,
+ 'title': title,
+ 'thumbnail': thumbnail,
+ 'formats': formats,
+ 'description': data.get('web_site_text'),
+ 'duration': int_or_none(data.get('duration')),
+ 'view_count': int_or_none(data.get('views')),
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/subtitles.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/subtitles.py
new file mode 100644
index 0000000000..59a51268d2
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/subtitles.py
@@ -0,0 +1,99 @@
+from __future__ import unicode_literals
+from .common import InfoExtractor
+
+from ..compat import compat_str
+from ..utils import (
+ ExtractorError,
+)
+
+
+class SubtitlesInfoExtractor(InfoExtractor):
+ @property
+ def _have_to_download_any_subtitles(self):
+ return any([self._downloader.params.get('writesubtitles', False),
+ self._downloader.params.get('writeautomaticsub')])
+
+ def _list_available_subtitles(self, video_id, webpage):
+ """ outputs the available subtitles for the video """
+ sub_lang_list = self._get_available_subtitles(video_id, webpage)
+ auto_captions_list = self._get_available_automatic_caption(video_id, webpage)
+ sub_lang = ",".join(list(sub_lang_list.keys()))
+ self.to_screen('%s: Available subtitles for video: %s' %
+ (video_id, sub_lang))
+ auto_lang = ",".join(auto_captions_list.keys())
+ self.to_screen('%s: Available automatic captions for video: %s' %
+ (video_id, auto_lang))
+
+ def extract_subtitles(self, video_id, webpage):
+ """
+ returns {sub_lang: sub} ,{} if subtitles not found or None if the
+ subtitles aren't requested.
+ """
+ if not self._have_to_download_any_subtitles:
+ return None
+ available_subs_list = {}
+ if self._downloader.params.get('writeautomaticsub', False):
+ available_subs_list.update(self._get_available_automatic_caption(video_id, webpage))
+ if self._downloader.params.get('writesubtitles', False):
+ available_subs_list.update(self._get_available_subtitles(video_id, webpage))
+
+ if not available_subs_list: # error, it didn't get the available subtitles
+ return {}
+ if self._downloader.params.get('allsubtitles', False):
+ sub_lang_list = available_subs_list
+ else:
+ if self._downloader.params.get('subtitleslangs', False):
+ requested_langs = self._downloader.params.get('subtitleslangs')
+ elif 'en' in available_subs_list:
+ requested_langs = ['en']
+ else:
+ requested_langs = [list(available_subs_list.keys())[0]]
+
+ sub_lang_list = {}
+ for sub_lang in requested_langs:
+ if sub_lang not in available_subs_list:
+ self._downloader.report_warning('no closed captions found in the specified language "%s"' % sub_lang)
+ continue
+ sub_lang_list[sub_lang] = available_subs_list[sub_lang]
+
+ subtitles = {}
+ for sub_lang, url in sub_lang_list.items():
+ subtitle = self._request_subtitle_url(sub_lang, url)
+ if subtitle:
+ subtitles[sub_lang] = subtitle
+ return subtitles
+
+ def _download_subtitle_url(self, sub_lang, url):
+ return self._download_webpage(url, None, note=False)
+
+ def _request_subtitle_url(self, sub_lang, url):
+ """ makes the http request for the subtitle """
+ try:
+ sub = self._download_subtitle_url(sub_lang, url)
+ except ExtractorError as err:
+ self._downloader.report_warning('unable to download video subtitles for %s: %s' % (sub_lang, compat_str(err)))
+ return
+ if not sub:
+ self._downloader.report_warning('Did not fetch video subtitles')
+ return
+ return sub
+
+ def _get_available_subtitles(self, video_id, webpage):
+ """
+ returns {sub_lang: url} or {} if not available
+ Must be redefined by the subclasses
+ """
+
+ # By default, allow implementations to simply pass in the result
+ assert isinstance(webpage, dict), \
+ '_get_available_subtitles not implemented'
+ return webpage
+
+ def _get_available_automatic_caption(self, video_id, webpage):
+ """
+ returns {sub_lang: url} or {} if not available
+ Must be redefined by the subclasses that support automatic captions,
+ otherwise it will return {}
+ """
+ self._downloader.report_warning('Automatic Captions not supported by this server')
+ return {}
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/sunporno.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/sunporno.py
new file mode 100644
index 0000000000..263f09b464
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/sunporno.py
@@ -0,0 +1,70 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+ parse_duration,
+ int_or_none,
+ qualities,
+ determine_ext,
+)
+
+
+class SunPornoIE(InfoExtractor):
+ _VALID_URL = r'https?://(?:www\.)?sunporno\.com/videos/(?P\d+)'
+ _TEST = {
+ 'url': 'http://www.sunporno.com/videos/807778/',
+ 'md5': '6457d3c165fd6de062b99ef6c2ff4c86',
+ 'info_dict': {
+ 'id': '807778',
+ 'ext': 'flv',
+ 'title': 'md5:0a400058e8105d39e35c35e7c5184164',
+ 'description': 'md5:a31241990e1bd3a64e72ae99afb325fb',
+ 'thumbnail': 're:^https?://.*\.jpg$',
+ 'duration': 302,
+ 'age_limit': 18,
+ }
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+
+ webpage = self._download_webpage(url, video_id)
+
+ title = self._html_search_regex(r'([^<]+) ', webpage, 'title')
+ description = self._html_search_meta('description', webpage, 'description')
+ thumbnail = self._html_search_regex(
+ r'poster="([^"]+)"', webpage, 'thumbnail', fatal=False)
+
+ duration = parse_duration(self._search_regex(
+ r'Duration:\s*(\d+:\d+)\s*<', webpage, 'duration', fatal=False))
+
+ view_count = int_or_none(self._html_search_regex(
+ r'class="views">\s*(\d+)\s*<', webpage, 'view count', fatal=False))
+ comment_count = int_or_none(self._html_search_regex(
+ r'(\d+) Comments?', webpage, 'comment count', fatal=False))
+
+ formats = []
+ quality = qualities(['mp4', 'flv'])
+ for video_url in re.findall(r'[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})'
+
+ _TESTS = [{
+ 'url': 'http://swrmediathek.de/player.htm?show=849790d0-dab8-11e3-a953-0026b975f2e6',
+ 'md5': '8c5f6f0172753368547ca8413a7768ac',
+ 'info_dict': {
+ 'id': '849790d0-dab8-11e3-a953-0026b975f2e6',
+ 'ext': 'mp4',
+ 'title': 'SWR odysso',
+ 'description': 'md5:2012e31baad36162e97ce9eb3f157b8a',
+ 'thumbnail': 're:^http:.*\.jpg$',
+ 'duration': 2602,
+ 'upload_date': '20140515',
+ 'uploader': 'SWR Fernsehen',
+ 'uploader_id': '990030',
+ },
+ }, {
+ 'url': 'http://swrmediathek.de/player.htm?show=0e1a8510-ddf2-11e3-9be3-0026b975f2e6',
+ 'md5': 'b10ab854f912eecc5a6b55cd6fc1f545',
+ 'info_dict': {
+ 'id': '0e1a8510-ddf2-11e3-9be3-0026b975f2e6',
+ 'ext': 'mp4',
+ 'title': 'Nachtcafц╘ - Alltagsdroge Alkohol - zwischen Sektempfang und Komasaufen',
+ 'description': 'md5:e0a3adc17e47db2c23aab9ebc36dbee2',
+ 'thumbnail': 're:http://.*\.jpg',
+ 'duration': 5305,
+ 'upload_date': '20140516',
+ 'uploader': 'SWR Fernsehen',
+ 'uploader_id': '990030',
+ },
+ }, {
+ 'url': 'http://swrmediathek.de/player.htm?show=bba23e10-cb93-11e3-bf7f-0026b975f2e6',
+ 'md5': '4382e4ef2c9d7ce6852535fa867a0dd3',
+ 'info_dict': {
+ 'id': 'bba23e10-cb93-11e3-bf7f-0026b975f2e6',
+ 'ext': 'mp3',
+ 'title': 'Saе║a Staniе║ic: Vor dem Fest',
+ 'description': 'md5:5b792387dc3fbb171eb709060654e8c9',
+ 'thumbnail': 're:http://.*\.jpg',
+ 'duration': 3366,
+ 'upload_date': '20140520',
+ 'uploader': 'SWR 2',
+ 'uploader_id': '284670',
+ }
+ }]
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+
+ video = self._download_json(
+ 'http://swrmediathek.de/AjaxEntry?ekey=%s' % video_id, video_id, 'Downloading video JSON')
+
+ attr = video['attr']
+ media_type = attr['entry_etype']
+
+ formats = []
+ for entry in video['sub']:
+ if entry['name'] != 'entry_media':
+ continue
+
+ entry_attr = entry['attr']
+ codec = entry_attr['val0']
+ quality = int(entry_attr['val1'])
+
+ fmt = {
+ 'url': entry_attr['val2'],
+ 'quality': quality,
+ }
+
+ if media_type == 'Video':
+ fmt.update({
+ 'format_note': ['144p', '288p', '544p', '720p'][quality - 1],
+ 'vcodec': codec,
+ })
+ elif media_type == 'Audio':
+ fmt.update({
+ 'acodec': codec,
+ })
+ formats.append(fmt)
+
+ self._sort_formats(formats)
+
+ return {
+ 'id': video_id,
+ 'title': attr['entry_title'],
+ 'description': attr['entry_descl'],
+ 'thumbnail': attr['entry_image_16_9'],
+ 'duration': parse_duration(attr['entry_durat']),
+ 'upload_date': attr['entry_pdatet'][:-4],
+ 'uploader': attr['channel_title'],
+ 'uploader_id': attr['channel_idkey'],
+ 'formats': formats,
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/syfy.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/syfy.py
new file mode 100644
index 0000000000..5ca079f880
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/syfy.py
@@ -0,0 +1,46 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+
+
+class SyfyIE(InfoExtractor):
+ _VALID_URL = r'https?://www\.syfy\.com/(?:videos/.+?vid:(?P[0-9]+)|(?!videos)(?P[^/]+)(?:$|[?#]))'
+
+ _TESTS = [{
+ 'url': 'http://www.syfy.com/videos/Robot%20Combat%20League/Behind%20the%20Scenes/vid:2631458',
+ 'info_dict': {
+ 'id': 'NmqMrGnXvmO1',
+ 'ext': 'flv',
+ 'title': 'George Lucas has Advice for his Daughter',
+ 'description': 'Listen to what insights George Lucas give his daughter Amanda.',
+ },
+ 'add_ie': ['ThePlatform'],
+ }, {
+ 'url': 'http://www.syfy.com/wilwheaton',
+ 'md5': '94dfa54ee3ccb63295b276da08c415f6',
+ 'info_dict': {
+ 'id': '4yoffOOXC767',
+ 'ext': 'flv',
+ 'title': 'The Wil Wheaton Project - Premiering May 27th at 10/9c.',
+ 'description': 'The Wil Wheaton Project premieres May 27th at 10/9c. Don\'t miss it.',
+ },
+ 'add_ie': ['ThePlatform'],
+ 'skip': 'Blocked outside the US',
+ }]
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_name = mobj.group('video_name')
+ if video_name:
+ generic_webpage = self._download_webpage(url, video_name)
+ video_id = self._search_regex(
+ r'',
+ generic_webpage, 'video ID')
+ url = 'http://www.syfy.com/videos/%s/%s/vid:%s' % (
+ video_name, video_name, video_id)
+ else:
+ video_id = mobj.group('id')
+ webpage = self._download_webpage(url, video_id)
+ return self.url_result(self._og_search_video_url(webpage))
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/sztvhu.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/sztvhu.py
new file mode 100644
index 0000000000..aa5964acb6
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/sztvhu.py
@@ -0,0 +1,41 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class SztvHuIE(InfoExtractor):
+ _VALID_URL = r'http://(?:(?:www\.)?sztv\.hu|www\.tvszombathely\.hu)/(?:[^/]+)/.+-(?P[0-9]+)'
+ _TEST = {
+ 'url': 'http://sztv.hu/hirek/cserkeszek-nepszerusitettek-a-kornyezettudatos-eletmodot-a-savaria-teren-20130909',
+ 'md5': 'a6df607b11fb07d0e9f2ad94613375cb',
+ 'info_dict': {
+ 'id': '20130909',
+ 'ext': 'mp4',
+ 'title': 'Cserkц╘szek nц╘pszerе╠sц╜tettц╘k a kц╤rnyezettudatos ц╘letmцЁdot a Savaria tц╘ren',
+ 'description': 'A zц╤ld nap jц║tц╘kos ismeretterjesztе▒ programjait a Magyar Cserkц╘sz Szц╤vetsц╘g szervezte, akik az orszц║g nyolc vц║rosц║ban adjц║k ц║t tudц║sukat az ц╘rdeklе▒dе▒knek. A PET...',
+ },
+ }
+
+ def _real_extract(self, url):
+ video_id = self._match_id(url)
+ webpage = self._download_webpage(url, video_id)
+ video_file = self._search_regex(
+ r'file: "...:(.*?)",', webpage, 'video file')
+ title = self._html_search_regex(
+ r'-?[0-9]+)\.html'
+
+ _TESTS = [{
+ 'url': 'http://www.tagesschau.de/multimedia/video/video1399128.html',
+ 'md5': 'bcdeac2194fb296d599ce7929dfa4009',
+ 'info_dict': {
+ 'id': '1399128',
+ 'ext': 'mp4',
+ 'title': 'Harald Range, Generalbundesanwalt, zu den Ermittlungen',
+ 'description': 'md5:69da3c61275b426426d711bde96463ab',
+ 'thumbnail': 're:^http:.*\.jpg$',
+ },
+ }, {
+ 'url': 'http://www.tagesschau.de/multimedia/sendung/ts-5727.html',
+ 'md5': '3c54c1f6243d279b706bde660ceec633',
+ 'info_dict': {
+ 'id': '5727',
+ 'ext': 'mp4',
+ 'description': 'md5:695c01bfd98b7e313c501386327aea59',
+ 'title': 'Sendung: tagesschau \t04.12.2014 20:00 Uhr',
+ 'thumbnail': 're:^http:.*\.jpg$',
+ }
+ }]
+
+ _FORMATS = {
+ 's': {'width': 256, 'height': 144, 'quality': 1},
+ 'm': {'width': 512, 'height': 288, 'quality': 2},
+ 'l': {'width': 960, 'height': 544, 'quality': 3},
+ }
+
+ def _real_extract(self, url):
+ video_id = self._match_id(url)
+ display_id = video_id.lstrip('-')
+ webpage = self._download_webpage(url, display_id)
+
+ player_url = self._html_search_meta(
+ 'twitter:player', webpage, 'player URL', default=None)
+ if player_url:
+ playerpage = self._download_webpage(
+ player_url, display_id, 'Downloading player page')
+
+ medias = re.findall(
+ r'"(http://media.+?)", type:"video/(.+?)", quality:"(.+?)"',
+ playerpage)
+ formats = []
+ for url, ext, res in medias:
+ f = {
+ 'format_id': res + '_' + ext,
+ 'url': url,
+ 'ext': ext,
+ }
+ f.update(self._FORMATS.get(res, {}))
+ formats.append(f)
+ thumbnail_fn = re.findall(r'"(/multimedia/.+?\.jpg)"', playerpage)[-1]
+ title = self._og_search_title(webpage).strip()
+ description = self._og_search_description(webpage).strip()
+ else:
+ download_text = self._search_regex(
+ r'(?s)Wir bieten dieses Video in folgenden Formaten zum Download an:
\s*(.*?)
\s*',
+ webpage, 'download links')
+ links = re.finditer(
+ r'
',
+ download_text)
+ formats = []
+ for l in links:
+ format_id = self._search_regex(
+ r'.*/[^/.]+\.([^/]+)\.[^/.]+', l.group('url'), 'format ID')
+ format = {
+ 'format_id': format_id,
+ 'url': l.group('url'),
+ 'format_name': l.group('name'),
+ }
+ m = re.match(
+ r'''(?x)
+ Video:\s*(?P[a-zA-Z0-9/._-]+)\s*&\#10;
+ (?P[0-9]+)x(?P[0-9]+)px&\#10;
+ (?P[0-9]+)kbps&\#10;
+ Audio:\s*(?P[0-9]+)kbps,\s*(?P[A-Za-z\.0-9]+)&\#10;
+ Größe:\s*(?P[0-9.,]+\s+[a-zA-Z]*B)''',
+ l.group('title'))
+ if m:
+ format.update({
+ 'format_note': m.group('audio_desc'),
+ 'vcodec': m.group('vcodec'),
+ 'width': int(m.group('width')),
+ 'height': int(m.group('height')),
+ 'abr': int(m.group('abr')),
+ 'vbr': int(m.group('vbr')),
+ 'filesize_approx': parse_filesize(m.group('filesize_approx')),
+ })
+ formats.append(format)
+ thumbnail_fn = self._search_regex(
+ r'(?s)(.*?) ', webpage, 'title')
+
+ self._sort_formats(formats)
+ thumbnail = 'http://www.tagesschau.de' + thumbnail_fn
+
+ return {
+ 'id': display_id,
+ 'title': title,
+ 'thumbnail': thumbnail,
+ 'formats': formats,
+ 'description': description,
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/tapely.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/tapely.py
new file mode 100644
index 0000000000..f1f43d0a71
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/tapely.py
@@ -0,0 +1,107 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..compat import (
+ compat_urllib_request,
+)
+from ..utils import (
+ clean_html,
+ ExtractorError,
+ float_or_none,
+ parse_iso8601,
+)
+
+
+class TapelyIE(InfoExtractor):
+ _VALID_URL = r'https?://(?:www\.)?tape\.ly/(?P[A-Za-z0-9\-_]+)(?:/(?P\d+))?'
+ _API_URL = 'http://tape.ly/showtape?id={0:}'
+ _S3_SONG_URL = 'http://mytape.s3.amazonaws.com/{0:}'
+ _SOUNDCLOUD_SONG_URL = 'http://api.soundcloud.com{0:}'
+ _TESTS = [
+ {
+ 'url': 'http://tape.ly/my-grief-as-told-by-water',
+ 'info_dict': {
+ 'id': 23952,
+ 'title': 'my grief as told by water',
+ 'thumbnail': 're:^https?://.*\.png$',
+ 'uploader_id': 16484,
+ 'timestamp': 1411848286,
+ 'description': 'For Robin and Ponkers, whom the tides of life have taken out to sea.',
+ },
+ 'playlist_count': 13,
+ },
+ {
+ 'url': 'http://tape.ly/my-grief-as-told-by-water/1',
+ 'md5': '79031f459fdec6530663b854cbc5715c',
+ 'info_dict': {
+ 'id': 258464,
+ 'title': 'Dreaming Awake (My Brightest Diamond)',
+ 'ext': 'm4a',
+ },
+ },
+ ]
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ display_id = mobj.group('id')
+
+ playlist_url = self._API_URL.format(display_id)
+ request = compat_urllib_request.Request(playlist_url)
+ request.add_header('X-Requested-With', 'XMLHttpRequest')
+ request.add_header('Accept', 'application/json')
+ request.add_header('Referer', url)
+
+ playlist = self._download_json(request, display_id)
+
+ tape = playlist['tape']
+
+ entries = []
+ for s in tape['songs']:
+ song = s['song']
+ entry = {
+ 'id': song['id'],
+ 'duration': float_or_none(song.get('songduration'), 1000),
+ 'title': song['title'],
+ }
+ if song['source'] == 'S3':
+ entry.update({
+ 'url': self._S3_SONG_URL.format(song['filename']),
+ })
+ entries.append(entry)
+ elif song['source'] == 'YT':
+ self.to_screen('YouTube video detected')
+ yt_id = song['filename'].replace('/youtube/', '')
+ entry.update(self.url_result(yt_id, 'Youtube', video_id=yt_id))
+ entries.append(entry)
+ elif song['source'] == 'SC':
+ self.to_screen('SoundCloud song detected')
+ sc_url = self._SOUNDCLOUD_SONG_URL.format(song['filename'])
+ entry.update(self.url_result(sc_url, 'Soundcloud'))
+ entries.append(entry)
+ else:
+ self.report_warning('Unknown song source: %s' % song['source'])
+
+ if mobj.group('songnr'):
+ songnr = int(mobj.group('songnr')) - 1
+ try:
+ return entries[songnr]
+ except IndexError:
+ raise ExtractorError(
+ 'No song with index: %s' % mobj.group('songnr'),
+ expected=True)
+
+ return {
+ '_type': 'playlist',
+ 'id': tape['id'],
+ 'display_id': display_id,
+ 'title': tape['name'],
+ 'entries': entries,
+ 'thumbnail': tape.get('image_url'),
+ 'description': clean_html(tape.get('subtext')),
+ 'like_count': tape.get('likescount'),
+ 'uploader_id': tape.get('user_id'),
+ 'timestamp': parse_iso8601(tape.get('published_at')),
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/tass.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/tass.py
new file mode 100644
index 0000000000..c4ef70778b
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/tass.py
@@ -0,0 +1,62 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+import json
+
+from .common import InfoExtractor
+from ..utils import (
+ js_to_json,
+ qualities,
+)
+
+
+class TassIE(InfoExtractor):
+ _VALID_URL = r'https?://(?:tass\.ru|itar-tass\.com)/[^/]+/(?P\d+)'
+ _TESTS = [
+ {
+ 'url': 'http://tass.ru/obschestvo/1586870',
+ 'md5': '3b4cdd011bc59174596b6145cda474a4',
+ 'info_dict': {
+ 'id': '1586870',
+ 'ext': 'mp4',
+ 'title': 'п÷п╬я│п╣я┌п╦я┌п╣п╩я▐п╪ п╪п╬я│п╨п╬п╡я│п╨п╬пЁп╬ п╥п╬п╬п©п╟я─п╨п╟ п©п╬п╨п╟п╥п╟п╩п╦ п╨я─п╟я│п╫я┐я▌ п©п╟п╫п╢я┐',
+ 'description': 'п÷я─п╦п╣я┘п╟п╡я┬я┐я▌ п╦п╥ п■я┐п╠п╩п╦п╫п╟ п≈п╣п╧п╫я┐ п╪п╬п╤п╫п╬ я┐п╡п╦п╢п╣я┌я▄ п╡ п©п╟п╡п╦п╩я▄п╬п╫п╣ "п п╬я┬п╨п╦ я┌я─п╬п©п╦п╨п╬п╡"',
+ 'thumbnail': 're:^https?://.*\.jpg$',
+ },
+ },
+ {
+ 'url': 'http://itar-tass.com/obschestvo/1600009',
+ 'only_matching': True,
+ },
+ ]
+
+ def _real_extract(self, url):
+ video_id = self._match_id(url)
+
+ webpage = self._download_webpage(url, video_id)
+
+ sources = json.loads(js_to_json(self._search_regex(
+ r'(?s)sources\s*:\s*(\[.+?\])', webpage, 'sources')))
+
+ quality = qualities(['sd', 'hd'])
+
+ formats = []
+ for source in sources:
+ video_url = source.get('file')
+ if not video_url or not video_url.startswith('http') or not video_url.endswith('.mp4'):
+ continue
+ label = source.get('label')
+ formats.append({
+ 'url': video_url,
+ 'format_id': label,
+ 'quality': quality(label),
+ })
+ self._sort_formats(formats)
+
+ return {
+ 'id': video_id,
+ 'title': self._og_search_title(webpage),
+ 'description': self._og_search_description(webpage),
+ 'thumbnail': self._og_search_thumbnail(webpage),
+ 'formats': formats,
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/teachertube.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/teachertube.py
new file mode 100644
index 0000000000..6c3445d792
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/teachertube.py
@@ -0,0 +1,133 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+ qualities,
+ determine_ext,
+)
+
+
+class TeacherTubeIE(InfoExtractor):
+ IE_NAME = 'teachertube'
+ IE_DESC = 'teachertube.com videos'
+
+ _VALID_URL = r'https?://(?:www\.)?teachertube\.com/(viewVideo\.php\?video_id=|music\.php\?music_id=|video/(?:[\da-z-]+-)?|audio/)(?P\d+)'
+
+ _TESTS = [{
+ 'url': 'http://www.teachertube.com/viewVideo.php?video_id=339997',
+ 'md5': 'f9434ef992fd65936d72999951ee254c',
+ 'info_dict': {
+ 'id': '339997',
+ 'ext': 'mp4',
+ 'title': 'Measures of dispersion from a frequency table',
+ 'description': 'Measures of dispersion from a frequency table',
+ 'thumbnail': 're:http://.*\.jpg',
+ },
+ }, {
+ 'url': 'http://www.teachertube.com/viewVideo.php?video_id=340064',
+ 'md5': '0d625ec6bc9bf50f70170942ad580676',
+ 'info_dict': {
+ 'id': '340064',
+ 'ext': 'mp4',
+ 'title': 'How to Make Paper Dolls _ Paper Art Projects',
+ 'description': 'Learn how to make paper dolls in this simple',
+ 'thumbnail': 're:http://.*\.jpg',
+ },
+ }, {
+ 'url': 'http://www.teachertube.com/music.php?music_id=8805',
+ 'md5': '01e8352006c65757caf7b961f6050e21',
+ 'info_dict': {
+ 'id': '8805',
+ 'ext': 'mp3',
+ 'title': 'PER ASPERA AD ASTRA',
+ 'description': 'RADIJSKA EMISIJA ZRAKOPLOVNE TEHNI?KE ?KOLE P',
+ },
+ }, {
+ 'url': 'http://www.teachertube.com/video/intro-video-schleicher-297790',
+ 'md5': '9c79fbb2dd7154823996fc28d4a26998',
+ 'info_dict': {
+ 'id': '297790',
+ 'ext': 'mp4',
+ 'title': 'Intro Video - Schleicher',
+ 'description': 'Intro Video - Why to flip, how flipping will',
+ },
+ }]
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+
+ webpage = self._download_webpage(url, video_id)
+
+ title = self._html_search_meta('title', webpage, 'title', fatal=True)
+ TITLE_SUFFIX = ' - TeacherTube'
+ if title.endswith(TITLE_SUFFIX):
+ title = title[:-len(TITLE_SUFFIX)].strip()
+
+ description = self._html_search_meta('description', webpage, 'description')
+ if description:
+ description = description.strip()
+
+ quality = qualities(['mp3', 'flv', 'mp4'])
+
+ media_urls = re.findall(r'data-contenturl="([^"]+)"', webpage)
+ media_urls.extend(re.findall(r'var\s+filePath\s*=\s*"([^"]+)"', webpage))
+ media_urls.extend(re.findall(r'\'file\'\s*:\s*["\']([^"\']+)["\'],', webpage))
+
+ formats = [
+ {
+ 'url': media_url,
+ 'quality': quality(determine_ext(media_url))
+ } for media_url in set(media_urls)
+ ]
+
+ self._sort_formats(formats)
+
+ return {
+ 'id': video_id,
+ 'title': title,
+ 'thumbnail': self._html_search_regex(r'\'image\'\s*:\s*["\']([^"\']+)["\']', webpage, 'thumbnail'),
+ 'formats': formats,
+ 'description': description,
+ }
+
+
+class TeacherTubeUserIE(InfoExtractor):
+ IE_NAME = 'teachertube:user:collection'
+ IE_DESC = 'teachertube.com user and collection videos'
+
+ _VALID_URL = r'https?://(?:www\.)?teachertube\.com/(user/profile|collection)/(?P[0-9a-zA-Z]+)/?'
+
+ _MEDIA_RE = r'''(?sx)
+ class="?sidebar_thumb_time"?>[0-9:]+
+ \s*
+ .+)'
+
+ _TEST = {
+ 'url': 'https://www.teachingchannel.org/videos/teacher-teaming-evolution',
+ 'info_dict': {
+ 'id': 'F3bnlzbToeI6pLEfRyrlfooIILUjz4nM',
+ 'ext': 'mp4',
+ 'title': 'A History of Teaming',
+ 'description': 'md5:2a9033db8da81f2edffa4c99888140b3',
+ },
+ 'params': {
+ # m3u8 download
+ 'skip_download': True,
+ },
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ title = mobj.group('title')
+ webpage = self._download_webpage(url, title)
+ ooyala_code = self._search_regex(
+ r'data-embed-code=\'(.+?)\'', webpage, 'ooyala code')
+
+ return OoyalaIE._build_url_result(ooyala_code)
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/teamcoco.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/teamcoco.py
new file mode 100644
index 0000000000..5fa67eb8d4
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/teamcoco.py
@@ -0,0 +1,84 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+
+
+class TeamcocoIE(InfoExtractor):
+ _VALID_URL = r'http://teamcoco\.com/video/(?P[0-9]+)?/?(?P.*)'
+ _TESTS = [
+ {
+ 'url': 'http://teamcoco.com/video/80187/conan-becomes-a-mary-kay-beauty-consultant',
+ 'file': '80187.mp4',
+ 'md5': '3f7746aa0dc86de18df7539903d399ea',
+ 'info_dict': {
+ 'title': 'Conan Becomes A Mary Kay Beauty Consultant',
+ 'description': 'Mary Kay is perhaps the most trusted name in female beauty, so of course Conan is a natural choice to sell their products.'
+ }
+ }, {
+ 'url': 'http://teamcoco.com/video/louis-ck-interview-george-w-bush',
+ 'file': '19705.mp4',
+ 'md5': 'cde9ba0fa3506f5f017ce11ead928f9a',
+ 'info_dict': {
+ "description": "Louis C.K. got starstruck by George W. Bush, so what? Part one.",
+ "title": "Louis C.K. Interview Pt. 1 11/3/11"
+ }
+ }
+ ]
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+
+ display_id = mobj.group('display_id')
+ webpage = self._download_webpage(url, display_id)
+
+ video_id = mobj.group("video_id")
+ if not video_id:
+ video_id = self._html_search_regex(
+ r'data-node-id="(\d+?)"',
+ webpage, 'video id')
+
+ data_url = 'http://teamcoco.com/cvp/2.0/%s.xml' % video_id
+ data = self._download_xml(
+ data_url, display_id, 'Downloading data webpage')
+
+ qualities = ['500k', '480p', '1000k', '720p', '1080p']
+ formats = []
+ for filed in data.findall('files/file'):
+ if filed.attrib.get('playmode') == 'all':
+ # it just duplicates one of the entries
+ break
+ file_url = filed.text
+ m_format = re.search(r'(\d+(k|p))\.mp4', file_url)
+ if m_format is not None:
+ format_id = m_format.group(1)
+ else:
+ format_id = filed.attrib['bitrate']
+ tbr = (
+ int(filed.attrib['bitrate'])
+ if filed.attrib['bitrate'].isdigit()
+ else None)
+
+ try:
+ quality = qualities.index(format_id)
+ except ValueError:
+ quality = -1
+ formats.append({
+ 'url': file_url,
+ 'ext': 'mp4',
+ 'tbr': tbr,
+ 'format_id': format_id,
+ 'quality': quality,
+ })
+
+ self._sort_formats(formats)
+
+ return {
+ 'id': video_id,
+ 'display_id': display_id,
+ 'formats': formats,
+ 'title': self._og_search_title(webpage),
+ 'thumbnail': self._og_search_thumbnail(webpage),
+ 'description': self._og_search_description(webpage),
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/techtalks.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/techtalks.py
new file mode 100644
index 0000000000..16e945d8e6
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/techtalks.py
@@ -0,0 +1,79 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+ get_element_by_attribute,
+ clean_html,
+)
+
+
+class TechTalksIE(InfoExtractor):
+ _VALID_URL = r'https?://techtalks\.tv/talks/[^/]*/(?P\d+)/'
+
+ _TEST = {
+ 'url': 'http://techtalks.tv/talks/learning-topic-models-going-beyond-svd/57758/',
+ 'info_dict': {
+ 'id': '57758',
+ 'title': 'Learning Topic Models --- Going beyond SVD',
+ },
+ 'playlist': [
+ {
+ 'info_dict': {
+ 'id': '57758',
+ 'ext': 'flv',
+ 'title': 'Learning Topic Models --- Going beyond SVD',
+ },
+ },
+ {
+ 'info_dict': {
+ 'id': '57758-slides',
+ 'ext': 'flv',
+ 'title': 'Learning Topic Models --- Going beyond SVD',
+ },
+ },
+ ],
+ 'params': {
+ # rtmp download
+ 'skip_download': True,
+ },
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ talk_id = mobj.group('id')
+ webpage = self._download_webpage(url, talk_id)
+ rtmp_url = self._search_regex(
+ r'netConnectionUrl: \'(.*?)\'', webpage, 'rtmp url')
+ play_path = self._search_regex(
+ r'href=\'(.*?)\' [^>]*id="flowplayer_presenter"',
+ webpage, 'presenter play path')
+ title = clean_html(get_element_by_attribute('class', 'title', webpage))
+ video_info = {
+ 'id': talk_id,
+ 'title': title,
+ 'url': rtmp_url,
+ 'play_path': play_path,
+ 'ext': 'flv',
+ }
+ m_slides = re.search(r'https?://)
+ (?Pwww|embed)(?P\.ted\.com/
+ (
+ (?Pplaylists(?:/\d+)?) # We have a playlist
+ |
+ ((?Ptalks)) # We have a simple talk
+ |
+ (?Pwatch)/[^/]+/[^/]+
+ )
+ (/lang/(.*?))? # The url may contain the language
+ /(?P[\w-]+) # Here goes the name and then ".html"
+ .*)$
+ '''
+ _TESTS = [{
+ 'url': 'http://www.ted.com/talks/dan_dennett_on_our_consciousness.html',
+ 'md5': 'fc94ac279feebbce69f21c0c6ee82810',
+ 'info_dict': {
+ 'id': '102',
+ 'ext': 'mp4',
+ 'title': 'The illusion of consciousness',
+ 'description': ('Philosopher Dan Dennett makes a compelling '
+ 'argument that not only don\'t we understand our own '
+ 'consciousness, but that half the time our brains are '
+ 'actively fooling us.'),
+ 'uploader': 'Dan Dennett',
+ 'width': 854,
+ 'duration': 1308,
+ }
+ }, {
+ 'url': 'http://www.ted.com/watch/ted-institute/ted-bcg/vishal-sikka-the-beauty-and-power-of-algorithms',
+ 'md5': '226f4fb9c62380d11b7995efa4c87994',
+ 'info_dict': {
+ 'id': 'vishal-sikka-the-beauty-and-power-of-algorithms',
+ 'ext': 'mp4',
+ 'title': 'Vishal Sikka: The beauty and power of algorithms',
+ 'thumbnail': 're:^https?://.+\.jpg',
+ 'description': 'Adaptive, intelligent, and consistent, algorithms are emerging as the ultimate app for everything from matching consumers to products to assessing medical diagnoses. Vishal Sikka shares his appreciation for the algorithm, charting both its inherent beauty and its growing power.',
+ }
+ }, {
+ 'url': 'http://www.ted.com/talks/gabby_giffords_and_mark_kelly_be_passionate_be_courageous_be_your_best',
+ 'info_dict': {
+ 'id': '1972',
+ 'ext': 'mp4',
+ 'title': 'Be passionate. Be courageous. Be your best.',
+ 'uploader': 'Gabby Giffords and Mark Kelly',
+ 'description': 'md5:5174aed4d0f16021b704120360f72b92',
+ 'duration': 1128,
+ },
+ }, {
+ 'url': 'http://www.ted.com/playlists/who_are_the_hackers',
+ 'info_dict': {
+ 'id': '10',
+ 'title': 'Who are the hackers?',
+ },
+ 'playlist_mincount': 6,
+ }, {
+ # contains a youtube video
+ 'url': 'https://www.ted.com/talks/douglas_adams_parrots_the_universe_and_everything',
+ 'add_ie': ['Youtube'],
+ 'info_dict': {
+ 'id': '_ZG8HBuDjgc',
+ 'ext': 'mp4',
+ 'title': 'Douglas Adams: Parrots the Universe and Everything',
+ 'description': 'md5:01ad1e199c49ac640cb1196c0e9016af',
+ 'uploader': 'University of California Television (UCTV)',
+ 'uploader_id': 'UCtelevision',
+ 'upload_date': '20080522',
+ },
+ 'params': {
+ 'skip_download': True,
+ },
+ }]
+
+ _NATIVE_FORMATS = {
+ 'low': {'preference': 1, 'width': 320, 'height': 180},
+ 'medium': {'preference': 2, 'width': 512, 'height': 288},
+ 'high': {'preference': 3, 'width': 854, 'height': 480},
+ }
+
+ def _extract_info(self, webpage):
+ info_json = self._search_regex(r'q\("\w+.init",({.+})\)',
+ webpage, 'info json')
+ return json.loads(info_json)
+
+ def _real_extract(self, url):
+ m = re.match(self._VALID_URL, url, re.VERBOSE)
+ if m.group('type') == 'embed':
+ desktop_url = m.group('proto') + 'www' + m.group('urlmain')
+ return self.url_result(desktop_url, 'TED')
+ name = m.group('name')
+ if m.group('type_talk'):
+ return self._talk_info(url, name)
+ elif m.group('type_watch'):
+ return self._watch_info(url, name)
+ else:
+ return self._playlist_videos_info(url, name)
+
+ def _playlist_videos_info(self, url, name):
+ '''Returns the videos of the playlist'''
+
+ webpage = self._download_webpage(url, name,
+ 'Downloading playlist webpage')
+ info = self._extract_info(webpage)
+ playlist_info = info['playlist']
+
+ playlist_entries = [
+ self.url_result('http://www.ted.com/talks/' + talk['slug'], self.ie_key())
+ for talk in info['talks']
+ ]
+ return self.playlist_result(
+ playlist_entries,
+ playlist_id=compat_str(playlist_info['id']),
+ playlist_title=playlist_info['title'])
+
+ def _talk_info(self, url, video_name):
+ webpage = self._download_webpage(url, video_name)
+ self.report_extraction(video_name)
+
+ talk_info = self._extract_info(webpage)['talks'][0]
+
+ if talk_info.get('external') is not None:
+ self.to_screen('Found video from %s' % talk_info['external']['service'])
+ return {
+ '_type': 'url',
+ 'url': talk_info['external']['uri'],
+ }
+
+ formats = [{
+ 'url': format_url,
+ 'format_id': format_id,
+ 'format': format_id,
+ } for (format_id, format_url) in talk_info['nativeDownloads'].items() if format_url is not None]
+ if formats:
+ for f in formats:
+ finfo = self._NATIVE_FORMATS.get(f['format_id'])
+ if finfo:
+ f.update(finfo)
+ else:
+ # Use rtmp downloads
+ formats = [{
+ 'format_id': f['name'],
+ 'url': talk_info['streamer'],
+ 'play_path': f['file'],
+ 'ext': 'flv',
+ 'width': f['width'],
+ 'height': f['height'],
+ 'tbr': f['bitrate'],
+ } for f in talk_info['resources']['rtmp']]
+ self._sort_formats(formats)
+
+ video_id = compat_str(talk_info['id'])
+ # subtitles
+ video_subtitles = self.extract_subtitles(video_id, talk_info)
+ if self._downloader.params.get('listsubtitles', False):
+ self._list_available_subtitles(video_id, talk_info)
+ return
+
+ thumbnail = talk_info['thumb']
+ if not thumbnail.startswith('http'):
+ thumbnail = 'http://' + thumbnail
+ return {
+ 'id': video_id,
+ 'title': talk_info['title'].strip(),
+ 'uploader': talk_info['speaker'],
+ 'thumbnail': thumbnail,
+ 'description': self._og_search_description(webpage),
+ 'subtitles': video_subtitles,
+ 'formats': formats,
+ 'duration': talk_info.get('duration'),
+ }
+
+ def _get_available_subtitles(self, video_id, talk_info):
+ languages = [lang['languageCode'] for lang in talk_info.get('languages', [])]
+ if languages:
+ sub_lang_list = {}
+ for l in languages:
+ url = 'http://www.ted.com/talks/subtitles/id/%s/lang/%s/format/srt' % (video_id, l)
+ sub_lang_list[l] = url
+ return sub_lang_list
+ else:
+ self._downloader.report_warning('video doesn\'t have subtitles')
+ return {}
+
+ def _watch_info(self, url, name):
+ webpage = self._download_webpage(url, name)
+
+ config_json = self._html_search_regex(
+ r'"pages\.jwplayer"\s*,\s*({.+?})\s*\)\s*',
+ webpage, 'config')
+ config = json.loads(config_json)['config']
+ video_url = config['video']['url']
+ thumbnail = config.get('image', {}).get('url')
+
+ title = self._html_search_regex(
+ r"(?s)(.+?)", webpage, 'title')
+ description = self._html_search_regex(
+ [
+ r'(?s).*? (.*?)',
+ r'(?s)About this talk: \s+(.*?)
',
+ ],
+ webpage, 'description', fatal=False)
+
+ return {
+ 'id': name,
+ 'url': video_url,
+ 'title': title,
+ 'thumbnail': thumbnail,
+ 'description': description,
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/telebruxelles.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/telebruxelles.py
new file mode 100644
index 0000000000..a3d05f97d6
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/telebruxelles.py
@@ -0,0 +1,60 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class TeleBruxellesIE(InfoExtractor):
+ _VALID_URL = r'https?://(?:www\.)?telebruxelles\.be/(news|sport|dernier-jt)/?(?P[^/#?]+)'
+ _TESTS = [{
+ 'url': 'http://www.telebruxelles.be/news/auditions-devant-parlement-francken-galant-tres-attendus/',
+ 'md5': '59439e568c9ee42fb77588b2096b214f',
+ 'info_dict': {
+ 'id': '11942',
+ 'display_id': 'auditions-devant-parlement-francken-galant-tres-attendus',
+ 'ext': 'flv',
+ 'title': 'Parlement : Francken et Galant rц╘pondent aux interpellations de lБ─≥opposition',
+ 'description': 're:Les auditions des ministres se poursuivent*'
+ },
+ 'params': {
+ 'skip_download': 'requires rtmpdump'
+ },
+ }, {
+ 'url': 'http://www.telebruxelles.be/sport/basket-brussels-bat-mons-80-74/',
+ 'md5': '181d3fbdcf20b909309e5aef5c6c6047',
+ 'info_dict': {
+ 'id': '10091',
+ 'display_id': 'basket-brussels-bat-mons-80-74',
+ 'ext': 'flv',
+ 'title': 'Basket : le Brussels bat Mons 80-74',
+ 'description': 're:^Ils l\u2019on fait ! En basket, le B*',
+ },
+ 'params': {
+ 'skip_download': 'requires rtmpdump'
+ },
+ }]
+
+ def _real_extract(self, url):
+ display_id = self._match_id(url)
+ webpage = self._download_webpage(url, display_id)
+
+ article_id = self._html_search_regex(
+ r"(.*?)', webpage, 'title')
+ description = self._og_search_description(webpage)
+
+ rtmp_url = self._html_search_regex(
+ r"file: \"(rtmp://\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{1,5}/vod/mp4:\" \+ \"\w+\" \+ \".mp4)\"",
+ webpage, 'RTMP url')
+ rtmp_url = rtmp_url.replace("\" + \"", "")
+
+ return {
+ 'id': article_id,
+ 'display_id': display_id,
+ 'title': title,
+ 'description': description,
+ 'url': rtmp_url,
+ 'ext': 'flv',
+ 'rtmp_live': True # if rtmpdump is not called with "--live" argument, the download is blocked and can be completed
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/telecinco.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/telecinco.py
new file mode 100644
index 0000000000..be3f72df7c
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/telecinco.py
@@ -0,0 +1,19 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .mitele import MiTeleIE
+
+
+class TelecincoIE(MiTeleIE):
+ IE_NAME = 'telecinco.es'
+ _VALID_URL = r'https?://www\.telecinco\.es/[^/]+/[^/]+/[^/]+/(?P.*?)\.html'
+
+ _TEST = {
+ 'url': 'http://www.telecinco.es/robinfood/temporada-01/t01xp14/Bacalao-cocochas-pil-pil_0_1876350223.html',
+ 'info_dict': {
+ 'id': 'MDSVID20141015_0058',
+ 'ext': 'mp4',
+ 'title': 'Con Martц╜n Berasategui, hacer un bacalao al ...',
+ 'duration': 662,
+ },
+ }
diff --git a/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/telemb.py b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/telemb.py
new file mode 100644
index 0000000000..1bbd0e7bdf
--- /dev/null
+++ b/couchpotato/core/media/movie/providers/trailer/youtube_dl/extractor/telemb.py
@@ -0,0 +1,78 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import remove_start
+
+
+class TeleMBIE(InfoExtractor):
+ _VALID_URL = r'https?://(?:www\.)?telemb\.be/(?P.+?)_d_(?P\d+)\.html'
+ _TESTS = [
+ {
+ 'url': 'http://www.telemb.be/mons-cook-with-danielle-des-cours-de-cuisine-en-anglais-_d_13466.html',
+ 'md5': 'f45ea69878516ba039835794e0f8f783',
+ 'info_dict': {
+ 'id': '13466',
+ 'display_id': 'mons-cook-with-danielle-des-cours-de-cuisine-en-anglais-',
+ 'ext': 'mp4',
+ 'title': 'Mons - Cook with Danielle : des cours de cuisine en anglais ! - Les reportages',
+ 'description': 'md5:bc5225f47b17c309761c856ad4776265',
+ 'thumbnail': 're:^http://.*\.(?:jpg|png)$',
+ }
+ },
+ {
+ # non-ASCII characters in download URL
+ 'url': 'http://telemb.be/les-reportages-havre-incendie-mortel_d_13514.html',
+ 'md5': '6e9682736e5ccd4eab7f21e855350733',
+ 'info_dict': {
+ 'id': '13514',
+ 'display_id': 'les-reportages-havre-incendie-mortel',
+ 'ext': 'mp4',
+ 'title': 'Havrц╘ - Incendie mortel - Les reportages',
+ 'description': 'md5:5e54cb449acb029c2b7734e2d946bd4a',
+ 'thumbnail': 're:^http://.*\.(?:jpg|png)$',
+ }
+ },
+ ]
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+ display_id = mobj.group('display_id')
+
+ webpage = self._download_webpage(url, display_id)
+
+ formats = []
+ for video_url in re.findall(r'file\s*:\s*"([^"]+)"', webpage):
+ fmt = {
+ 'url': video_url,
+ 'format_id': video_url.split(':')[0]
+ }
+ rtmp = re.search(r'^(?Prtmp://[^/]+/(?P.+))/(?Pmp4:.+)$', video_url)
+ if rtmp:
+ fmt.update({
+ 'play_path': rtmp.group('playpath'),
+ 'app': rtmp.group('app'),
+ 'player_url': 'http://p.jwpcdn.com/6/10/jwplayer.flash.swf',
+ 'page_url': 'http://www.telemb.be',
+ 'preference': -1,
+ })
+ formats.append(fmt)
+ self._sort_formats(formats)
+
+ title = remove_start(self._og_search_title(webpage), 'Tц╘lц╘MB : ')
+ description = self._html_search_regex(
+ r'