yt-dlp/youtube_dl/extractor/appletrailers.py

from __future__ import unicode_literals

import re
import json

from .common import InfoExtractor
from ..compat import compat_urlparse
from ..utils import (
    int_or_none,
)


class AppleTrailersIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?trailers\.apple\.com/(?:trailers|ca)/(?P<company>[^/]+)/(?P<movie>[^/]+)'
    _TESTS = [{
        'url': 'http://trailers.apple.com/trailers/wb/manofsteel/',
        'info_dict': {
            'id': 'manofsteel',
        },
        'playlist': [
            {
                'md5': 'd97a8e575432dbcb81b7c3acb741f8a8',
                'info_dict': {
                    'id': 'manofsteel-trailer4',
                    'ext': 'mov',
                    'duration': 111,
                    'title': 'Trailer 4',
                    'upload_date': '20130523',
                    'uploader_id': 'wb',
                },
            },
            {
                'md5': 'b8017b7131b721fb4e8d6f49e1df908c',
                'info_dict': {
                    'id': 'manofsteel-trailer3',
                    'ext': 'mov',
                    'duration': 182,
                    'title': 'Trailer 3',
                    'upload_date': '20130417',
                    'uploader_id': 'wb',
                },
            },
            {
                'md5': 'd0f1e1150989b9924679b441f3404d48',
                'info_dict': {
                    'id': 'manofsteel-trailer',
                    'ext': 'mov',
                    'duration': 148,
                    'title': 'Trailer',
                    'upload_date': '20121212',
                    'uploader_id': 'wb',
                },
            },
            {
                'md5': '5fe08795b943eb2e757fa95cb6def1cb',
                'info_dict': {
                    'id': 'manofsteel-teaser',
                    'ext': 'mov',
                    'duration': 93,
                    'title': 'Teaser',
                    'upload_date': '20120721',
                    'uploader_id': 'wb',
                },
            },
        ]
    }, {
        'url': 'http://trailers.apple.com/ca/metropole/autrui/',
        'only_matching': True,
    }]

    _JSON_RE = r'iTunes.playURL\((.*?)\);'

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        movie = mobj.group('movie')
        uploader_id = mobj.group('company')

        playlist_url = compat_urlparse.urljoin(url, 'includes/playlists/itunes.inc')

        def fix_html(s):
            s = re.sub(r'(?s)<script[^<]*?>.*?</script>', '', s)
            s = re.sub(r'<img ([^<]*?)>', r'<img \1/>', s)
            # The ' in the onClick attributes are not escaped, it couldn't be parsed
            # like: http://trailers.apple.com/trailers/wb/gravity/

            def _clean_json(m):
                return 'iTunes.playURL(%s);' % m.group(1).replace('\'', '&#39;')
            s = re.sub(self._JSON_RE, _clean_json, s)
            s = '<html>%s</html>' % s
            return s
        doc = self._download_xml(playlist_url, movie, transform_source=fix_html)

        playlist = []
        for li in doc.findall('./div/ul/li'):
            on_click = li.find('.//a').attrib['onClick']
            trailer_info_json = self._search_regex(self._JSON_RE,
                                                   on_click, 'trailer info')
            trailer_info = json.loads(trailer_info_json)
            title = trailer_info['title']
            video_id = movie + '-' + re.sub(r'[^a-zA-Z0-9]', '', title).lower()
            thumbnail = li.find('.//img').attrib['src']
            upload_date = trailer_info['posted'].replace('-', '')

            runtime = trailer_info['runtime']
            m = re.search(r'(?P<minutes>[0-9]+):(?P<seconds>[0-9]{1,2})', runtime)
            duration = None
            if m:
                duration = 60 * int(m.group('minutes')) + int(m.group('seconds'))

            first_url = trailer_info['url']
            trailer_id = first_url.split('/')[-1].rpartition('_')[0].lower()
            settings_json_url = compat_urlparse.urljoin(url, 'includes/settings/%s.json' % trailer_id)
            settings = self._download_json(settings_json_url, trailer_id, 'Downloading settings json')

            formats = []
            for format in settings['metadata']['sizes']:
                # The src is a file pointing to the real video file
                format_url = re.sub(r'_(\d*p.mov)', r'_h\1', format['src'])
                formats.append({
                    'url': format_url,
                    'format': format['type'],
                    'width': int_or_none(format['width']),
                    'height': int_or_none(format['height']),
                })

            self._sort_formats(formats)

            playlist.append({
                '_type': 'video',
                'id': video_id,
                'formats': formats,
                'title': title,
                'duration': duration,
                'thumbnail': thumbnail,
                'upload_date': upload_date,
                'uploader_id': uploader_id,
                'http_headers': {
                    'User-Agent': 'QuickTime compatible (youtube-dl)',
                },
            })

        return {
            '_type': 'playlist',
            'id': movie,
            'entries': playlist,
        }
More unicode literals 2014-01-07 10:04:48 +01:00			`from __future__ import unicode_literals`

[appletrailers] Add support 2013-08-28 02:18:44 +02:00			`import re`
[appletrailers] Rework extraction (fixes #1387) The exraction was broken: * The includes page contains img elements that need to be fixed. * Use the 'itunes.inc' page, it contains a json dictionary for each trailer with information. * Get the formats from 'includes/settings{trailer_name}.json' * Use urljoin to allow urls with a fragment identifier to work Removed the thumbnail urls from the tests, they are different now. 2013-09-29 20:49:58 +02:00			`import json`
[appletrailers] Add support 2013-08-28 02:18:44 +02:00
			`from .common import InfoExtractor`
Fix imports and general cleanup · Import from compat what comes from compat. Yes, some names are available in utils too, but that's an implementation detail. · Use _match_id consistently whenever possible · Fix some outdated tests · Use consistent valid URL (always match the whole protocol, no ^ at start required) · Use modern test definitions 2014-12-13 12:24:42 +01:00			`from ..compat import compat_urlparse`
[appletrailers] Add support 2013-08-28 02:18:44 +02:00			`from ..utils import (`
[appletrailers] Support height-less videos 2014-08-10 13:04:45 +02:00			`int_or_none,`
[appletrailers] Add support 2013-08-28 02:18:44 +02:00			`)`


			`class AppleTrailersIE(InfoExtractor):`
[appletrailers] Extend _VALID_URL (Closes #5027) 2015-02-22 14:53:32 +01:00			`_VALID_URL = r'https?://(?:www\.)?trailers\.apple\.com/(?:trailers\|ca)/(?P<company>[^/]+)/(?P<movie>[^/]+)'`
[appletrailers] Add test (#5027) 2015-02-22 14:58:39 +01:00			`_TESTS = [{`
[appletrailers] Quotes consistency 2015-10-05 16:21:53 +02:00			`'url': 'http://trailers.apple.com/trailers/wb/manofsteel/',`
Extend various playlist tests 2015-02-18 00:49:10 +01:00			`'info_dict': {`
			`'id': 'manofsteel',`
			`},`
[appletrailers] Quotes consistency 2015-10-05 16:21:53 +02:00			`'playlist': [`
[appletrailers] Add support 2013-08-28 02:18:44 +02:00			`{`
[appletrailers] Quotes consistency 2015-10-05 16:21:53 +02:00			`'md5': 'd97a8e575432dbcb81b7c3acb741f8a8',`
			`'info_dict': {`
			`'id': 'manofsteel-trailer4',`
			`'ext': 'mov',`
			`'duration': 111,`
			`'title': 'Trailer 4',`
			`'upload_date': '20130523',`
			`'uploader_id': 'wb',`
[appletrailers] Add support 2013-08-28 02:18:44 +02:00			`},`
			`},`
			`{`
[appletrailers] Quotes consistency 2015-10-05 16:21:53 +02:00			`'md5': 'b8017b7131b721fb4e8d6f49e1df908c',`
			`'info_dict': {`
			`'id': 'manofsteel-trailer3',`
			`'ext': 'mov',`
			`'duration': 182,`
			`'title': 'Trailer 3',`
			`'upload_date': '20130417',`
			`'uploader_id': 'wb',`
[appletrailers] Add support 2013-08-28 02:18:44 +02:00			`},`
			`},`
			`{`
[appletrailers] Quotes consistency 2015-10-05 16:21:53 +02:00			`'md5': 'd0f1e1150989b9924679b441f3404d48',`
			`'info_dict': {`
			`'id': 'manofsteel-trailer',`
			`'ext': 'mov',`
			`'duration': 148,`
			`'title': 'Trailer',`
			`'upload_date': '20121212',`
			`'uploader_id': 'wb',`
[appletrailers] Add support 2013-08-28 02:18:44 +02:00			`},`
			`},`
			`{`
[appletrailers] Quotes consistency 2015-10-05 16:21:53 +02:00			`'md5': '5fe08795b943eb2e757fa95cb6def1cb',`
			`'info_dict': {`
			`'id': 'manofsteel-teaser',`
			`'ext': 'mov',`
			`'duration': 93,`
			`'title': 'Teaser',`
			`'upload_date': '20120721',`
			`'uploader_id': 'wb',`
[appletrailers] Add support 2013-08-28 02:18:44 +02:00			`},`
[appletrailers] Modernize 2014-03-27 21:10:51 +01:00			`},`
[appletrailers] Add support 2013-08-28 02:18:44 +02:00			`]`
[appletrailers] Add test (#5027) 2015-02-22 14:58:39 +01:00			`}, {`
			`'url': 'http://trailers.apple.com/ca/metropole/autrui/',`
			`'only_matching': True,`
			`}]`
[appletrailers] Add support 2013-08-28 02:18:44 +02:00
[appletrailers] Rework extraction (fixes #1387) The exraction was broken: * The includes page contains img elements that need to be fixed. * Use the 'itunes.inc' page, it contains a json dictionary for each trailer with information. * Get the formats from 'includes/settings{trailer_name}.json' * Use urljoin to allow urls with a fragment identifier to work Removed the thumbnail urls from the tests, they are different now. 2013-09-29 20:49:58 +02:00			`_JSON_RE = r'iTunes.playURL\((.*?)\);'`

[appletrailers] Add support 2013-08-28 02:18:44 +02:00			`def _real_extract(self, url):`
			`mobj = re.match(self._VALID_URL, url)`
			`movie = mobj.group('movie')`
			`uploader_id = mobj.group('company')`

[appletrailers] Modernize 2014-03-27 21:10:51 +01:00			`playlist_url = compat_urlparse.urljoin(url, 'includes/playlists/itunes.inc')`
PEP8 applied 2014-11-23 20:41:03 +01:00
Use `_download_xml` in more extractors 2013-12-10 21:03:53 +01:00			`def fix_html(s):`
[appletrailers] Modernize 2014-03-27 21:10:51 +01:00			`s = re.sub(r'(?s)<script[^<]?>.?</script>', '', s)`
Use `_download_xml` in more extractors 2013-12-10 21:03:53 +01:00			`s = re.sub(r'<img ([^<]*?)>', r'<img \1/>', s)`
			`# The ' in the onClick attributes are not escaped, it couldn't be parsed`
			`# like: http://trailers.apple.com/trailers/wb/gravity/`
PEP8 applied 2014-11-23 20:41:03 +01:00
Use `_download_xml` in more extractors 2013-12-10 21:03:53 +01:00			`def _clean_json(m):`
[appletrailers] Modernize 2014-03-27 21:10:51 +01:00			`return 'iTunes.playURL(%s);' % m.group(1).replace('\'', ''')`
Use `_download_xml` in more extractors 2013-12-10 21:03:53 +01:00			`s = re.sub(self._JSON_RE, _clean_json, s)`
[appletrailers] Modernize 2014-11-26 12:41:24 +01:00			`s = '<html>%s</html>' % s`
Use `_download_xml` in more extractors 2013-12-10 21:03:53 +01:00			`return s`
			`doc = self._download_xml(playlist_url, movie, transform_source=fix_html)`
[appletrailers] Add support 2013-08-28 02:18:44 +02:00
			`playlist = []`
			`for li in doc.findall('./div/ul/li'):`
[appletrailers] Rework extraction (fixes #1387) The exraction was broken: * The includes page contains img elements that need to be fixed. * Use the 'itunes.inc' page, it contains a json dictionary for each trailer with information. * Get the formats from 'includes/settings{trailer_name}.json' * Use urljoin to allow urls with a fragment identifier to work Removed the thumbnail urls from the tests, they are different now. 2013-09-29 20:49:58 +02:00			`on_click = li.find('.//a').attrib['onClick']`
			`trailer_info_json = self._search_regex(self._JSON_RE,`
PEP8: applied even more rules 2014-11-23 21:39:15 +01:00			`on_click, 'trailer info')`
[appletrailers] Rework extraction (fixes #1387) The exraction was broken: * The includes page contains img elements that need to be fixed. * Use the 'itunes.inc' page, it contains a json dictionary for each trailer with information. * Get the formats from 'includes/settings{trailer_name}.json' * Use urljoin to allow urls with a fragment identifier to work Removed the thumbnail urls from the tests, they are different now. 2013-09-29 20:49:58 +02:00			`trailer_info = json.loads(trailer_info_json)`
			`title = trailer_info['title']`
[appletrailers] Add support 2013-08-28 02:18:44 +02:00			`video_id = movie + '-' + re.sub(r'[^a-zA-Z0-9]', '', title).lower()`
			`thumbnail = li.find('.//img').attrib['src']`
[appletrailers] Rework extraction (fixes #1387) The exraction was broken: * The includes page contains img elements that need to be fixed. * Use the 'itunes.inc' page, it contains a json dictionary for each trailer with information. * Get the formats from 'includes/settings{trailer_name}.json' * Use urljoin to allow urls with a fragment identifier to work Removed the thumbnail urls from the tests, they are different now. 2013-09-29 20:49:58 +02:00			`upload_date = trailer_info['posted'].replace('-', '')`
[appletrailers] Add support 2013-08-28 02:18:44 +02:00
[appletrailers] Rework extraction (fixes #1387) The exraction was broken: * The includes page contains img elements that need to be fixed. * Use the 'itunes.inc' page, it contains a json dictionary for each trailer with information. * Get the formats from 'includes/settings{trailer_name}.json' * Use urljoin to allow urls with a fragment identifier to work Removed the thumbnail urls from the tests, they are different now. 2013-09-29 20:49:58 +02:00			`runtime = trailer_info['runtime']`
			`m = re.search(r'(?P<minutes>[0-9]+):(?P<seconds>[0-9]{1,2})', runtime)`
[appletrailers] Add support 2013-08-28 02:18:44 +02:00			`duration = None`
			`if m:`
			`duration = 60 * int(m.group('minutes')) + int(m.group('seconds'))`

[appletrailers] Rework extraction (fixes #1387) The exraction was broken: * The includes page contains img elements that need to be fixed. * Use the 'itunes.inc' page, it contains a json dictionary for each trailer with information. * Get the formats from 'includes/settings{trailer_name}.json' * Use urljoin to allow urls with a fragment identifier to work Removed the thumbnail urls from the tests, they are different now. 2013-09-29 20:49:58 +02:00			`first_url = trailer_info['url']`
[appletrailers] The request for the settings must have the trailer name in lower case (fixes #1329) 2013-09-29 20:59:19 +02:00			`trailer_id = first_url.split('/')[-1].rpartition('_')[0].lower()`
[appletrailers] Rework extraction (fixes #1387) The exraction was broken: * The includes page contains img elements that need to be fixed. * Use the 'itunes.inc' page, it contains a json dictionary for each trailer with information. * Get the formats from 'includes/settings{trailer_name}.json' * Use urljoin to allow urls with a fragment identifier to work Removed the thumbnail urls from the tests, they are different now. 2013-09-29 20:49:58 +02:00			`settings_json_url = compat_urlparse.urljoin(url, 'includes/settings/%s.json' % trailer_id)`
[appletrailers] Modernize 2014-03-27 21:10:51 +01:00			`settings = self._download_json(settings_json_url, trailer_id, 'Downloading settings json')`
[appletrailers] Add support 2013-08-28 02:18:44 +02:00
[appletrailers] Rework extraction (fixes #1387) The exraction was broken: * The includes page contains img elements that need to be fixed. * Use the 'itunes.inc' page, it contains a json dictionary for each trailer with information. * Get the formats from 'includes/settings{trailer_name}.json' * Use urljoin to allow urls with a fragment identifier to work Removed the thumbnail urls from the tests, they are different now. 2013-09-29 20:49:58 +02:00			`formats = []`
			`for format in settings['metadata']['sizes']:`
			`# The src is a file pointing to the real video file`
			`format_url = re.sub(r'_(\d*p.mov)', r'_h\1', format['src'])`
			`formats.append({`
			`'url': format_url,`
			`'format': format['type'],`
[appletrailers] Support height-less videos 2014-08-10 13:04:45 +02:00			`'width': int_or_none(format['width']),`
			`'height': int_or_none(format['height']),`
[appletrailers] Rework extraction (fixes #1387) The exraction was broken: * The includes page contains img elements that need to be fixed. * Use the 'itunes.inc' page, it contains a json dictionary for each trailer with information. * Get the formats from 'includes/settings{trailer_name}.json' * Use urljoin to allow urls with a fragment identifier to work Removed the thumbnail urls from the tests, they are different now. 2013-09-29 20:49:58 +02:00			`})`
[appletrailers] Use centralized format selection 2013-12-25 15:24:41 +01:00
			`self._sort_formats(formats)`
[appletrailers] Add support 2013-08-28 02:18:44 +02:00
Remove the compatibility code used before the new format system was implemented 2013-12-03 14:21:06 +01:00			`playlist.append({`
[appletrailers] Add support 2013-08-28 02:18:44 +02:00			`'_type': 'video',`
			`'id': video_id,`
			`'formats': formats,`
			`'title': title,`
			`'duration': duration,`
			`'thumbnail': thumbnail,`
			`'upload_date': upload_date,`
			`'uploader_id': uploader_id,`
[extractors] Use http_headers for setting the User-Agent and the Referer 2015-01-24 18:19:58 +01:00			`'http_headers': {`
			`'User-Agent': 'QuickTime compatible (youtube-dl)',`
			`},`
Remove the compatibility code used before the new format system was implemented 2013-12-03 14:21:06 +01:00			`})`
[appletrailers] Add support 2013-08-28 02:18:44 +02:00
			`return {`
			`'_type': 'playlist',`
			`'id': movie,`
			`'entries': playlist,`
			`}`