mirror of
https://github.com/yt-dlp/yt-dlp
synced 2024-11-28 16:33:24 +01:00
227bf1a33b
New networking interface consists of a `RequestDirector` that directs each `Request` to appropriate `RequestHandler` and returns the `Response` or raises `RequestError`. The handlers define adapters to transform its internal Request/Response/Errors to our interfaces. User-facing changes: - Fix issues with per request proxies on redirects for urllib - Support for `ALL_PROXY` environment variable for proxy setting - Support for `socks5h` proxy - Closes https://github.com/yt-dlp/yt-dlp/issues/6325, https://github.com/ytdl-org/youtube-dl/issues/22618, https://github.com/ytdl-org/youtube-dl/pull/28093 - Raise error when using `https` proxy instead of silently converting it to `http` Authored by: coletdjnz
118 lines
3.5 KiB
Python
118 lines
3.5 KiB
Python
import collections
|
|
import random
|
|
import urllib.parse
|
|
import urllib.request
|
|
|
|
from ._utils import remove_start
|
|
|
|
|
|
def random_user_agent():
|
|
_USER_AGENT_TPL = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/%s Safari/537.36'
|
|
_CHROME_VERSIONS = (
|
|
'90.0.4430.212',
|
|
'90.0.4430.24',
|
|
'90.0.4430.70',
|
|
'90.0.4430.72',
|
|
'90.0.4430.85',
|
|
'90.0.4430.93',
|
|
'91.0.4472.101',
|
|
'91.0.4472.106',
|
|
'91.0.4472.114',
|
|
'91.0.4472.124',
|
|
'91.0.4472.164',
|
|
'91.0.4472.19',
|
|
'91.0.4472.77',
|
|
'92.0.4515.107',
|
|
'92.0.4515.115',
|
|
'92.0.4515.131',
|
|
'92.0.4515.159',
|
|
'92.0.4515.43',
|
|
'93.0.4556.0',
|
|
'93.0.4577.15',
|
|
'93.0.4577.63',
|
|
'93.0.4577.82',
|
|
'94.0.4606.41',
|
|
'94.0.4606.54',
|
|
'94.0.4606.61',
|
|
'94.0.4606.71',
|
|
'94.0.4606.81',
|
|
'94.0.4606.85',
|
|
'95.0.4638.17',
|
|
'95.0.4638.50',
|
|
'95.0.4638.54',
|
|
'95.0.4638.69',
|
|
'95.0.4638.74',
|
|
'96.0.4664.18',
|
|
'96.0.4664.45',
|
|
'96.0.4664.55',
|
|
'96.0.4664.93',
|
|
'97.0.4692.20',
|
|
)
|
|
return _USER_AGENT_TPL % random.choice(_CHROME_VERSIONS)
|
|
|
|
|
|
class HTTPHeaderDict(collections.UserDict, dict):
|
|
"""
|
|
Store and access keys case-insensitively.
|
|
The constructor can take multiple dicts, in which keys in the latter are prioritised.
|
|
"""
|
|
|
|
def __init__(self, *args, **kwargs):
|
|
super().__init__()
|
|
for dct in args:
|
|
if dct is not None:
|
|
self.update(dct)
|
|
self.update(kwargs)
|
|
|
|
def __setitem__(self, key, value):
|
|
super().__setitem__(key.title(), str(value))
|
|
|
|
def __getitem__(self, key):
|
|
return super().__getitem__(key.title())
|
|
|
|
def __delitem__(self, key):
|
|
super().__delitem__(key.title())
|
|
|
|
def __contains__(self, key):
|
|
return super().__contains__(key.title() if isinstance(key, str) else key)
|
|
|
|
|
|
std_headers = HTTPHeaderDict({
|
|
'User-Agent': random_user_agent(),
|
|
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
|
'Accept-Language': 'en-us,en;q=0.5',
|
|
'Sec-Fetch-Mode': 'navigate',
|
|
})
|
|
|
|
|
|
def clean_proxies(proxies: dict, headers: HTTPHeaderDict):
|
|
req_proxy = headers.pop('Ytdl-Request-Proxy', None)
|
|
if req_proxy:
|
|
proxies.clear() # XXX: compat: Ytdl-Request-Proxy takes preference over everything, including NO_PROXY
|
|
proxies['all'] = req_proxy
|
|
for proxy_key, proxy_url in proxies.items():
|
|
if proxy_url == '__noproxy__':
|
|
proxies[proxy_key] = None
|
|
continue
|
|
if proxy_key == 'no': # special case
|
|
continue
|
|
if proxy_url is not None:
|
|
# Ensure proxies without a scheme are http.
|
|
proxy_scheme = urllib.request._parse_proxy(proxy_url)[0]
|
|
if proxy_scheme is None:
|
|
proxies[proxy_key] = 'http://' + remove_start(proxy_url, '//')
|
|
|
|
replace_scheme = {
|
|
'socks5': 'socks5h', # compat: socks5 was treated as socks5h
|
|
'socks': 'socks4' # compat: non-standard
|
|
}
|
|
if proxy_scheme in replace_scheme:
|
|
proxies[proxy_key] = urllib.parse.urlunparse(
|
|
urllib.parse.urlparse(proxy_url)._replace(scheme=replace_scheme[proxy_scheme]))
|
|
|
|
|
|
def clean_headers(headers: HTTPHeaderDict):
|
|
if 'Youtubedl-No-Compression' in headers: # compat
|
|
del headers['Youtubedl-No-Compression']
|
|
headers['Accept-Encoding'] = 'identity'
|