Added support for urls from multiple sources

Also added database integration
This commit is contained in:
nathannathant 2021-03-21 20:25:33 -07:00
parent fd1fead8f8
commit 27695a96e9
8 changed files with 101 additions and 48 deletions

View File

@ -84,7 +84,6 @@ def download(ctx, items):
""" """
config = _get_config(ctx) config = _get_config(ctx)
core = QobuzDL(config) core = QobuzDL(config)
core.load_creds()
for item in items: for item in items:
try: try:
if os.path.isfile(item): if os.path.isfile(item):

View File

@ -124,7 +124,7 @@ class QobuzClient(ClientInterface):
logger.debug("Already logged in") logger.debug("Already logged in")
return return
if (kwargs.get("app_id") or kwargs.get("secrets")) in (None, [""]): if (kwargs.get("app_id") or kwargs.get("secrets")) in (None, [], ""):
logger.info("Fetching tokens from Qobuz") logger.info("Fetching tokens from Qobuz")
spoofer = Spoofer() spoofer = Spoofer()
kwargs["app_id"] = spoofer.get_app_id() kwargs["app_id"] = spoofer.get_app_id()

View File

@ -4,7 +4,7 @@ from pprint import pformat
from ruamel.yaml import YAML from ruamel.yaml import YAML
from .constants import FOLDER_FORMAT, TRACK_FORMAT from .constants import CONFIG_PATH, FOLDER_FORMAT, TRACK_FORMAT
from .exceptions import InvalidSourceError from .exceptions import InvalidSourceError
yaml = YAML() yaml = YAML()
@ -62,7 +62,10 @@ class Config:
} }
self.path_format = {"folder": folder_format, "track": track_format} self.path_format = {"folder": folder_format, "track": track_format}
self._path = path if path is None:
self._path = CONFIG_PATH
else:
self._path = path
if not os.path.exists(self._path): if not os.path.exists(self._path):
logger.debug(f"Creating yaml config file at {self._path}") logger.debug(f"Creating yaml config file at {self._path}")
@ -70,7 +73,7 @@ class Config:
else: else:
# sometimes the file gets erased, this will reset it # sometimes the file gets erased, this will reset it
with open(self._path) as f: with open(self._path) as f:
if f.read().strip() == '': if f.read().strip() == "":
logger.debug(f"Config file {self._path} corrupted, resetting.") logger.debug(f"Config file {self._path} corrupted, resetting.")
self.dump(self.info) self.dump(self.info)
else: else:
@ -139,7 +142,7 @@ class Config:
@property @property
def info(self): def info(self):
return {k: v for k, v in self.__dict__.items() if not k.startswith('_')} return {k: v for k, v in self.__dict__.items() if not k.startswith("_")}
@info.setter @info.setter
def info(self, val): def info(self, val):

View File

@ -128,10 +128,9 @@ FOLDER_FORMAT = (
) )
TRACK_FORMAT = "{tracknumber}. {artist} - {title}" TRACK_FORMAT = "{tracknumber}. {artist} - {title}"
QOBUZ_URL_REGEX = ( URL_REGEX = (
r"(?:https:\/\/(?:w{3}|open|play)\.qobuz\.com)?" r"https:\/\/(?:www|open|play)?\.?(\w+)\.com(?:(?:\/(track|playlist|album|"
r"(?:\/[a-z]{2}-[a-z]{2})?\/(album|artist|track|playlist|label)(?:" r"artist|label))|(?:\/[-\w]+?))+\/(\w+)"
r"\/[-\w\d]+)?\/([\w\d]+)"
) )

View File

@ -1,21 +1,25 @@
import hashlib
import inspect
import logging import logging
import os import os
import re import re
from typing import Generator, Optional, Sequence, Tuple, Union from getpass import getpass
from typing import Generator, Optional, Tuple, Union
import click import click
from .clients import DeezerClient, QobuzClient, TidalClient from .clients import DeezerClient, QobuzClient, TidalClient
from .config import Config from .config import Config
from .constants import CONFIG_PATH, DB_PATH, QOBUZ_URL_REGEX from .constants import CONFIG_PATH, DB_PATH, URL_REGEX
from .db import QobuzDB from .db import QobuzDB
from .downloader import Album, Artist, Playlist, Track from .downloader import Album, Artist, Playlist, Track, Label
from .exceptions import InvalidSourceError, ParsingError from .exceptions import AuthenticationError, ParsingError
from .utils import capitalize
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
MEDIA_CLASS = {"album": Album, "playlist": Playlist, "artist": Artist, "track": Track} MEDIA_CLASS = {"album": Album, "playlist": Playlist, "artist": Artist, "track": Track, "label": Label}
CLIENTS = {"qobuz": QobuzClient, "tidal": TidalClient, "deezer": DeezerClient} CLIENTS = {"qobuz": QobuzClient, "tidal": TidalClient, "deezer": DeezerClient}
Media = Union[Album, Playlist, Artist, Track] # type hint Media = Union[Album, Playlist, Artist, Track] # type hint
@ -26,20 +30,20 @@ class QobuzDL:
def __init__( def __init__(
self, self,
config: Optional[Config] = None, config: Optional[Config] = None,
source: str = "qobuz",
database: Optional[str] = None, database: Optional[str] = None,
): ):
logger.debug(locals()) logger.debug(locals())
self.source = source self.url_parse = re.compile(URL_REGEX)
self.url_parse = re.compile(QOBUZ_URL_REGEX)
self.config = config self.config = config
if self.config is None: if self.config is None:
self.config = Config(CONFIG_PATH) self.config = Config(CONFIG_PATH)
self.client = CLIENTS[source]() self.clients = {
"qobuz": QobuzClient(),
logger.debug("Using client: %s", self.client) "tidal": TidalClient(),
"deezer": DeezerClient(),
}
if database is None: if database is None:
self.db = QobuzDB(DB_PATH) self.db = QobuzDB(DB_PATH)
@ -47,17 +51,36 @@ class QobuzDL:
assert isinstance(database, QobuzDB) assert isinstance(database, QobuzDB)
self.db = database self.db = database
def load_creds(self): def prompt_creds(self, source: str):
if isinstance(self.client, (QobuzClient, TidalClient)): """Prompt the user for credentials.
creds = self.config.creds(self.source)
if not creds.get("app_id") and isinstance(self.client, QobuzClient): :param source:
self.client.login(**creds) :type source: str
app_id, secrets = self.client.get_tokens() """
self.config["qobuz"]["app_id"] = app_id click.secho(f"Enter {capitalize(source)} email:", fg="green")
self.config["qobuz"]["secrets"] = secrets self.config[source]["email"] = input()
self.config.save() click.secho(
else: f"Enter {capitalize(source)} password (will not show on screen):",
self.client.login(**creds) fg="green",
)
self.config[source]["password"] = getpass(
prompt=""
) # does hashing work for tidal?
self.config.save()
click.secho(f'Credentials saved to config file at "{self.config._path}"')
def assert_creds(self, source: str):
assert source in ("qobuz", "tidal", "deezer"), f"Invalid source {source}"
if source == "deezer":
# no login for deezer
return
if (
self.config[source]["email"] is None
or self.config[source]["password"] is None
):
self.prompt_creds(source)
def handle_url(self, url: str): def handle_url(self, url: str):
"""Download an url """Download an url
@ -67,23 +90,36 @@ class QobuzDL:
:raises InvalidSourceError :raises InvalidSourceError
:raises ParsingError :raises ParsingError
""" """
assert self.source in url, f"{url} is not a {self.source} source" source, url_type, item_id = self.parse_url(url)
url_type, item_id = self.parse_url(url)
if item_id in self.db: if item_id in self.db:
logger.info(f"{url} already downloaded, use --no-db to override.") logger.info(f"{url} already downloaded, use --no-db to override.")
return return
self.handle_item(url_type, item_id) self.handle_item(source, url_type, item_id)
def handle_item(self, source: str, media_type: str, item_id: str):
self.assert_creds(source)
def handle_item(self, media_type, item_id):
arguments = { arguments = {
"database": self.db,
"parent_folder": self.config.downloads["folder"], "parent_folder": self.config.downloads["folder"],
"quality": self.config.downloads["quality"], "quality": self.config.downloads["quality"],
"embed_cover": self.config.metadata["embed_cover"], "embed_cover": self.config.metadata["embed_cover"],
} }
item = MEDIA_CLASS[media_type](client=self.client, id=item_id) client = self.clients[source]
if not client.logged_in:
while True:
try:
client.login(**self.config.creds(source))
break
except AuthenticationError:
click.secho("Invalid credentials, try again.")
self.prompt_creds(source)
item = MEDIA_CLASS[media_type](client=client, id=item_id)
if isinstance(item, Artist): if isinstance(item, Artist):
keys = self.config.filters.keys() keys = self.config.filters.keys()
# TODO: move this to config.py
filters_ = tuple(key for key in keys if self.config.filters[key]) filters_ = tuple(key for key in keys if self.config.filters[key])
arguments["filters"] = filters_ arguments["filters"] = filters_
logger.debug("Added filter argument for artist/label: %s", filters_) logger.debug("Added filter argument for artist/label: %s", filters_)
@ -112,7 +148,7 @@ class QobuzDL:
if parsed is not None: if parsed is not None:
parsed = parsed.groups() parsed = parsed.groups()
if len(parsed) == 2: if len(parsed) == 3:
return tuple(parsed) # Convert from Seq for the sake of typing return tuple(parsed) # Convert from Seq for the sake of typing
raise ParsingError(f"Error parsing URL: `{url}`") raise ParsingError(f"Error parsing URL: `{url}`")

View File

@ -16,6 +16,8 @@ class QobuzDB:
:type db_path: Union[str, os.PathLike] :type db_path: Union[str, os.PathLike]
""" """
self.path = db_path self.path = db_path
if not os.path.exists(self.path):
self.create()
def create(self): def create(self):
"""Create a database at `self.path`""" """Create a database at `self.path`"""

View File

@ -1,8 +1,8 @@
import sys
import logging import logging
import os import os
import re import re
import shutil import shutil
import sys
from abc import ABC, abstractmethod from abc import ABC, abstractmethod
from pprint import pprint from pprint import pprint
from tempfile import gettempdir from tempfile import gettempdir
@ -23,6 +23,7 @@ from .constants import (
FOLDER_FORMAT, FOLDER_FORMAT,
TRACK_FORMAT, TRACK_FORMAT,
) )
from .db import QobuzDB
from .exceptions import ( from .exceptions import (
InvalidQuality, InvalidQuality,
InvalidSourceError, InvalidSourceError,
@ -138,6 +139,7 @@ class Track:
quality: int = 7, quality: int = 7,
parent_folder: str = "Downloads", parent_folder: str = "Downloads",
progress_bar: bool = True, progress_bar: bool = True,
database: QobuzDB = None,
): ):
""" """
Download the track. Download the track.
@ -157,13 +159,14 @@ class Track:
os.makedirs(self.folder, exist_ok=True) os.makedirs(self.folder, exist_ok=True)
if os.path.isfile(self.format_final_path()): assert database is not None # remove this later
if os.path.isfile(self.format_final_path()) or self.id in database:
self.__is_downloaded = True self.__is_downloaded = True
self.__is_tagged = True self.__is_tagged = True
click.secho(f"Track already downloaded: {self.final_path}", fg="green") click.secho(f"Track already downloaded: {self.final_path}", fg="green")
return False return False
if hasattr(self, "cover_url") and self.embed_cover: if hasattr(self, "cover_url"):
self.download_cover() self.download_cover()
dl_info = self.client.get_file_url(self.id, quality) # dict dl_info = self.client.get_file_url(self.id, quality) # dict
@ -196,6 +199,7 @@ class Track:
raise InvalidSourceError(self.client.source) raise InvalidSourceError(self.client.source)
shutil.move(temp_file, self.final_path) shutil.move(temp_file, self.final_path)
database.add(self.id)
logger.debug("Downloaded: %s -> %s", temp_file, self.final_path) logger.debug("Downloaded: %s -> %s", temp_file, self.final_path)
self.__is_downloaded = True self.__is_downloaded = True
@ -206,7 +210,7 @@ class Track:
assert hasattr(self, "cover_url"), "must pass cover_url parameter" assert hasattr(self, "cover_url"), "must pass cover_url parameter"
self.cover_path = os.path.join(self.folder, f"cover{hash(self.meta.title)}.jpg") self.cover_path = os.path.join(self.folder, f"cover{hash(self.cover_url)}.jpg")
logger.debug(f"Downloading cover from {self.cover_url}") logger.debug(f"Downloading cover from {self.cover_url}")
if not os.path.exists(self.cover_path): if not os.path.exists(self.cover_path):
tqdm_download(self.cover_url, self.cover_path) tqdm_download(self.cover_url, self.cover_path)
@ -721,6 +725,7 @@ class Album(Tracklist):
tag_tracks: bool = True, tag_tracks: bool = True,
cover_key: str = "large", cover_key: str = "large",
embed_cover: bool = False, embed_cover: bool = False,
database: QobuzDB = None,
): ):
"""Download all of the tracks in the album. """Download all of the tracks in the album.
@ -764,7 +769,7 @@ class Album(Tracklist):
for track in self: for track in self:
logger.debug("Downloading track to %s", folder) logger.debug("Downloading track to %s", folder)
track.download(quality, folder, progress_bar) track.download(quality, folder, progress_bar, database=database)
if tag_tracks and self.client.source != "deezer": if tag_tracks and self.client.source != "deezer":
track.tag(cover=cover, embed_cover=embed_cover) track.tag(cover=cover, embed_cover=embed_cover)
@ -780,10 +785,10 @@ class Album(Tracklist):
else: else:
dict_[key] = None dict_[key] = None
dict_['sampling_rate'] /= 1000 dict_["sampling_rate"] /= 1000
# 48.0kHz -> 48kHz, 44.1kHz -> 44.1kHz # 48.0kHz -> 48kHz, 44.1kHz -> 44.1kHz
if dict_['sampling_rate'] % 1 == 0.0: if dict_["sampling_rate"] % 1 == 0.0:
dict_['sampling_rate'] = int(dict_['sampling_rate']) dict_["sampling_rate"] = int(dict_["sampling_rate"])
return dict_ return dict_
@ -930,6 +935,7 @@ class Playlist(Tracklist):
quality: int = 6, quality: int = 6,
filters: Callable = None, filters: Callable = None,
embed_cover: bool = False, embed_cover: bool = False,
database: QobuzDB = None,
): ):
"""Download and tag all of the tracks. """Download and tag all of the tracks.
@ -944,7 +950,7 @@ class Playlist(Tracklist):
folder = os.path.join(parent_folder, folder) folder = os.path.join(parent_folder, folder)
for track in self: for track in self:
track.download(parent_folder=folder, quality=quality) track.download(parent_folder=folder, quality=quality, database=database)
if self.client.source != "deezer": if self.client.source != "deezer":
track.tag(embed_cover=embed_cover) track.tag(embed_cover=embed_cover)
@ -1046,6 +1052,7 @@ class Artist(Tracklist):
no_repeats: bool = False, no_repeats: bool = False,
quality: int = 6, quality: int = 6,
embed_cover: bool = False, embed_cover: bool = False,
database: QobuzDB = None,
): ):
"""Download all albums in the discography. """Download all albums in the discography.
@ -1088,7 +1095,10 @@ class Artist(Tracklist):
except NonStreamable: except NonStreamable:
logger.info("Skipping album, not available to stream.") logger.info("Skipping album, not available to stream.")
album.download( album.download(
parent_folder=folder, quality=quality, embed_cover=embed_cover parent_folder=folder,
quality=quality,
embed_cover=embed_cover,
database=database,
) )
logger.debug(f"{i} albums downloaded") logger.debug(f"{i} albums downloaded")

View File

@ -148,3 +148,7 @@ def init_log(
logging.getLogger("urllib3").setLevel(logging.WARNING) logging.getLogger("urllib3").setLevel(logging.WARNING)
logging.getLogger("tidal_api").setLevel(logging.WARNING) logging.getLogger("tidal_api").setLevel(logging.WARNING)
def capitalize(s: str) -> str:
return s[0].upper() + s[1:]