[movefiles] Fix bugs and make more robust

yt_dlp/YoutubeDL.py

@@ -2075,6 +2075,7 @@ class YoutubeDL(object):
 
         info_dict = self.pre_process(info_dict)
 
+        # info_dict['_filename'] needs to be set for backward compatibility
         info_dict['_filename'] = full_filename = self.prepare_filename(info_dict, warn=True)
         temp_filename = self.prepare_filename(info_dict, 'temp')
         files_to_move = {}
@@ -2159,6 +2160,7 @@ class YoutubeDL(object):
                 sub_filename_final = subtitles_filename(sub_fn, sub_lang, sub_format, info_dict.get('ext'))
                 if not self.params.get('overwrites', True) and os.path.exists(encodeFilename(sub_filename)):
                     self.to_screen('[info] Video subtitle %s.%s is already present' % (sub_lang, sub_format))
+                    sub_info['filepath'] = sub_filename
                     files_to_move[sub_filename] = sub_filename_final
                 else:
                     self.to_screen('[info] Writing video subtitles to: ' + sub_filename)
@@ -2168,13 +2170,15 @@ class YoutubeDL(object):
                             # See https://github.com/ytdl-org/youtube-dl/issues/10268
                             with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8', newline='') as subfile:
                                 subfile.write(sub_info['data'])
+                            sub_info['filepath'] = sub_filename
                             files_to_move[sub_filename] = sub_filename_final
                         except (OSError, IOError):
                             self.report_error('Cannot write subtitles file ' + sub_filename)
                             return
                     else:
                         try:
-                            dl(sub_filename, sub_info, subtitle=True)
+                            dl(sub_filename, sub_info.copy(), subtitle=True)
+                            sub_info['filepath'] = sub_filename
                             files_to_move[sub_filename] = sub_filename_final
                         except (ExtractorError, IOError, OSError, ValueError, compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
                             self.report_warning('Unable to download subtitle for "%s": %s' %
@@ -2223,7 +2227,7 @@ class YoutubeDL(object):
         for thumb_ext in self._write_thumbnails(info_dict, thumb_fn_temp):
             thumb_filename_temp = replace_extension(thumb_fn_temp, thumb_ext, info_dict.get('ext'))
             thumb_filename = replace_extension(thumbfn, thumb_ext, info_dict.get('ext'))
-            files_to_move[thumb_filename_temp] = info_dict['__thumbnail_filename'] = thumb_filename
+            files_to_move[thumb_filename_temp] = thumb_filename
 
         # Write internet shortcut files
         url_link = webloc_link = desktop_link = False
@@ -2529,15 +2533,17 @@ class YoutubeDL(object):
             (k, v) for k, v in info_dict.items()
             if (k[0] != '_' or k == '_type') and k not in fields_to_remove)
 
-    def run_pp(self, pp, infodict, files_to_move={}):
+    def run_pp(self, pp, infodict):
         files_to_delete = []
+        if '__files_to_move' not in infodict:
+            infodict['__files_to_move'] = {}
         files_to_delete, infodict = pp.run(infodict)
         if not files_to_delete:
-            return files_to_move, infodict
+            return infodict
 
         if self.params.get('keepvideo', False):
             for f in files_to_delete:
-                files_to_move.setdefault(f, '')
+                infodict['__files_to_move'].setdefault(f, '')
         else:
             for old_filename in set(files_to_delete):
                 self.to_screen('Deleting original file %s (pass -k to keep)' % old_filename)
@@ -2545,9 +2551,9 @@ class YoutubeDL(object):
                     os.remove(encodeFilename(old_filename))
                 except (IOError, OSError):
                     self.report_warning('Unable to remove downloaded original file')
-                if old_filename in files_to_move:
+                if old_filename in infodict['__files_to_move']:
-                    del files_to_move[old_filename]
+                    del infodict['__files_to_move'][old_filename]
-        return files_to_move, infodict
+        return infodict
 
     @staticmethod
     def post_extract(info_dict):
@@ -2570,20 +2576,21 @@ class YoutubeDL(object):
     def pre_process(self, ie_info):
         info = dict(ie_info)
         for pp in self._pps['beforedl']:
-            info = self.run_pp(pp, info)[1]
+            info = self.run_pp(pp, info)
         return info
 
-    def post_process(self, filename, ie_info, files_to_move={}):
+    def post_process(self, filename, ie_info, files_to_move=None):
         """Run all the postprocessors on the given file."""
         info = dict(ie_info)
         info['filepath'] = filename
-        info['__files_to_move'] = {}
+        info['__files_to_move'] = files_to_move or {}
 
         for pp in ie_info.get('__postprocessors', []) + self._pps['normal']:
-            files_to_move, info = self.run_pp(pp, info, files_to_move)
+            info = self.run_pp(pp, info)
-        info = self.run_pp(MoveFilesAfterDownloadPP(self, files_to_move), info)[1]
+        info = self.run_pp(MoveFilesAfterDownloadPP(self), info)
+        del info['__files_to_move']
         for pp in self._pps['aftermove']:
-            info = self.run_pp(pp, info, {})[1]
+            info = self.run_pp(pp, info)
 
     def _make_archive_id(self, info_dict):
         video_id = info_dict.get('id')
@@ -2951,7 +2958,7 @@ class YoutubeDL(object):
             thumb_ext = determine_ext(t['url'], 'jpg')
             suffix = '%s.' % t['id'] if multiple else ''
             thumb_display_id = '%s ' % t['id'] if multiple else ''
-            t['filename'] = thumb_filename = replace_extension(filename, suffix + thumb_ext, info_dict.get('ext'))
+            t['filepath'] = thumb_filename = replace_extension(filename, suffix + thumb_ext, info_dict.get('ext'))
 
             if not self.params.get('overwrites', True) and os.path.exists(encodeFilename(thumb_filename)):
                 ret.append(suffix + thumb_ext)

yt_dlp/postprocessor/embedthumbnail.py

@@ -47,7 +47,7 @@ class EmbedThumbnailPP(FFmpegPostProcessor):
             self.to_screen('There aren\'t any thumbnails to embed')
             return [], info
 
-        original_thumbnail = thumbnail_filename = info['thumbnails'][-1]['filename']
+        initial_thumbnail = original_thumbnail = thumbnail_filename = info['thumbnails'][-1]['filepath']
 
         if not os.path.exists(encodeFilename(thumbnail_filename)):
             self.report_warning('Skipping embedding the thumbnail because the file is missing.')
@@ -65,6 +65,8 @@ class EmbedThumbnailPP(FFmpegPostProcessor):
             if thumbnail_ext != 'webp' and is_webp(thumbnail_filename):
                 self.to_screen('Correcting extension to webp and escaping path for thumbnail "%s"' % thumbnail_filename)
                 thumbnail_webp_filename = replace_extension(thumbnail_filename, 'webp')
+                if os.path.exists(thumbnail_webp_filename):
+                    os.remove(thumbnail_webp_filename)
                 os.rename(encodeFilename(thumbnail_filename), encodeFilename(thumbnail_webp_filename))
                 original_thumbnail = thumbnail_filename = thumbnail_webp_filename
                 thumbnail_ext = 'webp'
@@ -194,7 +196,8 @@ class EmbedThumbnailPP(FFmpegPostProcessor):
         files_to_delete = [thumbnail_filename]
         if self._already_have_thumbnail:
             info['__files_to_move'][original_thumbnail] = replace_extension(
-                info['__thumbnail_filename'], os.path.splitext(original_thumbnail)[1][1:])
+                info['__files_to_move'][initial_thumbnail],
+                os.path.splitext(original_thumbnail)[1][1:])
             if original_thumbnail == thumbnail_filename:
                 files_to_delete = []
         elif original_thumbnail != thumbnail_filename:

yt_dlp/postprocessor/ffmpeg.py

@@ -486,7 +486,7 @@ class FFmpegEmbedSubtitlePP(FFmpegPostProcessor):
                 self.report_warning('JSON subtitles cannot be embedded')
             elif ext != 'webm' or ext == 'webm' and sub_ext == 'vtt':
                 sub_langs.append(lang)
-                sub_filenames.append(subtitles_filename(filename, lang, sub_ext, ext))
+                sub_filenames.append(sub_info['filepath'])
             else:
                 if not webm_vtt_warn and ext == 'webm' and sub_ext != 'vtt':
                     webm_vtt_warn = True
@@ -732,9 +732,9 @@ class FFmpegSubtitlesConvertorPP(FFmpegPostProcessor):
                     'You have requested to convert json subtitles into another format, '
                     'which is currently not possible')
                 continue
-            old_file = subtitles_filename(filename, lang, ext, info.get('ext'))
+            old_file = sub['filepath']
             sub_filenames.append(old_file)
-            new_file = subtitles_filename(filename, lang, new_ext, info.get('ext'))
+            new_file = replace_extension(old_file, new_ext)
 
             if ext in ('dfxp', 'ttml', 'tt'):
                 self.report_warning(
@@ -742,7 +742,7 @@ class FFmpegSubtitlesConvertorPP(FFmpegPostProcessor):
                     'which results in style information loss')
 
                 dfxp_file = old_file
-                srt_file = subtitles_filename(filename, lang, 'srt', info.get('ext'))
+                srt_file = replace_extension(old_file, 'srt')
 
                 with open(dfxp_file, 'rb') as f:
                     srt_data = dfxp2srt(f.read())
@@ -753,7 +753,8 @@ class FFmpegSubtitlesConvertorPP(FFmpegPostProcessor):
 
                 subs[lang] = {
                     'ext': 'srt',
-                    'data': srt_data
+                    'data': srt_data,
+                    'filepath': srt_file,
                 }
 
                 if new_ext == 'srt':
@@ -767,8 +768,12 @@ class FFmpegSubtitlesConvertorPP(FFmpegPostProcessor):
                 subs[lang] = {
                     'ext': new_ext,
                     'data': f.read(),
+                    'filepath': new_file,
                 }
 
+            info['__files_to_move'][new_file] = replace_extension(
+                info['__files_to_move'][old_file], new_ext)
+
         return sub_filenames, info
 
 
@@ -789,7 +794,7 @@ class FFmpegSplitChaptersPP(FFmpegPostProcessor):
         if not self._downloader._ensure_dir_exists(encodeFilename(destination)):
             return
 
-        chapter['_filename'] = destination
+        chapter['filepath'] = destination
         self.to_screen('Chapter %03d; Destination: %s' % (number, destination))
         return (
             destination,

yt_dlp/postprocessor/movefilesafterdownload.py

@@ -13,10 +13,6 @@ from ..utils import (
 
 class MoveFilesAfterDownloadPP(PostProcessor):
 
-    def __init__(self, downloader, files_to_move):
-        PostProcessor.__init__(self, downloader)
-        self.files_to_move = files_to_move
-
     @classmethod
     def pp_key(cls):
         return 'MoveFiles'
@@ -25,11 +21,10 @@ class MoveFilesAfterDownloadPP(PostProcessor):
         dl_path, dl_name = os.path.split(encodeFilename(info['filepath']))
         finaldir = info.get('__finaldir', dl_path)
         finalpath = os.path.join(finaldir, dl_name)
-        self.files_to_move.update(info['__files_to_move'])
+        info['__files_to_move'][info['filepath']] = decodeFilename(finalpath)
-        self.files_to_move[info['filepath']] = decodeFilename(finalpath)
 
         make_newfilename = lambda old: decodeFilename(os.path.join(finaldir, os.path.basename(encodeFilename(old))))
-        for oldfile, newfile in self.files_to_move.items():
+        for oldfile, newfile in info['__files_to_move'].items():
             if not newfile:
                 newfile = make_newfilename(oldfile)
             if os.path.abspath(encodeFilename(oldfile)) == os.path.abspath(encodeFilename(newfile)):