修正VIKI下载JS

修复part大于1时读取json混流文件的严重错误;自动去除优酷的广告分片及前情提要

N_m3u8DL-CLI.sln

@@ -1,12 +1,10 @@
 
 Microsoft Visual Studio Solution File, Format Version 12.00
-# Visual Studio 15
+# Visual Studio Version 16
-VisualStudioVersion = 15.0.27703.2000
+VisualStudioVersion = 16.0.29215.179
 MinimumVisualStudioVersion = 10.0.40219.1
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "N_m3u8DL-CLI", "N_m3u8DL-CLI\N_m3u8DL-CLI.csproj", "{4FB61439-B738-46AC-B3AF-2BF72150D057}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GUI-MainWindow", "GUI-MainWindow\GUI-MainWindow.csproj", "{FE91DB43-1F1F-4119-B808-A7F4795BB4D0}"
-EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -17,10 +15,6 @@ Global
 		{4FB61439-B738-46AC-B3AF-2BF72150D057}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{4FB61439-B738-46AC-B3AF-2BF72150D057}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{4FB61439-B738-46AC-B3AF-2BF72150D057}.Release|Any CPU.Build.0 = Release|Any CPU
-		{FE91DB43-1F1F-4119-B808-A7F4795BB4D0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{FE91DB43-1F1F-4119-B808-A7F4795BB4D0}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{FE91DB43-1F1F-4119-B808-A7F4795BB4D0}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{FE91DB43-1F1F-4119-B808-A7F4795BB4D0}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

N_m3u8DL-CLI/DownloadManager.cs

@@ -152,6 +152,7 @@ namespace N_m3u8DL_CLI
             //开始调用下载
             LOGGER.WriteLine("Start Downloading");
             LOGGER.PrintLine("开始下载文件", LOGGER.Warning);
+
             //下载MAP文件（若有）
             try
             {
@@ -225,6 +226,8 @@ namespace N_m3u8DL_CLI
             if (Global.HadReadInfo == false)
             {
                 string href = DownDir + "\\Part_" + 0.ToString(partsPadZero) + "\\" + firstSeg["index"].Value<int>().ToString(segsPadZero) + ".ts";
+                if (File.Exists(DownDir + "\\!MAP.ts"))
+                    href = DownDir + "\\!MAP.ts";
                 Global.GzipHandler(href);
                 bool flag = false;
                 foreach (string ss in (string[])Global.GetVideoInfo(href).ToArray(typeof(string)))
@@ -406,7 +409,8 @@ namespace N_m3u8DL_CLI
                                 //检测是否为MPEG-TS封装，不是的话就转换为TS封装
                                 foreach (string s in Global.GetFiles(DownDir + "\\Part_0", ".ts"))
                                 {
-                                    if (!FFmpeg.CheckMPEGTS(s))
+                                    //跳过有MAP的情况
+                                    if (!isVTT && !File.Exists(DownDir + "\\Part_0\\!MAP.ts") && !FFmpeg.CheckMPEGTS(s))
                                     {
                                         //转换
                                         LOGGER.PrintLine("将文件转换到 MPEG-TS 封装：" + Path.GetFileName(s));
@@ -521,7 +525,7 @@ namespace N_m3u8DL_CLI
 
                     FFmpeg.OutPutPath = Path.Combine(Directory.GetParent(DownDir).FullName, DownName);
                     FFmpeg.ReportFile = driverName + "\\:" + exePath.Remove(0, exePath.IndexOf(':') + 1).Replace("\\", "/") + "/Logs/" + Path.GetFileNameWithoutExtension(LOGGER.LOGFILE) + fflogName;
-
+                    
                     //合并分段
                     LOGGER.PrintLine("合并分段中...");
                     for (int i = 0; i < PartsCount; i++)
@@ -553,7 +557,8 @@ namespace N_m3u8DL_CLI
                             //检测是否为MPEG-TS封装，不是的话就转换为TS封装
                             foreach (string s in Global.GetFiles(DownDir, ".ts"))
                             {
-                                if (!FFmpeg.CheckMPEGTS(s))
+                                //跳过有MAP的情况
+                                if (!isVTT && !File.Exists(DownDir + "\\!MAP.ts") && !FFmpeg.CheckMPEGTS(s))
                                 {
                                     //转换
                                     LOGGER.PrintLine("将文件转换到 MPEG-TS 封装：" + Path.GetFileName(s));
@@ -570,7 +575,7 @@ namespace N_m3u8DL_CLI
                                 FFmpeg.Merge(Global.GetFiles(DownDir, ".ts"), MuxFormat, MuxFastStart);
                             else
                             {
-                                JObject json = JObject.Parse(MuxSetJson);
+                                JObject json = JObject.Parse(File.ReadAllText(MuxSetJson, Encoding.UTF8));
                                 string muxFormat = json["muxFormat"].Value<string>();
                                 bool fastStart = Convert.ToBoolean(json["fastStart"].Value<string>());
                                 string poster = json["poster"].Value<string>();

N_m3u8DL-CLI/Global.cs

@@ -30,8 +30,8 @@ namespace N_m3u8DL_CLI
 
 
         /*===============================================================================*/
-        static string nowVer = "2.4.1";
+        static string nowVer = "2.4.8";
-        static string nowDate = "20191130";
+        static string nowDate = "20200131";
         public static void WriteInit()
         {
             Console.Clear();
@@ -58,7 +58,10 @@ namespace N_m3u8DL_CLI
                         //尝试下载新版本(去码云)
                         string url = $"https://gitee.com/nilaoda/N_m3u8DL-CLI/raw/master/N_m3u8DL-CLI_v{latestVer}.exe";
                         if (File.Exists(Path.Combine(Path.GetDirectoryName(Process.GetCurrentProcess().MainModule.FileName), $"N_m3u8DL-CLI_v{latestVer}.exe")))
+                        {
+                            Console.Title = $"检测到更新，版本：{latestVer}! 新版下载成功，请您自行替换";
                             return;
+                        }
                         HttpDownloadFile(url, Path.Combine(Path.GetDirectoryName(Process.GetCurrentProcess().MainModule.FileName), $"N_m3u8DL-CLI_v{latestVer}.exe"));
                         if (File.Exists(Path.Combine(Path.GetDirectoryName(Process.GetCurrentProcess().MainModule.FileName), $"N_m3u8DL-CLI_v{latestVer}.exe")))
                             Console.Title = $"检测到更新，版本：{latestVer}! 新版下载成功，请您自行替换";
@@ -77,6 +80,16 @@ namespace N_m3u8DL_CLI
             }
         }
 
+        public static string GetValidFileName(string input, string re = ".")
+        {
+            string title = input;
+            foreach (char invalidChar in Path.GetInvalidFileNameChars())
+            {
+                title = title.Replace(invalidChar.ToString(), re);
+            }
+            return title;
+        }
+
         // parseInt(s, radix)
         public static int GetNum(string str, int numBase)
         {
@@ -699,6 +712,14 @@ namespace N_m3u8DL_CLI
                     {
                         VIDEO_TYPE = "DV";
                     }
+                    else if (res.Contains("Video hevc (Main 10) (dvh1"))  //优酷视频杜比视界
+                    {
+                        VIDEO_TYPE = "DV";
+                    }
+                    else if (res.Contains("Video hevc (dvh1"))  //优酷视频杜比视界
+                    {
+                        VIDEO_TYPE = "DV";
+                    }
                     else if (res.Contains("Video h264"))
                     {
                         VIDEO_TYPE = "H264";

N_m3u8DL-CLI/Parser.cs

@@ -48,6 +48,8 @@ namespace N_m3u8DL_CLI
         private static string durEnd = "";
         //是否自动清除优酷广告分片
         private static bool delAd = true;
+        //标记是否已清除优酷广告分片
+        private static bool hasAd = false;
 
         public string BaseUrl { get => baseUrl; set => baseUrl = value; }
         public string M3u8Url { get => m3u8Url; set => m3u8Url = value; }
@@ -109,6 +111,9 @@ namespace N_m3u8DL_CLI
             if (m3u8Content.Contains("qiqiuyun.net/") || m3u8Content.Contains("aliyunedu.net/") || m3u8Content.Contains("qncdn.edusoho.net/")) //气球云
                 isQiQiuYun = true;
 
+            if (M3u8Url.Contains("tlivecloud-playback-cdn.ysp.cctv.cn") && M3u8Url.Contains("endtime="))
+                isEndlist = true;
+
             //输出m3u8文件
             File.WriteAllText(m3u8SavePath, m3u8Content);
 
@@ -202,7 +207,16 @@ namespace N_m3u8DL_CLI
                     //解析不连续标记，需要单独合并（timestamp不同）
                     else if (line.StartsWith(HLSTags.ext_x_discontinuity))
                     {
-                        if (segments.Count > 1)
+                        //修复优酷去除广告后的遗留问题
+                        if (hasAd && parts.Count > 0)
+                        {
+                            segments = (JArray)parts[parts.Count - 1];
+                            parts.RemoveAt(parts.Count - 1);
+                            hasAd = false;
+                            continue;
+                        }
+                        //常规情况的#EXT-X-DISCONTINUITY标记，新建part
+                        if (!hasAd && segments.Count > 1)
                         {
                             parts.Add(segments);
                             segments = new JArray();
@@ -214,7 +228,7 @@ namespace N_m3u8DL_CLI
                     else if (line.StartsWith(HLSTags.ext_x_version)) ;
                     else if (line.StartsWith(HLSTags.ext_x_allow_cache)) ;
                     //解析KEY
-                    else if (line.StartsWith(HLSTags.ext_x_key) && string.IsNullOrEmpty(keyFile) && string.IsNullOrEmpty(keyBase64)) 
+                    else if (line.StartsWith(HLSTags.ext_x_key) && string.IsNullOrEmpty(keyFile) && string.IsNullOrEmpty(keyBase64))
                     {
                         m3u8CurrentKey = ParseKey(line);
                         //存储为上一行的key信息
@@ -265,7 +279,10 @@ namespace N_m3u8DL_CLI
                         if (Global.GetTagAttribute(line, "TYPE") == "AUDIO")
                             MEDIA_AUDIO.Add(Global.GetTagAttribute(line, "GROUP-ID"), CombineURL(BaseUrl, Global.GetTagAttribute(line, "URI")));
                         if (Global.GetTagAttribute(line, "TYPE") == "SUBTITLES")
-                            MEDIA_SUB.Add(Global.GetTagAttribute(line, "GROUP-ID"), CombineURL(BaseUrl, Global.GetTagAttribute(line, "URI")));
+                        {
+                            if (!MEDIA_SUB.ContainsKey(Global.GetTagAttribute(line, "GROUP-ID")))
+                                MEDIA_SUB.Add(Global.GetTagAttribute(line, "GROUP-ID"), CombineURL(BaseUrl, Global.GetTagAttribute(line, "URI")));
+                        }
                     }
                     else if (line.StartsWith(HLSTags.ext_x_playlist_type)) ;
                     else if (line.StartsWith(HLSTags.ext_i_frames_only))
@@ -279,7 +296,8 @@ namespace N_m3u8DL_CLI
                     //m3u8主体结束
                     else if (line.StartsWith(HLSTags.ext_x_endlist))
                     {
-                        parts.Add(segments);
+                        if (segments.Count > 0)
+                            parts.Add(segments);
                         segments = new JArray();
                         isEndlist = true;
                     }
@@ -300,7 +318,7 @@ namespace N_m3u8DL_CLI
                     else if (expectSegment)
                     {
                         segUrl = CombineURL(BaseUrl, line);
-                        if (M3u8Url.Contains("akamaized.net") && M3u8Url.Contains("?__gda__"))
+                        if (M3u8Url.Contains("?__gda__"))
                         {
                             segUrl += new Regex("\\?__gda__.*").Match(M3u8Url).Value;
                         }
@@ -308,10 +326,20 @@ namespace N_m3u8DL_CLI
                         segments.Add(segInfo);
                         segInfo = new JObject();
                         //优酷的广告分段则清除此分片
-                        if (DelAd && segUrl.Contains("ccode") && segUrl.Contains("/ad/") && segUrl.Contains("duration"))
+                        //需要注意，遇到广告说明程序对上文的#EXT-X-DISCONTINUITY做出的动作是不必要的，
+                        //其实上下文是同一种编码，需要恢复到原先的part上
+                        if (DelAd && segUrl.Contains("ccode=") && segUrl.Contains("/ad/") && segUrl.Contains("duration="))
                         {
                             segments.RemoveAt(segments.Count - 1);
                             segIndex--;
+                            hasAd = true;
+                        }
+                        //优酷广告(4K分辨率测试)
+                        if (DelAd && segUrl.Contains("ccode=0902") && segUrl.Contains("duration="))
+                        {
+                            segments.RemoveAt(segments.Count - 1);
+                            segIndex--;
+                            hasAd = true;
                         }
                         expectSegment = false;
                     }
@@ -320,7 +348,7 @@ namespace N_m3u8DL_CLI
                     {
                         string listUrl;
                         listUrl = CombineURL(BaseUrl, line);
-                        if (M3u8Url.Contains("akamaized.net") && M3u8Url.Contains("?__gda__"))
+                        if (M3u8Url.Contains("?__gda__"))
                         {
                             listUrl += new Regex("\\?__gda__.*").Match(M3u8Url).Value;
                         }

N_m3u8DL-CLI/Program.cs

@@ -5,7 +5,9 @@ using System.Diagnostics;
 using System.IO;
 using System.Linq;
 using System.Net;
+using System.Net.Security;
 using System.Runtime.InteropServices;
+using System.Security.Cryptography.X509Certificates;
 using System.Text;
 using System.Text.RegularExpressions;
 using System.Threading;
@@ -203,6 +205,23 @@ namespace N_m3u8DL_CLI.NetCore
     ///   - 增加disableIntegrityCheck选项
     /// 2019年10月24日
     ///   - 捕获Ctrl+C退出，移动光标到正确位置
+    /// 2019年11月30日
+    ///   - 完善芒果TV请求头的自动添加
+    /// 2019年12月16日
+    ///   - 处理文件名特殊字符
+    /// 2019年12月18日
+    ///   - 修复m3u8解析bug导致的无法合并问题
+    ///   - 增加杜比视界识别场景
+    ///   - 修复part大于1时读取json混流文件的严重错误
+    ///   - 自动去除优酷的广告分片及前情提要
+    ///   - 修复腾讯视频HDR10视频下载合并异常问题
+    /// 2020年1月26日
+    ///   - 在央视频回看链接且有endtime参数的情况下，不识别为直播流
+    /// 2020年1月29日
+    ///   - 修复识别大师列表的bug (多个字幕同一个GROUP-ID)
+    ///   - 修复vtt字幕无法正常合并的bug
+    /// 2020年1月31日
+    ///   - ?__gda__行为优化
     /// </summary>
     /// 
 
@@ -232,9 +251,17 @@ namespace N_m3u8DL_CLI.NetCore
             return false;
         }
 
+        private static bool ValidateServerCertificate(object sender, X509Certificate certificate, X509Chain chain, SslPolicyErrors sslPolicyErrors)
+        {
+            return true;
+        }
+
+
         static void Main(string[] args)
         {
             SetConsoleCtrlHandler(cancelHandler, true);
+            ServicePointManager.ServerCertificateValidationCallback = ValidateServerCertificate;
+
             try
             {
                 //goto httplitsen;
@@ -392,7 +419,7 @@ namespace N_m3u8DL_CLI.NetCore
                 }
                 if (arguments.Has("--saveName"))
                 {
-                    fileName = arguments.Get("--saveName").Next;
+                    fileName = Global.GetValidFileName(arguments.Get("--saveName").Next);
                 }
                 if (arguments.Has("--useKeyFile"))
                 {

README_ENG.md

@@ -0,0 +1,62 @@
+```
+
+███╗   ██╗        ███╗   ███╗██████╗ ██╗   ██╗ █████╗ ██████╗ ██╗       ██████╗██╗     ██╗
+████╗  ██║        ████╗ ████║╚════██╗██║   ██║██╔══██╗██╔══██╗██║      ██╔════╝██║     ██║
+██╔██╗ ██║        ██╔████╔██║ █████╔╝██║   ██║╚█████╔╝██║  ██║██║█████╗██║     ██║     ██║
+██║╚██╗██║        ██║╚██╔╝██║ ╚═══██╗██║   ██║██╔══██╗██║  ██║██║╚════╝██║     ██║     ██║
+██║ ╚████║███████╗██║ ╚═╝ ██║██████╔╝╚██████╔╝╚█████╔╝██████╔╝███████╗ ╚██████╗███████╗██║
+╚═╝  ╚═══╝╚══════╝╚═╝     ╚═╝╚═════╝  ╚═════╝  ╚════╝ ╚═════╝ ╚══════╝  ╚═════╝╚══════╝╚═╝
+                                                                                          
+```
+This is a m3u8 downloader.  
+## Summary
+Supports: 
+  * Auto deceypt for `AES-128`
+  * `Master List`
+  * Live stream recording(`BETA`)
+  * `Dolby Vision` from IQiYi, YouKu, Tencent
+  * Customize HTTP headers
+  * Auto merge clips(Binary or ffmpeg)
+  * Select save clip by `time code` or `index`
+  * Network driver on Windows OS
+  * Alternative audio/video track
+  * Mux without video track
+  * Auto use system proxy
+  * Optimization for Chinese streaming platform
+  
+  ![ScreenShot](https://nilaoda.github.io/N_m3u8DL-CLI/source/images/%E7%9B%B4%E6%8E%A5%E4%BD%BF%E7%94%A8.gif)  
+  
+## GUI
+  * Easy-to-use `GUI`
+  
+## Options
+```
+N_m3u8DL-CLI.exe <URL|JSON|FILE> [OPTIONS]  
+
+    --workDir    Directory      Set work dir (Video will be here)
+    --saveName   Filename       Set save name(Exclude extention)
+    --baseUrl    BaseUrl        Set Baseurl
+    --headers    headers        Set HTTP headers，format: key:value user | split all key&value
+    --maxThreads Thread         Set max thread(default: 32)
+    --minThreads Thread         Set min thread(default: 16)
+    --retryCount Count          Set retry times(default: 15)
+    --timeOut    Sec            Set timeout for http request(second，default: 10)
+    --muxSetJson File           Set a json file for mux
+    --useKeyFile File           Use 16 bytes file as KEY for AES-128 decryption
+    --useKeyBase64 Base64String Use Base64 String as KEY for AES-128 decryption
+    --downloadRange Range       Set range for a video
+    --stopSpeed  Number         Speed below this, retry(KB/s)
+    --maxSpeed   Number         Set max download speed(KB/s)
+    --enableDelAfterDone        Enable delete clips after download completed
+    --enableMuxFastStart        Enable fast start for mp4
+    --enableBinaryMerge         Enable use binary merge instead ffmpeg
+    --enableParseOnly           Enable parse mode
+    --enableAudioOnly           Enable only audio track when mux use ffmpeg
+    --disableDateInfo           Disable write date info when mux use ffmpeg
+    --noMerge                   Disable auto merge
+    --noProxy                   Disable use system proxy
+    --disableIntegrityCheck     Disable integrity check
+```
+  
+## Document
+  https://nilaoda.github.io/N_m3u8DL-CLI/