更新版本号

解决HTTPS协议自动重定向后，Referer丢失问题

.github/workflows/build_latest.yml

@@ -13,9 +13,13 @@ jobs:
      
     - name: Setup MSBuild Path
       uses: warrenbuckley/Setup-MSBuild@v1
+      env:
+        ACTIONS_ALLOW_UNSECURE_COMMANDS: 'true'
        
     - name: Setup NuGet
       uses: NuGet/setup-nuget@v1.0.2
+      env:
+        ACTIONS_ALLOW_UNSECURE_COMMANDS: 'true'
      
     - name: Restore NuGet Packages
       run: nuget restore N_m3u8DL-CLI.sln

N_m3u8DL-CLI/FFmpeg.cs

@@ -11,20 +11,19 @@ namespace N_m3u8DL_CLI
     class FFmpeg
     {
         public static string FFMPEG_PATH = "ffmpeg";
+        public static string REC_TIME = ""; //录制日期
 
-        private static string outPutPath = string.Empty;
-        private static string reportFile = string.Empty;
-        private static bool useAACFilter = false;  //是否启用滤镜
-        private static bool writeDate = true;  //是否写入录制日期
-        public static string OutPutPath { get => outPutPath; set => outPutPath = value; }
-        public static string ReportFile { get => reportFile; set => reportFile = value; }
-        public static bool UseAACFilter { get => useAACFilter; set => useAACFilter = value; }
-        public static bool WriteDate { get => writeDate; set => writeDate = value; }
+        public static string OutPutPath { get; set; } = string.Empty;
+        public static string ReportFile { get; set; } = string.Empty;
+        public static bool UseAACFilter { get; set; } = false;  //是否启用滤镜
+        public static bool WriteDate { get; set; } = true;  //是否写入录制日期
 
         public static void Merge(string[] files, string muxFormat, bool fastStart,
             string poster = "", string audioName = "", string title = "",
             string copyright = "", string comment = "", string encodingTool = "")
         {
+            string dateString = string.IsNullOrEmpty(REC_TIME) ? DateTime.Now.ToString("o") : REC_TIME;
+
             //同名文件已存在的共存策略
             if (File.Exists($"{OutPutPath}.{muxFormat.ToLower()}")) 
             {
@@ -52,7 +51,7 @@ namespace N_m3u8DL_CLI
                     command += " " + (string.IsNullOrEmpty(ddpAudio) ? "" : "-i \"" + ddpAudio + "\"");
                     command +=
                         $" -map 0:v? {(string.IsNullOrEmpty(ddpAudio) ? "-map 0:a?" : $"-map {(string.IsNullOrEmpty(poster) ? "1" : "2")}:a -map 0:a?")} -map 0:s? " + (string.IsNullOrEmpty(poster) ? "" : addPoster)
-                        + (writeDate ? " -metadata date=\"" + DateTime.Now.ToString("o") + "\"" : "") +
+                        + (WriteDate ? " -metadata date=\"" + dateString + "\"" : "") +
                         " -metadata encoding_tool=\"" + encodingTool + "\" -metadata title=\"" + title +
                         "\" -metadata copyright=\"" + copyright + "\" -metadata comment=\"" + comment +
                         $"\" -metadata:s:a:{(string.IsNullOrEmpty(ddpAudio) ? "0" : "1")} handler_name=\"" + audioName + $"\" -metadata:s:a:{(string.IsNullOrEmpty(ddpAudio) ? "0" : "1")} handler=\"" + audioName + "\" ";

N_m3u8DL-CLI/Global.cs

@@ -30,8 +30,8 @@ namespace N_m3u8DL_CLI
 
 
         /*===============================================================================*/
-        static string nowVer = "2.7.5";
-        static string nowDate = "20201014";
+        static string nowVer = "2.8.2";
+        static string nowDate = "20201122";
         public static void WriteInit()
         {
             Console.Clear();
@@ -101,18 +101,20 @@ namespace N_m3u8DL_CLI
         public static string GetWebSource(String url, string headers = "", int TimeOut = 60000)
         {
             string htmlCode = string.Empty;
-            for(int i = 0; i < 10; i++)
+            for (int i = 0; i < 10; i++)
             {
                 try
                 {
+                reProcess:
                     HttpWebRequest webRequest = (System.Net.HttpWebRequest)System.Net.WebRequest.Create(url);
                     webRequest.Method = "GET";
                     if (NoProxy) webRequest.Proxy = null;
-                    webRequest.UserAgent = "Mozilla/4.0";
-                    webRequest.Headers.Add("Accept-Encoding", "gzip, deflate");
+                    webRequest.UserAgent = "Mozilla/5.0 (Linux; U; Android 7.0; zh-cn; 15 Plus Build/NRD90M) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/66.0.3359.126 MQQBrowser/9.4 Mobile Safari/537.36";
+                    webRequest.Accept = "*/*";
+                    webRequest.Headers.Add("Accept-Encoding", "gzip, deflate, br");
                     webRequest.Timeout = TimeOut;  //设置超时
                     webRequest.KeepAlive = false;
-                    webRequest.AllowAutoRedirect = true;  //自动跳转
+                    webRequest.AllowAutoRedirect = false; //手动处理重定向，否则会丢失Referer
                     if (url.Contains("pcvideo") && url.Contains(".titan.mgtv.com"))
                     {
                         webRequest.UserAgent = "";
@@ -145,6 +147,18 @@ namespace N_m3u8DL_CLI
                         }
                     }
                     HttpWebResponse webResponse = (HttpWebResponse)webRequest.GetResponse();
+
+                    //302
+                    if (webResponse.Headers.Get("Location") != null)
+                    {
+                        url = webResponse.Headers.Get("Location");
+                        webResponse.Close();
+                        goto reProcess;
+                    }
+
+                    //文件过大则认为不是m3u8
+                    if (webResponse.ContentLength != -1 && webRequest.ContentLength > 50 * 1024 * 1024) return "";
+
                     if (webResponse.ContentEncoding != null
                         && webResponse.ContentEncoding.ToLower() == "gzip") //如果使用了GZip则先解压
                     {
@@ -402,14 +416,17 @@ namespace N_m3u8DL_CLI
                 }
             }
 
+        reProcess:
             byte[] arraryByte;
             HttpWebRequest req = (HttpWebRequest)WebRequest.Create(url);
             req.Method = "GET";
             req.Timeout = timeOut;
             req.ReadWriteTimeout = timeOut; //重要
+            req.AllowAutoRedirect = false; //手动处理重定向，否则会丢失Referer
             if (NoProxy) req.Proxy = null;
             req.Headers.Add("Accept-Encoding", "gzip, deflate");
             req.Accept = "*/*";
+            req.UserAgent = "Mozilla/5.0 (Linux; U; Android 7.0; zh-cn; 15 Plus Build/NRD90M) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/66.0.3359.126 MQQBrowser/9.4 Mobile Safari/537.36";
             //添加headers
             if (headers != "")
             {
@@ -437,6 +454,13 @@ namespace N_m3u8DL_CLI
 
             using (HttpWebResponse wr = (HttpWebResponse)req.GetResponse())
             {
+                //302
+                if (wr.Headers.Get("Location") != null)
+                {
+                    url = wr.Headers.Get("Location");
+                    wr.Close();
+                    goto reProcess;
+                }
                 if (wr.ContentEncoding != null && wr.ContentEncoding.ToLower() == "gzip") //如果使用了GZip则先解压
                 {
                     using (Stream streamReceive = wr.GetResponseStream())
@@ -497,10 +521,11 @@ namespace N_m3u8DL_CLI
                 if (shouldStop)
                     return;
 
+                reProcess:
                 HttpWebRequest request = WebRequest.Create(url) as HttpWebRequest;
                 request.Timeout = timeOut;
                 request.ReadWriteTimeout = timeOut; //重要
-                request.AllowAutoRedirect = true;
+                request.AllowAutoRedirect = false; //手动处理重定向，否则会丢失Referer
                 request.KeepAlive = false;
                 request.Method = "GET";
                 if (NoProxy) request.Proxy = null;
@@ -514,7 +539,7 @@ namespace N_m3u8DL_CLI
                     request.Headers.Add("Cookie", "MQGUID");
                 }
                 else
-                    request.UserAgent = "VLC/2.2.1 LibVLC/2.2.1";
+                    request.UserAgent = "Mozilla/5.0 (Linux; U; Android 7.0; zh-cn; 15 Plus Build/NRD90M) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/66.0.3359.126 MQQBrowser/9.4 Mobile Safari/537.36";
                 //下载部分字节
                 if (expectByte != -1)
                     request.AddRange("bytes", startByte, startByte + expectByte - 1);
@@ -548,6 +573,13 @@ namespace N_m3u8DL_CLI
                 bool pngHeader = false; //PNG HEADER检测
                 using (var response = (HttpWebResponse)request.GetResponse())
                 {
+                    //302
+                    if (response.Headers.Get("Location") != null)
+                    {
+                        url = response.Headers.Get("Location");
+                        response.Close();
+                        goto reProcess;
+                    }
                     using (var responseStream = response.GetResponseStream())
                     {
                         using (var stream = new FileStream(path, FileMode.Create, FileAccess.Write, FileShare.Write))
@@ -556,10 +588,17 @@ namespace N_m3u8DL_CLI
                             totalLen = response.ContentLength;
                             byte[] bArr = new byte[1024];
                             int size = responseStream.Read(bArr, 0, (int)bArr.Length);
-                            if (!pngHeader && size > 3 && 137 == bArr[0] && 80 == bArr[1] && 78 == bArr[2] && 71 == bArr[3]) ;
+                            if (!pngHeader && size > 3 && 137 == bArr[0] && 80 == bArr[1] && 78 == bArr[2] && 71 == bArr[3])
                             {
                                 pngHeader = true;
                             }
+                            //GIF HEADER检测
+                            if (!pngHeader && size > 3 && 0x47 == bArr[0] && 0x49 == bArr[1] && 0x46 == bArr[2] && 0x38 == bArr[3])
+                            {
+                                bArr = bArr.Skip(42).ToArray();
+                                size -= 42;
+                                downLen += 42;
+                            }
                             while (size > 0)
                             {
                                 stream.Write(bArr, 0, size);
@@ -582,10 +621,11 @@ namespace N_m3u8DL_CLI
                 }
                 if (shouldStop)
                     try { File.Delete(path); } catch (Exception) { }
-                if (totalLen != -1 && downLen != totalLen) 
+                if (totalLen != -1 && downLen != totalLen)
                     try { File.Delete(path); } catch (Exception) { }
                 if (pngHeader)
                     TrySkipPngHeader(path);
+
             }
             catch (Exception e)
             {
@@ -800,11 +840,17 @@ namespace N_m3u8DL_CLI
                     }
                 }
 
-                if(res.Contains("Audio aac"))
+                if (res.Contains("Audio aac"))
                 {
                     FFmpeg.UseAACFilter = true;
                 }
 
+                //有非AAC音轨则关闭UseAACFilter
+                if (res.Contains("Audio") && !res.Contains("Audio aac"))
+                {
+                    FFmpeg.UseAACFilter = false;
+                }
+
                 if ((VIDEO_TYPE == "" || VIDEO_TYPE == "IGNORE") && res.Contains("Audio eac3")) 
                 {
                     AUDIO_TYPE = "eac3";

N_m3u8DL-CLI/N_m3u8DL-CLI.csproj

@@ -73,6 +73,7 @@
     <Compile Include="HLSTags.cs" />
     <Compile Include="LOGGER.cs" />
     <Compile Include="DownloadManager.cs" />
+    <Compile Include="MPDParser.cs" />
     <Compile Include="Parser.cs" />
     <Compile Include="Program.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
@@ -129,4 +130,4 @@
     </EmbeddedResource>
   </ItemGroup>
   <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
-</Project>
+</Project>

N_m3u8DL-CLI/Parser.cs

@@ -17,15 +17,6 @@ namespace N_m3u8DL_CLI
         string[] m3u8CurrentKey = new string[] { "NONE", "", "" };
         private string m3u8SavePath = string.Empty;
         private string jsonSavePath = string.Empty;
-        private string headers = string.Empty;
-        private string baseUrl = string.Empty;
-        private string m3u8Url = string.Empty;
-        private string downDir = string.Empty;
-        private string downName = string.Empty;
-        private string keyFile = string.Empty;
-        private string keyBase64 = string.Empty;
-        private string keyIV = string.Empty;
-        private bool liveStream = false;
         private long bestBandwidth = 0;
         private string bestUrl = string.Empty;
         private string bestUrlAudio = string.Empty;
@@ -37,34 +28,31 @@ namespace N_m3u8DL_CLI
         //存放多轨道的信息
         private ArrayList extLists = new ArrayList();
         private static bool isQiQiuYun = false;
-        //存放Range信息，允许用户只下载部分视频
-        private static int rangeStart = 0;
-        private static int rangeEnd = -1;
-        //存放Range信息，允许用户只下载部分视频
-        private static string durStart = "";
-        private static string durEnd = "";
-        //是否自动清除优酷广告分片
-        private static bool delAd = true;
         //标记是否已清除优酷广告分片
         private static bool hasAd = false;
 
-        public string BaseUrl { get => baseUrl; set => baseUrl = value; }
-        public string M3u8Url { get => m3u8Url; set => m3u8Url = value; }
-        public string DownDir { get => downDir; set => downDir = value; }
-        public string DownName { get => downName; set => downName = value; }
-        public string Headers { get => headers; set => headers = value; }
-        public static int RangeStart { get => rangeStart; set => rangeStart = value; }
-        public static int RangeEnd { get => rangeEnd; set => rangeEnd = value; }
-        public static bool DelAd { get => delAd; set => delAd = value; }
-        public static string DurStart { get => durStart; set => durStart = value; }
-        public static string DurEnd { get => durEnd; set => durEnd = value; }
-        public string KeyFile { get => keyFile; set => keyFile = value; }
-        public string KeyBase64 { get => keyBase64; set => keyBase64 = value; }
-        public bool LiveStream { get => liveStream; set => liveStream = value; }
-        public string KeyIV { get => keyIV; set => keyIV = value; }
+        public string BaseUrl { get; set; } = string.Empty;
+        public string M3u8Url { get; set; } = string.Empty;
+        public string DownDir { get; set; } = string.Empty;
+        public string DownName { get; set; } = string.Empty;
+        public string Headers { get; set; } = string.Empty;
+        //存放Range信息，允许用户只下载部分视频
+        public static int RangeStart { get; set; } = 0;
+        public static int RangeEnd { get; set; } = -1;
+        //是否自动清除优酷广告分片
+        public static bool DelAd { get; set; } = true;
+        //存放Range信息，允许用户只下载部分视频
+        public static string DurStart { get; set; } = "";
+        public static string DurEnd { get; set; } = "";
+        public string KeyFile { get; set; } = string.Empty;
+        public string KeyBase64 { get; set; } = string.Empty;
+        public bool LiveStream { get; set; } = false;
+        public string KeyIV { get; set; } = string.Empty;
 
         public void Parse()
         {
+            FFmpeg.REC_TIME = "";
+
             m3u8SavePath = Path.Combine(DownDir, "raw.m3u8");
             jsonSavePath = Path.Combine(DownDir, "meta.json");
 
@@ -90,17 +78,17 @@ namespace N_m3u8DL_CLI
 
 
             //获取m3u8内容
-            if (!liveStream)
+            if (!LiveStream)
                 LOGGER.PrintLine(strings.downloadingM3u8, LOGGER.Warning);
 
             if (M3u8Url.StartsWith("http"))
             {
                 if (M3u8Url.Contains("nfmovies.com/hls"))
-                    m3u8Content = DecodeNfmovies.DecryptM3u8(Global.HttpDownloadFileToBytes(M3u8Url, headers));
+                    m3u8Content = DecodeNfmovies.DecryptM3u8(Global.HttpDownloadFileToBytes(M3u8Url, Headers));
                 else if (M3u8Url.Contains("hls.ddyunp.com/ddyun"))
-                    m3u8Content = DecodeDdyun.DecryptM3u8(Global.HttpDownloadFileToBytes(DecodeDdyun.GetVaildM3u8Url(M3u8Url), headers));
+                    m3u8Content = DecodeDdyun.DecryptM3u8(Global.HttpDownloadFileToBytes(DecodeDdyun.GetVaildM3u8Url(M3u8Url), Headers));
                 else
-                    m3u8Content = Global.GetWebSource(M3u8Url, headers);
+                    m3u8Content = Global.GetWebSource(M3u8Url, Headers);
             }
             else if (M3u8Url.StartsWith("file:"))
             {
@@ -130,6 +118,18 @@ namespace N_m3u8DL_CLI
                 m3u8Content = DecodeImooc.DecodeM3u8(m3u8Content);
             }
 
+            if (m3u8Content.Trim().StartsWith("<?xml version") && m3u8Content.Contains("<MPD"))
+            {
+                var mpdSavePath = Path.Combine(DownDir, "dash.mpd");
+                //输出mpd文件
+                File.WriteAllText(mpdSavePath, m3u8Content);
+                //分析mpd文件
+                M3u8Url = Global.Get302(M3u8Url, Headers);
+                var newUri = MPDParser.Parse(DownDir, M3u8Url, m3u8Content, BaseUrl);
+                M3u8Url = newUri;
+                m3u8Content = File.ReadAllText(new Uri(M3u8Url).LocalPath);
+            }
+
             //输出m3u8文件
             File.WriteAllText(m3u8SavePath, m3u8Content);
 
@@ -149,32 +149,32 @@ namespace N_m3u8DL_CLI
                 if (new Regex("#YUMING\\|(.*)").IsMatch(m3u8Content))
                     BaseUrl = new Regex("#YUMING\\|(.*)").Match(m3u8Content).Groups[1].Value;
                 else
-                    BaseUrl = GetBaseUrl(M3u8Url, headers);
+                    BaseUrl = GetBaseUrl(M3u8Url, Headers);
             }
 
-            if (!liveStream)
+            if (!LiveStream)
             {
                 LOGGER.WriteLine(strings.parsingM3u8);
                 LOGGER.PrintLine(strings.parsingM3u8);
             }
 
-            if (!string.IsNullOrEmpty(keyBase64))
+            if (!string.IsNullOrEmpty(KeyBase64))
             {
                 string line = "";
-                if (string.IsNullOrEmpty(keyIV))
-                    line = $"#EXT-X-KEY:METHOD=AES-128,URI=\"base64:{keyBase64}\"";
+                if (string.IsNullOrEmpty(KeyIV))
+                    line = $"#EXT-X-KEY:METHOD=AES-128,URI=\"base64:{KeyBase64}\"";
                 else
-                    line = $"#EXT-X-KEY:METHOD=AES-128,URI=\"base64:{keyBase64}\",IV=0x{keyIV.Replace("0x", "")}";
+                    line = $"#EXT-X-KEY:METHOD=AES-128,URI=\"base64:{KeyBase64}\",IV=0x{KeyIV.Replace("0x", "")}";
                 m3u8CurrentKey = ParseKey(line);
             }
-            if (!string.IsNullOrEmpty(keyFile))
+            if (!string.IsNullOrEmpty(KeyFile))
             {
                 string line = "";
-                Uri u = new Uri(keyFile);
-                if (string.IsNullOrEmpty(keyIV))
+                Uri u = new Uri(KeyFile);
+                if (string.IsNullOrEmpty(KeyIV))
                     line = $"#EXT-X-KEY:METHOD=AES-128,URI=\"{u.ToString()}\"";
                 else
-                    line = $"#EXT-X-KEY:METHOD=AES-128,URI=\"{u.ToString()}\",IV=0x{keyIV.Replace("0x", "")}";
+                    line = $"#EXT-X-KEY:METHOD=AES-128,URI=\"{u.ToString()}\",IV=0x{KeyIV.Replace("0x", "")}";
 
                 m3u8CurrentKey = ParseKey(line);
             }
@@ -241,7 +241,13 @@ namespace N_m3u8DL_CLI
                         startIndex = segIndex;
                     }
                     else if (line.StartsWith(HLSTags.ext_x_discontinuity_sequence)) ;
-                    else if (line.StartsWith(HLSTags.ext_x_program_date_time)) ;
+                    else if (line.StartsWith(HLSTags.ext_x_program_date_time))
+                    {
+                        if (string.IsNullOrEmpty(FFmpeg.REC_TIME))
+                        {
+                            FFmpeg.REC_TIME = line.Replace(HLSTags.ext_x_program_date_time + ":", "").Trim();
+                        }
+                    }
                     //解析不连续标记，需要单独合并（timestamp不同）
                     else if (line.StartsWith(HLSTags.ext_x_discontinuity))
                     {
@@ -269,7 +275,7 @@ namespace N_m3u8DL_CLI
                     else if (line.StartsWith(HLSTags.ext_x_key))
                     {
                         //自定义KEY情况 判断是否需要读取IV
-                        if (!string.IsNullOrEmpty(keyFile) || !string.IsNullOrEmpty(keyBase64))
+                        if (!string.IsNullOrEmpty(KeyFile) || !string.IsNullOrEmpty(KeyBase64))
                         {
                             if (m3u8CurrentKey[2] == "" && line.Contains("IV=0x"))
                             {
@@ -326,7 +332,7 @@ namespace N_m3u8DL_CLI
                     else if (line.StartsWith(HLSTags.ext_x_i_frame_stream_inf)) ;
                     else if (line.StartsWith(HLSTags.ext_x_media))
                     {
-                        if (Global.GetTagAttribute(line, "TYPE") == "AUDIO")
+                        if (Global.GetTagAttribute(line, "TYPE") == "AUDIO" && !MEDIA_AUDIO.ContainsKey(Global.GetTagAttribute(line, "GROUP-ID")))
                             MEDIA_AUDIO.Add(Global.GetTagAttribute(line, "GROUP-ID"), CombineURL(BaseUrl, Global.GetTagAttribute(line, "URI")));
                         if (Global.GetTagAttribute(line, "TYPE") == "SUBTITLES")
                         {
@@ -604,7 +610,7 @@ namespace N_m3u8DL_CLI
 
 
             //输出JSON文件
-            if (!liveStream)
+            if (!LiveStream)
             {
                 LOGGER.WriteLine(strings.wrtingMeta);
                 LOGGER.PrintLine(strings.wrtingMeta);