update

解决HTTPS协议自动重定向后，Referer丢失问题

.github/workflows/build_latest.yml

@@ -13,9 +13,13 @@ jobs:
      
     - name: Setup MSBuild Path
       uses: warrenbuckley/Setup-MSBuild@v1
+      env:
+        ACTIONS_ALLOW_UNSECURE_COMMANDS: 'true'
        
     - name: Setup NuGet
       uses: NuGet/setup-nuget@v1.0.2
+      env:
+        ACTIONS_ALLOW_UNSECURE_COMMANDS: 'true'
      
     - name: Restore NuGet Packages
       run: nuget restore N_m3u8DL-CLI.sln

N_m3u8DL-CLI/DecodeHuke88Key.cs

@@ -0,0 +1,53 @@
+using Newtonsoft.Json.Linq;
+using System;
+using System.Collections.Generic;
+using System.Security.Cryptography;
+using System.Text;
+using System.Text.RegularExpressions;
+
+namespace N_m3u8DL_CLI
+{
+    //https://js.huke88.com/assets/revision/js/plugins/tcplayer/tcplayer.v4.1.min.js?v=930
+    //https://js.huke88.com/assets/revision/js/plugins/tcplayer/libs/hls.min.0.13.2m.js?v=930
+    class DecodeHuke88Key
+    {
+        private static string[] GetOverlayInfo(string url)
+        {
+            var enc = new Regex("eyJ\\w{100,}").Match(url).Value;
+            var json = Encoding.UTF8.GetString(Convert.FromBase64String(enc));
+            JObject jObject = JObject.Parse(json);
+            var key = jObject["overlayKey"].ToString();
+            var iv = jObject["overlayIv"].ToString();
+            return new string[] { key, iv };
+        }
+
+        public static string DecodeKey(string url, byte[] data)
+        {
+            var info = GetOverlayInfo(url);
+            var overlayKey = info[0];
+            var overlayIv = info[1];
+            var l = new List<byte>();
+            var c = new List<byte>();
+            for (int h = 0; h < 16; h++)
+            {
+                var f = overlayKey.Substring(2 * h, 2);
+                var g = overlayIv.Substring(2 * h, 2);
+                l.Add(Convert.ToByte(f, 16));
+                c.Add(Convert.ToByte(g, 16));
+            }
+
+            var _lastCipherblock = c.ToArray();
+
+            var t = new byte[data.Length];
+            var r = data;
+            r = Decrypter.AES128Decrypt(data, l.ToArray(), Decrypter.HexStringToBytes("00000000000000000000000000000000"), CipherMode.CBC, PaddingMode.Zeros);
+
+            for (var o = 0; o < 16; o++)
+                t[o] = (byte)(r[o] ^ _lastCipherblock[o]);
+
+            var key = Convert.ToBase64String(t);
+
+            return key;
+        }
+    }
+}

N_m3u8DL-CLI/Decrypter.cs

@@ -1,16 +1,12 @@
 using System;
-using System.Collections.Generic;
 using System.IO;
-using System.Linq;
 using System.Security.Cryptography;
-using System.Text;
-using System.Threading.Tasks;
 
 namespace N_m3u8DL_CLI
 {
     class Decrypter
     {
-        public static byte[] AES128Decrypt(string filePath, byte[] keyByte, byte[] ivByte, CipherMode mode = CipherMode.CBC)
+        public static byte[] AES128Decrypt(string filePath, byte[] keyByte, byte[] ivByte, CipherMode mode = CipherMode.CBC, PaddingMode padding = PaddingMode.PKCS7)
         {
             FileStream fs = new FileStream(filePath, FileMode.Open);
             //获取文件大小
@@ -25,14 +21,14 @@ namespace N_m3u8DL_CLI
             dcpt.Key = keyByte;
             dcpt.IV = ivByte;
             dcpt.Mode = mode;
-            dcpt.Padding = PaddingMode.PKCS7;
+            dcpt.Padding = padding;
 
             ICryptoTransform cTransform = dcpt.CreateDecryptor();
             Byte[] resultArray = cTransform.TransformFinalBlock(inBuff, 0, inBuff.Length);
             return resultArray;
         }
 
-        public static byte[] AES128Decrypt(byte[] encryptedBuff, byte[] keyByte, byte[] ivByte, CipherMode mode = CipherMode.CBC)
+        public static byte[] AES128Decrypt(byte[] encryptedBuff, byte[] keyByte, byte[] ivByte, CipherMode mode = CipherMode.CBC, PaddingMode padding = PaddingMode.PKCS7)
         {
             byte[] inBuff = encryptedBuff;
 
@@ -42,7 +38,7 @@ namespace N_m3u8DL_CLI
             dcpt.Key = keyByte;
             dcpt.IV = ivByte;
             dcpt.Mode = mode;
-            dcpt.Padding = PaddingMode.PKCS7;
+            dcpt.Padding = padding;
 
             ICryptoTransform cTransform = dcpt.CreateDecryptor();
             Byte[] resultArray = cTransform.TransformFinalBlock(inBuff, 0, inBuff.Length);
@@ -56,7 +52,7 @@ namespace N_m3u8DL_CLI
                 return new byte[0];
             }
 
-            if (hexStr.StartsWith("0x") || hexStr.StartsWith("0X")) 
+            if (hexStr.StartsWith("0x") || hexStr.StartsWith("0X"))
             {
                 hexStr = hexStr.Remove(0, 2);
             }

N_m3u8DL-CLI/FFmpeg.cs

@@ -10,19 +10,20 @@ namespace N_m3u8DL_CLI
 {
     class FFmpeg
     {
-        private static string outPutPath = string.Empty;
-        private static string reportFile = string.Empty;
-        private static bool useAACFilter = false;  //是否启用滤镜
-        private static bool writeDate = true;  //是否写入录制日期
-        public static string OutPutPath { get => outPutPath; set => outPutPath = value; }
-        public static string ReportFile { get => reportFile; set => reportFile = value; }
-        public static bool UseAACFilter { get => useAACFilter; set => useAACFilter = value; }
-        public static bool WriteDate { get => writeDate; set => writeDate = value; }
+        public static string FFMPEG_PATH = "ffmpeg";
+        public static string REC_TIME = ""; //录制日期
+
+        public static string OutPutPath { get; set; } = string.Empty;
+        public static string ReportFile { get; set; } = string.Empty;
+        public static bool UseAACFilter { get; set; } = false;  //是否启用滤镜
+        public static bool WriteDate { get; set; } = true;  //是否写入录制日期
 
         public static void Merge(string[] files, string muxFormat, bool fastStart,
             string poster = "", string audioName = "", string title = "",
             string copyright = "", string comment = "", string encodingTool = "")
         {
+            string dateString = string.IsNullOrEmpty(REC_TIME) ? DateTime.Now.ToString("o") : REC_TIME;
+
             //同名文件已存在的共存策略
             if (File.Exists($"{OutPutPath}.{muxFormat.ToLower()}")) 
             {
@@ -50,7 +51,7 @@ namespace N_m3u8DL_CLI
                     command += " " + (string.IsNullOrEmpty(ddpAudio) ? "" : "-i \"" + ddpAudio + "\"");
                     command +=
                         $" -map 0:v? {(string.IsNullOrEmpty(ddpAudio) ? "-map 0:a?" : $"-map {(string.IsNullOrEmpty(poster) ? "1" : "2")}:a -map 0:a?")} -map 0:s? " + (string.IsNullOrEmpty(poster) ? "" : addPoster)
-                        + (writeDate ? " -metadata date=\"" + DateTime.Now.ToString("o") + "\"" : "") +
+                        + (WriteDate ? " -metadata date=\"" + dateString + "\"" : "") +
                         " -metadata encoding_tool=\"" + encodingTool + "\" -metadata title=\"" + title +
                         "\" -metadata copyright=\"" + copyright + "\" -metadata comment=\"" + comment +
                         $"\" -metadata:s:a:{(string.IsNullOrEmpty(ddpAudio) ? "0" : "1")} handler_name=\"" + audioName + $"\" -metadata:s:a:{(string.IsNullOrEmpty(ddpAudio) ? "0" : "1")} handler=\"" + audioName + "\" ";
@@ -83,7 +84,7 @@ namespace N_m3u8DL_CLI
 
             }
 
-            Run("ffmpeg", command, Path.GetDirectoryName(files[0]));
+            Run(FFMPEG_PATH, command, Path.GetDirectoryName(files[0]));
             LOGGER.WriteLine(strings.ffmpegDone);
             //Console.WriteLine(command);
         }
@@ -92,7 +93,7 @@ namespace N_m3u8DL_CLI
         {
             if (Global.VIDEO_TYPE == "H264")
             {
-                Run("ffmpeg",
+                Run(FFMPEG_PATH,
                     "-loglevel quiet -i \"" + file + "\" -map 0 -c copy -copy_unknown -f mpegts -bsf:v h264_mp4toannexb \""
                     + Path.GetFileNameWithoutExtension(file) + "[MPEGTS].ts\"", 
                     Path.GetDirectoryName(file));
@@ -104,7 +105,7 @@ namespace N_m3u8DL_CLI
             }
             else if (Global.VIDEO_TYPE == "H265")
             {
-                Run("ffmpeg",
+                Run(FFMPEG_PATH,
                     "-loglevel quiet -i \"" + file + "\" -map 0 -c copy -copy_unknown -f mpegts -bsf:v hevc_mp4toannexb \""
                     + Path.GetFileNameWithoutExtension(file) + "[MPEGTS].ts\"",
                     Path.GetDirectoryName(file));

N_m3u8DL-CLI/Global.cs

@@ -30,8 +30,9 @@ namespace N_m3u8DL_CLI
 
 
         /*===============================================================================*/
-        static string nowVer = "2.7.4";
-        static string nowDate = "20200920";
+        static Version ver = System.Reflection.Assembly.GetExecutingAssembly().GetName().Version;
+        static string nowVer = $"{ver.Major}.{ver.Minor}.{ver.Build}";
+        static string nowDate = "20210124";
         public static void WriteInit()
         {
             Console.Clear();
@@ -101,23 +102,25 @@ namespace N_m3u8DL_CLI
         public static string GetWebSource(String url, string headers = "", int TimeOut = 60000)
         {
             string htmlCode = string.Empty;
-            for(int i = 0; i < 10; i++)
+            for (int i = 0; i < 10; i++)
             {
                 try
                 {
+                reProcess:
                     HttpWebRequest webRequest = (System.Net.HttpWebRequest)System.Net.WebRequest.Create(url);
                     webRequest.Method = "GET";
                     if (NoProxy) webRequest.Proxy = null;
-                    webRequest.UserAgent = "Mozilla/4.0";
-                    webRequest.Headers.Add("Accept-Encoding", "gzip, deflate");
+                    webRequest.UserAgent = "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36";
+                    webRequest.Accept = "*/*";
+                    webRequest.Headers.Add("Accept-Encoding", "gzip, deflate, br");
                     webRequest.Timeout = TimeOut;  //设置超时
                     webRequest.KeepAlive = false;
-                    webRequest.AllowAutoRedirect = true;  //自动跳转
+                    webRequest.AllowAutoRedirect = false; //手动处理重定向，否则会丢失Referer
                     if (url.Contains("pcvideo") && url.Contains(".titan.mgtv.com"))
                     {
                         webRequest.UserAgent = "";
                         if (!url.Contains("/internettv/"))
-                            webRequest.Referer = "https://player.mgtv.com/mgtv_v6_player/PlayerCore.swf";
+                            webRequest.Referer = "https://www.mgtv.com";
                         webRequest.Headers.Add("Cookie", "MQGUID");
                     }
                     //添加headers
@@ -145,6 +148,18 @@ namespace N_m3u8DL_CLI
                         }
                     }
                     HttpWebResponse webResponse = (HttpWebResponse)webRequest.GetResponse();
+
+                    //302
+                    if (webResponse.Headers.Get("Location") != null)
+                    {
+                        url = webResponse.Headers.Get("Location");
+                        webResponse.Close();
+                        goto reProcess;
+                    }
+
+                    //文件过大则认为不是m3u8
+                    if (webResponse.ContentLength != -1 && webResponse.ContentLength > 50 * 1024 * 1024) return "";
+
                     if (webResponse.ContentEncoding != null
                         && webResponse.ContentEncoding.ToLower() == "gzip") //如果使用了GZip则先解压
                     {
@@ -402,14 +417,17 @@ namespace N_m3u8DL_CLI
                 }
             }
 
+        reProcess:
             byte[] arraryByte;
             HttpWebRequest req = (HttpWebRequest)WebRequest.Create(url);
             req.Method = "GET";
             req.Timeout = timeOut;
             req.ReadWriteTimeout = timeOut; //重要
+            req.AllowAutoRedirect = false; //手动处理重定向，否则会丢失Referer
             if (NoProxy) req.Proxy = null;
             req.Headers.Add("Accept-Encoding", "gzip, deflate");
             req.Accept = "*/*";
+            req.UserAgent = "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36";
             //添加headers
             if (headers != "")
             {
@@ -437,6 +455,13 @@ namespace N_m3u8DL_CLI
 
             using (HttpWebResponse wr = (HttpWebResponse)req.GetResponse())
             {
+                //302
+                if (wr.Headers.Get("Location") != null)
+                {
+                    url = wr.Headers.Get("Location");
+                    wr.Close();
+                    goto reProcess;
+                }
                 if (wr.ContentEncoding != null && wr.ContentEncoding.ToLower() == "gzip") //如果使用了GZip则先解压
                 {
                     using (Stream streamReceive = wr.GetResponseStream())
@@ -497,10 +522,11 @@ namespace N_m3u8DL_CLI
                 if (shouldStop)
                     return;
 
+                reProcess:
                 HttpWebRequest request = WebRequest.Create(url) as HttpWebRequest;
                 request.Timeout = timeOut;
                 request.ReadWriteTimeout = timeOut; //重要
-                request.AllowAutoRedirect = true;
+                request.AllowAutoRedirect = false; //手动处理重定向，否则会丢失Referer
                 request.KeepAlive = false;
                 request.Method = "GET";
                 if (NoProxy) request.Proxy = null;
@@ -510,11 +536,11 @@ namespace N_m3u8DL_CLI
                 {
                     request.UserAgent = "";
                     if (!url.Contains("/internettv/"))
-                        request.Referer = "https://player.mgtv.com/mgtv_v6_player/PlayerCore.swf";
+                        request.Referer = "https://www.mgtv.com";
                     request.Headers.Add("Cookie", "MQGUID");
                 }
                 else
-                    request.UserAgent = "VLC/2.2.1 LibVLC/2.2.1";
+                    request.UserAgent = "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36";
                 //下载部分字节
                 if (expectByte != -1)
                     request.AddRange("bytes", startByte, startByte + expectByte - 1);
@@ -548,6 +574,13 @@ namespace N_m3u8DL_CLI
                 bool pngHeader = false; //PNG HEADER检测
                 using (var response = (HttpWebResponse)request.GetResponse())
                 {
+                    //302
+                    if (response.Headers.Get("Location") != null)
+                    {
+                        url = response.Headers.Get("Location");
+                        response.Close();
+                        goto reProcess;
+                    }
                     using (var responseStream = response.GetResponseStream())
                     {
                         using (var stream = new FileStream(path, FileMode.Create, FileAccess.Write, FileShare.Write))
@@ -556,10 +589,17 @@ namespace N_m3u8DL_CLI
                             totalLen = response.ContentLength;
                             byte[] bArr = new byte[1024];
                             int size = responseStream.Read(bArr, 0, (int)bArr.Length);
-                            if (!pngHeader && size > 3 && 137 == bArr[0] && 80 == bArr[1] && 78 == bArr[2] && 71 == bArr[3]) ;
+                            if (!pngHeader && size > 3 && 137 == bArr[0] && 80 == bArr[1] && 78 == bArr[2] && 71 == bArr[3])
                             {
                                 pngHeader = true;
                             }
+                            //GIF HEADER检测
+                            if (!pngHeader && size > 3 && 0x47 == bArr[0] && 0x49 == bArr[1] && 0x46 == bArr[2] && 0x38 == bArr[3])
+                            {
+                                bArr = bArr.Skip(42).ToArray();
+                                size -= 42;
+                                downLen += 42;
+                            }
                             while (size > 0)
                             {
                                 stream.Write(bArr, 0, size);
@@ -582,10 +622,11 @@ namespace N_m3u8DL_CLI
                 }
                 if (shouldStop)
                     try { File.Delete(path); } catch (Exception) { }
-                if (totalLen != -1 && downLen != totalLen) 
+                if (totalLen != -1 && downLen != totalLen)
                     try { File.Delete(path); } catch (Exception) { }
                 if (pngHeader)
                     TrySkipPngHeader(path);
+
             }
             catch (Exception e)
             {
@@ -800,11 +841,17 @@ namespace N_m3u8DL_CLI
                     }
                 }
 
-                if(res.Contains("Audio aac"))
+                if (res.Contains("Audio aac"))
                 {
                     FFmpeg.UseAACFilter = true;
                 }
 
+                //有非AAC音轨则关闭UseAACFilter
+                if (res.Contains("Audio") && !res.Contains("Audio aac"))
+                {
+                    FFmpeg.UseAACFilter = false;
+                }
+
                 if ((VIDEO_TYPE == "" || VIDEO_TYPE == "IGNORE") && res.Contains("Audio eac3")) 
                 {
                     AUDIO_TYPE = "eac3";

N_m3u8DL-CLI/N_m3u8DL-CLI.csproj

@@ -1,5 +1,6 @@
 <?xml version="1.0" encoding="utf-8"?>
 <Project ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <Import Project="..\packages\Costura.Fody.4.1.0\build\Costura.Fody.props" Condition="Exists('..\packages\Costura.Fody.4.1.0\build\Costura.Fody.props')" />
   <Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
   <PropertyGroup>
     <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
@@ -12,6 +13,8 @@
     <FileAlignment>512</FileAlignment>
     <AutoGenerateBindingRedirects>true</AutoGenerateBindingRedirects>
     <TargetFrameworkProfile />
+    <NuGetPackageImportStamp>
+    </NuGetPackageImportStamp>
   </PropertyGroup>
   <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
     <PlatformTarget>x86</PlatformTarget>
@@ -38,10 +41,13 @@
     <ApplicationIcon>logo_3Iv_icon.ico</ApplicationIcon>
   </PropertyGroup>
   <ItemGroup>
+    <Reference Include="Costura, Version=4.1.0.0, Culture=neutral, PublicKeyToken=9919ef960d84173d, processorArchitecture=MSIL">
+      <HintPath>..\packages\Costura.Fody.4.1.0\lib\net40\Costura.dll</HintPath>
+    </Reference>
     <Reference Include="Microsoft.JScript" />
     <Reference Include="netstandard, Version=2.0.0.0, Culture=neutral, PublicKeyToken=cc7b13ffcd2ddd51" />
-    <Reference Include="Newtonsoft.Json">
-      <HintPath>..\..\Newtonsoft.Json.dll</HintPath>
+    <Reference Include="Newtonsoft.Json, Version=12.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
+      <HintPath>..\packages\Newtonsoft.Json.12.0.3\lib\net45\Newtonsoft.Json.dll</HintPath>
     </Reference>
     <Reference Include="NiL.JS, Version=2.5.1428.0, Culture=neutral, PublicKeyToken=fa941a7c2a4de689, processorArchitecture=MSIL">
       <HintPath>..\packages\NiL.JS.2.5.1428\lib\net45\NiL.JS.dll</HintPath>
@@ -63,15 +69,18 @@
     <Compile Include="CommandLineArgument.cs" />
     <Compile Include="CommandLineArgumentParser.cs" />
     <Compile Include="Decode51CtoKey.cs" />
+    <Compile Include="DecodeDdyun.cs" />
+    <Compile Include="DecodeHuke88Key.cs" />
     <Compile Include="DecodeImooc.cs" />
+    <Compile Include="DecodeNfmovies.cs" />
     <Compile Include="Decrypter.cs" />
-    <Compile Include="DecryptNfmovies.cs" />
     <Compile Include="FFmpeg.cs" />
     <Compile Include="Global.cs" />
     <Compile Include="HLSLiveDownloader.cs" />
     <Compile Include="HLSTags.cs" />
     <Compile Include="LOGGER.cs" />
     <Compile Include="DownloadManager.cs" />
+    <Compile Include="MPDParser.cs" />
     <Compile Include="Parser.cs" />
     <Compile Include="Program.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
@@ -99,9 +108,6 @@
     </None>
     <None Include="packages.config" />
   </ItemGroup>
-  <ItemGroup>
-    <None Include="bin\Debug\Newtonsoft.Json.dll" />
-  </ItemGroup>
   <ItemGroup>
     <COMReference Include="Scripting">
       <Guid>{420B2830-E718-11CF-893D-00A0C9054228}</Guid>
@@ -131,4 +137,12 @@
     </EmbeddedResource>
   </ItemGroup>
   <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
-</Project>
+  <Import Project="..\packages\Fody.6.0.0\build\Fody.targets" Condition="Exists('..\packages\Fody.6.0.0\build\Fody.targets')" />
+  <Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
+    <PropertyGroup>
+      <ErrorText>这台计算机上缺少此项目引用的 NuGet 程序包。使用“NuGet 程序包还原”可下载这些程序包。有关更多信息，请参见 http://go.microsoft.com/fwlink/?LinkID=322105。缺少的文件是 {0}。</ErrorText>
+    </PropertyGroup>
+    <Error Condition="!Exists('..\packages\Fody.6.0.0\build\Fody.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Fody.6.0.0\build\Fody.targets'))" />
+    <Error Condition="!Exists('..\packages\Costura.Fody.4.1.0\build\Costura.Fody.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Costura.Fody.4.1.0\build\Costura.Fody.props'))" />
+  </Target>
+</Project>

N_m3u8DL-CLI/packages.config

@@ -1,4 +1,7 @@
 <?xml version="1.0" encoding="utf-8"?>
 <packages>
+  <package id="Fody" version="6.0.0" targetFramework="net46" developmentDependency="true" />
+  <package id="Costura.Fody" version="4.1.0" targetFramework="net46" />
+  <package id="Newtonsoft.Json" version="12.0.3" targetFramework="net46" />
   <package id="NiL.JS" version="2.5.1428" targetFramework="net46" />
 </packages>