update

解决HTTPS协议自动重定向后，Referer丢失问题

N_m3u8DL-CLI/DecodeHuke88Key.cs

@@ -0,0 +1,53 @@
+using Newtonsoft.Json.Linq;
+using System;
+using System.Collections.Generic;
+using System.Security.Cryptography;
+using System.Text;
+using System.Text.RegularExpressions;
+
+namespace N_m3u8DL_CLI
+{
+    //https://js.huke88.com/assets/revision/js/plugins/tcplayer/tcplayer.v4.1.min.js?v=930
+    //https://js.huke88.com/assets/revision/js/plugins/tcplayer/libs/hls.min.0.13.2m.js?v=930
+    class DecodeHuke88Key
+    {
+        private static string[] GetOverlayInfo(string url)
+        {
+            var enc = new Regex("eyJ\\w{100,}").Match(url).Value;
+            var json = Encoding.UTF8.GetString(Convert.FromBase64String(enc));
+            JObject jObject = JObject.Parse(json);
+            var key = jObject["overlayKey"].ToString();
+            var iv = jObject["overlayIv"].ToString();
+            return new string[] { key, iv };
+        }
+
+        public static string DecodeKey(string url, byte[] data)
+        {
+            var info = GetOverlayInfo(url);
+            var overlayKey = info[0];
+            var overlayIv = info[1];
+            var l = new List<byte>();
+            var c = new List<byte>();
+            for (int h = 0; h < 16; h++)
+            {
+                var f = overlayKey.Substring(2 * h, 2);
+                var g = overlayIv.Substring(2 * h, 2);
+                l.Add(Convert.ToByte(f, 16));
+                c.Add(Convert.ToByte(g, 16));
+            }
+
+            var _lastCipherblock = c.ToArray();
+
+            var t = new byte[data.Length];
+            var r = data;
+            r = Decrypter.AES128Decrypt(data, l.ToArray(), Decrypter.HexStringToBytes("00000000000000000000000000000000"), CipherMode.CBC, PaddingMode.Zeros);
+
+            for (var o = 0; o < 16; o++)
+                t[o] = (byte)(r[o] ^ _lastCipherblock[o]);
+
+            var key = Convert.ToBase64String(t);
+
+            return key;
+        }
+    }
+}

N_m3u8DL-CLI/Decrypter.cs

@@ -1,16 +1,12 @@
 using System;
-using System.Collections.Generic;
 using System.IO;
-using System.Linq;
 using System.Security.Cryptography;
-using System.Text;
-using System.Threading.Tasks;
 
 namespace N_m3u8DL_CLI
 {
     class Decrypter
     {
-        public static byte[] AES128Decrypt(string filePath, byte[] keyByte, byte[] ivByte, CipherMode mode = CipherMode.CBC)
+        public static byte[] AES128Decrypt(string filePath, byte[] keyByte, byte[] ivByte, CipherMode mode = CipherMode.CBC, PaddingMode padding = PaddingMode.PKCS7)
         {
             FileStream fs = new FileStream(filePath, FileMode.Open);
             //获取文件大小
@@ -25,14 +21,14 @@ namespace N_m3u8DL_CLI
             dcpt.Key = keyByte;
             dcpt.IV = ivByte;
             dcpt.Mode = mode;
-            dcpt.Padding = PaddingMode.PKCS7;
+            dcpt.Padding = padding;
 
             ICryptoTransform cTransform = dcpt.CreateDecryptor();
             Byte[] resultArray = cTransform.TransformFinalBlock(inBuff, 0, inBuff.Length);
             return resultArray;
         }
 
-        public static byte[] AES128Decrypt(byte[] encryptedBuff, byte[] keyByte, byte[] ivByte, CipherMode mode = CipherMode.CBC)
+        public static byte[] AES128Decrypt(byte[] encryptedBuff, byte[] keyByte, byte[] ivByte, CipherMode mode = CipherMode.CBC, PaddingMode padding = PaddingMode.PKCS7)
         {
             byte[] inBuff = encryptedBuff;
 
@@ -42,7 +38,7 @@ namespace N_m3u8DL_CLI
             dcpt.Key = keyByte;
             dcpt.IV = ivByte;
             dcpt.Mode = mode;
-            dcpt.Padding = PaddingMode.PKCS7;
+            dcpt.Padding = padding;
 
             ICryptoTransform cTransform = dcpt.CreateDecryptor();
             Byte[] resultArray = cTransform.TransformFinalBlock(inBuff, 0, inBuff.Length);
@@ -56,7 +52,7 @@ namespace N_m3u8DL_CLI
                 return new byte[0];
             }
 
-            if (hexStr.StartsWith("0x") || hexStr.StartsWith("0X")) 
+            if (hexStr.StartsWith("0x") || hexStr.StartsWith("0X"))
             {
                 hexStr = hexStr.Remove(0, 2);
             }

N_m3u8DL-CLI/Global.cs

@@ -30,8 +30,9 @@ namespace N_m3u8DL_CLI
 
 
         /*===============================================================================*/
-        static string nowVer = "2.8.1";
-        static string nowDate = "20201121";
+        static Version ver = System.Reflection.Assembly.GetExecutingAssembly().GetName().Version;
+        static string nowVer = $"{ver.Major}.{ver.Minor}.{ver.Build}";
+        static string nowDate = "20210124";
         public static void WriteInit()
         {
             Console.Clear();
@@ -101,23 +102,25 @@ namespace N_m3u8DL_CLI
         public static string GetWebSource(String url, string headers = "", int TimeOut = 60000)
         {
             string htmlCode = string.Empty;
-            for(int i = 0; i < 10; i++)
+            for (int i = 0; i < 10; i++)
             {
                 try
                 {
+                reProcess:
                     HttpWebRequest webRequest = (System.Net.HttpWebRequest)System.Net.WebRequest.Create(url);
                     webRequest.Method = "GET";
                     if (NoProxy) webRequest.Proxy = null;
-                    webRequest.UserAgent = "Mozilla/5.0 (Linux; U; Android 7.0; zh-cn; 15 Plus Build/NRD90M) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/66.0.3359.126 MQQBrowser/9.4 Mobile Safari/537.36";
-                    webRequest.Headers.Add("Accept-Encoding", "gzip, deflate");
+                    webRequest.UserAgent = "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36";
+                    webRequest.Accept = "*/*";
+                    webRequest.Headers.Add("Accept-Encoding", "gzip, deflate, br");
                     webRequest.Timeout = TimeOut;  //设置超时
                     webRequest.KeepAlive = false;
-                    webRequest.AllowAutoRedirect = true;  //自动跳转
+                    webRequest.AllowAutoRedirect = false; //手动处理重定向，否则会丢失Referer
                     if (url.Contains("pcvideo") && url.Contains(".titan.mgtv.com"))
                     {
                         webRequest.UserAgent = "";
                         if (!url.Contains("/internettv/"))
-                            webRequest.Referer = "https://player.mgtv.com/mgtv_v6_player/PlayerCore.swf";
+                            webRequest.Referer = "https://www.mgtv.com";
                         webRequest.Headers.Add("Cookie", "MQGUID");
                     }
                     //添加headers
@@ -146,8 +149,16 @@ namespace N_m3u8DL_CLI
                     }
                     HttpWebResponse webResponse = (HttpWebResponse)webRequest.GetResponse();
 
+                    //302
+                    if (webResponse.Headers.Get("Location") != null)
+                    {
+                        url = webResponse.Headers.Get("Location");
+                        webResponse.Close();
+                        goto reProcess;
+                    }
+
                     //文件过大则认为不是m3u8
-                    if (webResponse.ContentLength != -1 && webRequest.ContentLength > 50 * 1024 * 1024) return "";
+                    if (webResponse.ContentLength != -1 && webResponse.ContentLength > 50 * 1024 * 1024) return "";
 
                     if (webResponse.ContentEncoding != null
                         && webResponse.ContentEncoding.ToLower() == "gzip") //如果使用了GZip则先解压
@@ -406,14 +417,17 @@ namespace N_m3u8DL_CLI
                 }
             }
 
+        reProcess:
             byte[] arraryByte;
             HttpWebRequest req = (HttpWebRequest)WebRequest.Create(url);
             req.Method = "GET";
             req.Timeout = timeOut;
             req.ReadWriteTimeout = timeOut; //重要
+            req.AllowAutoRedirect = false; //手动处理重定向，否则会丢失Referer
             if (NoProxy) req.Proxy = null;
             req.Headers.Add("Accept-Encoding", "gzip, deflate");
             req.Accept = "*/*";
+            req.UserAgent = "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36";
             //添加headers
             if (headers != "")
             {
@@ -441,6 +455,13 @@ namespace N_m3u8DL_CLI
 
             using (HttpWebResponse wr = (HttpWebResponse)req.GetResponse())
             {
+                //302
+                if (wr.Headers.Get("Location") != null)
+                {
+                    url = wr.Headers.Get("Location");
+                    wr.Close();
+                    goto reProcess;
+                }
                 if (wr.ContentEncoding != null && wr.ContentEncoding.ToLower() == "gzip") //如果使用了GZip则先解压
                 {
                     using (Stream streamReceive = wr.GetResponseStream())
@@ -501,10 +522,11 @@ namespace N_m3u8DL_CLI
                 if (shouldStop)
                     return;
 
+                reProcess:
                 HttpWebRequest request = WebRequest.Create(url) as HttpWebRequest;
                 request.Timeout = timeOut;
                 request.ReadWriteTimeout = timeOut; //重要
-                request.AllowAutoRedirect = true;
+                request.AllowAutoRedirect = false; //手动处理重定向，否则会丢失Referer
                 request.KeepAlive = false;
                 request.Method = "GET";
                 if (NoProxy) request.Proxy = null;
@@ -514,11 +536,11 @@ namespace N_m3u8DL_CLI
                 {
                     request.UserAgent = "";
                     if (!url.Contains("/internettv/"))
-                        request.Referer = "https://player.mgtv.com/mgtv_v6_player/PlayerCore.swf";
+                        request.Referer = "https://www.mgtv.com";
                     request.Headers.Add("Cookie", "MQGUID");
                 }
                 else
-                    request.UserAgent = "Mozilla/5.0 (Linux; U; Android 7.0; zh-cn; 15 Plus Build/NRD90M) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/66.0.3359.126 MQQBrowser/9.4 Mobile Safari/537.36";
+                    request.UserAgent = "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36";
                 //下载部分字节
                 if (expectByte != -1)
                     request.AddRange("bytes", startByte, startByte + expectByte - 1);
@@ -552,6 +574,13 @@ namespace N_m3u8DL_CLI
                 bool pngHeader = false; //PNG HEADER检测
                 using (var response = (HttpWebResponse)request.GetResponse())
                 {
+                    //302
+                    if (response.Headers.Get("Location") != null)
+                    {
+                        url = response.Headers.Get("Location");
+                        response.Close();
+                        goto reProcess;
+                    }
                     using (var responseStream = response.GetResponseStream())
                     {
                         using (var stream = new FileStream(path, FileMode.Create, FileAccess.Write, FileShare.Write))
@@ -560,7 +589,7 @@ namespace N_m3u8DL_CLI
                             totalLen = response.ContentLength;
                             byte[] bArr = new byte[1024];
                             int size = responseStream.Read(bArr, 0, (int)bArr.Length);
-                            if (!pngHeader && size > 3 && 137 == bArr[0] && 80 == bArr[1] && 78 == bArr[2] && 71 == bArr[3]) 
+                            if (!pngHeader && size > 3 && 137 == bArr[0] && 80 == bArr[1] && 78 == bArr[2] && 71 == bArr[3])
                             {
                                 pngHeader = true;
                             }
@@ -593,10 +622,11 @@ namespace N_m3u8DL_CLI
                 }
                 if (shouldStop)
                     try { File.Delete(path); } catch (Exception) { }
-                if (totalLen != -1 && downLen != totalLen) 
+                if (totalLen != -1 && downLen != totalLen)
                     try { File.Delete(path); } catch (Exception) { }
                 if (pngHeader)
                     TrySkipPngHeader(path);
+
             }
             catch (Exception e)
             {

N_m3u8DL-CLI/N_m3u8DL-CLI.csproj

@@ -1,5 +1,6 @@
 <?xml version="1.0" encoding="utf-8"?>
 <Project ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <Import Project="..\packages\Costura.Fody.4.1.0\build\Costura.Fody.props" Condition="Exists('..\packages\Costura.Fody.4.1.0\build\Costura.Fody.props')" />
   <Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
   <PropertyGroup>
     <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
@@ -12,6 +13,8 @@
     <FileAlignment>512</FileAlignment>
     <AutoGenerateBindingRedirects>true</AutoGenerateBindingRedirects>
     <TargetFrameworkProfile />
+    <NuGetPackageImportStamp>
+    </NuGetPackageImportStamp>
   </PropertyGroup>
   <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
     <PlatformTarget>x86</PlatformTarget>
@@ -38,6 +41,9 @@
     <ApplicationIcon>logo_3Iv_icon.ico</ApplicationIcon>
   </PropertyGroup>
   <ItemGroup>
+    <Reference Include="Costura, Version=4.1.0.0, Culture=neutral, PublicKeyToken=9919ef960d84173d, processorArchitecture=MSIL">
+      <HintPath>..\packages\Costura.Fody.4.1.0\lib\net40\Costura.dll</HintPath>
+    </Reference>
     <Reference Include="Microsoft.JScript" />
     <Reference Include="netstandard, Version=2.0.0.0, Culture=neutral, PublicKeyToken=cc7b13ffcd2ddd51" />
     <Reference Include="Newtonsoft.Json, Version=12.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
@@ -64,6 +70,7 @@
     <Compile Include="CommandLineArgumentParser.cs" />
     <Compile Include="Decode51CtoKey.cs" />
     <Compile Include="DecodeDdyun.cs" />
+    <Compile Include="DecodeHuke88Key.cs" />
     <Compile Include="DecodeImooc.cs" />
     <Compile Include="DecodeNfmovies.cs" />
     <Compile Include="Decrypter.cs" />
@@ -130,4 +137,12 @@
     </EmbeddedResource>
   </ItemGroup>
   <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
+  <Import Project="..\packages\Fody.6.0.0\build\Fody.targets" Condition="Exists('..\packages\Fody.6.0.0\build\Fody.targets')" />
+  <Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
+    <PropertyGroup>
+      <ErrorText>这台计算机上缺少此项目引用的 NuGet 程序包。使用“NuGet 程序包还原”可下载这些程序包。有关更多信息，请参见 http://go.microsoft.com/fwlink/?LinkID=322105。缺少的文件是 {0}。</ErrorText>
+    </PropertyGroup>
+    <Error Condition="!Exists('..\packages\Fody.6.0.0\build\Fody.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Fody.6.0.0\build\Fody.targets'))" />
+    <Error Condition="!Exists('..\packages\Costura.Fody.4.1.0\build\Costura.Fody.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Costura.Fody.4.1.0\build\Costura.Fody.props'))" />
+  </Target>
 </Project>

N_m3u8DL-CLI/changelog.txt

@@ -284,4 +284,32 @@
 2020年11月20日
   - 识别大部分mpd地址，自动转换为m3u8并下载
   - GIF HEADER检测
-  - 修复BUG
+  - 修复BUG
+2020年11月22日
+  - 解决HTTPS协议自动重定向后，Referer丢失问题
+  - 新的任务速度监控逻辑
+2020年11月23日
+  - 将默认UA修改为 Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36
+  - 修改芒果TV请求头
+2020年11月25日
+  - 修正MPD判断最高清晰度的逻辑
+  - 在MPD输入下支持选择音轨
+  - 修复BUG
+2020年11月26日
+  - 优化MPD识别方案
+  - 修复MPD情况下时间戳溢出问题
+2020年12月2日
+  - FIX Language Bug
+2020年12月6日
+  - 使用手机UA请求气球云密钥服务器
+2020年12月12日
+  - 修复MPD下同一个ID分散在不同Period导致下载不完全问题
+2020年12月20日
+  - 支持解密虎课网
+2021年1月18日
+  - 完善MPD下载相关
+  - 重新打包多语言资源
+2021年1月24日
+  - 适配Disney+资源
+  - MPD选择流行为优化
+  - 修复二进制合并时vtt字幕被合并为ts后缀问题

N_m3u8DL-CLI/packages.config

@@ -1,5 +1,7 @@
 <?xml version="1.0" encoding="utf-8"?>
 <packages>
+  <package id="Fody" version="6.0.0" targetFramework="net46" developmentDependency="true" />
+  <package id="Costura.Fody" version="4.1.0" targetFramework="net46" />
   <package id="Newtonsoft.Json" version="12.0.3" targetFramework="net46" />
   <package id="NiL.JS" version="2.5.1428" targetFramework="net46" />
 </packages>