ModNet.vb 102 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187
  1. Imports System.Drawing
  2. Imports System.Net.Http
  3. Imports System.Net.Http.Headers
  4. Imports System.Threading.Tasks
  5. Imports Newtonsoft.Json
  6. Public Module ModNet
  7. Public Const NetDownloadEnd As String = ".PCLDownloading"
  8. Public ReadOnly HttpProxy As New HttpWebProxyFactory()
  9. Private ClientFactory As New HttpClientFactory()
  10. Private Request As New HttpRequest()
  11. Public Class HttpRequest
  12. Public Sub New()
  13. End Sub
  14. Public Function GetRequestMessage(RequestUrl As String,
  15. Optional Method As String = "GET",
  16. Optional Headers As Dictionary(Of String, String) = Nothing,
  17. Optional Data As Byte() = Nothing,
  18. Optional UseBrowserUA As Boolean = False,
  19. Optional RequireHeaderSign As Boolean = False,
  20. Optional RequireCdnSign As Boolean = False) As HttpRequestMessage
  21. Dim Watcher As New Stopwatch()
  22. Watcher.Start()
  23. Dim RequestMethod As HttpMethod = GetHttpMethod(Method)
  24. If RequireCdnSign Then RequestUrl = SecretCdnSign(RequestUrl)
  25. Dim RequestMessage As New HttpRequestMessage(RequestMethod, RequestUrl)
  26. '基本参数验证
  27. If Data IsNot Nothing AndAlso Not {HttpMethod.Get, HttpMethod.Head}.Contains(RequestMethod) Then
  28. RequestMessage.Content = New ByteArrayContent(Data)
  29. RequestMessage.Content.Headers.ContentLength = Data.Length
  30. End If
  31. If Headers IsNot Nothing Then
  32. For Each Header In Headers
  33. If Header.Key.ContainsF("Content") AndAlso RequestMessage.Content IsNot Nothing Then
  34. 'FixContentHeader 往下挪一行就 NullReference 了
  35. FixContentHeader:
  36. If Data Is Nothing OrElse RequestMessage.Content Is Nothing Then Continue For
  37. If RequestMessage.Content.Headers.Contains(Header.Key) Then Continue For
  38. If Not RequestMessage.Content.Headers.Contains(Header.Key) Then
  39. If Header.Key.ToLower() = "Content-Type" Then RequestMessage.Content.Headers.ContentType = New MediaTypeHeaderValue(Header.Value)
  40. RequestMessage.Content.Headers.TryAddWithoutValidation(Header.Key, Header.Value)
  41. End If
  42. Else
  43. '不知道为什么 Content-Type 标头会通过字符串验证然后加入不应该被加入的 HttpRequestMessage Headers
  44. '这会导致 HttpRequestMessage 内部的标头验证失败,并抛出一个 Exception 导致网络请求失败
  45. If Header.Key.ContainsF("Content") Then GoTo FixContentHeader
  46. If RequestMessage.Headers.Contains(Header.Key) Then Continue For
  47. RequestMessage.Headers.Add(Header.Key, Header.Value)
  48. End If
  49. Next
  50. End If
  51. If RequireHeaderSign Then
  52. SecretHeadersSign(RequestUrl, requestMessage, UseBrowserUA)
  53. End If
  54. Watcher.Stop()
  55. Log($"[Test] GetRequestMessage 执行用时:{Watcher.ElapsedMilliseconds} 毫秒")
  56. Return RequestMessage
  57. End Function
  58. ' 发送请求
  59. Public Function SendRequest(Req As HttpRequestMessage,
  60. Optional RequireDecode As Boolean = True,
  61. Optional Encode As Encoding = Nothing,
  62. Optional Timeout As Integer = 25000,
  63. Optional RequireReturnRespObj As Boolean = False)
  64. Dim Watcher As New Stopwatch()
  65. Watcher.Start()
  66. Using CTS As New CancellationTokenSource(Timeout)
  67. Dim Client = ClientFactory.GetHttpClient()
  68. Dim Resp As HttpResponseMessage = Client.SendAsync(Req, HttpCompletionOption.ResponseHeadersRead, CTS.Token).GetAwaiter().GetResult()
  69. If RequireReturnRespObj Then Return Resp
  70. Resp.EnsureSuccessStatusCode()
  71. CTS.CancelAfter(Timeout)
  72. Dim ResultStream = Resp.Content.ReadAsStreamAsync().GetAwaiter().GetResult()
  73. Using DataStream As New MemoryStream()
  74. ResultStream.CopyTo(DataStream)
  75. Dim Bytes = DataStream.ToArray()
  76. Resp.Dispose()
  77. Req.Dispose()
  78. Watcher.Stop()
  79. Log($"[Test] SendRequest 执行用时:{Watcher.ElapsedMilliseconds} 毫秒")
  80. If RequireDecode Then
  81. If Encode Is Nothing Then Encode = Encoding.UTF8
  82. Return Encode.GetString(Bytes)
  83. Else
  84. Return Bytes
  85. End If
  86. End Using
  87. End Using
  88. End Function
  89. ' 获取HttpMethod
  90. Private Function GetHttpMethod(Method As String) As HttpMethod
  91. Select Case Method?.Trim().ToUpperInvariant()
  92. Case "HEAD"
  93. Return HttpMethod.Head
  94. Case "POST"
  95. Return HttpMethod.Post
  96. Case "PUT"
  97. Return HttpMethod.Put
  98. Case "DELETE"
  99. Return HttpMethod.Delete
  100. Case Else
  101. Return HttpMethod.Get
  102. End Select
  103. End Function
  104. End Class
  105. Public Class HttpWebProxyFactory
  106. Implements IWebProxy
  107. Public DisableHttpProxy As Boolean = True
  108. Public ProxyAddress As String
  109. Private CurrentProxy As WebProxy
  110. Public RequiredReloadProxy As Boolean
  111. Public RefreshSyncLock As New Object
  112. Public SystemProxy As New WebProxy()
  113. Public Sub New()
  114. End Sub
  115. Public Property Credentials As ICredentials Implements IWebProxy.Credentials
  116. Get
  117. Return AuthCredentials
  118. End Get
  119. Set(value As ICredentials)
  120. AuthCredentials = value
  121. End Set
  122. End Property
  123. Private AuthCredentials As ICredentials
  124. Public Function GetProxy(destination As Uri) As Uri Implements IWebProxy.GetProxy
  125. Return GetProxy(destination.AbsoluteUri).Address
  126. End Function
  127. Public Function GetProxy(destination As String) As WebProxy
  128. If CurrentProxy IsNot Nothing AndAlso Not RequiredReloadProxy Then Return CurrentProxy
  129. If RequiredReloadProxy OrElse CurrentProxy Is Nothing Then
  130. If ProxyAddress IsNot Nothing OrElse
  131. Not String.IsNullOrWhiteSpace(ProxyAddress) Then
  132. SyncLock RefreshSyncLock
  133. CurrentProxy = New WebProxy(ProxyAddress, True)
  134. If AuthCredentials IsNot Nothing Then CurrentProxy.Credentials = AuthCredentials
  135. Return CurrentProxy
  136. End SyncLock
  137. Else
  138. SyncLock RefreshSyncLock
  139. CurrentProxy = New WebProxy()
  140. If AuthCredentials IsNot Nothing Then CurrentProxy.Credentials = AuthCredentials
  141. Return CurrentProxy
  142. End SyncLock
  143. End If
  144. End If
  145. Return Nothing
  146. End Function
  147. Public Function IsBypassed(host As Uri) As Boolean Implements IWebProxy.IsBypassed
  148. If DisableHttpProxy Then Return True
  149. If CurrentProxy Is Nothing Then
  150. SyncLock RefreshSyncLock
  151. CurrentProxy = GetProxy(host.AbsoluteUri)
  152. End SyncLock
  153. End If
  154. If CurrentProxy.IsBypassed(host) Then Return True
  155. Return False
  156. End Function
  157. End Class
  158. Public Class HttpClientFactory
  159. Private PreviousClient As HttpClient
  160. Private CurrentClient As HttpClient
  161. Private HttpClientLock As New Object
  162. Public ClientHandler As New HttpClientHandler() With {
  163. .UseProxy = True,
  164. .Proxy = HttpProxy,
  165. .AutomaticDecompression = DecompressionMethods.Deflate Or DecompressionMethods.GZip Or DecompressionMethods.None
  166. }
  167. Public Sub New()
  168. HttpClientReload()
  169. End Sub
  170. Public Function GetHttpClient(Optional Handler As HttpClientHandler = Nothing) As HttpClient
  171. SyncLock HttpClientLock
  172. If Handler IsNot Nothing Then
  173. ClientHandler = Handler
  174. CurrentClient.Dispose()
  175. CurrentClient = New HttpClient(Handler)
  176. Return CurrentClient
  177. End If
  178. If CurrentClient IsNot Nothing Then Return CurrentClient
  179. CurrentClient = New HttpClient(ClientHandler) With {.Timeout = Timeout.InfiniteTimeSpan}
  180. Return CurrentClient
  181. End SyncLock
  182. End Function
  183. Public Sub HttpClientReload()
  184. RunInNewThread(
  185. Sub()
  186. While True
  187. Thread.Sleep(TimeSpan.FromHours(6))
  188. If PreviousClient IsNot Nothing Then PreviousClient.Dispose()
  189. PreviousClient = CurrentClient
  190. CurrentClient = GetHttpClient()
  191. End While
  192. End Sub
  193. )
  194. End Sub
  195. End Class
  196. ''' <summary>
  197. ''' 测试 Ping。失败则返回 -1。
  198. ''' </summary>
  199. '''
  200. Public Function Ping(Ip As String, Optional Timeout As Integer = 10000, Optional MakeLog As Boolean = True) As Integer
  201. Dim PingResult As NetworkInformation.PingReply
  202. Try
  203. PingResult = (New NetworkInformation.Ping).Send(Ip)
  204. Catch ex As Exception
  205. If MakeLog Then Log("[Net] Ping " & Ip & " 失败:" & ex.Message)
  206. Return -1
  207. End Try
  208. If PingResult.Status = NetworkInformation.IPStatus.Success Then
  209. If MakeLog Then Log("[Net] Ping " & Ip & " 结束:" & PingResult.RoundtripTime & "ms")
  210. Return PingResult.RoundtripTime
  211. Else
  212. If MakeLog Then Log("[Net] Ping " & Ip & " 失败")
  213. Return -1
  214. End If
  215. End Function
  216. ''' <summary>
  217. ''' 以 WebClient 获取网页源代码。会进行至多 45 秒 3 次的尝试,允许最长 30s 的超时。
  218. ''' </summary>
  219. ''' <param name="Url">网页的 Url。</param>
  220. ''' <param name="Encoding">网页的编码,通常为 UTF-8。</param>
  221. Public Function NetGetCodeByClient(Url As String, Encoding As Encoding, Optional Accept As String = "application/json, text/javascript, */*; q=0.01", Optional UseBrowserUserAgent As Boolean = False) As String
  222. Dim RetryCount As Integer = 0
  223. Dim RetryException As Exception = Nothing
  224. Dim StartTime As Long = GetTimeTick()
  225. Try
  226. Retry:
  227. Select Case RetryCount
  228. Case 0 '正常尝试
  229. Return NetGetCodeByClient(Url, Encoding, 10000, Accept, UseBrowserUserAgent)
  230. Case 1 '慢速重试
  231. Thread.Sleep(500)
  232. Return NetGetCodeByClient(Url, Encoding, 30000, Accept, UseBrowserUserAgent)
  233. Case Else '快速重试
  234. If GetTimeTick() - StartTime > 5500 Then
  235. '若前两次加载耗费 5 秒以上,才进行重试
  236. Thread.Sleep(500)
  237. Return NetGetCodeByClient(Url, Encoding, 4000, Accept, UseBrowserUserAgent)
  238. Else
  239. Throw RetryException
  240. End If
  241. End Select
  242. Catch ex As Exception
  243. Select Case RetryCount
  244. Case 0
  245. RetryException = ex
  246. RetryCount += 1
  247. GoTo Retry
  248. Case 1
  249. RetryCount += 1
  250. GoTo Retry
  251. Case Else
  252. Throw
  253. End Select
  254. End Try
  255. End Function
  256. Public Function NetGetCodeByClient(Url As String, Encoding As Encoding, Timeout As Integer, Accept As String, Optional UseBrowserUserAgent As Boolean = False) As String
  257. Log("[Net] 获取客户端网络结果:" & Url & ",最大超时 " & Timeout)
  258. Try
  259. Dim Headers As New Dictionary(Of String, String) From {
  260. {"Accept", Accept}
  261. }
  262. Using Req As HttpRequestMessage = Request.GetRequestMessage(Url, Headers:=Headers, RequireHeaderSign:=True, RequireCdnSign:=True, UseBrowserUA:=UseBrowserUserAgent)
  263. Dim Resp As String = Request.SendRequest(Req, Encode:=Encoding, Timeout:=Timeout)
  264. Return Resp
  265. End Using
  266. Catch ex As TaskCanceledException
  267. Throw New TaskCanceledException("连接服务器超时(" & Url & ")", ex)
  268. Catch ex As Exception
  269. Throw New WebException("获取结果失败," & ex.Message & "(" & Url & ")", ex)
  270. End Try
  271. End Function
  272. ''' <summary>
  273. ''' 以 WebRequest 获取网页源代码或 Json。会进行至多 45 秒 3 次的尝试,允许最长 30s 的超时。
  274. ''' </summary>
  275. ''' <param name="Url">网页的 Url。</param>
  276. ''' <param name="Encode">网页的编码,通常为 UTF-8。</param>
  277. ''' <param name="BackupUrl">如果第一次尝试失败,换用的备用 URL。</param>
  278. Public Function NetGetCodeByRequestRetry(Url As String, Optional Encode As Encoding = Nothing, Optional Accept As String = "",
  279. Optional IsJson As Boolean = False, Optional BackupUrl As String = Nothing, Optional UseBrowserUserAgent As Boolean = False)
  280. Dim RetryCount As Integer = 0
  281. Dim RetryException As Exception = Nothing
  282. Dim StartTime As Long = GetTimeTick()
  283. Try
  284. Retry:
  285. Select Case RetryCount
  286. Case 0 '正常尝试
  287. Return NetGetCodeByRequestOnce(Url, Encode, 10000, IsJson, Accept, UseBrowserUserAgent)
  288. Case 1 '慢速重试
  289. Thread.Sleep(500)
  290. Return NetGetCodeByRequestOnce(If(BackupUrl, Url), Encode, 30000, IsJson, Accept, UseBrowserUserAgent)
  291. Case Else '快速重试
  292. If GetTimeTick() - StartTime > 5500 Then
  293. '若前两次加载耗费 5 秒以上,才进行重试
  294. Thread.Sleep(500)
  295. Return NetGetCodeByRequestOnce(If(BackupUrl, Url), Encode, 4000, IsJson, Accept, UseBrowserUserAgent)
  296. Else
  297. Throw RetryException
  298. End If
  299. End Select
  300. Catch ex As ThreadInterruptedException
  301. Throw
  302. Catch ex As Exception
  303. Select Case RetryCount
  304. Case 0
  305. RetryException = ex
  306. RetryCount += 1
  307. GoTo Retry
  308. Case 1
  309. RetryCount += 1
  310. GoTo Retry
  311. Case Else
  312. Throw
  313. End Select
  314. End Try
  315. End Function
  316. ''' <summary>
  317. ''' 以 WebRequest 获取网页源代码或 Json。会逐渐生成 4 个尝试线程,并在 60s 后超时。
  318. ''' </summary>
  319. ''' <param name="Url">网页的 Url。</param>
  320. ''' <param name="Encode">网页的编码,通常为 UTF-8。</param>
  321. Public Function NetGetCodeByRequestMultiple(Url As String, Optional Encode As Encoding = Nothing, Optional Accept As String = "", Optional IsJson As Boolean = False)
  322. Dim Threads As New List(Of Thread)
  323. Dim RequestResult = Nothing
  324. Dim RequestEx As Exception = Nothing
  325. Dim FailCount As Integer = 0
  326. For i = 1 To 4
  327. Dim th As New Thread(
  328. Sub()
  329. Try
  330. RequestResult = NetGetCodeByRequestOnce(Url, Encode, 30000, IsJson, Accept)
  331. Catch ex As Exception
  332. FailCount += 1
  333. RequestEx = ex
  334. End Try
  335. End Sub)
  336. th.Start()
  337. Threads.Add(th)
  338. Thread.Sleep(i * 250)
  339. If RequestResult IsNot Nothing Then GoTo RequestFinished
  340. Next
  341. Do While True
  342. If RequestResult IsNot Nothing Then
  343. RequestFinished:
  344. Try
  345. For Each th In Threads
  346. If th.IsAlive Then th.Interrupt()
  347. Next
  348. Catch
  349. End Try
  350. Return RequestResult
  351. ElseIf FailCount = 4 Then
  352. Try
  353. For Each th In Threads
  354. If th.IsAlive Then th.Interrupt()
  355. Next
  356. Catch
  357. End Try
  358. Throw RequestEx
  359. End If
  360. Thread.Sleep(20)
  361. Loop
  362. Throw New Exception("未知错误")
  363. End Function
  364. Public Function NetGetCodeByRequestOnce(Url As String, Optional Encode As Encoding = Nothing, Optional Timeout As Integer = 30000, Optional IsJson As Boolean = False, Optional Accept As String = "", Optional UseBrowserUserAgent As Boolean = False)
  365. If RunInUi() AndAlso Not Url.Contains("//127.") Then Throw New Exception("在 UI 线程执行了网络请求")
  366. Log($"[Net] 获取网络结果:{Url},超时 {Timeout}ms{If(IsJson, ",要求 json", "")}")
  367. Try
  368. Dim Headers As New Dictionary(Of String, String) From {
  369. {"Accept", If(Accept = "", "*/*", Accept)}
  370. }
  371. Using Req As HttpRequestMessage = Request.GetRequestMessage(Url, Headers:=Headers, UseBrowserUA:=UseBrowserUserAgent, RequireHeaderSign:=True, RequireCdnSign:=True)
  372. Dim Resp As String = Request.SendRequest(Req, Encode:=Encode, Timeout:=Timeout)
  373. Return If(IsJson, GetJson(Resp), Resp)
  374. End Using
  375. Catch ex As ThreadInterruptedException
  376. Throw
  377. Catch ex As TaskCanceledException
  378. Throw New TimeoutException($"获取结果失败({ex.Message},{Url})", ex)
  379. Catch ex As HttpRequestException
  380. Throw New WebException($"获取结果失败({ex.Message},{Url})", ex)
  381. End Try
  382. End Function
  383. ''' <summary>
  384. ''' 以多线程下载网页文件的方式获取网页源代码。
  385. ''' </summary>
  386. ''' <param name="Url">网页的 Url。</param>
  387. Public Function NetGetCodeByLoader(Url As String, Optional Timeout As Integer = 45000, Optional IsJson As Boolean = False, Optional UseBrowserUserAgent As Boolean = False) As String
  388. Dim Temp As String = RequestTaskTempFolder() & "download.txt"
  389. Dim NewTask As New LoaderDownload("源码获取 " & GetUuid() & "#", New List(Of NetFile) From {New NetFile({Url}, Temp, New FileChecker With {.IsJson = IsJson}, UseBrowserUserAgent)})
  390. Try
  391. NewTask.WaitForExitTime(Timeout, TimeoutMessage:="连接服务器超时(" & Url & ")")
  392. NetGetCodeByLoader = ReadFile(Temp)
  393. File.Delete(Temp)
  394. Finally
  395. NewTask.Abort()
  396. End Try
  397. End Function
  398. ''' <summary>
  399. ''' 以多线程下载网页文件的方式获取网页源代码。
  400. ''' </summary>
  401. ''' <param name="Urls">网页的 Url 列表。</param>
  402. Public Function NetGetCodeByLoader(Urls As IEnumerable(Of String), Optional Timeout As Integer = 45000, Optional IsJson As Boolean = False, Optional UseBrowserUserAgent As Boolean = False) As String
  403. Dim Temp As String = RequestTaskTempFolder() & "download.txt"
  404. Dim NewTask As New LoaderDownload("源码获取 " & GetUuid() & "#", New List(Of NetFile) From {New NetFile(Urls, Temp, New FileChecker With {.IsJson = IsJson}, UseBrowserUserAgent)})
  405. Try
  406. NewTask.WaitForExitTime(Timeout, TimeoutMessage:="连接服务器超时(第一下载源:" & Urls.First & ")")
  407. NetGetCodeByLoader = ReadFile(Temp)
  408. File.Delete(Temp)
  409. Finally
  410. NewTask.Abort()
  411. End Try
  412. End Function
  413. ''' <summary>
  414. ''' 使用 WebClient 从网络中下载文件。这不能下载 CDN 中的文件。
  415. ''' </summary>
  416. ''' <param name="Url">网络 Url。</param>
  417. ''' <param name="LocalFile">下载的本地地址。</param>
  418. Public Sub NetDownloadByClient(Url As String, LocalFile As String, Optional UseBrowserUserAgent As Boolean = False)
  419. Log("[Net] 直接下载文件:" & Url)
  420. '初始化
  421. Try
  422. '建立目录
  423. Directory.CreateDirectory(GetPathFromFullPath(LocalFile))
  424. '尝试删除原文件
  425. File.Delete(LocalFile)
  426. Catch ex As Exception
  427. Throw New WebException($"预处理下载文件路径失败({LocalFile})", ex)
  428. End Try
  429. '下载
  430. Using Client As New WebClient
  431. Try
  432. 'SecretHeadersSign(Url, Client, UseBrowserUserAgent)
  433. Client.DownloadFile(Url, LocalFile)
  434. Catch ex As Exception
  435. File.Delete(LocalFile)
  436. Throw New WebException($"直接下载文件失败({Url})", ex)
  437. End Try
  438. End Using
  439. End Sub
  440. ''' <summary>
  441. ''' 简单的多线程下载文件。可以下载 CDN 中的文件。
  442. ''' </summary>
  443. ''' <param name="Url">文件的 Url。</param>
  444. ''' <param name="LocalFile">下载的本地地址。</param>
  445. Public Sub NetDownloadByLoader(Url As String, LocalFile As String, Optional LoaderToSyncProgress As LoaderBase = Nothing, Optional Check As FileChecker = Nothing, Optional UseBrowserUserAgent As Boolean = False)
  446. Dim NewTask As New LoaderDownload("文件下载 " & GetUuid() & "#", New List(Of NetFile) From {New NetFile({Url}, LocalFile, Check, UseBrowserUserAgent)})
  447. Try
  448. NewTask.WaitForExit(LoaderToSyncProgress:=LoaderToSyncProgress)
  449. Catch ex As Exception
  450. Throw New WebException($"多线程直接下载文件失败({Url})", ex)
  451. Finally
  452. NewTask.Abort()
  453. End Try
  454. End Sub
  455. ''' <summary>
  456. ''' 简单的多线程下载文件。可以下载 CDN 中的文件。
  457. ''' </summary>
  458. ''' <param name="Urls">文件的 Url 列表。</param>
  459. ''' <param name="LocalFile">下载的本地地址。</param>
  460. Public Sub NetDownloadByLoader(Urls As IEnumerable(Of String), LocalFile As String, Optional LoaderToSyncProgress As LoaderBase = Nothing, Optional Check As FileChecker = Nothing, Optional UseBrowserUserAgent As Boolean = False)
  461. Dim NewTask As New LoaderDownload("文件下载 " & GetUuid() & "#", New List(Of NetFile) From {New NetFile(Urls, LocalFile, Check, UseBrowserUserAgent)})
  462. Try
  463. NewTask.WaitForExit(LoaderToSyncProgress:=LoaderToSyncProgress)
  464. Catch ex As Exception
  465. Throw New WebException($"多线程直接下载文件失败(第一下载源:" & Urls.First() & ")", ex)
  466. Finally
  467. NewTask.Abort()
  468. End Try
  469. End Sub
  470. ''' <summary>
  471. ''' 发送一个网络请求并获取返回内容,会重试三次并在最长 45s 后超时。
  472. ''' </summary>
  473. ''' <param name="Url">请求的服务器地址。</param>
  474. ''' <param name="Method">请求方式(POST 或 GET)。</param>
  475. ''' <param name="Data">请求的内容。</param>
  476. ''' <param name="ContentType">请求的套接字类型。</param>
  477. ''' <param name="DontRetryOnRefused">当返回 40x 时不重试。</param>
  478. Public Function NetRequestRetry(Url As String, Method As String, Data As Object, ContentType As String, Optional DontRetryOnRefused As Boolean = True, Optional Headers As Dictionary(Of String, String) = Nothing, Optional RequireReturnRespObj As Boolean = False) As String
  479. Dim RetryCount As Integer = 0
  480. Dim RetryException As Exception = Nothing
  481. Dim StartTime As Long = GetTimeTick()
  482. Try
  483. Retry:
  484. Select Case RetryCount
  485. Case 0 '正常尝试
  486. Return NetRequestOnce(Url, Method, Data, ContentType, 15000, Headers, RequiredReturnRespObj:=RequireReturnRespObj)
  487. Case 1 '慢速重试
  488. Thread.Sleep(500)
  489. Return NetRequestOnce(Url, Method, Data, ContentType, 25000, Headers, RequiredReturnRespObj:=RequireReturnRespObj)
  490. Case Else '快速重试
  491. If GetTimeTick() - StartTime > 5500 Then
  492. '若前两次加载耗费 5 秒以上,才进行重试
  493. Thread.Sleep(500)
  494. Return NetRequestOnce(Url, Method, Data, ContentType, 4000, Headers,RequiredReturnRespObj:=RequireReturnRespObj)
  495. Else
  496. Throw RetryException
  497. End If
  498. End Select
  499. Catch ex As ThreadInterruptedException
  500. Throw
  501. Catch ex As Exception
  502. If ex.InnerException IsNot Nothing AndAlso ex.InnerException.Message.Contains("(40") AndAlso DontRetryOnRefused Then Throw
  503. Select Case RetryCount
  504. Case 0
  505. If ModeDebug Then Log(ex, "[Net] 网络请求第一次失败(" & Url & ")")
  506. RetryException = ex
  507. RetryCount += 1
  508. GoTo Retry
  509. Case 1
  510. If ModeDebug Then Log(ex, "[Net] 网络请求第二次失败(" & Url & ")")
  511. RetryCount += 1
  512. GoTo Retry
  513. Case Else
  514. Throw
  515. End Select
  516. End Try
  517. End Function
  518. ''' <summary>
  519. ''' 同时发送多个网络请求并要求返回内容。
  520. ''' </summary>
  521. Public Function NetRequestMultiple(Url As String, Method As String, Data As Object, ContentType As String, Optional RequestCount As Integer = 4, Optional Headers As Dictionary(Of String, String) = Nothing, Optional MakeLog As Boolean = True)
  522. Dim Threads As New List(Of Thread)
  523. Dim RequestResult = Nothing
  524. Dim RequestEx As Exception = Nothing
  525. Dim FailCount As Integer = 0
  526. For i = 1 To RequestCount
  527. Dim th As New Thread(
  528. Sub()
  529. Try
  530. RequestResult = NetRequestOnce(Url, Method, Data, ContentType, 30000, Headers, MakeLog)
  531. Catch ex As Exception
  532. FailCount += 1
  533. RequestEx = ex
  534. End Try
  535. End Sub)
  536. th.Start()
  537. Threads.Add(th)
  538. Thread.Sleep(i * 250)
  539. If RequestResult IsNot Nothing Then GoTo RequestFinished
  540. Next
  541. Do While True
  542. If RequestResult IsNot Nothing Then
  543. RequestFinished:
  544. For Each th In Threads
  545. If th.IsAlive Then th.Interrupt()
  546. Next
  547. Return RequestResult
  548. ElseIf FailCount = RequestCount Then
  549. For Each th In Threads
  550. If th.IsAlive Then th.Interrupt()
  551. Next
  552. Throw RequestEx
  553. End If
  554. Thread.Sleep(20)
  555. Loop
  556. Throw New Exception("未知错误")
  557. End Function
  558. ''' <summary>
  559. ''' 发送一次网络请求并获取返回内容。
  560. ''' </summary>
  561. Public Function NetRequestOnce(Url As String, Method As String, Data As Object, ContentType As String, Optional Timeout As Integer = 25000, Optional Headers As Dictionary(Of String, String) = Nothing, Optional MakeLog As Boolean = True, Optional UseBrowserUserAgent As Boolean = False, Optional RequiredReturnRespObj As Boolean = False) As Object
  562. If RunInUi() AndAlso Not Url.Contains("//127.") Then Throw New Exception("在 UI 线程执行了网络请求")
  563. If MakeLog Then Log("[Net] 发起网络请求(" & Method & "," & Url & "),最大超时 " & Timeout)
  564. Try
  565. Dim ReqHeaders As New Dictionary(Of String, String)
  566. If ContentType IsNot Nothing AndAlso Data IsNot Nothing Then
  567. ' 可能指定了 Content-Type 但是 Headers 是 Nothing
  568. If Headers IsNot Nothing Then
  569. If Not Headers.ContainsKey("Content-Type") Then Headers.Add("Content-Type", ContentType)
  570. Else
  571. If Not ReqHeaders.ContainsKey("Content-Type") Then ReqHeaders.Add("Content-Type", ContentType)
  572. End If
  573. End If
  574. Dim ReqData As Byte() = If(TypeOf Data Is String, Encoding.UTF8.GetBytes(Data), Data)
  575. Dim Req As HttpRequestMessage = Request.GetRequestMessage(Url, Method, If(Headers, ReqHeaders), If(Data IsNot Nothing, ReqData, Nothing), UseBrowserUserAgent, True, True)
  576. Return Request.SendRequest(Req, If(RequiredReturnRespObj, False, True), Timeout:=Timeout, RequireReturnRespObj:=RequiredReturnRespObj)
  577. Catch ex As ThreadInterruptedException
  578. Throw
  579. Catch ex As TaskCanceledException
  580. Return New WebException($"连接服务器超时,请检查你的网络环境是否良好({ex.Message},{Url})", ex)
  581. Catch ex As HttpRequestException
  582. If MakeLog Then Log(ex, "NetRequestOnce 失败", LogLevel.Developer)
  583. Throw New WebException(ex.Message, ex)
  584. Catch ex As Exception
  585. ex = New WebException("网络请求失败(" & Url & ")", ex)
  586. If MakeLog Then Log(ex, "NetRequestOnce 失败", LogLevel.Developer)
  587. Throw ex
  588. End Try
  589. End Function
  590. Public Class ResponsedWebException
  591. Inherits WebException
  592. ''' <summary>
  593. ''' 远程服务器给予的回复。
  594. ''' </summary>
  595. Public Overloads Property Response As String
  596. Public Sub New(Message As String, Response As String, InnerException As Exception)
  597. MyBase.New(Message, InnerException)
  598. Me.Response = Response
  599. End Sub
  600. End Class
  601. ''' <summary>
  602. ''' 最大线程数。
  603. ''' </summary>
  604. Public NetTaskThreadLimit As Integer
  605. ''' <summary>
  606. ''' 速度下限。
  607. ''' </summary>
  608. Public NetTaskSpeedLimitLow As Long = 256 * 1024L '256K/s
  609. ''' <summary>
  610. ''' 速度上限。若无限制则为 -1。
  611. ''' </summary>
  612. Public NetTaskSpeedLimitHigh As Long = -1
  613. ''' <summary>
  614. ''' 基于限速,当前可以下载的剩余量。
  615. ''' </summary>
  616. Public NetTaskSpeedLimitLeft As Long = -1
  617. Private ReadOnly NetTaskSpeedLimitLeftLock As New Object
  618. Private NetTaskSpeedLimitLeftLast As Long
  619. ''' <summary>
  620. ''' 正在运行中的线程数。
  621. ''' </summary>
  622. Public NetTaskThreadCount As Integer = 0
  623. Private ReadOnly NetTaskThreadCountLock As New Object
  624. ''' <summary>
  625. ''' 下载源。
  626. ''' </summary>
  627. Public Class NetSource
  628. Public Id As Integer
  629. Public Url As String
  630. Public FailCount As Integer
  631. Public Ex As Exception
  632. Public Thread As NetThread
  633. Public IsFailed As Boolean
  634. Public Overrides Function ToString() As String
  635. Return Url
  636. End Function
  637. End Class
  638. ''' <summary>
  639. ''' 下载进度标示。
  640. ''' </summary>
  641. Public Enum NetState
  642. ''' <summary>
  643. ''' 尚未进行已存在检查。
  644. ''' </summary>
  645. WaitForCheck = -1
  646. ''' <summary>
  647. ''' 尚未开始。
  648. ''' </summary>
  649. WaitForDownload = 0
  650. ''' <summary>
  651. ''' 正在连接,尚未获取文件大小。
  652. ''' </summary>
  653. Connect = 1
  654. ''' <summary>
  655. ''' 已获取文件大小,尚未有有效下载。
  656. ''' </summary>
  657. [Get] = 2
  658. ''' <summary>
  659. ''' 正在下载。
  660. ''' </summary>
  661. Download = 3
  662. ''' <summary>
  663. ''' 正在合并文件。
  664. ''' </summary>
  665. Merge = 4
  666. ''' <summary>
  667. ''' 不进行下载,因为已发现现存的文件。
  668. ''' </summary>
  669. WaitForCopy = 5
  670. ''' <summary>
  671. ''' 已完成。
  672. ''' </summary>
  673. Finish = 6
  674. ''' <summary>
  675. ''' 已失败或中断。
  676. ''' </summary>
  677. [Error] = 7
  678. End Enum
  679. ''' <summary>
  680. ''' 预下载检查行为。
  681. ''' </summary>
  682. Public Enum NetPreDownloadBehaviour
  683. ''' <summary>
  684. ''' 当文件已存在时,显示提示以提醒用户是否继续下载。
  685. ''' </summary>
  686. HintWhileExists
  687. ''' <summary>
  688. ''' 当文件已存在或正在下载时,直接退出下载函数执行,不对用户进行提示。
  689. ''' </summary>
  690. ExitWhileExistsOrDownloading
  691. ''' <summary>
  692. ''' 不进行已存在检查。
  693. ''' </summary>
  694. IgnoreCheck
  695. End Enum
  696. ''' <summary>
  697. ''' 下载线程。
  698. ''' </summary>
  699. Public Class NetThread
  700. Implements IEnumerable(Of NetThread)
  701. ''' <summary>
  702. ''' 对应的下载任务。
  703. ''' </summary>
  704. Public Task As NetFile
  705. ''' <summary>
  706. ''' 对应的线程。
  707. ''' </summary>
  708. Public Thread As Thread
  709. ''' <summary>
  710. ''' 链表中的下一个线程。
  711. ''' </summary>
  712. Public NextThread As NetThread
  713. Private ReadOnly Iterator Property [Next]() As IEnumerable(Of NetThread)
  714. Get
  715. Dim CurrentChain As NetThread = Me
  716. While CurrentChain IsNot Nothing
  717. Yield CurrentChain
  718. CurrentChain = CurrentChain.NextThread
  719. End While
  720. End Get
  721. End Property
  722. Public Function GetEnumerator() As IEnumerator(Of NetThread) Implements IEnumerable(Of NetThread).GetEnumerator
  723. Return [Next].GetEnumerator()
  724. End Function
  725. Private Function IEnumerable_GetEnumerator() As IEnumerator Implements IEnumerable.GetEnumerator
  726. Return [Next].GetEnumerator()
  727. End Function
  728. ''' <summary>
  729. ''' 分配给任务中每个线程(无论其是否失败)的编号。
  730. ''' </summary>
  731. Public Uuid As Integer
  732. ''' <summary>
  733. ''' 是否为第一个线程。
  734. ''' </summary>
  735. Public ReadOnly Property IsFirstThread As Boolean
  736. Get
  737. Return DownloadStart = 0 AndAlso Task.FileSize = -2
  738. End Get
  739. End Property
  740. ''' <summary>
  741. ''' 该线程的缓存文件。
  742. ''' </summary>
  743. Public Temp As String
  744. ''' <summary>
  745. ''' 线程下载起始位置。
  746. ''' </summary>
  747. Public DownloadStart As Long
  748. ''' <summary>
  749. ''' 线程下载结束位置。
  750. ''' </summary>
  751. Public ReadOnly Property DownloadEnd As Long
  752. Get
  753. SyncLock Task.LockChain
  754. If NextThread Is Nothing Then
  755. If Task.IsUnknownSize Then
  756. Return 5 * 1024 * 1024 * 1024L '5G
  757. Else
  758. Return Task.FileSize - 1
  759. End If
  760. Else
  761. Return NextThread.DownloadStart - 1
  762. End If
  763. End SyncLock
  764. End Get
  765. End Property
  766. ''' <summary>
  767. ''' 线程未下载的文件大小。
  768. ''' </summary>
  769. Public ReadOnly Property DownloadUndone As Long
  770. Get
  771. Return DownloadEnd - (DownloadStart + DownloadDone) + 1
  772. End Get
  773. End Property
  774. ''' <summary>
  775. ''' 线程已下载的文件大小。
  776. ''' </summary>
  777. Public DownloadDone As Long = 0
  778. ''' <summary>
  779. ''' 上次记速时的时间。
  780. ''' </summary>
  781. Private SpeedLastTime As Long = GetTimeTick()
  782. ''' <summary>
  783. ''' 上次记速时的已下载大小。
  784. ''' </summary>
  785. Private SpeedLastDone As Long = 0
  786. ''' <summary>
  787. ''' 当前的下载速度,单位为 Byte / 秒。
  788. ''' </summary>
  789. Public ReadOnly Property Speed As Long
  790. Get
  791. If GetTimeTick() - SpeedLastTime > 200 Then
  792. Dim DeltaTime As Long = GetTimeTick() - SpeedLastTime
  793. _Speed = (DownloadDone - SpeedLastDone) / (DeltaTime / 1000)
  794. SpeedLastDone = DownloadDone
  795. SpeedLastTime += DeltaTime
  796. End If
  797. Return _Speed
  798. End Get
  799. End Property
  800. Private _Speed As Long = 0
  801. ''' <summary>
  802. ''' 线程初始化时的时间。
  803. ''' </summary>
  804. Public InitTime As Long = GetTimeTick()
  805. ''' <summary>
  806. ''' 上次接受到有效数据的时间,-1 表示尚未有有效数据。
  807. ''' </summary>
  808. Public LastReceiveTime As Long = -1
  809. ''' <summary>
  810. ''' 当前线程的状态。
  811. ''' </summary>
  812. Public State As NetState = NetState.WaitForDownload
  813. ''' <summary>
  814. ''' 是否已经结束。
  815. ''' </summary>
  816. Public ReadOnly Property IsEnded As Boolean
  817. Get
  818. Return State = NetState.Finish OrElse State = NetState.Error
  819. End Get
  820. End Property
  821. ''' <summary>
  822. ''' 当前选取的是哪一个 Url。
  823. ''' </summary>
  824. Public Source As NetSource
  825. End Class
  826. ''' <summary>
  827. ''' 下载单个文件。
  828. ''' </summary>
  829. Public Class NetFile
  830. #Region "属性"
  831. ''' <summary>
  832. ''' 所属的文件列表任务。
  833. ''' </summary>
  834. Public Tasks As New SafeList(Of LoaderDownload)
  835. ''' <summary>
  836. ''' 所有下载源。
  837. ''' </summary>
  838. Public Sources As SafeList(Of NetSource)
  839. ''' <summary>
  840. ''' 用于在第一个线程出错时切换下载源。
  841. ''' </summary>
  842. Private FirstThreadSource As Integer = 0
  843. ''' <summary>
  844. ''' 所有已经被标记为失败的,但未完整尝试过的,不允许断点续传的下载源。
  845. ''' </summary>
  846. Public SourcesOnce As New SafeList(Of NetSource)
  847. ''' <summary>
  848. ''' 获取从某个源开始,第一个可用的源。
  849. ''' </summary>
  850. Private Function GetSource(Optional Id As Integer = 0) As NetSource
  851. If Id >= Sources.Count OrElse Id < 0 Then Id = 0
  852. SyncLock LockSource
  853. If Not IsSourceFailed(False) Then
  854. '存在多线程可用源
  855. Dim CurrentSource As NetSource = Sources(Id)
  856. While CurrentSource.IsFailed
  857. Id += 1
  858. If Id >= Sources.Count Then Id = 0
  859. CurrentSource = Sources(Id)
  860. End While
  861. Return CurrentSource
  862. ElseIf SourcesOnce.Any Then
  863. '仅存在单线程可用源
  864. Return SourcesOnce(0)
  865. Else
  866. '没有可用源
  867. Return Nothing
  868. End If
  869. End SyncLock
  870. End Function
  871. ''' <summary>
  872. ''' 是否已经没有可用源了。
  873. ''' </summary>
  874. Public Function IsSourceFailed(Optional AllowOnceSource As Boolean = True) As Boolean
  875. If AllowOnceSource AndAlso SourcesOnce.Any Then Return False
  876. SyncLock LockSource
  877. For Each Source As NetSource In Sources
  878. If Not Source.IsFailed Then Return False
  879. Next
  880. End SyncLock
  881. Return True
  882. End Function
  883. ''' <summary>
  884. ''' 存储在本地的带文件名的地址。
  885. ''' </summary>
  886. Public LocalPath As String = Nothing
  887. ''' <summary>
  888. ''' 存储在本地的文件名。
  889. ''' </summary>
  890. Public LocalName As String = Nothing
  891. ''' <summary>
  892. ''' 当前的下载状态。
  893. ''' </summary>
  894. Public State As NetState = NetState.WaitForCheck
  895. ''' <summary>
  896. ''' 导致下载失败的原因。
  897. ''' </summary>
  898. Public Ex As New List(Of Exception)
  899. ''' <summary>
  900. ''' 作为文件组成部分的线程链表。
  901. ''' 如果没有线程,可以为 Nothing。
  902. ''' </summary>
  903. Public Threads As NetThread
  904. ''' <summary>
  905. ''' 文件的总大小。若为 -2 则为未获取,若为 -1 则为无法获取准确大小。
  906. ''' </summary>
  907. Public FileSize As Long = -2
  908. ''' <summary>
  909. ''' 该文件是否无法获取准确大小。
  910. ''' </summary>
  911. Public IsUnknownSize As Boolean = False
  912. ''' <summary>
  913. ''' 该文件是否不需要分割。
  914. ''' </summary>
  915. Public ReadOnly Property IsNoSplit As Boolean
  916. Get
  917. Return IsUnknownSize OrElse FileSize < FilePieceLimit
  918. End Get
  919. End Property
  920. ''' <summary>
  921. ''' 为不需要分割的小文件进行临时存储。
  922. ''' </summary>
  923. Private SmailFileCache As Queue(Of Byte)
  924. ''' <summary>
  925. ''' 文件的已下载大小。
  926. ''' </summary>
  927. Public DownloadDone As Long = 0
  928. Private ReadOnly LockDone As New Object
  929. ''' <summary>
  930. ''' 文件的校验规则。
  931. ''' </summary>
  932. Public Check As FileChecker
  933. ''' <summary>
  934. ''' 下载时是否添加浏览器 UA。
  935. ''' </summary>
  936. Public UseBrowserUserAgent As Boolean
  937. ''' <summary>
  938. ''' 上次记速时的时间。
  939. ''' </summary>
  940. Private SpeedLastTime As Long = GetTimeTick()
  941. ''' <summary>
  942. ''' 上次记速时的已下载大小。
  943. ''' </summary>
  944. Private SpeedLastDone As Long = 0
  945. ''' <summary>
  946. ''' 当前的下载速度,单位为 Byte / 秒。
  947. ''' </summary>
  948. Public ReadOnly Property Speed As Long
  949. Get
  950. If GetTimeTick() - SpeedLastTime > 200 Then
  951. Dim DeltaTime As Long = GetTimeTick() - SpeedLastTime
  952. _Speed = (DownloadDone - SpeedLastDone) / (DeltaTime / 1000)
  953. SpeedLastDone = DownloadDone
  954. SpeedLastTime += DeltaTime
  955. End If
  956. Return _Speed
  957. End Get
  958. End Property
  959. Private _Speed As Long = 0
  960. ''' <summary>
  961. ''' 该文件是否由本地文件直接拷贝完成。
  962. ''' </summary>
  963. Public IsCopy As Boolean = False
  964. ''' <summary>
  965. ''' 本文件的显示进度。
  966. ''' </summary>
  967. Public ReadOnly Property Progress As Double
  968. Get
  969. Select Case State
  970. Case NetState.WaitForCheck
  971. Return 0
  972. Case NetState.WaitForCopy
  973. Return 0.2
  974. Case NetState.WaitForDownload
  975. Return 0.01
  976. Case NetState.Connect
  977. Return 0.02
  978. Case NetState.Get
  979. Return 0.04
  980. Case NetState.Download
  981. '正在下载中,对应 5% ~ 98%
  982. Dim OriginalProgress As Double = If(IsUnknownSize, 0.5, DownloadDone / Math.Max(FileSize, 1))
  983. OriginalProgress = 1 - (1 - OriginalProgress) ^ 0.9
  984. Return OriginalProgress * 0.93 + 0.05
  985. Case NetState.Merge
  986. Return 0.99
  987. Case NetState.Finish, NetState.Error
  988. Return 1
  989. Case Else
  990. Throw New ArgumentOutOfRangeException("文件状态未知:" & State)
  991. End Select
  992. End Get
  993. End Property
  994. ''' <summary>
  995. ''' 各个线程建立连接成功的总次数。
  996. ''' </summary>
  997. Private ConnectCount As Integer = 0
  998. ''' <summary>
  999. ''' 各个线程建立连接成功的总时间。
  1000. ''' </summary>
  1001. Private ConnectTime As Long = 0
  1002. ''' <summary>
  1003. ''' 各个线程建立连接成功的平均时间,单位为毫秒,-1 代表尚未有成功连接。
  1004. ''' </summary>
  1005. Private ReadOnly Property ConnectAverage As Integer
  1006. Get
  1007. SyncLock LockCount
  1008. Return If(ConnectCount = 0, -1, ConnectTime / ConnectCount)
  1009. End SyncLock
  1010. End Get
  1011. End Property
  1012. Private Const FilePieceLimit As Long = 256 * 1024
  1013. Public ReadOnly LockCount As New Object
  1014. Public ReadOnly LockState As New Object
  1015. Public ReadOnly LockChain As New Object
  1016. Public ReadOnly LockSource As New Object
  1017. Public ReadOnly Uuid As Integer = GetUuid()
  1018. Public Overrides Function Equals(obj As Object) As Boolean
  1019. Dim file = TryCast(obj, NetFile)
  1020. Return file IsNot Nothing AndAlso Uuid = file.Uuid
  1021. End Function
  1022. #End Region
  1023. ''' <summary>
  1024. ''' 新建一个需要下载的文件。
  1025. ''' </summary>
  1026. ''' <param name="LocalPath">包含文件名的本地地址。</param>
  1027. Public Sub New(Urls As IEnumerable(Of String), LocalPath As String, Optional Check As FileChecker = Nothing, Optional UseBrowserUserAgent As Boolean = False)
  1028. Dim Sources As New List(Of NetSource)
  1029. Dim Count As Integer = 0
  1030. Urls = Urls.Distinct.ToArray
  1031. For Each Source As String In Urls
  1032. Sources.Add(New NetSource With {.FailCount = 0, .Url = SecretCdnSign(Source.Replace(vbCr, "").Replace(vbLf, "").Trim), .Id = Count, .IsFailed = False, .Ex = Nothing})
  1033. Count += 1
  1034. Next
  1035. Me.Sources = Sources
  1036. Me.LocalPath = LocalPath
  1037. Me.Check = Check
  1038. Me.UseBrowserUserAgent = UseBrowserUserAgent
  1039. Me.LocalName = GetFileNameFromPath(LocalPath)
  1040. End Sub
  1041. ''' <summary>
  1042. ''' 尝试开始一个新的下载线程。
  1043. ''' 如果失败,返回 Nothing。
  1044. ''' </summary>
  1045. Public Function TryBeginThread() As NetThread
  1046. Try
  1047. '条件检测
  1048. If NetTaskThreadCount >= NetTaskThreadLimit OrElse IsSourceFailed() OrElse
  1049. (IsNoSplit AndAlso Threads IsNot Nothing AndAlso Threads.State <> NetState.Error) Then Return Nothing
  1050. If State >= NetState.Merge OrElse State = NetState.WaitForCheck Then Return Nothing
  1051. SyncLock LockState
  1052. If State < NetState.Connect Then State = NetState.Connect
  1053. End SyncLock
  1054. '初始化参数
  1055. Dim StartPosition As Long, StartSource As NetSource = Nothing
  1056. Dim Th As Thread, ThreadInfo As NetThread
  1057. SyncLock LockChain
  1058. '获取线程起点与下载源
  1059. '不分割
  1060. If IsNoSplit Then GoTo Capture
  1061. '单线程
  1062. If IsSourceFailed(False) Then
  1063. '确认没有其他线程正使用此点
  1064. If SourcesOnce(0).Thread IsNot Nothing AndAlso SourcesOnce(0).Thread.State <> NetState.Error Then Return Nothing
  1065. '占用此点
  1066. Capture:
  1067. SmailFileCache = Nothing
  1068. Threads = Nothing
  1069. NetManager.DownloadDone -= DownloadDone
  1070. SyncLock LockDone
  1071. DownloadDone = 0
  1072. End SyncLock
  1073. SpeedLastDone = 0
  1074. State = NetState.Get
  1075. End If
  1076. '首个开始点
  1077. If Threads Is Nothing Then
  1078. StartPosition = 0
  1079. StartSource = GetSource(FirstThreadSource)
  1080. FirstThreadSource = StartSource.Id + 1
  1081. GoTo StartThread
  1082. End If
  1083. '寻找失败点
  1084. For Each Thread As NetThread In Threads
  1085. If Thread.State = NetState.Error AndAlso Thread.DownloadUndone > 0 Then
  1086. StartPosition = Thread.DownloadStart + Thread.DownloadDone
  1087. StartSource = GetSource(Thread.Source.Id + 1)
  1088. GoTo StartThread
  1089. End If
  1090. Next
  1091. '是否禁用多线程,以及规定碎片大小
  1092. Dim TargetUrl As String = GetSource().Url
  1093. If TargetUrl.Contains("pcl2-server") OrElse TargetUrl.Contains("bmclapi") OrElse TargetUrl.Contains("github.com") OrElse
  1094. TargetUrl.Contains("optifine.net") OrElse TargetUrl.Contains("modrinth") Then Return Nothing
  1095. '寻找最大碎片
  1096. 'FUTURE: 下载引擎重做,计算下载源平均链接时间和线程下载速度,按最高时间节省来开启多线程
  1097. Dim FilePieceMax As NetThread = Threads
  1098. For Each Thread As NetThread In Threads
  1099. If Thread.DownloadUndone > FilePieceMax.DownloadUndone Then FilePieceMax = Thread
  1100. Next
  1101. If FilePieceMax Is Nothing OrElse FilePieceMax.DownloadUndone < FilePieceLimit Then Return Nothing
  1102. StartPosition = FilePieceMax.DownloadEnd - FilePieceMax.DownloadUndone * 0.4
  1103. StartSource = GetSource()
  1104. '开始线程
  1105. StartThread:
  1106. If (StartPosition > FileSize AndAlso FileSize >= 0 AndAlso Not IsUnknownSize) OrElse StartPosition < 0 OrElse IsNothing(StartSource) Then Return Nothing
  1107. '构建线程
  1108. Dim ThreadUuid As Integer = GetUuid()
  1109. If Not Tasks.Any() Then Return Nothing '由于中断,已没有可用任务
  1110. Th = New Thread(AddressOf Thread) With {.Name = $"NetTask {Tasks(0).Uuid}/{Uuid} Download {ThreadUuid}#", .Priority = ThreadPriority.BelowNormal}
  1111. ThreadInfo = New NetThread With {.Uuid = ThreadUuid, .DownloadStart = StartPosition, .Thread = Th, .Source = StartSource, .Task = Me, .State = NetState.WaitForDownload}
  1112. '链表处理
  1113. If ThreadInfo.IsFirstThread OrElse Threads Is Nothing Then
  1114. Threads = ThreadInfo
  1115. Else
  1116. Dim CurrentChain As NetThread = Threads
  1117. While CurrentChain.DownloadEnd <= StartPosition
  1118. CurrentChain = CurrentChain.NextThread
  1119. End While
  1120. ThreadInfo.NextThread = CurrentChain.NextThread
  1121. CurrentChain.NextThread = ThreadInfo
  1122. End If
  1123. End SyncLock
  1124. '开始线程
  1125. SyncLock NetTaskThreadCountLock
  1126. NetTaskThreadCount += 1
  1127. End SyncLock
  1128. SyncLock LockSource
  1129. If IsSourceFailed(False) Then SourcesOnce(0).Thread = ThreadInfo
  1130. End SyncLock
  1131. Th.Start(ThreadInfo)
  1132. Return ThreadInfo
  1133. Catch ex As Exception
  1134. Log(ex, "尝试开始下载线程失败(" & If(LocalName, "Nothing") & ")", LogLevel.Hint)
  1135. Return Nothing
  1136. End Try
  1137. End Function
  1138. ''' <summary>
  1139. ''' 每个下载线程执行的代码。
  1140. ''' </summary>
  1141. Private Sub Thread(Info As NetThread)
  1142. If ModeDebug OrElse Info.DownloadStart = 0 Then Log("[Download] " & LocalName & " " & Info.Uuid & "#:开始,起始点 " & Info.DownloadStart & "," & Info.Source.Url)
  1143. Dim HttpRequest As HttpRequestMessage
  1144. Dim ResultStream As Stream = Nothing
  1145. Dim HttpResponse As HttpResponseMessage
  1146. '部分下载源真的特别慢,并且只需要一个请求,例如 Ping 为 20s,如果增长太慢,就会造成类似 2.5s 5s 7.5s 10s 12.5s... 的极大延迟
  1147. '延迟过长会导致某些特别慢的链接迟迟不被掐死
  1148. Dim Timeout As Integer = Math.Min(Math.Max(ConnectAverage, 6000) * (1 + Info.Source.FailCount), 30000)
  1149. Info.State = NetState.Connect
  1150. Try
  1151. Dim HttpDataCount As Integer = 0
  1152. If SourcesOnce.Contains(Info.Source) AndAlso Not Info.Equals(Info.Source.Thread) Then GoTo SourceBreak
  1153. HttpRequest = Request.GetRequestMessage(Info.Source.Url,
  1154. RequireHeaderSign:=True)
  1155. '可能出现结束减去起始等于或者小于 0 的情况,导致下载抛出 ArgumentOutOfRangeException
  1156. If Not Info.DownloadEnd - Info.DownloadEnd <= 0 Then HttpRequest.Headers.Range = New RangeHeaderValue(Info.DownloadStart, Info.DownloadEnd)
  1157. HttpResponse = Request.SendRequest(HttpRequest, False, Timeout:=Timeout, RequireReturnRespObj:=True)
  1158. Dim ContentLength As Long = 0
  1159. Using HttpResponse
  1160. If State = NetState.Error Then GoTo SourceBreak '快速中断
  1161. If ModeDebug AndAlso HttpResponse.RequestMessage.RequestUri.ToString <> Info.Source.Url Then
  1162. Log($"[Download] {LocalName} {Info.Uuid}#:重定向至 {HttpResponse.RequestMessage.RequestUri.ToString}")
  1163. End If
  1164. ''从响应头获取文件名
  1165. 'If Info.IsFirstThread Then
  1166. ' Dim FileName As String = GetFileNameFromResponse(HttpResponse)
  1167. ' If ModeDebug Then Log($"[Download] {LocalName} {Info.Uuid}#:远程文件名:{If(FileName, "未提供")}")
  1168. ' If FileName IsNot Nothing AndAlso LocalName = "待定" Then
  1169. ' LocalName = FileName
  1170. ' Log($"[Download] {LocalName} {Info.Uuid}#:从响应头获取到文件名")
  1171. ' End If
  1172. 'End If
  1173. '文件大小校验
  1174. ContentLength = HttpResponse.Content.Headers.ContentLength.GetValueOrDefault(-1)
  1175. If ContentLength = -1 Then
  1176. If FileSize > 1 Then
  1177. If Info.DownloadStart = 0 Then
  1178. Log($"[Download] {LocalName} {Info.Uuid}#:文件大小未知,但已从其他下载源获取,不作处理")
  1179. Else
  1180. Log($"[Download] {LocalName} {Info.Uuid}#:ContentLength 返回了 -1,无法确定是否支持分段下载,视作不支持")
  1181. GoTo NotSupportRange
  1182. End If
  1183. Else
  1184. FileSize = -1 : IsUnknownSize = True
  1185. Log($"[Download] {LocalName} {Info.Uuid}#:文件大小未知")
  1186. End If
  1187. ElseIf ContentLength < 0 Then
  1188. Throw New Exception("获取片大小失败,结果为 " & ContentLength & "。")
  1189. ElseIf Info.IsFirstThread Then
  1190. If Check IsNot Nothing Then
  1191. If ContentLength < Check.MinSize AndAlso Check.MinSize > 0 Then
  1192. Throw New Exception($"文件大小不足,获取结果为 {ContentLength},要求至少为 {Check.MinSize}。")
  1193. End If
  1194. If ContentLength <> Check.ActualSize AndAlso Check.ActualSize > 0 Then
  1195. 'Throw New Exception($"文件大小不一致,获取结果为 {ContentLength},要求必须为 {Check.ActualSize}。")
  1196. End If
  1197. End If
  1198. FileSize = ContentLength : IsUnknownSize = False
  1199. Log($"[Download] {LocalName} {Info.Uuid}#:文件大小 {ContentLength}({GetString(ContentLength)})")
  1200. '若文件大小大于 50 M,进行剩余磁盘空间校验
  1201. If ContentLength > 50 * 1024 * 1024 Then
  1202. For Each Drive As DriveInfo In DriveInfo.GetDrives
  1203. Dim DriveName As String = Drive.Name.First.ToString
  1204. Dim RequiredSpace = If(PathTemp.StartsWithF(DriveName), ContentLength * 1.1, 0) +
  1205. If(LocalPath.StartsWithF(DriveName), ContentLength + 5 * 1024 * 1024, 0)
  1206. If Drive.TotalFreeSpace < RequiredSpace Then
  1207. Throw New Exception(DriveName & " 盘空间不足,无法进行下载。" & vbCrLf & "需要至少 " & GetString(RequiredSpace) & " 空间,但当前仅剩余 " & GetString(Drive.TotalFreeSpace) & "。" &
  1208. If(PathTemp.StartsWithF(DriveName), vbCrLf & vbCrLf & "下载时需要与文件同等大小的空间存放缓存,你可以在设置中调整缓存文件夹的位置。", ""))
  1209. End If
  1210. Next
  1211. End If
  1212. ElseIf FileSize < 0 Then
  1213. Throw New Exception("非首线程运行时,尚未获取文件大小")
  1214. ElseIf Info.DownloadStart > 0 AndAlso ContentLength = FileSize Then
  1215. NotSupportRange:
  1216. SyncLock LockSource
  1217. If SourcesOnce.Contains(Info.Source) Then
  1218. GoTo SourceBreak
  1219. Else
  1220. SourcesOnce.Add(Info.Source)
  1221. End If
  1222. End SyncLock
  1223. Throw New WebException($"该下载源不支持分段下载:Range 起始于 {Info.DownloadStart},预期 ContentLength 为 {FileSize - Info.DownloadStart},返回 ContentLength 为 {ContentLength},总文件大小 {FileSize}")
  1224. ElseIf Not FileSize - Info.DownloadStart = ContentLength Then
  1225. 'Throw New WebException($"获取到的分段大小不一致:Range 起始于 {Info.DownloadStart},预期 ContentLength 为 {FileSize - Info.DownloadStart},返回 ContentLength 为 {ContentLength},总文件大小 {FileSize}")
  1226. End If
  1227. 'Log($"[Download] {LocalName} {Info.Uuid}#:通过大小检查,文件大小 {FileSize},起始点 {Info.DownloadStart},ContentLength {ContentLength}")
  1228. Info.State = NetState.Get
  1229. SyncLock LockState
  1230. If State < NetState.Get Then State = NetState.Get
  1231. End SyncLock
  1232. '创建缓存文件
  1233. If IsNoSplit Then
  1234. Info.Temp = Nothing
  1235. SmailFileCache = New Queue(Of Byte)
  1236. Else
  1237. Info.Temp = $"{PathTemp}Download\{Uuid}_{Info.Uuid}_{RandomInteger(0, 999999)}.tmp"
  1238. ResultStream = New FileStream(Info.Temp, FileMode.Create, FileAccess.Write, FileShare.Read)
  1239. End If
  1240. '开始下载
  1241. Using HttpStream = HttpResponse.Content.ReadAsStreamAsync().GetAwaiter().GetResult()
  1242. If Setup.Get("SystemDebugDelay") Then Threading.Thread.Sleep(RandomInteger(50, 3000))
  1243. Dim HttpData As Byte() = New Byte(16384) {}
  1244. HttpDataCount = HttpStream.Read(HttpData, 0, 16384)
  1245. While (IsUnknownSize OrElse Info.DownloadUndone > 0) AndAlso '判断是否下载完成
  1246. HttpDataCount > 0 AndAlso Not IsProgramEnded AndAlso State < NetState.Merge AndAlso (Not Info.Source.IsFailed OrElse Info.Equals(Info.Source.Thread))
  1247. '限速
  1248. While NetTaskSpeedLimitHigh > 0 AndAlso NetTaskSpeedLimitLeft <= 0
  1249. Threading.Thread.Sleep(16)
  1250. End While
  1251. Dim RealDataCount As Integer = If(IsUnknownSize, HttpDataCount, Math.Min(HttpDataCount, Info.DownloadUndone))
  1252. SyncLock NetTaskSpeedLimitLeftLock
  1253. If NetTaskSpeedLimitHigh > 0 Then NetTaskSpeedLimitLeft -= RealDataCount
  1254. End SyncLock
  1255. Dim DeltaTime = GetTimeTick() - Info.LastReceiveTime
  1256. If DeltaTime > 1000000 Then DeltaTime = 1 '时间刻反转导致出现极大值
  1257. If RealDataCount > 0 Then
  1258. '有数据
  1259. If Info.DownloadDone = 0 Then
  1260. '第一次接受到数据
  1261. Info.State = NetState.Download
  1262. SyncLock LockState
  1263. If State < NetState.Download Then State = NetState.Download
  1264. End SyncLock
  1265. SyncLock LockCount
  1266. ConnectCount += 1
  1267. ConnectTime += GetTimeTick() - Info.InitTime
  1268. End SyncLock
  1269. End If
  1270. SyncLock LockCount
  1271. Info.Source.FailCount = 0
  1272. For Each Task In Tasks
  1273. Task.FailCount = 0
  1274. Next
  1275. End SyncLock
  1276. NetManager.DownloadDone += RealDataCount
  1277. SyncLock LockDone
  1278. DownloadDone += RealDataCount
  1279. End SyncLock
  1280. Info.DownloadDone += RealDataCount
  1281. If IsNoSplit Then
  1282. If HttpData.Count = RealDataCount Then
  1283. 'SmailFileCache.AddRange(HttpData)
  1284. For Each B In HttpData
  1285. SmailFileCache.Enqueue(B)
  1286. Next
  1287. Else
  1288. 'SmailFileCache.AddRange(HttpData.ToList.GetRange(0, RealDataCount))
  1289. For i = 0 To RealDataCount - 1
  1290. SmailFileCache.Enqueue(HttpData(i))
  1291. Next
  1292. End If
  1293. Else
  1294. ResultStream.Write(HttpData, 0, RealDataCount)
  1295. End If
  1296. '检查速度是否过慢
  1297. If DeltaTime > 1500 AndAlso DeltaTime > RealDataCount Then '数据包间隔大于 1.5s,且速度小于 1.5K/s
  1298. Throw New TimeoutException("由于速度过慢断开链接,下载 " & RealDataCount & " B,消耗 " & DeltaTime & " ms。")
  1299. End If
  1300. Info.LastReceiveTime = GetTimeTick()
  1301. '已完成
  1302. If Info.DownloadUndone = 0 AndAlso Not IsUnknownSize Then Exit While
  1303. ElseIf Info.LastReceiveTime > 0 AndAlso DeltaTime > Timeout Then
  1304. '无数据,且已超时
  1305. Throw New TimeoutException("操作超时,无数据。")
  1306. End If
  1307. HttpDataCount = HttpStream.Read(HttpData, 0, 16384)
  1308. End While
  1309. End Using
  1310. End Using
  1311. SourceBreak:
  1312. If State = NetState.Error OrElse Info.Source.IsFailed OrElse (Info.DownloadUndone > 0 AndAlso Not IsUnknownSize) Then
  1313. '被外部中断
  1314. Info.State = NetState.Error
  1315. Log($"[Download] {LocalName} {Info.Uuid}#:中断")
  1316. ElseIf HttpDataCount = 0 AndAlso Info.DownloadUndone > 0 AndAlso Not IsUnknownSize Then
  1317. '服务器无返回数据
  1318. Throw New Exception($"返回的 ContentLength 过多:ContentLength 为 {ContentLength},但获取到的总数据量仅为 {Info.DownloadDone}(全文件总数据量 {DownloadDone})")
  1319. Else
  1320. '本线程完成
  1321. Info.State = NetState.Finish
  1322. If ModeDebug Then Log($"[Download] {LocalName} {Info.Uuid}#:完成,已下载 {Info.DownloadDone}")
  1323. End If
  1324. Catch ex As Exception
  1325. '状态变更
  1326. SyncLock LockCount
  1327. Info.Source.FailCount += 1
  1328. For Each Task In Tasks
  1329. Task.FailCount += 1
  1330. Next
  1331. End SyncLock
  1332. Dim IsTimeoutString As String = GetExceptionSummary(ex).ToLower.Replace(" ", "")
  1333. Dim IsTimeout As Boolean = IsTimeoutString.Contains("由于连接方在一段时间后没有正确答复或连接的主机没有反应") OrElse
  1334. IsTimeoutString.Contains("超时") OrElse IsTimeoutString.Contains("timeout") OrElse IsTimeoutString.Contains("timedout")
  1335. Log("[Download] " & LocalName & " " & Info.Uuid & If(IsTimeout, "#:超时(" & (Timeout * 0.001) & "s)", "#:出错," & GetExceptionDetail(ex)))
  1336. Info.State = NetState.Error
  1337. ''使用该下载源的线程是否没有速度
  1338. ''下载超时也会导致没有速度,容易误判下载失败,所以已弃用此方法
  1339. 'Dim IsNoSpeed As Boolean = True
  1340. 'SyncLock LockChain
  1341. ' If Threads IsNot Nothing Then
  1342. ' For Each Thread As NetThread In Threads
  1343. ' If Thread.Source.Id = Info.Source.Id AndAlso Thread.Speed > 0 Then
  1344. ' IsNoSpeed = False
  1345. ' Exit For
  1346. ' End If
  1347. ' Next
  1348. ' End If
  1349. 'End SyncLock
  1350. Info.Source.Ex = ex
  1351. '根据情况判断,是否在多线程下禁用下载源(连续错误过多,或不支持断点续传)
  1352. If ex.Message.Contains("该下载源不支持") OrElse ex.Message.Contains("未能解析") OrElse ex.Message.Contains("(404)") OrElse
  1353. ex.Message.Contains("(502)") OrElse ex.Message.Contains("无返回数据") OrElse ex.Message.Contains("空间不足") OrElse ex.Message.Contains("获取到的分段大小不一致") OrElse
  1354. (ex.Message.Contains("(403)") AndAlso Not Info.Source.Url.ContainsF("bmclapi")) OrElse 'BMCLAPI 的部分源在高频率请求下会返回 403,所以不应因此禁用下载源
  1355. (Info.Source.FailCount >= MathClamp(NetTaskThreadLimit, 5, 30) AndAlso DownloadDone < 1) OrElse
  1356. Info.Source.FailCount > NetTaskThreadLimit + 2 Then
  1357. Dim IsThisFail As Boolean = False
  1358. SyncLock LockSource
  1359. If Info.Source.Thread IsNot Nothing AndAlso Info.Source.Thread.Equals(Info) Then
  1360. '单线程下,本线程出错
  1361. SourcesOnce.RemoveAt(0)
  1362. GoTo Wrong
  1363. ElseIf Not Info.Source.IsFailed Then
  1364. '多线程下,本线程出错
  1365. Wrong:
  1366. Info.Source.IsFailed = True
  1367. IsThisFail = True
  1368. End If
  1369. End SyncLock
  1370. '本线程引发下载源被禁用
  1371. If IsThisFail Then
  1372. Log($"[Download] {LocalName} {Info.Uuid}#:下载源被禁用({Info.Source.Id}):{Info.Source.Url}")
  1373. Log(ex, "下载源 " & Info.Source.Id & " 已被禁用", If(ex.Message.Contains("不支持分段下载") OrElse ex.Message.Contains("(404)") OrElse ex.Message.Contains("(416)"), LogLevel.Developer, LogLevel.Debug))
  1374. If IsSourceFailed() Then
  1375. '没有可用源
  1376. Log("[Download] 文件 " & LocalName & " 已无可用下载源")
  1377. Dim ExampleEx As Exception = Nothing
  1378. SyncLock LockSource
  1379. For Each Source As NetSource In Sources
  1380. Log("[Download] 已禁用的下载源:" & Source.Url)
  1381. If Source.Ex IsNot Nothing Then
  1382. ExampleEx = Source.Ex
  1383. Log(Source.Ex, "下载源禁用原因", LogLevel.Developer)
  1384. End If
  1385. Next
  1386. End SyncLock
  1387. Fail(ExampleEx)
  1388. ElseIf ex.Message.Contains("空间不足") Then
  1389. '没有空间
  1390. Fail(ex)
  1391. End If
  1392. End If
  1393. End If
  1394. '首线程错误
  1395. If FileSize = -2 Then
  1396. SyncLock LockChain
  1397. Threads = Nothing
  1398. End SyncLock
  1399. End If
  1400. Finally
  1401. If ResultStream IsNot Nothing Then ResultStream.Dispose()
  1402. SyncLock NetTaskThreadCountLock
  1403. NetTaskThreadCount -= 1
  1404. End SyncLock
  1405. '可能在没有下载完的时候开始合并文件了,这造成了大多数合并失败
  1406. If ((FileSize >= 0 AndAlso DownloadDone >= FileSize) OrElse (FileSize = -1 AndAlso DownloadDone > 0)) AndAlso State < NetState.Merge Then Merge()
  1407. End Try
  1408. End Sub
  1409. ''' <summary>
  1410. ''' 从 HTTP 响应头中获取文件名。
  1411. ''' 如果没有,返回 Nothing。
  1412. ''' </summary>
  1413. Private Function GetFileNameFromResponse(response As HttpWebResponse) As String
  1414. Dim header As String = response.Headers("Content-Disposition")
  1415. If String.IsNullOrEmpty(header) Then Return Nothing
  1416. 'attachment; filename="filename.ext"
  1417. If Not header.Contains("filename=") Then Return Nothing
  1418. Return header.AfterLast("filename=").Trim(""""c, " "c).BeforeFirst(";")
  1419. End Function
  1420. '下载文件的最终收束事件
  1421. ''' <summary>
  1422. ''' 下载完成。合并文件。
  1423. ''' </summary>
  1424. Private Sub Merge()
  1425. '状态判断
  1426. SyncLock LockState
  1427. If State < NetState.Merge Then
  1428. State = NetState.Merge
  1429. Else
  1430. Exit Sub
  1431. End If
  1432. End SyncLock
  1433. Dim RetryCount As Integer = 0
  1434. Dim MergeFile As Stream = Nothing, AddWriter As BinaryWriter = Nothing
  1435. Try
  1436. Retry:
  1437. SyncLock LockChain
  1438. '创建文件夹
  1439. If File.Exists(LocalPath) Then File.Delete(LocalPath)
  1440. Dim Info As New FileInfo(LocalPath)
  1441. Info.Directory.Create()
  1442. '合并文件
  1443. If IsNoSplit Then
  1444. '仅有一个线程,从缓存中输出
  1445. If ModeDebug Then Log($"[Download] {LocalName}:下载结束,从缓存输出文件,长度:" & SmailFileCache.Count)
  1446. MergeFile = New FileStream(LocalPath, FileMode.Create)
  1447. AddWriter = New BinaryWriter(MergeFile)
  1448. AddWriter.Write(SmailFileCache.ToArray)
  1449. AddWriter.Dispose() : AddWriter = Nothing
  1450. MergeFile.Dispose() : MergeFile = Nothing
  1451. ElseIf Threads.DownloadDone = DownloadDone AndAlso Threads.Temp IsNot Nothing Then
  1452. '仅有一个文件,直接复制
  1453. If ModeDebug Then Log($"[Download] {LocalName}:下载结束,仅有一个文件,无需合并")
  1454. CopyFile(Threads.Temp, LocalPath)
  1455. Else
  1456. '有多个线程,合并
  1457. If ModeDebug Then Log($"[Download] {LocalName}:下载结束,开始合并文件")
  1458. MergeFile = New FileStream(LocalPath, FileMode.Create)
  1459. AddWriter = New BinaryWriter(MergeFile)
  1460. For Each Thread As NetThread In Threads
  1461. If Thread.DownloadDone = 0 OrElse Thread.Temp Is Nothing Then Continue For
  1462. Using fs As New FileStream(Thread.Temp, FileMode.Open, FileAccess.Read, FileShare.Read)
  1463. Using TempReader As New BinaryReader(fs)
  1464. AddWriter.Write(TempReader.ReadBytes(Thread.DownloadDone))
  1465. End Using
  1466. End Using
  1467. Next
  1468. AddWriter.Dispose() : AddWriter = Nothing
  1469. MergeFile.Dispose() : MergeFile = Nothing
  1470. End If
  1471. '写入大小要求
  1472. If Not IsUnknownSize AndAlso Check IsNot Nothing Then
  1473. If Check.ActualSize = -1 Then
  1474. Check.ActualSize = FileSize
  1475. ElseIf Check.ActualSize <> FileSize Then
  1476. Throw New Exception($"文件大小不一致:任务要求为 {Check.ActualSize} B,网络获取结果为 {FileSize}B")
  1477. End If
  1478. End If
  1479. '检查文件
  1480. Dim CheckResult As String = Check?.Check(LocalPath)
  1481. If CheckResult IsNot Nothing Then
  1482. Log($"[Download] {LocalName} 文件校验失败,下载线程细节:")
  1483. For Each Th As NetThread In Threads
  1484. Log($"[Download] {Th.Uuid}#,状态 {GetStringFromEnum(Th.State)},范围 {Th.DownloadStart}~{Th.DownloadStart + Th.DownloadDone},完成 {Th.DownloadDone},剩余 {Th.DownloadUndone}")
  1485. Next
  1486. Throw New Exception(CheckResult)
  1487. End If
  1488. '后处理
  1489. If IsNoSplit Then
  1490. SmailFileCache = Nothing
  1491. Else
  1492. For Each Thread As NetThread In Threads
  1493. If Thread.Temp IsNot Nothing Then File.Delete(Thread.Temp)
  1494. Next
  1495. End If
  1496. Finish()
  1497. End SyncLock
  1498. Catch ex As Exception
  1499. Log(ex, "合并文件出错(" & LocalName & ")")
  1500. If MergeFile IsNot Nothing Then
  1501. MergeFile.Dispose() : MergeFile = Nothing
  1502. End If
  1503. If AddWriter IsNot Nothing Then
  1504. AddWriter.Dispose() : AddWriter = Nothing
  1505. End If
  1506. '重试
  1507. If RetryCount <= 3 Then
  1508. Threading.Thread.Sleep(RandomInteger(500, 1000))
  1509. RetryCount += 1
  1510. GoTo Retry
  1511. End If
  1512. Fail(ex)
  1513. End Try
  1514. End Sub
  1515. ''' <summary>
  1516. ''' 下载失败。
  1517. ''' </summary>
  1518. Private Sub Fail(Optional RaiseEx As Exception = Nothing)
  1519. SyncLock LockState
  1520. If State >= NetState.Finish Then Exit Sub
  1521. If RaiseEx IsNot Nothing Then Ex.Add(RaiseEx)
  1522. '凉凉
  1523. State = NetState.Error
  1524. End SyncLock
  1525. InterruptAndDelete()
  1526. For Each Task In Tasks
  1527. Task.OnFileFail(Me)
  1528. Next
  1529. End Sub
  1530. ''' <summary>
  1531. ''' 下载中断。
  1532. ''' </summary>
  1533. Public Sub Abort(CausedByTask As LoaderDownload)
  1534. '从特定任务中移除,如果它还属于其他任务,则继续下载
  1535. Tasks.Remove(CausedByTask)
  1536. If Tasks.Any Then Exit Sub
  1537. '确认中断
  1538. SyncLock LockState
  1539. If State >= NetState.Finish Then Exit Sub
  1540. State = NetState.Error
  1541. End SyncLock
  1542. InterruptAndDelete()
  1543. End Sub
  1544. Private Sub InterruptAndDelete()
  1545. On Error Resume Next
  1546. If File.Exists(LocalPath) Then File.Delete(LocalPath)
  1547. SyncLock NetManager.LockRemain
  1548. NetManager.FileRemain -= 1
  1549. Log($"[Download] {LocalName}:状态 {State},剩余文件 {NetManager.FileRemain}")
  1550. End SyncLock
  1551. End Sub
  1552. '状态改变接口
  1553. ''' <summary>
  1554. ''' 将该文件设置为已下载完成。
  1555. ''' </summary>
  1556. Public Sub Finish(Optional PrintLog As Boolean = True)
  1557. SyncLock LockState
  1558. If State >= NetState.Finish Then Exit Sub
  1559. State = NetState.Finish
  1560. End SyncLock
  1561. SyncLock NetManager.LockRemain
  1562. NetManager.FileRemain -= 1
  1563. If PrintLog Then Log("[Download] " & LocalName & ":已完成,剩余文件 " & NetManager.FileRemain)
  1564. End SyncLock
  1565. For Each Task In Tasks
  1566. Task.OnFileFinish(Me)
  1567. Next
  1568. End Sub
  1569. End Class
  1570. ''' <summary>
  1571. ''' 下载一系列文件的加载器。
  1572. ''' </summary>
  1573. Public Class LoaderDownload
  1574. Inherits LoaderBase
  1575. #Region "属性"
  1576. ''' <summary>
  1577. ''' 需要下载的文件。
  1578. ''' </summary>
  1579. Public Files As SafeList(Of NetFile)
  1580. ''' <summary>
  1581. ''' 剩余未完成的文件数。(用于减轻 FilesLock 的占用)
  1582. ''' </summary>
  1583. Private FileRemain As Integer
  1584. Private ReadOnly FileRemainLock As New Object
  1585. ''' <summary>
  1586. ''' 用于显示的百分比进度。
  1587. ''' </summary>
  1588. Public Overrides Property Progress As Double
  1589. Get
  1590. If State >= LoadState.Finished Then Return 1
  1591. If Not Files.Any() Then Return 0 '必须返回 0,否则在获取列表的时候会错觉已经下载完了
  1592. Return _Progress
  1593. End Get
  1594. Set(value As Double)
  1595. Throw New Exception("文件下载不允许指定进度")
  1596. End Set
  1597. End Property
  1598. Private _Progress As Double = 0
  1599. ''' <summary>
  1600. ''' 任务中的文件的连续失败计数。
  1601. ''' </summary>
  1602. Public Property FailCount As Integer
  1603. Get
  1604. Return _FailCount
  1605. End Get
  1606. Set(value As Integer)
  1607. _FailCount = value
  1608. If State = LoadState.Loading AndAlso value >= Math.Min(10000, Math.Max(FileRemain * 5.5, NetTaskThreadLimit * 5.5 + 3)) Then
  1609. Log("[Download] 由于同加载器中失败次数过多引发强制失败:连续失败了 " & value & " 次", LogLevel.Debug)
  1610. On Error Resume Next
  1611. Dim ExList As New List(Of Exception)
  1612. For Each File In Files
  1613. For Each Source In File.Sources
  1614. If Source.Ex IsNot Nothing Then
  1615. ExList.Add(Source.Ex)
  1616. If ExList.Count > 10 Then GoTo FinishExCatch
  1617. End If
  1618. Next
  1619. Next
  1620. FinishExCatch:
  1621. OnFail(ExList)
  1622. End If
  1623. End Set
  1624. End Property
  1625. Private _FailCount As Integer = 0
  1626. #End Region
  1627. ''' <summary>
  1628. ''' 刷新公开属性。由 NetManager 每 0.1 秒调用一次。
  1629. ''' </summary>
  1630. Public Sub RefreshStat()
  1631. '计算进度
  1632. Dim NewProgress As Double = 0
  1633. Dim TotalProgress As Double = 0
  1634. For Each File In Files
  1635. If File.IsCopy Then
  1636. NewProgress += File.Progress * 0.2
  1637. TotalProgress += 0.2
  1638. Else
  1639. NewProgress += File.Progress
  1640. TotalProgress += 1
  1641. End If
  1642. Next
  1643. If TotalProgress > 0 AndAlso Not Double.IsNaN(TotalProgress) Then NewProgress /= TotalProgress
  1644. '刷新进度
  1645. _Progress = NewProgress
  1646. End Sub
  1647. Public Sub New(Name As String, FileTasks As List(Of NetFile))
  1648. Me.Name = Name
  1649. Files = New SafeList(Of NetFile)(FileTasks)
  1650. End Sub
  1651. Public Overrides Sub Start(Optional Input As Object = Nothing, Optional IsForceRestart As Boolean = False)
  1652. If Input IsNot Nothing Then Files = New SafeList(Of NetFile)(Input)
  1653. '去重
  1654. Dim ResultArray As New SafeList(Of NetFile)
  1655. For i = 0 To Files.Count - 1
  1656. For ii = i + 1 To Files.Count - 1
  1657. If Files(i).LocalPath = Files(ii).LocalPath Then GoTo NextElement
  1658. Next
  1659. ResultArray.Add(Files(i))
  1660. NextElement:
  1661. Next
  1662. Files = ResultArray
  1663. '设置剩余文件数
  1664. SyncLock FileRemainLock
  1665. For Each File In Files
  1666. If File.State <> NetState.Finish Then FileRemain += 1
  1667. Next
  1668. End SyncLock
  1669. State = LoadState.Loading
  1670. '开始执行
  1671. RunInNewThread(
  1672. Sub()
  1673. Try
  1674. '输入检测
  1675. If Not Files.Any() Then
  1676. OnFinish()
  1677. Exit Sub
  1678. End If
  1679. For Each File As NetFile In Files
  1680. If File Is Nothing Then Throw New ArgumentException("存在空文件请求!")
  1681. For Each Source As NetSource In File.Sources
  1682. If Not (Source.Url.StartsWithF("https://", True) OrElse Source.Url.StartsWithF("http://", True)) Then
  1683. Source.Ex = New ArgumentException("输入的下载链接不正确!")
  1684. Source.IsFailed = True
  1685. End If
  1686. Next
  1687. If File.IsSourceFailed() Then Throw New ArgumentException("输入的下载链接不正确!")
  1688. If Not File.LocalPath.ToLower.Contains(":\") Then Throw New ArgumentException("输入的本地文件地址不正确!")
  1689. If File.LocalPath.EndsWithF("\") Then Throw New ArgumentException("请输入含文件名的完整文件路径!")
  1690. '文件夹检测
  1691. Dim DirPath As String = New FileInfo(File.LocalPath).Directory.FullName
  1692. If Not Directory.Exists(DirPath) Then Directory.CreateDirectory(DirPath)
  1693. Next
  1694. '接入下载管理器
  1695. NetManager.Start(Me)
  1696. '将文件分配给多个线程以进行已存在查找
  1697. Dim Folders As New List(Of String) '可能会用于已存在查找的文件夹列表
  1698. Dim FoldersFinal As New List(Of String) '最终用于查找的列表
  1699. If Not Setup.Get("SystemDebugSkipCopy") Then '在设置中禁用复制
  1700. Folders.Add(Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData) & "\.minecraft\") '总是添加官启文件夹,因为 HMCL 会把所有文件存在这里
  1701. For Each Folder In McFolderList
  1702. Folders.Add(Folder.Path)
  1703. Next
  1704. Folders = Folders.Distinct.ToList
  1705. For Each Folder In Folders
  1706. If Folder <> PathMcFolder AndAlso Directory.Exists(Folder) Then FoldersFinal.Add(Folder)
  1707. Next
  1708. End If
  1709. '最多 5 个线程,最少每个线程分配 10 个文件
  1710. Dim FilesPerThread As Integer = Math.Max(5, Files.Count / 10 + 1)
  1711. Dim FilesInThread As New List(Of NetFile)
  1712. For Each File In Files
  1713. FilesInThread.Add(File)
  1714. If FilesInThread.Count = FilesPerThread Then
  1715. Dim FilesToRun As New List(Of NetFile)
  1716. FilesToRun.AddRange(FilesInThread)
  1717. RunInNewThread(Sub() StartCopy(FilesToRun, FoldersFinal), "NetTask FileCopy " & Uuid)
  1718. FilesInThread.Clear()
  1719. End If
  1720. Next
  1721. If FilesInThread.Any Then
  1722. Dim FilesToRun As New List(Of NetFile)
  1723. FilesToRun.AddRange(FilesInThread)
  1724. RunInNewThread(Sub() StartCopy(FilesToRun, FoldersFinal), "NetTask FileCopy " & Uuid)
  1725. FilesInThread.Clear()
  1726. End If
  1727. Catch ex As Exception
  1728. OnFail(New List(Of Exception) From {ex})
  1729. End Try
  1730. End Sub, "NetTask " & Uuid & " Main")
  1731. End Sub
  1732. Private Sub StartCopy(Files As List(Of NetFile), FolderList As List(Of String))
  1733. Try
  1734. If ModeDebug Then Log("[Download] 检查线程分配文件数:" & Files.Count & ",线程名:" & Thread.CurrentThread.Name)
  1735. '试图从已存在的 Minecraft 文件夹中寻找目标文件
  1736. Dim ExistFiles As New List(Of KeyValuePair(Of NetFile, String)) '{NetFile, Target As String}
  1737. For Each File As NetFile In Files
  1738. Dim ExistFilePath As String = Nothing
  1739. '判断是否有已存在的文件
  1740. If File.Check IsNot Nothing AndAlso McFolderList IsNot Nothing AndAlso PathMcFolder IsNot Nothing AndAlso File.Check.CanUseExistsFile AndAlso File.LocalPath.StartsWithF(PathMcFolder) Then
  1741. Dim Relative = File.LocalPath.Replace(PathMcFolder, "")
  1742. For Each Folder In FolderList
  1743. Dim Target = Folder & Relative
  1744. If File.Check.Check(Target) Is Nothing Then
  1745. ExistFilePath = Target
  1746. Exit For
  1747. End If
  1748. Next
  1749. End If
  1750. '若存在,则改变状态
  1751. SyncLock LockState
  1752. If ExistFilePath IsNot Nothing Then
  1753. File.State = NetState.WaitForCopy
  1754. File.IsCopy = True
  1755. ExistFiles.Add(New KeyValuePair(Of NetFile, String)(File, ExistFilePath))
  1756. Else
  1757. File.State = NetState.WaitForDownload
  1758. File.IsCopy = False
  1759. End If
  1760. End SyncLock
  1761. Next
  1762. '复制已存在的文件
  1763. For Each FileToken In ExistFiles
  1764. Dim File As NetFile = FileToken.Key
  1765. SyncLock LockState
  1766. If File.State > NetState.WaitForCopy Then Exit Sub
  1767. End SyncLock
  1768. Dim LocalPath As String = FileToken.Value
  1769. Dim RetryCount As Integer = 0
  1770. Retry:
  1771. Try
  1772. Log("[Download] 复制已存在的文件(" & LocalPath & ")")
  1773. CopyFile(LocalPath, File.LocalPath)
  1774. File.Finish(False)
  1775. Catch ex As Exception
  1776. RetryCount += 1
  1777. Log(ex, String.Format("复制已存在的文件失败,重试第 {2} 次({0} -> {1})", LocalPath, File.LocalPath, RetryCount))
  1778. If RetryCount < 3 Then
  1779. Thread.Sleep(200)
  1780. GoTo Retry
  1781. End If
  1782. File.State = NetState.WaitForDownload
  1783. File.IsCopy = False
  1784. End Try
  1785. Next
  1786. Catch ex As Exception
  1787. Log(ex, "下载已存在文件查找失败", LogLevel.Feedback)
  1788. End Try
  1789. End Sub
  1790. Public Sub OnFileFinish(File As NetFile)
  1791. '要求全部文件完成
  1792. SyncLock FileRemainLock
  1793. FileRemain -= 1
  1794. If FileRemain > 0 Then Exit Sub
  1795. End SyncLock
  1796. OnFinish()
  1797. End Sub
  1798. Public Sub OnFinish()
  1799. RaisePreviewFinish()
  1800. SyncLock LockState
  1801. If State > LoadState.Loading Then Exit Sub
  1802. State = LoadState.Finished
  1803. End SyncLock
  1804. End Sub
  1805. Public Sub OnFileFail(File As NetFile)
  1806. '将下载源的错误加入主错误列表
  1807. For Each Source In File.Sources
  1808. If Not IsNothing(Source.Ex) Then File.Ex.Add(Source.Ex)
  1809. Next
  1810. OnFail(File.Ex)
  1811. End Sub
  1812. Public Sub OnFail(ExList As List(Of Exception))
  1813. SyncLock LockState
  1814. If State > LoadState.Loading Then Exit Sub
  1815. If ExList Is Nothing OrElse Not ExList.Any() Then ExList = New List(Of Exception) From {New Exception("未知错误!")}
  1816. '寻找第一个不是 404 的下载源
  1817. Dim UsefulExs = ExList.Where(Function(e) Not e.Message.Contains("(404)")).ToList
  1818. [Error] = If(UsefulExs.Any, UsefulExs(0), ExList(0))
  1819. '获取实际失败的文件
  1820. For Each File In Files
  1821. If File.State = NetState.Error Then
  1822. [Error] = New Exception("文件下载失败:" & File.LocalPath & vbCrLf & Join(
  1823. File.Sources.Select(Function(s) If(s.Ex Is Nothing, s.Url, s.Ex.Message & "(" & s.Url & ")")), vbCrLf), [Error])
  1824. Exit For
  1825. End If
  1826. Next
  1827. '在设置 Error 对象后再更改为失败,避免 WaitForExit 无法捕获错误
  1828. State = LoadState.Failed
  1829. End SyncLock
  1830. '中断所有文件
  1831. For Each TaskFile In Files
  1832. If TaskFile.State < NetState.Merge Then TaskFile.State = NetState.Error
  1833. Next
  1834. '在退出同步锁后再进行日志输出
  1835. Dim ErrOutput As New List(Of String)
  1836. For Each Ex As Exception In ExList
  1837. ErrOutput.Add(GetExceptionDetail(Ex))
  1838. Next
  1839. Log("[Download] " & Join(ErrOutput.Distinct.ToArray, vbCrLf))
  1840. End Sub
  1841. Public Overrides Sub Abort()
  1842. SyncLock LockState
  1843. If State >= LoadState.Finished Then Exit Sub
  1844. State = LoadState.Aborted
  1845. End SyncLock
  1846. Log("[Download] " & Name & " 已取消!")
  1847. '中断所有文件
  1848. For Each TaskFile In Files
  1849. TaskFile.Abort(Me)
  1850. Next
  1851. End Sub
  1852. End Class
  1853. Public NetManager As New NetManagerClass
  1854. ''' <summary>
  1855. ''' 下载文件管理。
  1856. ''' </summary>
  1857. Public Class NetManagerClass
  1858. #Region "属性"
  1859. ''' <summary>
  1860. ''' 需要下载的文件。为“本地地址 - 文件对象”键值对。
  1861. ''' </summary>
  1862. Public Files As New Dictionary(Of String, NetFile)
  1863. Public ReadOnly LockFiles As New Object
  1864. ''' <summary>
  1865. ''' 当前的所有下载任务。
  1866. ''' </summary>
  1867. Public Tasks As New SafeList(Of LoaderDownload)
  1868. ''' <summary>
  1869. ''' 已下载完成的大小。
  1870. ''' </summary>
  1871. Public Property DownloadDone As Long
  1872. Get
  1873. Return _DownloadDone
  1874. End Get
  1875. Set(value As Long)
  1876. SyncLock LockDone
  1877. _DownloadDone = value
  1878. End SyncLock
  1879. End Set
  1880. End Property
  1881. Private _DownloadDone As Long = 0
  1882. Private ReadOnly LockDone As New Object
  1883. ''' <summary>
  1884. ''' 尚未完成下载的文件数。
  1885. ''' </summary>
  1886. Public FileRemain As Integer = 0
  1887. Public ReadOnly LockRemain As New Object
  1888. ''' <summary>
  1889. ''' 上次记速时的已下载大小。
  1890. ''' </summary>
  1891. Private SpeedLastDone As Long = 0
  1892. ''' <summary>
  1893. ''' 至多最近 30 次下载速度的记录,较新的在前面。
  1894. ''' </summary>
  1895. Private SpeedLast As New List(Of Long)
  1896. '这些属性由 RefreshStat 刷新
  1897. ''' <summary>
  1898. ''' 当前的全局下载速度,单位为 Byte / 秒。
  1899. ''' </summary>
  1900. Public Speed As Long = 0
  1901. Public ReadOnly Uuid As Integer = GetUuid()
  1902. #End Region
  1903. ''' <summary>
  1904. ''' 进度与下载速度由下载管理线程每隔约 0.1 秒刷新一次。
  1905. ''' </summary>
  1906. Private Sub RefreshStat()
  1907. Try
  1908. Dim DeltaTime As Long = GetTimeTick() - RefreshStatLast
  1909. If DeltaTime = 0 Then Return
  1910. RefreshStatLast += DeltaTime
  1911. #Region "刷新整体速度"
  1912. '计算瞬时速度
  1913. Dim ActualSpeed As Double = Math.Max(0, (DownloadDone - SpeedLastDone) / (DeltaTime / 1000))
  1914. SpeedLast.Insert(0, ActualSpeed)
  1915. If SpeedLast.Count >= 31 Then SpeedLast.RemoveAt(30)
  1916. SpeedLastDone = DownloadDone
  1917. '计算用于显示的速度
  1918. Dim SpeedSum As Long = 0, SpeedDiv As Long = 0, Weight = SpeedLast.Count
  1919. For Each SpeedRecord In SpeedLast
  1920. SpeedSum += SpeedRecord * Weight
  1921. SpeedDiv += Weight
  1922. Weight -= 1
  1923. Next
  1924. Speed = If(SpeedDiv > 0, SpeedSum / SpeedDiv, 0)
  1925. '计算新的速度下限
  1926. Dim Limit As Long = 0
  1927. If SpeedLast.Count >= 10 Then Limit = SpeedLast.Take(10).Average * 0.85 '取近 1 秒的平均速度的 85%
  1928. If Limit > NetTaskSpeedLimitLow Then
  1929. NetTaskSpeedLimitLow = Limit
  1930. Log("[Download] " & "速度下限已提升到 " & GetString(Limit))
  1931. End If
  1932. #End Region
  1933. #Region "刷新下载任务属性"
  1934. For Each Task In Tasks
  1935. Task.RefreshStat()
  1936. Next
  1937. #End Region
  1938. Catch ex As Exception
  1939. Log(ex, "刷新下载公开属性失败")
  1940. End Try
  1941. End Sub
  1942. Private RefreshStatLast As Long
  1943. ''' <summary>
  1944. ''' 启动监控线程,用于新增下载线程。
  1945. ''' </summary>
  1946. Private Sub StartManager()
  1947. If IsManagerStarted Then Exit Sub
  1948. IsManagerStarted = True
  1949. Dim ThreadStarter =
  1950. Sub(Id As Integer) '0 或 1
  1951. Try
  1952. While True
  1953. Thread.Sleep(20)
  1954. '获取文件列表
  1955. Dim AllFiles As List(Of NetFile)
  1956. SyncLock LockFiles
  1957. If Id = 0 AndAlso FileRemain = 0 AndAlso Files.Any() Then Files.Clear() '若已完成,则清空
  1958. AllFiles = Files.Values.ToList()
  1959. End SyncLock
  1960. Dim WaitingFiles As New List(Of NetFile)
  1961. Dim OngoingFiles As New List(Of NetFile)
  1962. For Each File As NetFile In AllFiles
  1963. If File.Uuid Mod 2 = Id Then Continue For
  1964. If File.State = NetState.WaitForDownload Then
  1965. WaitingFiles.Add(File)
  1966. ElseIf File.State < NetState.Merge Then
  1967. OngoingFiles.Add(File)
  1968. End If
  1969. Next
  1970. '为等待中的文件开始线程
  1971. For Each File As NetFile In WaitingFiles
  1972. If NetTaskThreadCount >= NetTaskThreadLimit Then Continue While '最大线程数检查
  1973. Dim NewThread = File.TryBeginThread()
  1974. If NewThread IsNot Nothing AndAlso NewThread.Source.Url.Contains("bmclapi") Then Thread.Sleep(30) '减少 BMCLAPI 请求频率(目前每分钟限制 4000 次)
  1975. Next
  1976. '为进行中的文件追加线程
  1977. If Speed >= NetTaskSpeedLimitLow Then Continue While '下载速度足够,无需新增
  1978. For Each File As NetFile In OngoingFiles
  1979. If NetTaskThreadCount >= NetTaskThreadLimit Then Continue While '最大线程数检查
  1980. '线程种类计数
  1981. Dim PreparingCount = 0, DownloadingCount = 0
  1982. If File.Threads IsNot Nothing Then
  1983. For Each Thread As NetThread In File.Threads.ToList
  1984. If Thread.State < NetState.Download Then
  1985. PreparingCount += 1
  1986. ElseIf Thread.State = NetState.Download Then
  1987. DownloadingCount += 1
  1988. End If
  1989. Next
  1990. End If
  1991. '新增线程
  1992. If PreparingCount > DownloadingCount Then Continue For '准备中的线程已多于下载中的线程,不再新增
  1993. Dim NewThread = File.TryBeginThread()
  1994. If NewThread IsNot Nothing AndAlso NewThread.Source.Url.Contains("bmclapi") Then Thread.Sleep(30) '减少 BMCLAPI 请求频率(目前每分钟限制 4000 次)
  1995. Next
  1996. End While
  1997. Catch ex As Exception
  1998. Log(ex, $"下载管理启动线程 {Id} 出错", LogLevel.Critical)
  1999. End Try
  2000. End Sub
  2001. RunInNewThread(Sub() ThreadStarter(0), "NetManager ThreadStarter 0")
  2002. RunInNewThread(Sub() ThreadStarter(1), "NetManager ThreadStarter 1")
  2003. RunInNewThread(
  2004. Sub()
  2005. Try
  2006. Dim LastLoopTime As Long
  2007. NetTaskSpeedLimitLeftLast = GetTimeTick()
  2008. While True
  2009. Dim TimeNow = GetTimeTick()
  2010. LastLoopTime = TimeNow
  2011. '增加限速余量
  2012. If NetTaskSpeedLimitHigh > 0 Then NetTaskSpeedLimitLeft = NetTaskSpeedLimitHigh / 1000 * (TimeNow - NetTaskSpeedLimitLeftLast)
  2013. NetTaskSpeedLimitLeftLast = TimeNow
  2014. '刷新公开属性
  2015. RefreshStat()
  2016. '等待直至 80 ms
  2017. Do While GetTimeTick() - LastLoopTime < 80
  2018. Thread.Sleep(10)
  2019. Loop
  2020. End While
  2021. Catch ex As Exception
  2022. Log(ex, "下载管理刷新线程出错", LogLevel.Critical)
  2023. End Try
  2024. End Sub, "NetManager StatRefresher")
  2025. End Sub
  2026. Private IsManagerStarted As Boolean = False
  2027. 'Public FileRemainList As New List(Of String)
  2028. Private IsDownloadCacheCleared As Boolean = False
  2029. ''' <summary>
  2030. ''' 开始一个下载任务。
  2031. ''' </summary>
  2032. Public Sub Start(Task As LoaderDownload)
  2033. StartManager()
  2034. '清理缓存
  2035. If Not IsDownloadCacheCleared Then
  2036. Try
  2037. DeleteDirectory(PathTemp & "Download")
  2038. Catch ex As Exception
  2039. Log(ex, "清理下载缓存失败")
  2040. End Try
  2041. IsDownloadCacheCleared = True
  2042. End If
  2043. Directory.CreateDirectory(PathTemp & "Download")
  2044. '文件处理
  2045. SyncLock LockFiles
  2046. '添加每个文件
  2047. For i = 0 To Task.Files.Count - 1
  2048. Dim File = Task.Files(i)
  2049. If Files.ContainsKey(File.LocalPath) Then
  2050. '已有该文件
  2051. If Files(File.LocalPath).State >= NetState.Finish Then
  2052. '该文件已经下载过一次,且下载完成
  2053. '将已下载的文件替换成当前文件,重新下载
  2054. File.Tasks.Add(Task)
  2055. Files(File.LocalPath) = File
  2056. SyncLock LockRemain
  2057. FileRemain += 1
  2058. If ModeDebug Then Log("[Download] " & File.LocalName & ":已替换列表,剩余文件 " & FileRemain)
  2059. 'FileRemainList.Add(File.LocalPath)
  2060. End SyncLock
  2061. Else
  2062. '该文件正在下载中
  2063. '将当前文件替换成下载中的文件,即两个任务指向同一个文件
  2064. File = Files(File.LocalPath)
  2065. File.Tasks.Add(Task)
  2066. End If
  2067. Else
  2068. '没有该文件
  2069. File.Tasks.Add(Task)
  2070. Files.Add(File.LocalPath, File)
  2071. SyncLock LockRemain
  2072. FileRemain += 1
  2073. If ModeDebug Then Log("[Download] " & File.LocalName & ":已加入列表,剩余文件 " & FileRemain)
  2074. 'FileRemainList.Add(File.LocalPath)
  2075. End SyncLock
  2076. End If
  2077. Task.Files(i) = File '回设
  2078. Next
  2079. End SyncLock
  2080. Tasks.Add(Task)
  2081. End Sub
  2082. End Class
  2083. ''' <summary>
  2084. ''' 是否有正在进行中、需要在下载管理页面显示的下载任务?
  2085. ''' </summary>
  2086. Public Function HasDownloadingTask(Optional IgnoreCustomDownload As Boolean = False) As Boolean
  2087. For Each Task In LoaderTaskbar.ToList()
  2088. If (Task.Show AndAlso Task.State = LoadState.Loading) AndAlso
  2089. (Not IgnoreCustomDownload OrElse Not Task.Name.ToString.Contains("自定义下载")) Then
  2090. Return True
  2091. End If
  2092. Next
  2093. Return False
  2094. End Function
  2095. End Module