From c197f23b3bdb0ff67f5fe62974dcb7fe74553b49 Mon Sep 17 00:00:00 2001 From: Paul Hebble Date: Sat, 6 Jan 2024 13:48:17 -0600 Subject: [PATCH] Refactor DownloadTarget and DownloadPart --- Core/Net/AutoUpdate.cs | 11 +- Core/Net/Net.cs | 54 +----- Core/Net/NetAsyncDownloader.DownloadPart.cs | 114 +++++++++++++ Core/Net/NetAsyncDownloader.DownloadTarget.cs | 42 +++++ Core/Net/NetAsyncDownloader.cs | 155 +++++------------- Core/Net/NetAsyncModulesDownloader.cs | 6 +- Core/Repositories/RepositoryDataManager.cs | 16 +- Tests/Core/Net/NetAsyncDownloaderTests.cs | 9 +- 8 files changed, 223 insertions(+), 184 deletions(-) create mode 100644 Core/Net/NetAsyncDownloader.DownloadPart.cs create mode 100644 Core/Net/NetAsyncDownloader.DownloadTarget.cs diff --git a/Core/Net/AutoUpdate.cs b/Core/Net/AutoUpdate.cs index 1eef7b8c79..7f297415aa 100644 --- a/Core/Net/AutoUpdate.cs +++ b/Core/Net/AutoUpdate.cs @@ -1,5 +1,4 @@ using System; -using System.Collections.Generic; using System.IO; using System.Diagnostics; using System.Reflection; @@ -146,15 +145,15 @@ public void StartUpdateProcess(bool launchCKANAfterUpdate, IUser user = null) // download updater app and new ckan.exe string updaterFilename = Path.GetTempPath() + Guid.NewGuid().ToString() + ".exe"; string ckanFilename = Path.GetTempPath() + Guid.NewGuid().ToString() + ".exe"; - Net.DownloadWithProgress( + NetAsyncDownloader.DownloadWithProgress( new[] { - new Net.DownloadTarget( - new List { latestUpdate.UpdaterDownload }, + new NetAsyncDownloader.DownloadTarget( + latestUpdate.UpdaterDownload, updaterFilename, latestUpdate.UpdaterSize), - new Net.DownloadTarget( - new List { latestUpdate.ReleaseDownload }, + new NetAsyncDownloader.DownloadTarget( + latestUpdate.ReleaseDownload, ckanFilename, latestUpdate.ReleaseSize), }, diff --git a/Core/Net/Net.cs b/Core/Net/Net.cs index 4c0608f487..9aef1dbc3c 100644 --- a/Core/Net/Net.cs +++ b/Core/Net/Net.cs @@ -21,6 +21,7 @@ namespace CKAN public static class Net { // The user agent that we report to web sites + // Maybe overwritten by command line args public static string UserAgentString = "Mozilla/4.0 (compatible; CKAN)"; private const int MaxRetries = 3; @@ -76,14 +77,14 @@ public static string Download(string url, string filename = null, IUser user = n public static string Download(string url, out string etag, string filename = null, IUser user = null) { - TxFileManager FileTransaction = new TxFileManager(); - user = user ?? new NullUser(); user.RaiseMessage(Properties.Resources.NetDownloading, url); + var FileTransaction = new TxFileManager(); // Generate a temporary file if none is provided. if (filename == null) { + filename = FileTransaction.GetTempFileName(); } @@ -141,55 +142,6 @@ public static string Download(string url, out string etag, string filename = nul return filename; } - public class DownloadTarget - { - public List urls { get; private set; } - public string filename { get; private set; } - public long size { get; set; } - public string mimeType { get; private set; } - - public DownloadTarget(List urls, string filename = null, long size = 0, string mimeType = "") - { - TxFileManager FileTransaction = new TxFileManager(); - - this.urls = urls; - this.filename = string.IsNullOrEmpty(filename) - ? FileTransaction.GetTempFileName() - : filename; - this.size = size; - this.mimeType = mimeType; - } - } - - public static string DownloadWithProgress(string url, string filename = null, IUser user = null) - => DownloadWithProgress(new Uri(url), filename, user); - - public static string DownloadWithProgress(Uri url, string filename = null, IUser user = null) - { - var targets = new[] { - new DownloadTarget(new List { url }, filename) - }; - DownloadWithProgress(targets, user); - return targets.First().filename; - } - - public static void DownloadWithProgress(IList downloadTargets, IUser user = null) - { - var downloader = new NetAsyncDownloader(user ?? new NullUser()); - downloader.onOneCompleted += (url, filename, error, etag) => - { - if (error != null) - { - user?.RaiseError(error.ToString()); - } - else - { - File.Move(filename, downloadTargets.First(p => p.urls.Contains(url)).filename); - } - }; - downloader.DownloadAndWait(downloadTargets); - } - /// /// Download a string from a URL /// diff --git a/Core/Net/NetAsyncDownloader.DownloadPart.cs b/Core/Net/NetAsyncDownloader.DownloadPart.cs new file mode 100644 index 0000000000..b49f6434e7 --- /dev/null +++ b/Core/Net/NetAsyncDownloader.DownloadPart.cs @@ -0,0 +1,114 @@ +using System; +using System.IO; +using System.ComponentModel; + +using Autofac; + +using CKAN.Configuration; + +namespace CKAN +{ + public partial class NetAsyncDownloader + { + // Private utility class for tracking downloads + private class DownloadPart + { + public readonly DownloadTarget target; + public readonly string path; + + public DateTime lastProgressUpdateTime; + public long lastProgressUpdateSize; + public long bytesLeft; + public long size; + public long bytesPerSecond; + public Exception error; + + // Number of target URLs already tried and failed + private int triedDownloads; + + /// + /// Percentage, bytes received, total bytes to receive + /// + public event Action Progress; + public event Action Done; + + private string mimeType => target.mimeType; + private ResumingWebClient agent; + + public DownloadPart(DownloadTarget target) + { + this.target = target; + path = target.filename ?? Path.GetTempFileName(); + size = bytesLeft = target.size; + lastProgressUpdateTime = DateTime.Now; + triedDownloads = 0; + } + + public void Download(Uri url, string path) + { + ResetAgent(); + // Check whether to use an auth token for this host + if (url.IsAbsoluteUri + && ServiceLocator.Container.Resolve().TryGetAuthToken(url.Host, out string token) + && !string.IsNullOrEmpty(token)) + { + log.InfoFormat("Using auth token for {0}", url.Host); + // Send our auth token to the GitHub API (or whoever else needs one) + agent.Headers.Add("Authorization", $"token {token}"); + } + agent.DownloadFileAsyncWithResume(url, path); + } + + public Uri CurrentUri => target.urls[triedDownloads]; + + public bool HaveMoreUris => triedDownloads + 1 < target.urls.Count; + + public void NextUri() + { + if (HaveMoreUris) + { + ++triedDownloads; + } + } + + public void Abort() + { + agent?.CancelAsyncOverridden(); + } + + private void ResetAgent() + { + // This WebClient child class does some complicated stuff, let's keep using it for now + #pragma warning disable SYSLIB0014 + agent = new ResumingWebClient(); + #pragma warning restore SYSLIB0014 + + agent.Headers.Add("User-Agent", Net.UserAgentString); + + // Tell the server what kind of files we want + if (!string.IsNullOrEmpty(mimeType)) + { + log.InfoFormat("Setting MIME type {0}", mimeType); + agent.Headers.Add("Accept", mimeType); + } + + // Forward progress and completion events to our listeners + agent.DownloadProgressChanged += (sender, args) => + { + Progress?.Invoke(args.ProgressPercentage, args.BytesReceived, args.TotalBytesToReceive); + }; + agent.DownloadProgress += (percent, bytesReceived, totalBytesToReceive) => + { + Progress?.Invoke(percent, bytesReceived, totalBytesToReceive); + }; + agent.DownloadFileCompleted += (sender, args) => + { + Done?.Invoke(sender, args, + args.Cancelled || args.Error != null + ? null + : agent.ResponseHeaders?.Get("ETag")?.Replace("\"", "")); + }; + } + } + } +} diff --git a/Core/Net/NetAsyncDownloader.DownloadTarget.cs b/Core/Net/NetAsyncDownloader.DownloadTarget.cs new file mode 100644 index 0000000000..b00787be30 --- /dev/null +++ b/Core/Net/NetAsyncDownloader.DownloadTarget.cs @@ -0,0 +1,42 @@ +using System; +using System.Collections.Generic; + +using ChinhDo.Transactions.FileManager; + +namespace CKAN +{ + public partial class NetAsyncDownloader + { + public class DownloadTarget + { + public List urls { get; private set; } + public string filename { get; private set; } + public long size { get; set; } + public string mimeType { get; private set; } + + public DownloadTarget(List urls, + string filename = null, + long size = 0, + string mimeType = "") + { + var FileTransaction = new TxFileManager(); + + this.urls = urls; + this.filename = string.IsNullOrEmpty(filename) + ? FileTransaction.GetTempFileName() + : filename; + this.size = size; + this.mimeType = mimeType; + } + + public DownloadTarget(Uri url, + string filename = null, + long size = 0, + string mimeType = "") + : this(new List { url }, + filename, size, mimeType) + { + } + } + } +} diff --git a/Core/Net/NetAsyncDownloader.cs b/Core/Net/NetAsyncDownloader.cs index ebaa513c54..88c3d107fd 100644 --- a/Core/Net/NetAsyncDownloader.cs +++ b/Core/Net/NetAsyncDownloader.cs @@ -1,6 +1,5 @@ using System; using System.Collections.Generic; -using System.ComponentModel; using System.IO; using System.Linq; using System.Net; @@ -10,115 +9,13 @@ using Autofac; using log4net; -using CKAN.Configuration; - namespace CKAN { /// /// Download lots of files at once! /// - public class NetAsyncDownloader + public partial class NetAsyncDownloader { - // Private utility class for tracking downloads - private class NetAsyncDownloaderDownloadPart - { - public readonly Net.DownloadTarget target; - public DateTime lastProgressUpdateTime; - public long lastProgressUpdateSize; - public readonly string path; - public long bytesLeft; - public long size; - public long bytesPerSecond; - public Exception error; - - // Number of target URLs already tried and failed - private int triedDownloads; - - /// - /// Percentage, bytes received, total bytes to receive - /// - public event Action Progress; - public event Action Done; - - private string mimeType => target.mimeType; - private ResumingWebClient agent; - - public NetAsyncDownloaderDownloadPart(Net.DownloadTarget target) - { - this.target = target; - path = target.filename ?? Path.GetTempFileName(); - size = bytesLeft = target.size; - lastProgressUpdateTime = DateTime.Now; - triedDownloads = 0; - } - - public void Download(Uri url, string path) - { - ResetAgent(); - // Check whether to use an auth token for this host - if (url.IsAbsoluteUri - && ServiceLocator.Container.Resolve().TryGetAuthToken(url.Host, out string token) - && !string.IsNullOrEmpty(token)) - { - log.InfoFormat("Using auth token for {0}", url.Host); - // Send our auth token to the GitHub API (or whoever else needs one) - agent.Headers.Add("Authorization", $"token {token}"); - } - agent.DownloadFileAsyncWithResume(url, path); - } - - public Uri CurrentUri => target.urls[triedDownloads]; - - public bool HaveMoreUris => triedDownloads + 1 < target.urls.Count; - - public void NextUri() - { - if (HaveMoreUris) - { - ++triedDownloads; - } - } - - public void Abort() - { - agent?.CancelAsyncOverridden(); - } - - private void ResetAgent() - { - // This WebClient child class does some complicated stuff, let's keep using it for now - #pragma warning disable SYSLIB0014 - agent = new ResumingWebClient(); - #pragma warning restore SYSLIB0014 - - agent.Headers.Add("User-Agent", Net.UserAgentString); - - // Tell the server what kind of files we want - if (!string.IsNullOrEmpty(mimeType)) - { - log.InfoFormat("Setting MIME type {0}", mimeType); - agent.Headers.Add("Accept", mimeType); - } - - // Forward progress and completion events to our listeners - agent.DownloadProgressChanged += (sender, args) => - { - Progress?.Invoke(args.ProgressPercentage, args.BytesReceived, args.TotalBytesToReceive); - }; - agent.DownloadProgress += (percent, bytesReceived, totalBytesToReceive) => - { - Progress?.Invoke(percent, bytesReceived, totalBytesToReceive); - }; - agent.DownloadFileCompleted += (sender, args) => - { - Done?.Invoke(sender, args, - args.Cancelled || args.Error != null - ? null - : agent.ResponseHeaders?.Get("ETag")?.Replace("\"", "")); - }; - } - } - private static readonly ILog log = LogManager.GetLogger(typeof(NetAsyncDownloader)); public readonly IUser User; @@ -126,13 +23,13 @@ private void ResetAgent() /// /// Raised when data arrives for a download /// - public event Action Progress; + public event Action Progress; private readonly object dlMutex = new object(); // NOTE: Never remove anything from this, because closures have indexes into it! // (Clearing completely after completion is OK) - private readonly List downloads = new List(); - private readonly List queuedDownloads = new List(); + private readonly List downloads = new List(); + private readonly List queuedDownloads = new List(); private int completed_downloads; // For inter-thread communication @@ -150,20 +47,50 @@ public NetAsyncDownloader(IUser user) complete_or_canceled = new ManualResetEvent(false); } + public static string DownloadWithProgress(string url, string filename = null, IUser user = null) + => DownloadWithProgress(new Uri(url), filename, user); + + public static string DownloadWithProgress(Uri url, string filename = null, IUser user = null) + { + var targets = new[] + { + new DownloadTarget(url, filename) + }; + DownloadWithProgress(targets, user); + return targets.First().filename; + } + + public static void DownloadWithProgress(IList downloadTargets, IUser user = null) + { + var downloader = new NetAsyncDownloader(user ?? new NullUser()); + downloader.onOneCompleted += (url, filename, error, etag) => + { + if (error != null) + { + user?.RaiseError(error.ToString()); + } + else + { + File.Move(filename, downloadTargets.First(p => p.urls.Contains(url)).filename); + } + }; + downloader.DownloadAndWait(downloadTargets); + } + /// /// Start a new batch of downloads /// /// The downloads to begin - public void DownloadAndWait(IList targets) + public void DownloadAndWait(IList targets) { lock (dlMutex) { if (downloads.Count + queuedDownloads.Count > completed_downloads) { // Some downloads are still in progress, add to the current batch - foreach (Net.DownloadTarget target in targets) + foreach (DownloadTarget target in targets) { - DownloadModule(new NetAsyncDownloaderDownloadPart(target)); + DownloadModule(new DownloadPart(target)); } // Wait for completion along with original caller // so we can handle completion tasks for the added mods @@ -282,17 +209,17 @@ public void CancelDownload() /// Downloads our files. /// /// A collection of DownloadTargets - private void Download(ICollection targets) + private void Download(ICollection targets) { downloads.Clear(); queuedDownloads.Clear(); foreach (var t in targets) { - DownloadModule(new NetAsyncDownloaderDownloadPart(t)); + DownloadModule(new DownloadPart(t)); } } - private void DownloadModule(NetAsyncDownloaderDownloadPart dl) + private void DownloadModule(DownloadPart dl) { if (shouldQueue(dl.CurrentUri)) { @@ -372,7 +299,7 @@ private void triggerCompleted() /// The total amount of bytes we expect to download private void FileProgressReport(int index, long bytesDownloaded, long bytesToDownload) { - NetAsyncDownloaderDownloadPart download = downloads[index]; + DownloadPart download = downloads[index]; DateTime now = DateTime.Now; TimeSpan timeSpan = now - download.lastProgressUpdateTime; diff --git a/Core/Net/NetAsyncModulesDownloader.cs b/Core/Net/NetAsyncModulesDownloader.cs index 0d79fee14e..70c3e86330 100644 --- a/Core/Net/NetAsyncModulesDownloader.cs +++ b/Core/Net/NetAsyncModulesDownloader.cs @@ -42,14 +42,14 @@ public NetAsyncModulesDownloader(IUser user, NetModuleCache cache) this.cache = cache; } - internal Net.DownloadTarget TargetFromModuleGroup(HashSet group, + internal NetAsyncDownloader.DownloadTarget TargetFromModuleGroup(HashSet group, string[] preferredHosts) => TargetFromModuleGroup(group, group.OrderBy(m => m.identifier).First(), preferredHosts); - private Net.DownloadTarget TargetFromModuleGroup(HashSet group, + private NetAsyncDownloader.DownloadTarget TargetFromModuleGroup(HashSet group, CkanModule first, string[] preferredHosts) - => new Net.DownloadTarget( + => new NetAsyncDownloader.DownloadTarget( group.SelectMany(mod => mod.download) .Concat(group.Select(mod => mod.InternetArchiveDownload) .Where(uri => uri != null) diff --git a/Core/Repositories/RepositoryDataManager.cs b/Core/Repositories/RepositoryDataManager.cs index 2b33d75755..3cf20d9c35 100644 --- a/Core/Repositories/RepositoryDataManager.cs +++ b/Core/Repositories/RepositoryDataManager.cs @@ -7,9 +7,7 @@ using ChinhDo.Transactions.FileManager; using log4net; -#if NETFRAMEWORK using CKAN.Extensions; -#endif using CKAN.Games; namespace CKAN @@ -160,7 +158,7 @@ public UpdateResult Update(Repository[] repos, try { // Download metadata - var targets = toUpdate.Select(r => new Net.DownloadTarget(new List() { r.uri })) + var targets = toUpdate.Select(r => new NetAsyncDownloader.DownloadTarget(r.uri)) .ToArray(); downloader.DownloadAndWait(targets); @@ -170,17 +168,20 @@ public UpdateResult Update(Repository[] repos, var progress = new ProgressFilesOffsetsToPercent( new Progress(p => user.RaiseProgress(msg, p)), targets.Select(t => new FileInfo(t.filename).Length)); - foreach ((Repository repo, Net.DownloadTarget target) in toUpdate.Zip(targets)) + foreach ((Repository repo, NetAsyncDownloader.DownloadTarget target) in toUpdate.Zip(targets)) { var file = target.filename; msg = string.Format(Properties.Resources.NetRepoLoadingModulesFromRepo, repo.name); // Load the file, save to in memory cache + log.InfoFormat("Loading {0}...", file); var repoData = repositoriesData[repo] = RepositoryData.FromDownload(file, game, progress); // Save parsed data to disk + log.DebugFormat("Saving data for {0} repo...", repo.name); repoData.SaveTo(GetRepoDataPath(repo)); // Delete downloaded archive + log.DebugFormat("Deleting {0}...", file); File.Delete(file); progress.NextFile(); } @@ -200,8 +201,13 @@ public UpdateResult Update(Repository[] repos, kvp.Value)) .ToList()); } - catch + catch (Exception exc) { + foreach (var e in exc.TraverseNodes(ex => ex.InnerException) + .Reverse()) + { + log.Error("Repository update failed", e); + } // Reset etags on errors loadETags(); throw; diff --git a/Tests/Core/Net/NetAsyncDownloaderTests.cs b/Tests/Core/Net/NetAsyncDownloaderTests.cs index 9e1f8795b3..fab3e2d76a 100644 --- a/Tests/Core/Net/NetAsyncDownloaderTests.cs +++ b/Tests/Core/Net/NetAsyncDownloaderTests.cs @@ -1,6 +1,5 @@ using System; using System.IO; -using System.Collections.Generic; using System.Linq; using NUnit.Framework; @@ -25,9 +24,9 @@ public void DownloadAndWait_WithValidFileUrl_SetsTargetSize(string pathWithinTes // Arrange var downloader = new NetAsyncDownloader(new NullUser()); var fromPath = TestData.DataDir(pathWithinTestData); - var target = new CKAN.Net.DownloadTarget(new List { new Uri(fromPath) }, + var target = new NetAsyncDownloader.DownloadTarget(new Uri(fromPath), Path.GetTempFileName()); - var targets = new CKAN.Net.DownloadTarget[] { target }; + var targets = new NetAsyncDownloader.DownloadTarget[] { target }; var origSize = new FileInfo(fromPath).Length; // Act @@ -71,7 +70,7 @@ public void DownloadAndWait_WithValidFileUrls_SetsTargetsSize(params string[] pa // Arrange var downloader = new NetAsyncDownloader(new NullUser()); var fromPaths = pathsWithinTestData.Select(p => TestData.DataDir(p)).ToArray(); - var targets = fromPaths.Select(p => new CKAN.Net.DownloadTarget(new List { new Uri(p) }, + var targets = fromPaths.Select(p => new NetAsyncDownloader.DownloadTarget(new Uri(p), Path.GetTempFileName())) .ToArray(); var origSizes = fromPaths.Select(p => new FileInfo(p).Length).ToArray(); @@ -187,7 +186,7 @@ public void DownloadAndWait_WithSomeInvalidUrls_ThrowsDownloadErrorsKraken( // Arrange var downloader = new NetAsyncDownloader(new NullUser()); var fromPaths = pathsWithinTestData.Select(p => Path.GetFullPath(TestData.DataDir(p))).ToArray(); - var targets = fromPaths.Select(p => new CKAN.Net.DownloadTarget(new List { new Uri(p) }, + var targets = fromPaths.Select(p => new NetAsyncDownloader.DownloadTarget(new Uri(p), Path.GetTempFileName())) .ToArray(); var badIndices = fromPaths.Select((p, i) => new Tuple(i, File.Exists(p)))