From 99e04055e6a84c3f87b3f3213ab2f7e805d9d5e1 Mon Sep 17 00:00:00 2001 From: Michael Hallett Date: Wed, 7 Feb 2024 23:02:25 -0500 Subject: [PATCH] Added support for downloading the Game & Watch ROMs (#235) GlobalHelper: - made the ArchiveFiles property populate on first use. - added GameAndWatchArchiveFiles Archive: - changed files to be a List<> vs an array so items could be easily removed - updated some variable names to camel casing Config: -added new config setting to set the archive name for game and watch Program - awaited the Global Helper Initialize as it wasn't done populating when testing updating the game and watch core - removed commented out code Core: - moved ad hoc string concats to string interpolation - removed unused method ReadPlatformFile - made Download Asset more generic so it can support downloading the ROMs for Game & Watch - added code to download the Game & Watch ROMs from ArchiveOrg - made Check CRC more generic so it can support Game & Watch - replaced "" with string.Empty - collapsed Build Asset Url code into Download Asset to make it easier to be more generic --- src/Program.cs | 4 +- src/helpers/GlobalHelper.cs | 69 +++++++++---- src/models/Archive/Archive.cs | 6 +- src/models/Core.cs | 187 +++++++++++++++++++++------------- src/models/Settings/Config.cs | 1 + 5 files changed, 173 insertions(+), 94 deletions(-) diff --git a/src/Program.cs b/src/Program.cs index 5e453d73..f58e6542 100644 --- a/src/Program.cs +++ b/src/Program.cs @@ -183,7 +183,7 @@ private static async Task Main(string[] args) #endregion - GlobalHelper.Initialize(path); + await GlobalHelper.Initialize(path); if (!CLI_MODE) { @@ -259,8 +259,6 @@ private static async Task Main(string[] args) coreUpdater.DownloadAssets(GlobalHelper.SettingsManager.GetConfig().download_assets); coreUpdater.BackupSaves(GlobalHelper.SettingsManager.GetConfig().backup_saves, GlobalHelper.SettingsManager.GetConfig().backup_saves_location); - //await coreUpdater.Initialize(); - // If we have any missing cores, handle them. if (GlobalHelper.SettingsManager.GetMissingCores().Any()) { diff --git a/src/helpers/GlobalHelper.cs b/src/helpers/GlobalHelper.cs index a2e18370..5f692ca0 100644 --- a/src/helpers/GlobalHelper.cs +++ b/src/helpers/GlobalHelper.cs @@ -6,7 +6,55 @@ namespace Pannella.Helpers; public static class GlobalHelper { - public static Archive ArchiveFiles { get; private set; } + private static Archive archiveFiles; + + public static Archive ArchiveFiles + { + get + { + if (archiveFiles == null) + { + Console.WriteLine("Loading Assets Index..."); + + if (SettingsManager.GetConfig().use_custom_archive) + { + var custom = SettingsManager.GetConfig().custom_archive; + Uri baseUrl = new Uri(custom["url"]); + Uri url = new Uri(baseUrl, custom["index"]); + + archiveFiles = ArchiveService.GetFilesCustom(url.ToString()).Result; + } + else + { + archiveFiles = ArchiveService.GetFiles(SettingsManager.GetConfig().archive_name).Result; + } + } + + return archiveFiles; + } + } + + private static Archive gameAndWatchArchiveFiles; + + public static Archive GameAndWatchArchiveFiles + { + get + { + if (gameAndWatchArchiveFiles == null) + { + Console.WriteLine("Loading Game and Watch Assets Index..."); + + gameAndWatchArchiveFiles = ArchiveService.GetFiles(SettingsManager.GetConfig().gnw_archive_name).Result; + + // remove the metadata files since we're processing the entire json list + gameAndWatchArchiveFiles.files.RemoveAll(file => + Path.GetExtension(file.name) is ".sqlite" or ".torrent" or ".xml"); + } + + return gameAndWatchArchiveFiles; + } + } + public static SettingsManager SettingsManager { get; private set ;} public static string UpdateDirectory { get; private set; } public static string[] Blacklist { get; private set; } @@ -15,7 +63,7 @@ public static class GlobalHelper private static bool isInitialized; - public static async void Initialize(string path) + public static async Task Initialize(string path) { if (!isInitialized) { @@ -26,23 +74,6 @@ public static async void Initialize(string path) SettingsManager.InitializeCoreSettings(Cores); RefreshInstalledCores(); Blacklist = await AssetsService.GetBlacklist(); - - Console.WriteLine("Loading Assets Index..."); - - if (SettingsManager.GetConfig().use_custom_archive) - { - var custom = SettingsManager.GetConfig().custom_archive; - Uri baseUrl = new Uri(custom["url"]); - Uri url = new Uri(baseUrl, custom["index"]); - - ArchiveFiles = await ArchiveService.GetFilesCustom(url.ToString()); - } - else - { - ArchiveFiles = await ArchiveService.GetFiles(SettingsManager.GetConfig().archive_name); - } - - RefreshInstalledCores(); } } diff --git a/src/models/Archive/Archive.cs b/src/models/Archive/Archive.cs index ad3a1673..118c7e93 100644 --- a/src/models/Archive/Archive.cs +++ b/src/models/Archive/Archive.cs @@ -4,11 +4,11 @@ public class Archive { public int files_count { get; set; } public int item_last_updated { get; set; } - public File[] files { get; set; } + public List files { get; set; } - public File GetFile(string filename) + public File GetFile(string fileName) { - File file = this.files.FirstOrDefault(file => file.name == filename); + File file = this.files.FirstOrDefault(file => file.name == fileName); return file; } diff --git a/src/models/Core.cs b/src/models/Core.cs index 3554662d..2f7f322f 100644 --- a/src/models/Core.cs +++ b/src/models/Core.cs @@ -81,7 +81,7 @@ private async Task InstallGithubAsset(bool preservePlatformsFolder) return false; } - WriteMessage("Downloading file " + this.download_url + "..."); + WriteMessage($"Downloading file {this.download_url}..."); string zipPath = Path.Combine(GlobalHelper.UpdateDirectory, ZIP_FILE_NAME); string extractPath = GlobalHelper.UpdateDirectory; @@ -162,31 +162,14 @@ public void Uninstall(bool nuke = false) Divide(); } - public Platform ReadPlatformFile() - { - var info = this.GetConfig(); - - if (info == null) - { - return this.platform; - } - - string updateDirectory = GlobalHelper.UpdateDirectory; - // cores with multiple platforms won't work...not sure any exist right now? - string platformsFolder = Path.Combine(updateDirectory, "Platforms"); - string dataFile = Path.Combine(platformsFolder, info.metadata.platform_ids[0] + ".json"); - var p = JsonSerializer.Deserialize>(File.ReadAllText(dataFile)); - - return p["platform"]; - } - public async Task> DownloadAssets() { List installed = new List(); List skipped = new List(); bool missingBetaKey = false; - if (!this.download_assets || !GlobalHelper.SettingsManager.GetCoreSettings(this.identifier).download_assets) + if (!GlobalHelper.SettingsManager.GetConfig().download_assets || + !GlobalHelper.SettingsManager.GetCoreSettings(this.identifier).download_assets) { return new Dictionary { @@ -202,6 +185,7 @@ public async Task> DownloadAssets() string updateDirectory = GlobalHelper.UpdateDirectory; // cores with multiple platforms won't work...not sure any exist right now? string instancesDirectory = Path.Combine(updateDirectory, "Assets", info.metadata.platform_ids[0], this.identifier); + string path = Path.Combine(updateDirectory, "Assets", info.metadata.platform_ids[0]); var options = new JsonSerializerOptions { Converters = { new StringConverter() } }; DataJSON dataJson = ReadDataJSON(); @@ -218,8 +202,6 @@ public async Task> DownloadAssets() if (slot.filename != null && !slot.filename.EndsWith(".sav") && !GlobalHelper.Blacklist.Contains(slot.filename)) { - string path = Path.Combine(updateDirectory, "Assets", info.metadata.platform_ids[0]); - if (slot.IsCoreSpecific()) { path = Path.Combine(path, this.identifier); @@ -240,19 +222,26 @@ public async Task> DownloadAssets() { string filepath = Path.Combine(path, f); - if (File.Exists(filepath) && CheckCRC(filepath)) + if (File.Exists(filepath) && CheckCRC(filepath, GlobalHelper.ArchiveFiles)) { - WriteMessage("Already installed: " + f); + WriteMessage($"Already installed: {f}"); } else { - if (await DownloadAsset(f, filepath)) + bool result = await DownloadAsset( + f, + filepath, + GlobalHelper.ArchiveFiles, + GlobalHelper.SettingsManager.GetConfig().archive_name, + GlobalHelper.SettingsManager.GetConfig().use_custom_archive); + + if (result) { - installed.Add(filepath.Replace(updateDirectory, "")); + installed.Add(filepath.Replace(updateDirectory, string.Empty)); } else { - skipped.Add(filepath.Replace(updateDirectory, "")); + skipped.Add(filepath.Replace(updateDirectory, string.Empty)); } } } @@ -270,6 +259,51 @@ public async Task> DownloadAssets() }; } + if (this.identifier is "agg23.GameAndWatch") + { + string commonPath = Path.Combine(path, "common"); + + foreach (var f in GlobalHelper.GameAndWatchArchiveFiles.files) + { + string filePath = Path.Combine(commonPath, f.name); + string subDirectory = Path.GetDirectoryName(f.name); + + if (!string.IsNullOrEmpty(subDirectory)) + { + Directory.CreateDirectory(Path.Combine(commonPath, subDirectory)); + } + + if (File.Exists(filePath) && CheckCRC(filePath, GlobalHelper.GameAndWatchArchiveFiles)) + { + WriteMessage($"Already installed: {f.name}"); + } + else + { + bool result = await DownloadAsset( + f.name, + filePath, + GlobalHelper.GameAndWatchArchiveFiles, + GlobalHelper.SettingsManager.GetConfig().gnw_archive_name); + + if (result) + { + installed.Add(filePath.Replace(updateDirectory, string.Empty)); + } + else + { + skipped.Add(filePath.Replace(updateDirectory, string.Empty)); + } + } + } + + return new Dictionary + { + { "installed", installed }, + { "skipped", skipped }, + { "missingBetaKey", false } + }; + } + if (CheckInstancePackager()) { BuildInstanceJSONs(); @@ -318,25 +352,30 @@ public async Task> DownloadAssets() missingBetaKey = true; } - if (!GlobalHelper.Blacklist.Contains(slot.filename) && - !slot.filename.EndsWith(".sav")) + if (!GlobalHelper.Blacklist.Contains(slot.filename) && !slot.filename.EndsWith(".sav")) { - string path = Path.Combine(updateDirectory, "Assets", info.metadata.platform_ids[0], - "common", dataPath, slot.filename); + string slotPath = Path.Combine(path, "common", dataPath, slot.filename); - if (File.Exists(path) && CheckCRC(path)) + if (File.Exists(slotPath) && CheckCRC(slotPath, GlobalHelper.ArchiveFiles)) { - WriteMessage("Already installed: " + slot.filename); + WriteMessage($"Already installed: {slot.filename}"); } else { - if (await DownloadAsset(slot.filename, path)) + bool result = await DownloadAsset( + slot.filename, + slotPath, + GlobalHelper.ArchiveFiles, + GlobalHelper.SettingsManager.GetConfig().archive_name, + GlobalHelper.SettingsManager.GetConfig().use_custom_archive); + + if (result) { - installed.Add(path.Replace(updateDirectory, "")); + installed.Add(slotPath.Replace(updateDirectory, string.Empty)); } else { - skipped.Add(path.Replace(updateDirectory, "")); + skipped.Add(slotPath.Replace(updateDirectory, string.Empty)); } } } @@ -345,7 +384,7 @@ public async Task> DownloadAssets() } catch (Exception e) { - WriteMessage("Error while processing " + file); + WriteMessage($"Error while processing '{file}'"); WriteMessage(e.Message); } } @@ -357,6 +396,7 @@ public async Task> DownloadAssets() { "skipped", skipped }, { "missingBetaKey", missingBetaKey } }; + return results; } @@ -404,76 +444,85 @@ public bool IsInstalled() return File.Exists(localCoreFile); } - private async Task DownloadAsset(string filename, string destination) + private async Task DownloadAsset(string fileName, string destination, Archive.Archive archive, + string archiveName, bool useCustomArchive = false) { - if (GlobalHelper.ArchiveFiles != null) + if (archive != null) { - ArchiveFile file = GlobalHelper.ArchiveFiles.GetFile(filename); + ArchiveFile file = archive.GetFile(fileName); if (file == null) { - WriteMessage("Unable to find " + filename + " in archive"); + WriteMessage($"Unable to find '{fileName}' in archive"); return false; } } try { - string url = BuildAssetUrl(filename); + string url; + + if (useCustomArchive) + { + var custom = GlobalHelper.SettingsManager.GetConfig().custom_archive; + Uri baseUri = new Uri(custom["url"]); + Uri uri = new Uri(baseUri, fileName); + + url = uri.ToString(); + } + else + { + url = $"{ARCHIVE_BASE_URL}/{archiveName}/{fileName}"; + } + int count = 0; do { - WriteMessage("Downloading " + filename); + WriteMessage($"Downloading '{fileName}'"); await HttpHelper.Instance.DownloadFileAsync(url, destination, 600); - WriteMessage("Finished downloading " + filename); + WriteMessage($"Finished downloading '{fileName}'"); count++; } - while (count < 3 && !CheckCRC(destination)); + while (count < 3 && !CheckCRC(destination, GlobalHelper.ArchiveFiles)); } catch (HttpRequestException e) { if (e.StatusCode == HttpStatusCode.NotFound) { - WriteMessage("Unable to find " + filename + " in archive"); + WriteMessage($"Unable to find '{fileName}' in archive"); } else { - WriteMessage("There was a problem downloading " + filename); + WriteMessage($"There was a problem downloading '{fileName}'"); } return false; } - - return true; - } - - private static string BuildAssetUrl(string filename) - { - if (GlobalHelper.SettingsManager.GetConfig().use_custom_archive) + catch (Exception e) { - var custom = GlobalHelper.SettingsManager.GetConfig().custom_archive; - Uri baseUrl = new Uri(custom["url"]); - Uri url = new Uri(baseUrl, filename); - return url.ToString(); + WriteMessage($"Something went wrong with '{fileName}'"); + WriteMessage(e.ToString()); + + return false; } - return ARCHIVE_BASE_URL + "/" + GlobalHelper.SettingsManager.GetConfig().archive_name + "/" + filename; + return true; } - private bool CheckCRC(string filepath) + private bool CheckCRC(string filepath, Archive.Archive archive) { - if (GlobalHelper.ArchiveFiles == null || !GlobalHelper.SettingsManager.GetConfig().crc_check) + if (archive == null || !GlobalHelper.SettingsManager.GetConfig().crc_check) { return true; } string filename = Path.GetFileName(filepath); - ArchiveFile file = GlobalHelper.ArchiveFiles.GetFile(filename); + ArchiveFile file = archive.GetFile(filename); if (file == null) { - return true; //no checksum to compare to + return true; // no checksum to compare to } if (Util.CompareChecksum(filepath, file.crc32)) @@ -481,7 +530,7 @@ private bool CheckCRC(string filepath) return true; } - WriteMessage(filename + ": Bad checksum!"); + WriteMessage($"{filename}: Bad checksum!"); return false; } @@ -527,7 +576,7 @@ public void BuildInstanceJSONs(bool overwrite = true) try { - instance.data_path = dir.Replace(commonPath + Path.DirectorySeparatorChar, "") + "/"; + instance.data_path = dir.Replace(commonPath + Path.DirectorySeparatorChar, string.Empty) + "/"; List slots = new(); string jsonFileName = dirName + ".json"; @@ -581,7 +630,7 @@ public void BuildInstanceJSONs(bool overwrite = true) if (slots.Count == 0 || (jsonPackager.slot_limit != null && slots.Count > limit.GetInt32())) { - WriteMessage("Unable to build " + jsonFileName); + WriteMessage($"Unable to build {jsonFileName}"); warning = true; continue; } @@ -603,13 +652,13 @@ public void BuildInstanceJSONs(bool overwrite = true) if (!overwrite && File.Exists(outputFile)) { - WriteMessage(jsonFileName + " already exists."); + WriteMessage($"{jsonFileName} already exists."); } else { string json = JsonSerializer.Serialize(simpleInstanceJson, options); - WriteMessage("Saving " + jsonFileName); + WriteMessage($"Saving {jsonFileName}"); FileInfo file = new FileInfo(outputFile); @@ -624,7 +673,7 @@ public void BuildInstanceJSONs(bool overwrite = true) } catch (Exception) { - WriteMessage("Unable to build " + dirName); + WriteMessage($"Unable to build {dirName}"); } } diff --git a/src/models/Settings/Config.cs b/src/models/Settings/Config.cs index 0772406c..69cbf25c 100644 --- a/src/models/Settings/Config.cs +++ b/src/models/Settings/Config.cs @@ -4,6 +4,7 @@ public class Config { public bool download_assets { get; set; } = true; public string archive_name { get; set; } = "openFPGA-Files"; + public string gnw_archive_name { get; set; } = "fpga-gnw-opt"; public string github_token { get; set; } = string.Empty; public bool download_firmware { get; set; } = true; public bool core_selector { get; set; } = true;