From 58722ef428bea8eb55c4d4476da57e6c62c76a85 Mon Sep 17 00:00:00 2001 From: Paul Fenwick Date: Sun, 19 Oct 2014 23:38:41 +1100 Subject: [PATCH 1/4] First working copy of the BootKAN! --- .gitignore | 2 +- BootKAN/CrewFiles.ckan | 11 ++++++ BootKAN/FerramAerospaceResearch.ckan | 28 ++++++++++++++++ BootKAN/README.md | 6 ++++ BootKAN/RemoteTech.ckan | 19 +++++++++++ BootKAN/kOS.ckan | 20 +++++++++++ CKAN/KerbalStuff/KerbalStuff.csproj | 2 +- CKAN/KerbalStuff/MainClass.cs | 26 +++++++++++++-- KSMods.json | 10 ++++++ bin/build | 15 ++++++++- bin/build.lean | 15 ++++++++- bin/kspan | 50 ++++++++++++++++++++++++++++ 12 files changed, 198 insertions(+), 6 deletions(-) create mode 100644 BootKAN/CrewFiles.ckan create mode 100644 BootKAN/FerramAerospaceResearch.ckan create mode 100644 BootKAN/README.md create mode 100644 BootKAN/RemoteTech.ckan create mode 100644 BootKAN/kOS.ckan create mode 100644 KSMods.json create mode 100755 bin/kspan diff --git a/.gitignore b/.gitignore index f6fd8f8c13..7c6efdbecd 100644 --- a/.gitignore +++ b/.gitignore @@ -7,9 +7,9 @@ test-results TestResult.xml StyleCop.Cache ckan.exe +ks2ckan.exe .*.swp .fatten.out *.resources *.suo *.orig -*.ckan diff --git a/BootKAN/CrewFiles.ckan b/BootKAN/CrewFiles.ckan new file mode 100644 index 0000000000..3ed19b995c --- /dev/null +++ b/BootKAN/CrewFiles.ckan @@ -0,0 +1,11 @@ +{ + "spec_version" : 1, + "identifier" : "CrewFiles", + "$kref" : "#/ckan/kerbalstuff", + "install" : [ + { + "file" : "GameData/CrewFiles", + "install_to" : "GameData" + } + ] +} diff --git a/BootKAN/FerramAerospaceResearch.ckan b/BootKAN/FerramAerospaceResearch.ckan new file mode 100644 index 0000000000..d66434a1e6 --- /dev/null +++ b/BootKAN/FerramAerospaceResearch.ckan @@ -0,0 +1,28 @@ +{ + "spec_version": 1, + "identifier": "FerramAerospaceResearch", + "$kref" : "#/ckan/kerbalstuff", + "license" : "GPL-3.0", + "release_status" : "stable", + "comment" : "Setting to *no* github releases so the code doesn't try to confuse NEAR with FAR", + "resources" : { + "github" : { + "url" : "https://github.com/ferram4/Ferram-Aerospace-Research", + "releases" : false + } + }, + "depends" : [ { "name" : "ModuleManager" } ], + "install" : [ + { + "file" : "GameData/FerramAerospaceResearch", + "install_to" : "GameData" + }, + { + "file" : "Ships", + "install_to" : "Ships", + "optional" : true, + "description" : "FAR example craft" + } + + ] +} diff --git a/BootKAN/README.md b/BootKAN/README.md new file mode 100644 index 0000000000..c2197d0ee1 --- /dev/null +++ b/BootKAN/README.md @@ -0,0 +1,6 @@ +The BootKAN are CKAN fragments that exist for mods that will support +the CKAN, but haven't yet released fragments in their own distributions +yet. + +It allows us to boot up the CKAN, using bots to generate metadata, but +without requiring everyone make a new release of their mods. diff --git a/BootKAN/RemoteTech.ckan b/BootKAN/RemoteTech.ckan new file mode 100644 index 0000000000..6a96939fd9 --- /dev/null +++ b/BootKAN/RemoteTech.ckan @@ -0,0 +1,19 @@ +{ + "spec_version" : 1, + "identifier" : "RemoteTech", + "$kref" : "#/ckan/kerbalstuff", + "license" : "restricted", + "release_status" : "stable", + "resources" : { + "github" : { + "url" : "https://github.com/RemoteTechnologiesGroup/RemoteTech" + } + }, + "depends": [ { "name" : "ModuleManager" } ], + "install" : [ + { + "file" : "GameData/RemoteTech", + "install_to" : "GameData" + } + ] +} diff --git a/BootKAN/kOS.ckan b/BootKAN/kOS.ckan new file mode 100644 index 0000000000..27b960a6d2 --- /dev/null +++ b/BootKAN/kOS.ckan @@ -0,0 +1,20 @@ +{ + "spec_version" : 1, + "identifier" : "kOS", + "$kref" : "#/ckan/kerbalstuff", + "license" : "GPL-3.0", + "release_status" : "stable", + "resources" : { + "manual" : "http://ksp-kos.github.io/KOS_DOC/", + "github" : { + "url" : "https://github.com/KSP-KOS/KOS", + "releases" : true + } + }, + "install" : [ + { + "file" : "GameData/kOS", + "install_to" : "GameData" + } + ] +} diff --git a/CKAN/KerbalStuff/KerbalStuff.csproj b/CKAN/KerbalStuff/KerbalStuff.csproj index f168b1c977..c1ab91e8c9 100644 --- a/CKAN/KerbalStuff/KerbalStuff.csproj +++ b/CKAN/KerbalStuff/KerbalStuff.csproj @@ -8,7 +8,7 @@ {4336F356-33DB-442A-BF74-5E89AF47A5B9} Exe CKAN.KerbalStuff - KerbalStuff + ks2ckan CKAN.KerbalStuff.MainClass diff --git a/CKAN/KerbalStuff/MainClass.cs b/CKAN/KerbalStuff/MainClass.cs index 7ac8e6323e..1e0afbcccc 100644 --- a/CKAN/KerbalStuff/MainClass.cs +++ b/CKAN/KerbalStuff/MainClass.cs @@ -36,6 +36,7 @@ public static int Main(string[] args) string identifier = args[0]; int ksid = Convert.ToInt32(args[1]); + string output_path = args[2] ?? "."; log.InfoFormat("Processing {0}", identifier); @@ -47,7 +48,16 @@ public static int Main(string[] args) KSVersion latest = ks.versions[0]; string filename = latest.Download(identifier); - JObject metadata = ExtractCkanInfo(filename); + JObject metadata = null; + try + { + metadata = ExtractCkanInfo(filename); + } + catch (MetadataNotFoundKraken) + { + log.WarnFormat ("Reading BootKAN metadata for {0}", filename); + metadata = BootKAN (identifier); + } // Check if we should auto-inflate! if ((string) metadata[expand_token] == ks_expand_path) @@ -72,7 +82,9 @@ public static int Main(string[] args) // All done! Write it out! - File.WriteAllText(String.Format("{0}-{1}.ckan", mod.identifier, mod.version), metadata.ToString()); + string final_path = Path.Combine(output_path, String.Format ("{0}-{1}.ckan", mod.identifier, mod.version)); + + File.WriteAllText(final_path, metadata.ToString()); return EXIT_OK; } @@ -117,6 +129,16 @@ private static JObject ExtractCkanInfo(string filename) throw new MetadataNotFoundKraken(filename); } + + /// + /// Returns the metadata fragment found in the BootKAN/ directory for + /// this identifier, if it exists. + /// + internal static JObject BootKAN(string identifier) + { + string fragment = File.ReadAllText(Path.Combine ("BootKAN", identifier + ".ckan")); + return JObject.Parse(fragment); + } } internal class MetadataNotFoundKraken : Exception diff --git a/KSMods.json b/KSMods.json new file mode 100644 index 0000000000..c7e9beca2d --- /dev/null +++ b/KSMods.json @@ -0,0 +1,10 @@ +{ + "comment" : "Here be all the mods we process from KerbalStuff, in identifier => ksid pairs", + "modlist" : { + "CrewFiles" : 39, + "DogeCoinFlag" : 269, + "kOS" : 86, + "FerramAerospaceResearch" : 52, + "RemoteTech" : 134 + } +} diff --git a/bin/build b/bin/build index 6c2ec7241c..5c5baf58ca 100755 --- a/bin/build +++ b/bin/build @@ -1190,6 +1190,8 @@ say "\n\n=== Repacking ===\n\n"; chdir("$Bin/.."); +# Repack ckan.exe + my @cmd = ( $REPACK, "--out:ckan.exe", @@ -1199,12 +1201,23 @@ my @cmd = ( "build/CmdLine/bin/$TARGET/CKAN-GUI.exe", # Yes, bundle the .exe as a .dll ); -say "@cmd"; +system(@cmd); + +# Repack ks2ckan + +my @cmd = ( + $REPACK, + "--out:ks2ckan.exe", + "--lib:build/KerbalStuff/bin/$TARGET", + "build/KerbalStuff/bin/$TARGET/ks2ckan.exe", + glob("build/KerbalStuff/bin/$TARGET/*.dll"), +); system(@cmd); say "\n\n=== Tidying up===\n\n"; unlink("$OUTNAME.mdb"); +unlink("ks2ckan.exe.mdb"); say "Done!"; diff --git a/bin/build.lean b/bin/build.lean index 8843cbddff..a31adfe7ab 100755 --- a/bin/build.lean +++ b/bin/build.lean @@ -54,6 +54,8 @@ say "\n\n=== Repacking ===\n\n"; chdir("$Bin/.."); +# Repack ckan.exe + my @cmd = ( $REPACK, "--out:ckan.exe", @@ -63,12 +65,23 @@ my @cmd = ( "build/CmdLine/bin/$TARGET/CKAN-GUI.exe", # Yes, bundle the .exe as a .dll ); -say "@cmd"; +system(@cmd); + +# Repack ks2ckan + +my @cmd = ( + $REPACK, + "--out:ks2ckan.exe", + "--lib:build/KerbalStuff/bin/$TARGET", + "build/KerbalStuff/bin/$TARGET/ks2ckan.exe", + glob("build/KerbalStuff/bin/$TARGET/*.dll"), +); system(@cmd); say "\n\n=== Tidying up===\n\n"; unlink("$OUTNAME.mdb"); +unlink("ks2ckan.exe.mdb"); say "Done!"; diff --git a/bin/kspan b/bin/kspan new file mode 100755 index 0000000000..1ebb40c005 --- /dev/null +++ b/bin/kspan @@ -0,0 +1,50 @@ +#!/usr/bin/perl +use 5.010; +use strict; +use warnings; +use autodie qw(:all); +use JSON::PP qw(decode_json); +use FindBin qw($Bin); +use File::Copy qw(move); + +use constant INDEX => "$Bin/../KSMods.json"; +use constant KS2CKAN => "$Bin/../ks2ckan.exe"; +use constant CKANMETA => "$Bin/../../CKAN-meta/"; + +# Generate everything we need for the KerbalStuffPAN (ks2ckan) + +# Hop over to our project root. +chdir("$Bin/.."); + +my $index = decode_json( slurp(INDEX) ); + +my @modlist; + +# Process mods listed on the commandline, or the whole modlist by default. +if (@ARGV) { + @modlist = @ARGV; +} +else { + @modlist = keys %{ $index->{modlist} }; +} + +foreach my $mod (@modlist) { + say "$mod..."; + + $index->{modlist}{$mod} or die "Cannot find data for $mod\n"; + + # Convert our mod + system(KS2CKAN, $mod, $index->{modlist}{$mod}, CKANMETA); + + # TODO: Automatically submit changes to git. +} + +sub slurp { + my ($filename) = @_; + + open(my $fh, '<', $filename); + + local $/; # Slurp mode! + + return <$fh>; +} From 512cf3efff0dd569921abc7f01b2daa30abe1d4e Mon Sep 17 00:00:00 2001 From: Paul Fenwick Date: Sun, 19 Oct 2014 23:50:24 +1100 Subject: [PATCH 2/4] Fixup CrewFiles path in BootKAN. --- BootKAN/CrewFiles.ckan | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/BootKAN/CrewFiles.ckan b/BootKAN/CrewFiles.ckan index 3ed19b995c..60f29efb09 100644 --- a/BootKAN/CrewFiles.ckan +++ b/BootKAN/CrewFiles.ckan @@ -4,7 +4,7 @@ "$kref" : "#/ckan/kerbalstuff", "install" : [ { - "file" : "GameData/CrewFiles", + "file" : "CrewFiles/GameData/CrewFiles", "install_to" : "GameData" } ] From 67dffcefbded1e1f3609757988fd8542614f09d7 Mon Sep 17 00:00:00 2001 From: Paul Fenwick Date: Mon, 20 Oct 2014 00:47:15 +1100 Subject: [PATCH 3/4] ks2ckan now fills in author info. Also updated Module.cs to allow author info to be read as either a string or a list. ;) --- CKAN/CKAN/CKAN.csproj | 1 + CKAN/CKAN/Module.cs | 6 ++- CKAN/CKAN/Types/JsonSingleOrArrayConverter.cs | 41 +++++++++++++++++++ CKAN/KerbalStuff/KSMod.cs | 2 + 4 files changed, 49 insertions(+), 1 deletion(-) create mode 100644 CKAN/CKAN/Types/JsonSingleOrArrayConverter.cs diff --git a/CKAN/CKAN/CKAN.csproj b/CKAN/CKAN/CKAN.csproj index 569fff5071..646b1724b8 100644 --- a/CKAN/CKAN/CKAN.csproj +++ b/CKAN/CKAN/CKAN.csproj @@ -64,6 +64,7 @@ + diff --git a/CKAN/CKAN/Module.cs b/CKAN/CKAN/Module.cs index c7de9d6608..81afa04fe6 100644 --- a/CKAN/CKAN/Module.cs +++ b/CKAN/CKAN/Module.cs @@ -1,6 +1,7 @@ using System; using System.IO; using System.Runtime.Serialization; +using System.Collections.Generic; using log4net; using Newtonsoft.Json; using Newtonsoft.Json.Linq; @@ -24,7 +25,10 @@ public class Module [JsonProperty("abstract")] public string @abstract; - [JsonProperty("author")] public string[] author; + [JsonProperty("author")] + [JsonConverter(typeof(JsonSingleOrArrayConverter))] + public List author; + [JsonProperty("comment")] public string comment; [JsonProperty("conflicts")] public dynamic[] conflicts; [JsonProperty("depends")] public dynamic[] depends; diff --git a/CKAN/CKAN/Types/JsonSingleOrArrayConverter.cs b/CKAN/CKAN/Types/JsonSingleOrArrayConverter.cs new file mode 100644 index 0000000000..78624e3f21 --- /dev/null +++ b/CKAN/CKAN/Types/JsonSingleOrArrayConverter.cs @@ -0,0 +1,41 @@ +using System; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using Newtonsoft.Json.Converters; +using System.Collections.Generic; + +namespace CKAN +{ + + // With thanks to + // https://stackoverflow.com/questions/18994685/how-to-handle-both-a-single-item-and-an-array-for-the-same-property-using-json-n + + public class JsonSingleOrArrayConverter : JsonConverter + { + public override bool CanConvert(Type object_type) + { + return(object_type == typeof(List)); + } + + public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer) + { + JToken token = JToken.Load(reader); + if (token.Type == JTokenType.Array) + { + return token.ToObject>(); + } + return new List { token.ToObject() }; + } + + public override bool CanWrite + { + get { return false; } + } + + public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer) + { + throw new NotImplementedException(); + } + } +} + diff --git a/CKAN/KerbalStuff/KSMod.cs b/CKAN/KerbalStuff/KSMod.cs index 1aa6846592..c499379751 100644 --- a/CKAN/KerbalStuff/KSMod.cs +++ b/CKAN/KerbalStuff/KSMod.cs @@ -13,6 +13,7 @@ public class KSMod public string license; public string name; public string short_description; + public string author; public KSVersion[] versions; public string website; @@ -42,6 +43,7 @@ public void InflateMetadata(JObject metadata, KSVersion version) Inflate(metadata, "name", name); Inflate(metadata, "license", license); Inflate(metadata, "abstract", short_description); + Inflate(metadata, "author", author); Inflate(metadata, "version", version.friendly_version.ToString()); Inflate(metadata, "download", Uri.EscapeUriString(version.download_path)); Inflate(metadata, "comment", "Generated by ks2ckan"); From 9576a00fd7475302bb682a6fb1dd93e2b79ad7db Mon Sep 17 00:00:00 2001 From: Paul Fenwick Date: Mon, 20 Oct 2014 01:10:24 +1100 Subject: [PATCH 4/4] Add `download_size` field. For KerbalStuff, spec, and schema. Closes #77. --- CKAN.schema | 4 ++++ CKAN/CKAN/Module.cs | 3 +++ CKAN/KerbalStuff/KSMod.cs | 17 ++++++++++++++++- CKAN/KerbalStuff/MainClass.cs | 2 +- Spec.md | 7 +++++++ 5 files changed, 31 insertions(+), 2 deletions(-) diff --git a/CKAN.schema b/CKAN.schema index bc7c1ebd6f..51ebcbe19b 100644 --- a/CKAN.schema +++ b/CKAN.schema @@ -41,6 +41,10 @@ "type" : "string", "format" : "uri" }, + "download_size" : { + "description" : "The size of the download in bytes", + "type" : "integer" + }, "license" : { "description" : "Machine readable license, or array of licenses", "$ref" : "#/definitions/licenses" diff --git a/CKAN/CKAN/Module.cs b/CKAN/CKAN/Module.cs index 81afa04fe6..59c1c0a20a 100644 --- a/CKAN/CKAN/Module.cs +++ b/CKAN/CKAN/Module.cs @@ -29,6 +29,9 @@ public class Module [JsonConverter(typeof(JsonSingleOrArrayConverter))] public List author; + [JsonProperty("download_size")] + public long download_size; + [JsonProperty("comment")] public string comment; [JsonProperty("conflicts")] public dynamic[] conflicts; [JsonProperty("depends")] public dynamic[] depends; diff --git a/CKAN/KerbalStuff/KSMod.cs b/CKAN/KerbalStuff/KSMod.cs index c499379751..ff79790c77 100644 --- a/CKAN/KerbalStuff/KSMod.cs +++ b/CKAN/KerbalStuff/KSMod.cs @@ -1,4 +1,5 @@ using System; +using System.IO; using log4net; using Newtonsoft.Json.Linq; @@ -26,8 +27,12 @@ public override string ToString() /// Takes a JObject and inflates it with KS metadata. /// This will not overwrite fields that already exist. /// - public void InflateMetadata(JObject metadata, KSVersion version) + public void InflateMetadata(JObject metadata, KSVersion version, string filename) { + + // Check how big our file is + long download_size = (new FileInfo (filename)).Length; + // Make sure resources exist. if (metadata["resources"] == null) { @@ -47,6 +52,7 @@ public void InflateMetadata(JObject metadata, KSVersion version) Inflate(metadata, "version", version.friendly_version.ToString()); Inflate(metadata, "download", Uri.EscapeUriString(version.download_path)); Inflate(metadata, "comment", "Generated by ks2ckan"); + Inflate(metadata, "download_size", download_size); Inflate((JObject) metadata["resources"], "homepage", website); Inflate((JObject) metadata["resources"]["kerbalstuff"], "url", KSHome()); Inflate(metadata, "ksp_version", version.KSP_version.ToString()); @@ -70,5 +76,14 @@ internal static void Inflate(JObject metadata, string key, string value) log.DebugFormat("Leaving {0} as {1}", key, metadata[key]); } } + + internal static void Inflate(JObject metadata, string key, long value) + { + if (metadata[key] == null) + { + metadata[key] = value; + } + } + } } \ No newline at end of file diff --git a/CKAN/KerbalStuff/MainClass.cs b/CKAN/KerbalStuff/MainClass.cs index 1e0afbcccc..23d146cc47 100644 --- a/CKAN/KerbalStuff/MainClass.cs +++ b/CKAN/KerbalStuff/MainClass.cs @@ -63,7 +63,7 @@ public static int Main(string[] args) if ((string) metadata[expand_token] == ks_expand_path) { log.InfoFormat("Inflating..."); - ks.InflateMetadata(metadata, latest); + ks.InflateMetadata(metadata, latest, filename); metadata.Remove(expand_token); } diff --git a/Spec.md b/Spec.md index 39e79b413c..4427476009 100644 --- a/Spec.md +++ b/Spec.md @@ -466,6 +466,13 @@ and the old name of the mod is listed in the `provides` field. This allows for mods to be renamed without updating all other mods which depend upon it. +##### download_size + +If supplied, `download_size` is the number of bytes to expect when +downloading from the `download` URL. It is recommended this this field +only be generated by automated tools (where it is encouraged), +and not filled in by hand. + #### Extensions Any field starting with `x_` (an x, followed by an underscore) is considered