diff --git a/Cargo.lock b/Cargo.lock index 52bc649fe..6d23a51af 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -154,9 +154,9 @@ checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" [[package]] name = "castaway" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a17ed5635fc8536268e5d4de1e22e81ac34419e5f052d4d51f4e01dcc263fcc" +checksum = "0abae9be0aaf9ea96a3b1b8b1b55c602ca751eba1b1500220cea4ecbafe7c0d5" dependencies = [ "rustversion", ] @@ -314,6 +314,31 @@ dependencies = [ "libc", ] +[[package]] +name = "crossterm" +version = "0.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "829d955a0bb380ef178a640b91779e3987da38c9aea133b20614cfed8cdea9c6" +dependencies = [ + "bitflags 2.4.2", + "crossterm_winapi", + "mio", + "parking_lot", + "rustix", + "signal-hook", + "signal-hook-mio", + "winapi", +] + +[[package]] +name = "crossterm_winapi" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acdd7c62a3665c7f6830a51635d9ac9b23ed385797f70a83bb8bafe9c572ab2b" +dependencies = [ + "winapi", +] + [[package]] name = "crypto-common" version = "0.1.6" @@ -387,6 +412,7 @@ dependencies = [ "ansiterm", "clap", "cli-table", + "crossterm", "dotlock", "fs2", "hex", @@ -394,6 +420,8 @@ dependencies = [ "indoc", "miette", "nix", + "petgraph", + "pretty_assertions", "regex", "reqwest", "schemars", @@ -403,6 +431,10 @@ dependencies = [ "serde_yaml", "sha2", "tempdir", + "tempfile", + "test-log", + "thiserror", + "tokio", "tracing", "which", "whoami", @@ -416,8 +448,15 @@ dependencies = [ "clap", "devenv", "tempdir", + "tokio", ] +[[package]] +name = "diff" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" + [[package]] name = "digest" version = "0.10.7" @@ -479,6 +518,27 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "env_filter" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f2c92ceda6ceec50f43169f9ee8424fe2db276791afde7b2cd8bc084cb376ab" +dependencies = [ + "log", +] + +[[package]] +name = "env_logger" +version = "0.11.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13fa619b91fb2381732789fc5de83b45675e882f66623b7d8cb4f643017018d" +dependencies = [ + "anstream", + "anstyle", + "env_filter", + "log", +] + [[package]] name = "equivalent" version = "1.0.1" @@ -501,6 +561,12 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" +[[package]] +name = "fixedbitset" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" + [[package]] name = "fnv" version = "1.0.7" @@ -678,6 +744,12 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "hermit-abi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" + [[package]] name = "hex" version = "0.4.3" @@ -863,7 +935,7 @@ checksum = "85c833ca1e66078851dba29046874e38f08b2c883700aa29a03ddd3b23814ee8" dependencies = [ "bitflags 2.4.2", "libc", - "redox_syscall", + "redox_syscall 0.4.1", ] [[package]] @@ -872,11 +944,21 @@ version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" +[[package]] +name = "lock_api" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" +dependencies = [ + "autocfg", + "scopeguard", +] + [[package]] name = "log" -version = "0.4.20" +version = "0.4.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" +checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" [[package]] name = "markdown" @@ -887,6 +969,15 @@ dependencies = [ "unicode-id", ] +[[package]] +name = "matchers" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" +dependencies = [ + "regex-automata 0.1.10", +] + [[package]] name = "memchr" version = "2.7.1" @@ -941,13 +1032,15 @@ dependencies = [ [[package]] name = "mio" -version = "0.8.10" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f3d0b296e374a4e6f3c7b0a1f5a51d748a0d34c85e7dc48fc3fa9a87657fe09" +checksum = "80e04d1dcff3aae0704555fe5fee3bcfaf3d1fdf8a7e521d5b9d2b42acb52cec" dependencies = [ + "hermit-abi", "libc", + "log", "wasi", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -980,6 +1073,16 @@ dependencies = [ "libc", ] +[[package]] +name = "nu-ansi-term" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" +dependencies = [ + "overload", + "winapi", +] + [[package]] name = "object" version = "0.32.2" @@ -1045,18 +1148,57 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" +[[package]] +name = "overload" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" + [[package]] name = "owo-colors" version = "4.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "caff54706df99d2a78a5a4e3455ff45448d81ef1bb63c22cd14052ca0e993a3f" +[[package]] +name = "parking_lot" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall 0.5.3", + "smallvec", + "windows-targets 0.52.6", +] + [[package]] name = "percent-encoding" version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" +[[package]] +name = "petgraph" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" +dependencies = [ + "fixedbitset", + "indexmap", +] + [[package]] name = "pin-project-lite" version = "0.2.13" @@ -1075,6 +1217,16 @@ version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" +[[package]] +name = "pretty_assertions" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af7cee1a6c8a5b9208b3cb1061f10c0cb689087b3d8ce85fb9d2dd7a29b6ba66" +dependencies = [ + "diff", + "yansi", +] + [[package]] name = "proc-macro2" version = "1.0.78" @@ -1139,6 +1291,15 @@ dependencies = [ "bitflags 1.3.2", ] +[[package]] +name = "redox_syscall" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a908a6e00f1fdd0dfd9c0eb08ce85126f6d8bbda50017e74bc4a4b7d4a926a4" +dependencies = [ + "bitflags 2.4.2", +] + [[package]] name = "redox_users" version = "0.4.4" @@ -1158,8 +1319,17 @@ checksum = "b62dbe01f0b06f9d8dc7d49e05a0785f153b00b2c227856282f671e0318c9b15" dependencies = [ "aho-corasick", "memchr", - "regex-automata", - "regex-syntax", + "regex-automata 0.4.5", + "regex-syntax 0.8.2", +] + +[[package]] +name = "regex-automata" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" +dependencies = [ + "regex-syntax 0.6.29", ] [[package]] @@ -1170,9 +1340,15 @@ checksum = "5bb987efffd3c6d0d8f5f89510bb458559eab11e4f869acb20bf845e016259cd" dependencies = [ "aho-corasick", "memchr", - "regex-syntax", + "regex-syntax 0.8.2", ] +[[package]] +name = "regex-syntax" +version = "0.6.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" + [[package]] name = "regex-syntax" version = "0.8.2" @@ -1242,9 +1418,9 @@ checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" [[package]] name = "rustix" -version = "0.38.31" +version = "0.38.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ea3e1a662af26cd7a3ba09c0297a31af215563ecf42817c98df621387f4e949" +checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" dependencies = [ "bitflags 2.4.2", "errno", @@ -1349,6 +1525,12 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f3adfbe1c90a6a9643433e490ef1605c6a99f93be37e4c83fe5149fca9698c6" +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + [[package]] name = "security-framework" version = "2.9.2" @@ -1461,6 +1643,45 @@ dependencies = [ "digest", ] +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "signal-hook" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8621587d4798caf8eb44879d42e56b9a93ea5dcd315a6487c357130095b62801" +dependencies = [ + "libc", + "signal-hook-registry", +] + +[[package]] +name = "signal-hook-mio" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34db1a06d485c9142248b7a054f034b349b212551f3dfd19c94d45a754a217cd" +dependencies = [ + "libc", + "mio", + "signal-hook", +] + +[[package]] +name = "signal-hook-registry" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9e9e0b4211b72e7b8b6e85c807d36c212bdb33ea8587f7569562a84df5465b1" +dependencies = [ + "libc", +] + [[package]] name = "slab" version = "0.4.9" @@ -1591,6 +1812,16 @@ dependencies = [ "libc", ] +[[package]] +name = "tasks" +version = "0.1.0" +dependencies = [ + "clap", + "devenv", + "serde_json", + "tokio", +] + [[package]] name = "tempdir" version = "0.3.7" @@ -1603,14 +1834,15 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.10.1" +version = "3.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1" +checksum = "04cbcdd0c794ebb0d4cf35e88edd2f7d2c4c3e9a5a6dab322839b321c6a87a64" dependencies = [ "cfg-if", "fastrand", + "once_cell", "rustix", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -1632,6 +1864,28 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "test-log" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3dffced63c2b5c7be278154d76b479f9f9920ed34e7574201407f0b14e2bbb93" +dependencies = [ + "env_logger", + "test-log-macros", + "tracing-subscriber", +] + +[[package]] +name = "test-log-macros" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5999e24eaa32083191ba4e425deb75cdf25efefabe5aaccb7446dd0d4122a3f5" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.51", +] + [[package]] name = "textwrap" version = "0.16.1" @@ -1645,24 +1899,34 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.57" +version = "1.0.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e45bcbe8ed29775f228095caf2cd67af7a4ccf756ebff23a306bf3e8b47b24b" +checksum = "c0342370b38b6a11b6cc11d6a805569958d54cfa061a29969c3b5ce2ea405724" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.57" +version = "1.0.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a953cb265bef375dae3de6663da4d3804eee9682ea80d8e2542529b73c531c81" +checksum = "a4558b58466b9ad7ca0f102865eccc95938dca1a74a856f2b57b6629050da261" dependencies = [ "proc-macro2", "quote", "syn 2.0.51", ] +[[package]] +name = "thread_local" +version = "1.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c" +dependencies = [ + "cfg-if", + "once_cell", +] + [[package]] name = "tinyvec" version = "1.6.0" @@ -1680,17 +1944,30 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.36.0" +version = "1.39.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61285f6515fa018fb2d1e46eb21223fff441ee8db5d0f1435e8ab4f5cdb80931" +checksum = "9babc99b9923bfa4804bd74722ff02c0381021eafa4db9949217e3be8e84fff5" dependencies = [ "backtrace", "bytes", "libc", "mio", "pin-project-lite", + "signal-hook-registry", "socket2", - "windows-sys 0.48.0", + "tokio-macros", + "windows-sys 0.52.0", +] + +[[package]] +name = "tokio-macros" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.51", ] [[package]] @@ -1752,6 +2029,35 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" dependencies = [ "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex", + "sharded-slab", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", ] [[package]] @@ -1807,9 +2113,9 @@ checksum = "d4c87d22b6e3f4a18d4d40ef354e97c90fcb14dd91d7dc0aa9d8a1172ebf7202" [[package]] name = "unicode-width" -version = "0.1.11" +version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85" +checksum = "0336d538f7abc86d282a4189614dfaa90810dfc2c6f6427eaf88e16311dd225d" [[package]] name = "unsafe-libyaml" @@ -1834,6 +2140,12 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" +[[package]] +name = "valuable" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" + [[package]] name = "vcpkg" version = "0.2.15" @@ -1962,7 +2274,7 @@ version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a44ab49fad634e88f55bf8f9bb3abd2f27d7204172a112c7c9987e01c1c94ea9" dependencies = [ - "redox_syscall", + "redox_syscall 0.4.1", "wasite", "web-sys", ] @@ -2013,7 +2325,16 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets 0.52.3", + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets 0.52.6", ] [[package]] @@ -2033,17 +2354,18 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.52.3" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d380ba1dc7187569a8a9e91ed34b8ccfc33123bbacb8c0aed2d1ad7f3ef2dc5f" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "windows_aarch64_gnullvm 0.52.3", - "windows_aarch64_msvc 0.52.3", - "windows_i686_gnu 0.52.3", - "windows_i686_msvc 0.52.3", - "windows_x86_64_gnu 0.52.3", - "windows_x86_64_gnullvm 0.52.3", - "windows_x86_64_msvc 0.52.3", + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", ] [[package]] @@ -2054,9 +2376,9 @@ checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" [[package]] name = "windows_aarch64_gnullvm" -version = "0.52.3" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68e5dcfb9413f53afd9c8f86e56a7b4d86d9a2fa26090ea2dc9e40fba56c6ec6" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" [[package]] name = "windows_aarch64_msvc" @@ -2066,9 +2388,9 @@ checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" [[package]] name = "windows_aarch64_msvc" -version = "0.52.3" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8dab469ebbc45798319e69eebf92308e541ce46760b49b18c6b3fe5e8965b30f" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" [[package]] name = "windows_i686_gnu" @@ -2078,9 +2400,15 @@ checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" [[package]] name = "windows_i686_gnu" -version = "0.52.3" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a4e9b6a7cac734a8b4138a4e1044eac3404d8326b6c0f939276560687a033fb" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" [[package]] name = "windows_i686_msvc" @@ -2090,9 +2418,9 @@ checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" [[package]] name = "windows_i686_msvc" -version = "0.52.3" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28b0ec9c422ca95ff34a78755cfa6ad4a51371da2a5ace67500cf7ca5f232c58" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" [[package]] name = "windows_x86_64_gnu" @@ -2102,9 +2430,9 @@ checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" [[package]] name = "windows_x86_64_gnu" -version = "0.52.3" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "704131571ba93e89d7cd43482277d6632589b18ecf4468f591fbae0a8b101614" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" [[package]] name = "windows_x86_64_gnullvm" @@ -2114,9 +2442,9 @@ checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" [[package]] name = "windows_x86_64_gnullvm" -version = "0.52.3" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42079295511643151e98d61c38c0acc444e52dd42ab456f7ccfd5152e8ecf21c" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" [[package]] name = "windows_x86_64_msvc" @@ -2126,9 +2454,9 @@ checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" [[package]] name = "windows_x86_64_msvc" -version = "0.52.3" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0770833d60a970638e989b3fa9fd2bb1aaadcf88963d1659fd7d9990196ed2d6" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "winreg" @@ -2156,3 +2484,9 @@ dependencies = [ "devenv", "miette", ] + +[[package]] +name = "yansi" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09041cd90cf85f7f8b2df60c646f853b7f535ce68f85244eb6731cf89fa498ec" diff --git a/Cargo.toml b/Cargo.toml index e793c48ac..053ba5d61 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,10 +1,6 @@ [workspace] resolver = "2" -members = [ - "devenv", - "devenv-run-tests", - "xtask", -] +members = ["devenv", "devenv-run-tests", "xtask", "tasks"] [workspace.package] edition = "2021" @@ -25,7 +21,12 @@ miette = { version = "7.1.0", features = ["fancy"] } nix = { version = "0.28.0", features = ["signal"] } regex = "1.10.3" reqwest = "0.11.26" -schematic = { version = "0.14.3", features = ["schema", "yaml", "renderer_template", "renderer_json_schema"] } +schematic = { version = "0.14.3", features = [ + "schema", + "yaml", + "renderer_template", + "renderer_json_schema", +] } serde = "1.0.197" serde_json = "1.0.114" serde_yaml = "0.9.32" @@ -35,5 +36,5 @@ tracing = "0.1.40" which = "6.0.0" whoami = "1.5.1" xdg = "2.5.2" - +tokio = "1.39.3" schemars = "0.8.16" diff --git a/devenv-run-tests/Cargo.toml b/devenv-run-tests/Cargo.toml index 761473b2a..b13c40344 100644 --- a/devenv-run-tests/Cargo.toml +++ b/devenv-run-tests/Cargo.toml @@ -9,3 +9,4 @@ clap.workspace = true tempdir.workspace = true devenv= { path = "../devenv" } +tokio = "1.39.3" diff --git a/devenv-run-tests/src/main.rs b/devenv-run-tests/src/main.rs index b6e7cb345..2e81ba677 100644 --- a/devenv-run-tests/src/main.rs +++ b/devenv-run-tests/src/main.rs @@ -32,7 +32,9 @@ struct TestResult { passed: bool, } -fn run_tests_in_directory(args: &Args) -> Result, Box> { +async fn run_tests_in_directory( + args: &Args, +) -> Result, Box> { let logger = Logger::new(Level::Info); logger.info("Running Tests"); @@ -118,11 +120,13 @@ fn run_tests_in_directory(args: &Args) -> Result, Box Result, Box Result<(), Box> { +#[tokio::main] +async fn main() -> Result<(), Box> { let args = Args::parse(); - let test_results = run_tests_in_directory(&args)?; + let executable_path = std::env::current_exe()?; + let executable_dir = executable_path.parent().unwrap(); + std::env::set_var( + "PATH", + format!( + "{}:{}", + executable_dir.display(), + std::env::var("PATH").unwrap_or_default() + ), + ); + + let test_results = run_tests_in_directory(&args).await?; let num_tests = test_results.len(); let num_failed_tests = test_results.iter().filter(|r| !r.passed).count(); diff --git a/devenv.nix b/devenv.nix index 25961a272..31e49014b 100644 --- a/devenv.nix +++ b/devenv.nix @@ -2,6 +2,8 @@ env.DEVENV_NIX = inputs.nix.packages.${pkgs.stdenv.system}.nix; # ignore annoying browserlists warning that breaks pre-commit hooks env.BROWSERSLIST_IGNORE_OLD_DATA = "1"; + env.RUST_LOG = "devenv=debug"; + env.RUST_LOG_SPAN_EVENTS = "full"; packages = [ pkgs.cairo diff --git a/devenv/Cargo.toml b/devenv/Cargo.toml index ee0ecf2ed..0db421a0b 100644 --- a/devenv/Cargo.toml +++ b/devenv/Cargo.toml @@ -13,6 +13,7 @@ default-run = "devenv" ansiterm.workspace = true clap.workspace = true cli-table.workspace = true +crossterm = "0.28.1" dotlock.workspace = true fs2.workspace = true hex.workspace = true @@ -20,6 +21,8 @@ include_dir.workspace = true indoc.workspace = true miette.workspace = true nix.workspace = true +petgraph = "0.6.5" +pretty_assertions = { version = "1.4.0", features = ["unstable"] } regex.workspace = true reqwest.workspace = true schemars.workspace = true @@ -29,6 +32,14 @@ serde_json.workspace = true serde_yaml.workspace = true sha2.workspace = true tempdir.workspace = true +tempfile = "3.12.0" +test-log = { version = "0.2.16", features = ["trace"] } +thiserror = "1.0.63" +tokio = { version = "1.39.3", features = [ + "process", + "macros", + "rt-multi-thread", +] } tracing.workspace = true which.workspace = true whoami.workspace = true diff --git a/devenv/init/devenv.nix b/devenv/init/devenv.nix index 5a693e8b7..884e6047b 100644 --- a/devenv/init/devenv.nix +++ b/devenv/init/devenv.nix @@ -26,6 +26,12 @@ git --version ''; + # https://devenv.sh/tasks/ + # tasks = { + # "myproj:setup".exec = "mytool build"; + # "devenv:enterShell".after = [ "myproj:setup" ]; + # }; + # https://devenv.sh/tests/ enterTest = '' echo "Running tests" diff --git a/devenv/src/cli.rs b/devenv/src/cli.rs index a9ecb0999..a6d1ef503 100644 --- a/devenv/src/cli.rs +++ b/devenv/src/cli.rs @@ -165,6 +165,12 @@ pub(crate) enum Commands { command: ProcessesCommand, }, + #[command(about = "Run tasks. https://devenv.sh/tasks/")] + Tasks { + #[command(subcommand)] + command: TasksCommand, + }, + #[command(about = "Run tests. http://devenv.sh/tests/", alias = "ci")] Test { #[arg(short, long, help = "Don't override .devenv to a temporary directory.")] @@ -241,6 +247,13 @@ pub(crate) enum ProcessesCommand { // TODO: Status/Attach } +#[derive(Subcommand, Clone)] +#[clap(about = "Run tasks. https://devenv.sh/tasks/")] +pub(crate) enum TasksCommand { + #[command(about = "Run tasks.")] + Run { tasks: Vec }, +} + #[derive(Subcommand, Clone)] #[clap( about = "Build, copy, or run a container. https://devenv.sh/containers/", diff --git a/devenv/src/cnix.rs b/devenv/src/cnix.rs new file mode 100644 index 000000000..96e2132c7 --- /dev/null +++ b/devenv/src/cnix.rs @@ -0,0 +1,708 @@ +use crate::{cli, config, log}; +use miette::{bail, IntoDiagnostic, Result, WrapErr}; +use serde::Deserialize; +use std::cell::{Ref, RefCell}; +use std::collections::HashMap; +use std::env; +use std::fs; +use std::os::unix::fs::symlink; +use std::os::unix::process::CommandExt; +use std::path::{Path, PathBuf}; +use std::process; +use std::time::{SystemTime, UNIX_EPOCH}; + +pub struct Nix<'a> { + logger: log::Logger, + pub options: Options<'a>, + // TODO: all these shouldn't be here + config: config::Config, + global_options: cli::GlobalOptions, + cachix_caches: RefCell>, + cachix_trusted_keys: PathBuf, + devenv_home_gc: PathBuf, + devenv_dot_gc: PathBuf, + devenv_root: PathBuf, +} + +#[derive(Clone)] +pub struct Options<'a> { + pub replace_shell: bool, + pub logging: bool, + pub logging_stdout: bool, + pub nix_flags: &'a [&'a str], +} + +impl<'a> Nix<'a> { + pub fn new>( + logger: log::Logger, + config: config::Config, + global_options: cli::GlobalOptions, + cachix_trusted_keys: P, + devenv_home_gc: P, + devenv_dot_gc: P, + devenv_root: P, + ) -> Self { + let cachix_trusted_keys = cachix_trusted_keys.as_ref().to_path_buf(); + let devenv_home_gc = devenv_home_gc.as_ref().to_path_buf(); + let devenv_dot_gc = devenv_dot_gc.as_ref().to_path_buf(); + let devenv_root = devenv_root.as_ref().to_path_buf(); + + let cachix_caches = RefCell::new(None); + + Self { + logger, + cachix_caches, + config, + global_options, + options: Options { + replace_shell: false, + logging: true, + logging_stdout: false, + nix_flags: &[ + "--show-trace", + "--extra-experimental-features", + "nix-command", + "--extra-experimental-features", + "flakes", + "--option", + "warn-dirty", + "false", + "--keep-going", + ], + }, + cachix_trusted_keys, + devenv_home_gc, + devenv_dot_gc, + devenv_root, + } + } + + pub async fn develop(&self, args: &[&str], replace_shell: bool) -> Result { + let options = Options { + logging_stdout: true, + replace_shell, + ..self.options + }; + self.run_nix_with_substituters("nix", args, &options).await + } + + pub async fn dev_env(&self, json: bool, gc_root: &PathBuf) -> Result> { + let gc_root_str = gc_root.to_str().expect("gc root should be utf-8"); + let mut args: Vec<&str> = vec!["print-dev-env", "--profile", gc_root_str]; + if json { + args.push("--json"); + } + + let options = Options { ..self.options }; + let env = self + .run_nix_with_substituters("nix", &args, &options) + .await?; + + let options = Options { + logging: false, + ..self.options + }; + + let args: Vec<&str> = vec!["-p", gc_root_str, "--delete-generations", "old"]; + self.run_nix("nix-env", &args, &options)?; + let now_ns = get_now_with_nanoseconds(); + let target = format!("{}-shell", now_ns); + symlink_force( + &self.logger, + &fs::canonicalize(gc_root).expect("to resolve gc_root"), + &self.devenv_home_gc.join(target), + ); + Ok(env.stdout) + } + + pub fn add_gc(&self, name: &str, path: &Path) -> Result<()> { + self.run_nix( + "nix-store", + &[ + "--add-root", + self.devenv_dot_gc.join(name).to_str().unwrap(), + "-r", + path.to_str().unwrap(), + ], + &self.options, + )?; + let link_path = self + .devenv_dot_gc + .join(format!("{}-{}", name, get_now_with_nanoseconds())); + symlink_force(&self.logger, path, &link_path); + Ok(()) + } + + pub fn repl(&self) -> Result<()> { + let mut cmd = self.prepare_command("nix", &["repl", "."], &self.options)?; + cmd.exec(); + Ok(()) + } + + pub async fn build(&self, attributes: &[&str]) -> Result> { + if !attributes.is_empty() { + // TODO: use eval underneath + let mut args: Vec = vec![ + "build".to_string(), + "--no-link".to_string(), + "--print-out-paths".to_string(), + ]; + args.extend(attributes.iter().map(|attr| format!(".#{}", attr))); + let args_str: Vec<&str> = args.iter().map(AsRef::as_ref).collect(); + let output = self + .run_nix_with_substituters("nix", &args_str, &self.options) + .await?; + Ok(String::from_utf8_lossy(&output.stdout) + .to_string() + .split_whitespace() + .map(|s| PathBuf::from(s.to_string())) + .collect()) + } else { + Ok(Vec::new()) + } + } + + pub async fn eval(&self, attributes: &[&str]) -> Result { + let mut args: Vec = vec!["eval", "--json"] + .into_iter() + .map(String::from) + .collect(); + args.extend(attributes.iter().map(|attr| format!(".#{}", attr))); + let args = &args.iter().map(|s| s.as_str()).collect::>(); + let result = self.run_nix("nix", args, &self.options)?; + String::from_utf8(result.stdout) + .map_err(|err| miette::miette!("Failed to parse command output as UTF-8: {}", err)) + } + + pub fn update(&self, input_name: &Option) -> Result<()> { + match input_name { + Some(input_name) => { + self.run_nix( + "nix", + &["flake", "lock", "--update-input", input_name], + &self.options, + )?; + } + None => { + self.run_nix("nix", &["flake", "update"], &self.options)?; + } + } + Ok(()) + } + + pub fn metadata(&self) -> Result { + // TODO: use --json + let metadata = self.run_nix("nix", &["flake", "metadata"], &self.options)?; + + let re = regex::Regex::new(r"(Inputs:.+)$").unwrap(); + let metadata_str = String::from_utf8_lossy(&metadata.stdout); + let inputs = match re.captures(&metadata_str) { + Some(captures) => captures.get(1).unwrap().as_str(), + None => "", + }; + + let info_ = self.run_nix("nix", &["eval", "--raw", ".#info"], &self.options)?; + Ok(format!( + "{}\n{}", + inputs, + &String::from_utf8_lossy(&info_.stdout) + )) + } + + pub async fn search(&self, name: &str) -> Result { + self.run_nix_with_substituters( + "nix", + &["search", "--inputs-from", ".", "--json", "nixpkgs", name], + &self.options, + ) + .await + } + + pub fn gc(&self, paths: Vec) -> Result<()> { + let paths: std::collections::HashSet<&str> = paths + .iter() + .filter_map(|path_buf| path_buf.to_str()) + .collect(); + for path in paths { + self.logger.info(&format!("Deleting {}...", path)); + let args: Vec<&str> = ["store", "delete", path].to_vec(); + let cmd = self.prepare_command("nix", &args, &self.options); + // we ignore if this command fails, because root might be in use + let _ = cmd?.output(); + } + Ok(()) + } + + // Run Nix with debugger capability and return the output + pub fn run_nix( + &self, + command: &str, + args: &[&str], + options: &Options<'a>, + ) -> Result { + let cmd = self.prepare_command(command, args, options)?; + self.run_nix_command(cmd, options) + } + + pub async fn run_nix_with_substituters( + &self, + command: &str, + args: &[&str], + options: &Options<'a>, + ) -> Result { + let cmd = self + .prepare_command_with_substituters(command, args, options) + .await?; + self.run_nix_command(cmd, options) + } + + fn run_nix_command( + &self, + mut cmd: std::process::Command, + options: &Options<'a>, + ) -> Result { + let mut logger = self.logger.clone(); + + if !options.logging { + logger.level = log::Level::Error; + } + + if options.replace_shell { + if self.global_options.nix_debugger + && cmd.get_program().to_string_lossy().ends_with("bin/nix") + { + cmd.arg("--debugger"); + } + let error = cmd.exec(); + self.logger.error(&format!( + "Failed to replace shell with `{}`: {error}", + display_command(&cmd), + )); + bail!("Failed to replace shell") + } else { + if options.logging { + cmd.stdin(process::Stdio::inherit()) + .stderr(process::Stdio::inherit()); + if options.logging_stdout { + cmd.stdout(std::process::Stdio::inherit()); + } + } + + let result = cmd + .output() + .into_diagnostic() + .wrap_err_with(|| format!("Failed to run command `{}`", display_command(&cmd)))?; + + if !result.status.success() { + let code = match result.status.code() { + Some(code) => format!("with exit code {}", code), + None => "without exit code".to_string(), + }; + if options.logging { + eprintln!(); + self.logger.error(&format!( + "Command produced the following output:\n{}\n{}", + String::from_utf8_lossy(&result.stdout), + String::from_utf8_lossy(&result.stderr), + )); + } + if self.global_options.nix_debugger + && cmd.get_program().to_string_lossy().ends_with("bin/nix") + { + self.logger.info("Starting Nix debugger ..."); + cmd.arg("--debugger").exec(); + } + bail!(format!( + "Command `{}` failed with {code}", + display_command(&cmd) + )) + } else { + Ok(result) + } + } + } + + // We have a separate function to avoid recursion as this needs to call self.prepare_command + pub async fn prepare_command_with_substituters( + &self, + command: &str, + args: &[&str], + options: &Options<'a>, + ) -> Result { + let mut final_args = Vec::new(); + let known_keys; + let pull_caches; + let mut push_cache = None; + + if !self.global_options.offline { + let cachix_caches = self.get_cachix_caches().await; + + match cachix_caches { + Err(e) => { + self.logger + .warn("Failed to get cachix caches due to evaluation error"); + self.logger.debug(&format!("{}", e)); + } + Ok(cachix_caches) => { + push_cache = cachix_caches.caches.push.clone(); + // handle cachix.pull + pull_caches = cachix_caches + .caches + .pull + .iter() + .map(|cache| format!("https://{}.cachix.org", cache)) + .collect::>() + .join(" "); + final_args.extend_from_slice(&["--option", "extra-substituters", &pull_caches]); + known_keys = cachix_caches + .known_keys + .values() + .cloned() + .collect::>() + .join(" "); + final_args.extend_from_slice(&[ + "--option", + "extra-trusted-public-keys", + &known_keys, + ]); + } + } + } + + final_args.extend(args.iter().copied()); + let cmd = self.prepare_command(command, &final_args, options)?; + + // handle cachix.push + if let Some(push_cache) = push_cache { + if env::var("CACHIX_AUTH_TOKEN").is_ok() { + let original_command = cmd.get_program().to_string_lossy().to_string(); + let mut new_cmd = std::process::Command::new("cachix"); + let push_args = vec![ + "watch-exec".to_string(), + push_cache.clone(), + "--".to_string(), + original_command, + ]; + new_cmd.args(&push_args); + new_cmd.args(cmd.get_args()); + // make sure to copy all env vars + for (key, value) in cmd.get_envs() { + if let Some(value) = value { + new_cmd.env(key, value); + } + } + new_cmd.current_dir(cmd.get_current_dir().unwrap_or_else(|| Path::new("."))); + return Ok(new_cmd); + } else { + self.logger.warn(&format!( + "CACHIX_AUTH_TOKEN is not set, but required to push to {}.", + push_cache + )); + } + } + Ok(cmd) + } + + pub fn prepare_command( + &self, + command: &str, + args: &[&str], + options: &Options<'a>, + ) -> Result { + let mut flags = options.nix_flags.to_vec(); + flags.push("--max-jobs"); + let max_jobs = self.global_options.max_jobs.to_string(); + flags.push(&max_jobs); + + flags.push("--option"); + flags.push("eval-cache"); + let eval_cache = self.global_options.eval_cache.to_string(); + flags.push(&eval_cache); + + // handle --nix-option key value + for chunk in self.global_options.nix_option.chunks_exact(2) { + flags.push("--option"); + flags.push(&chunk[0]); + flags.push(&chunk[1]); + } + + flags.extend_from_slice(args); + + let mut cmd = match env::var("DEVENV_NIX") { + Ok(devenv_nix) => std::process::Command::new(format!("{devenv_nix}/bin/{command}")), + Err(_) => { + self.logger.error( + "$DEVENV_NIX is not set, but required as devenv doesn't work without a few Nix patches." + ); + self.logger + .error("Please follow https://devenv.sh/getting-started/ to install devenv."); + bail!("$DEVENV_NIX is not set") + } + }; + + if self.global_options.offline && command == "nix" { + flags.push("--offline"); + } + + if self.global_options.impure || self.config.impure { + // only pass the impure option to the nix command that supports it. + // avoid passing it to the older utilities, e.g. like `nix-store` when creating GC roots. + if command == "nix" + && args + .iter() + .any(|&arg| arg == "build" || arg == "eval" || arg == "print-dev-env") + { + flags.push("--no-pure-eval"); + } + // set a dummy value to overcome https://github.com/NixOS/nix/issues/10247 + cmd.env("NIX_PATH", ":"); + } + cmd.args(flags); + cmd.current_dir(&self.devenv_root); + + if self.global_options.verbose { + self.logger + .debug(&format!("Running command: {}", display_command(&cmd))); + } + Ok(cmd) + } + + async fn get_cachix_caches(&self) -> Result> { + if self.cachix_caches.borrow().is_none() { + let no_logging = Options { + logging: false, + ..self.options + }; + let caches_raw = self.eval(&["devenv.cachix"]).await?; + let cachix = serde_json::from_str(&caches_raw).expect("Failed to parse JSON"); + let known_keys = if let Ok(known_keys) = + std::fs::read_to_string(self.cachix_trusted_keys.as_path()) + { + serde_json::from_str(&known_keys).expect("Failed to parse JSON") + } else { + HashMap::new() + }; + + let mut caches = CachixCaches { + caches: cachix, + known_keys, + }; + + let mut new_known_keys: HashMap = HashMap::new(); + let client = reqwest::Client::new(); + for name in caches.caches.pull.iter() { + if !caches.known_keys.contains_key(name) { + let mut request = + client.get(&format!("https://cachix.org/api/v1/cache/{}", name)); + if let Ok(ret) = env::var("CACHIX_AUTH_TOKEN") { + request = request.bearer_auth(ret); + } + let resp = request.send().await.expect("Failed to get cache"); + if resp.status().is_client_error() { + self.logger.error(&format!( + "Cache {} does not exist or you don't have a CACHIX_AUTH_TOKEN configured.", + name + )); + self.logger + .error("To create a cache, go to https://app.cachix.org/."); + bail!("Cache does not exist or you don't have a CACHIX_AUTH_TOKEN configured.") + } else { + let resp_json = + serde_json::from_slice::(&resp.bytes().await.unwrap()) + .expect("Failed to parse JSON"); + new_known_keys + .insert(name.clone(), resp_json.public_signing_keys[0].clone()); + } + } + } + + if !caches.caches.pull.is_empty() { + let store = self.run_nix("nix", &["store", "ping", "--json"], &no_logging)?; + let trusted = serde_json::from_slice::(&store.stdout) + .expect("Failed to parse JSON") + .trusted; + if trusted.is_none() { + self.logger.warn( + "You're using very old version of Nix, please upgrade and restart nix-daemon.", + ); + } + let restart_command = if cfg!(target_os = "linux") { + "sudo systemctl restart nix-daemon" + } else { + "sudo launchctl kickstart -k system/org.nixos.nix-daemon" + }; + + self.logger + .info(&format!("Using Cachix: {}", caches.caches.pull.join(", "))); + if !new_known_keys.is_empty() { + for (name, pubkey) in new_known_keys.iter() { + self.logger.info(&format!( + "Trusting {}.cachix.org on first use with the public key {}", + name, pubkey + )); + } + caches.known_keys.extend(new_known_keys); + } + + std::fs::write( + self.cachix_trusted_keys.as_path(), + serde_json::to_string(&caches.known_keys).unwrap(), + ) + .expect("Failed to write cachix caches to file"); + + if trusted == Some(0) { + if !Path::new("/etc/NIXOS").exists() { + self.logger.error(&indoc::formatdoc!( + "You're not a trusted user of the Nix store. You have the following options: + + a) Add yourself to the trusted-users list in /etc/nix/nix.conf for devenv to manage caches for you. + + trusted-users = root {} + + Restart nix-daemon with: + + $ {restart_command} + + b) Add binary caches to /etc/nix/nix.conf yourself: + + extra-substituters = {} + extra-trusted-public-keys = {} + + And disable automatic cache configuration in `devenv.nix`: + + {{ + cachix.enable = false; + }} + ", whoami::username() + , caches.caches.pull.iter().map(|cache| format!("https://{}.cachix.org", cache)).collect::>().join(" ") + , caches.known_keys.values().cloned().collect::>().join(" ") + )); + } else { + self.logger.error(&indoc::formatdoc!( + "You're not a trusted user of the Nix store. You have the following options: + + a) Add yourself to the trusted-users list in /etc/nix/nix.conf by editing configuration.nix for devenv to manage caches for you. + + {{ + nix.extraOptions = '' + trusted-users = root {} + ''; + }} + + b) Add binary caches to /etc/nix/nix.conf yourself by editing configuration.nix: + {{ + nix.extraOptions = '' + extra-substituters = {}; + extra-trusted-public-keys = {}; + ''; + }} + + Lastly rebuild your system + + $ sudo nixos-rebuild switch + ", whoami::username() + , caches.caches.pull.iter().map(|cache| format!("https://{}.cachix.org", cache)).collect::>().join(" ") + , caches.known_keys.values().cloned().collect::>().join(" ") + )); + } + bail!("You're not a trusted user of the Nix store.") + } + } + + *self.cachix_caches.borrow_mut() = Some(caches); + } + + Ok(Ref::map(self.cachix_caches.borrow(), |option| { + option.as_ref().unwrap() + })) + } +} + +fn symlink_force(logger: &log::Logger, link_path: &Path, target: &Path) { + let _lock = dotlock::Dotlock::create(target.with_extension("lock")).unwrap(); + logger.debug(&format!( + "Creating symlink {} -> {}", + link_path.display(), + target.display() + )); + + if target.exists() { + fs::remove_file(target).unwrap_or_else(|_| panic!("Failed to remove {}", target.display())); + } + + symlink(link_path, target).unwrap_or_else(|_| { + panic!( + "Failed to create symlink: {} -> {}", + link_path.display(), + target.display() + ) + }); +} + +fn get_now_with_nanoseconds() -> String { + let now = SystemTime::now(); + let duration = now.duration_since(UNIX_EPOCH).expect("Time went backwards"); + let secs = duration.as_secs(); + let nanos = duration.subsec_nanos(); + format!("{}.{}", secs, nanos) +} + +// Display a command as a pretty string. +fn display_command(cmd: &std::process::Command) -> String { + let command = cmd.get_program().to_string_lossy(); + let args = cmd + .get_args() + .map(|arg| arg.to_str().unwrap()) + .collect::>() + .join(" "); + format!("{command} {args}") +} + +#[derive(Deserialize, Clone)] +pub struct Cachix { + pull: Vec, + push: Option, +} + +#[derive(Deserialize, Clone)] +pub struct CachixCaches { + caches: Cachix, + known_keys: HashMap, +} + +#[derive(Deserialize, Clone)] +struct CachixResponse { + #[serde(rename = "publicSigningKeys")] + public_signing_keys: Vec, +} + +#[derive(Deserialize, Clone)] +struct StorePing { + trusted: Option, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_trusted() { + let store_ping = r#"{"trusted":1,"url":"daemon","version":"2.18.1"}"#; + let store_ping: StorePing = serde_json::from_str(store_ping).unwrap(); + assert_eq!(store_ping.trusted, Some(1)); + } + + #[test] + fn test_no_trusted() { + let store_ping = r#"{"url":"daemon","version":"2.18.1"}"#; + let store_ping: StorePing = serde_json::from_str(store_ping).unwrap(); + assert_eq!(store_ping.trusted, None); + } + + #[test] + fn test_not_trusted() { + let store_ping = r#"{"trusted":0,"url":"daemon","version":"2.18.1"}"#; + let store_ping: StorePing = serde_json::from_str(store_ping).unwrap(); + assert_eq!(store_ping.trusted, Some(0)); + } +} diff --git a/devenv/src/command.rs b/devenv/src/command.rs deleted file mode 100644 index 3680975a6..000000000 --- a/devenv/src/command.rs +++ /dev/null @@ -1,454 +0,0 @@ -use crate::devenv::Devenv; -use miette::{bail, IntoDiagnostic, Result, WrapErr}; -use serde::Deserialize; -use std::collections::HashMap; -use std::env; -use std::os::unix::process::CommandExt; -use std::path::Path; - -const NIX_FLAGS: [&str; 9] = [ - "--show-trace", - "--extra-experimental-features", - "nix-command", - "--extra-experimental-features", - "flakes", - // remove unnecessary warnings - "--option", - "warn-dirty", - "false", - // always build all dependencies and report errors at the end - "--keep-going", -]; - -pub struct Options { - pub replace_shell: bool, - pub logging: bool, -} - -impl Default for Options { - fn default() -> Self { - Options { - replace_shell: false, - logging: true, - } - } -} - -impl Devenv { - pub fn run_nix( - &mut self, - command: &str, - args: &[&str], - options: &Options, - ) -> Result { - let prev_logging = self.logger.clone(); - if !options.logging { - self.logger = crate::log::Logger::new(crate::log::Level::Error); - } - - let mut cmd = self.prepare_command(command, args)?; - - if options.replace_shell { - if self.global_options.nix_debugger && command.ends_with("bin/nix") { - cmd.arg("--debugger"); - } - let error = cmd.exec(); - self.logger.error(&format!( - "Failed to replace shell with `{}`: {error}", - display_command(&cmd), - )); - bail!("Failed to replace shell") - } else { - if options.logging { - cmd.stdin(std::process::Stdio::inherit()) - .stderr(std::process::Stdio::inherit()); - } - - let result = cmd - .output() - .into_diagnostic() - .wrap_err_with(|| format!("Failed to run command `{}`", display_command(&cmd)))?; - - if !result.status.success() { - let code = match result.status.code() { - Some(code) => format!("with exit code {}", code), - None => "without exit code".to_string(), - }; - if options.logging { - eprintln!(); - self.logger.error(&format!( - "Command produced the following output:\n{}\n{}", - String::from_utf8_lossy(&result.stdout), - String::from_utf8_lossy(&result.stderr), - )); - } - if self.global_options.nix_debugger && command.ends_with("bin/nix") { - self.logger.info("Starting Nix debugger ..."); - cmd.arg("--debugger").exec(); - } - bail!(format!( - "Command `{}` failed with {code}", - display_command(&cmd) - )) - } else { - self.logger = prev_logging; - Ok(result) - } - } - } - - pub fn prepare_command( - &mut self, - command: &str, - args: &[&str], - ) -> Result { - let mut cmd = if command.starts_with("nix") { - let mut flags = NIX_FLAGS.to_vec(); - flags.push("--max-jobs"); - let max_jobs = self.global_options.max_jobs.to_string(); - flags.push(&max_jobs); - - flags.push("--option"); - flags.push("eval-cache"); - let eval_cache = self.global_options.eval_cache.to_string(); - flags.push(&eval_cache); - - // handle --nix-option key value - for chunk in self.global_options.nix_option.chunks_exact(2) { - flags.push("--option"); - flags.push(&chunk[0]); - flags.push(&chunk[1]); - } - - flags.extend_from_slice(args); - - let mut cmd = match env::var("DEVENV_NIX") { - Ok(devenv_nix) => std::process::Command::new(format!("{devenv_nix}/bin/{command}")), - Err(_) => { - self.logger.error( - "$DEVENV_NIX is not set, but required as devenv doesn't work without a few Nix patches." - ); - self.logger.error( - "Please follow https://devenv.sh/getting-started/ to install devenv.", - ); - bail!("$DEVENV_NIX is not set") - } - }; - - if self.global_options.offline && command == "nix" { - flags.push("--offline"); - } - - if self.global_options.impure || self.config.impure { - // only pass the impure option to the nix command that supports it. - // avoid passing it to the older utilities, e.g. like `nix-store` when creating GC roots. - if command == "nix" - && args - .first() - .map(|arg| arg == &"build" || arg == &"eval" || arg == &"print-dev-env") - .unwrap_or(false) - { - flags.push("--no-pure-eval"); - } - // set a dummy value to overcome https://github.com/NixOS/nix/issues/10247 - cmd.env("NIX_PATH", ":"); - } - cmd.args(flags); - - if args - .first() - .map(|arg| arg == &"build" || arg == &"print-dev-env" || arg == &"search") - .unwrap_or(false) - && !self.global_options.offline - { - let cachix_caches = self.get_cachix_caches(); - - match cachix_caches { - Err(e) => { - self.logger - .warn("Failed to get cachix caches due to evaluation error"); - self.logger.debug(&format!("{}", e)); - } - Ok(cachix_caches) => { - // handle cachix.pull - let pull_caches = cachix_caches - .caches - .pull - .iter() - .map(|cache| format!("https://{}.cachix.org", cache)) - .collect::>() - .join(" "); - cmd.arg("--option"); - cmd.arg("extra-substituters"); - cmd.arg(pull_caches); - cmd.arg("--option"); - cmd.arg("extra-trusted-public-keys"); - cmd.arg( - cachix_caches - .known_keys - .values() - .cloned() - .collect::>() - .join(" "), - ); - - // handle cachix.push - if let Some(push_cache) = &cachix_caches.caches.push { - if env::var("CACHIX_AUTH_TOKEN").is_ok() { - let args = cmd - .get_args() - .map(|arg| arg.to_str().unwrap()) - .collect::>(); - let envs = cmd.get_envs().collect::>(); - let command_name = cmd.get_program().to_string_lossy(); - let mut newcmd = std::process::Command::new("cachix"); - newcmd - .args(["watch-exec", &push_cache, "--"]) - .arg(command_name.as_ref()) - .args(args); - for (key, value) in envs { - if let Some(value) = value { - newcmd.env(key, value); - } - } - cmd = newcmd; - } else { - self.logger.warn(&format!( - "CACHIX_AUTH_TOKEN is not set, but required to push to {}.", - push_cache - )); - } - } - } - } - } - cmd - } else { - let mut cmd = std::process::Command::new(command); - cmd.args(args); - cmd - }; - - cmd.current_dir(self.devenv_root()); - - if self.global_options.verbose { - self.logger - .debug(&format!("Running command: {}", display_command(&cmd))); - } - - Ok(cmd) - } - - fn get_cachix_caches(&mut self) -> Result { - match &self.cachix_caches { - Some(caches) => Ok(caches.clone()), - None => { - let no_logging = Options { - logging: false, - ..Default::default() - }; - - let caches_raw = - self.run_nix("nix", &["eval", ".#devenv.cachix", "--json"], &no_logging)?; - - let cachix = - serde_json::from_slice(&caches_raw.stdout).expect("Failed to parse JSON"); - - let known_keys = - if let Ok(known_keys) = std::fs::read_to_string(&self.cachix_trusted_keys) { - serde_json::from_str(&known_keys).expect("Failed to parse JSON") - } else { - HashMap::new() - }; - - let mut caches = CachixCaches { - caches: cachix, - known_keys, - }; - - let mut new_known_keys: HashMap = HashMap::new(); - let client = reqwest::blocking::Client::new(); - for name in caches.caches.pull.iter() { - if !caches.known_keys.contains_key(name) { - let mut request = - client.get(&format!("https://cachix.org/api/v1/cache/{}", name)); - if let Ok(ret) = env::var("CACHIX_AUTH_TOKEN") { - request = request.bearer_auth(ret); - } - let resp = request.send().expect("Failed to get cache"); - if resp.status().is_client_error() { - self.logger.error(&format!( - "Cache {} does not exist or you don't have a CACHIX_AUTH_TOKEN configured.", - name - )); - self.logger - .error("To create a cache, go to https://app.cachix.org/."); - bail!("Cache does not exist or you don't have a CACHIX_AUTH_TOKEN configured.") - } else { - let resp_json = - serde_json::from_slice::(&resp.bytes().unwrap()) - .expect("Failed to parse JSON"); - new_known_keys - .insert(name.clone(), resp_json.public_signing_keys[0].clone()); - } - } - } - - if !caches.caches.pull.is_empty() { - let store = self.run_nix("nix", &["store", "ping", "--json"], &no_logging)?; - let trusted = serde_json::from_slice::(&store.stdout) - .expect("Failed to parse JSON") - .trusted; - if trusted.is_none() { - self.logger - .warn("You're using very old version of Nix, please upgrade and restart nix-daemon."); - } - let restart_command = if cfg!(target_os = "linux") { - "sudo systemctl restart nix-daemon" - } else { - "sudo launchctl kickstart -k system/org.nixos.nix-daemon" - }; - - self.logger - .info(&format!("Using Cachix: {}", caches.caches.pull.join(", "))); - if !new_known_keys.is_empty() { - for (name, pubkey) in new_known_keys.iter() { - self.logger.info(&format!( - "Trusting {}.cachix.org on first use with the public key {}", - name, pubkey - )); - } - caches.known_keys.extend(new_known_keys); - } - - std::fs::write( - &self.cachix_trusted_keys, - serde_json::to_string(&caches.known_keys).unwrap(), - ) - .expect("Failed to write cachix caches to file"); - - if trusted == Some(0) { - if !Path::new("/etc/NIXOS").exists() { - self.logger.error(&indoc::formatdoc!( - "You're not a trusted user of the Nix store. You have the following options: - - a) Add yourself to the trusted-users list in /etc/nix/nix.conf for devenv to manage caches for you. - - trusted-users = root {} - - Restart nix-daemon with: - - $ {restart_command} - - b) Add binary caches to /etc/nix/nix.conf yourself: - - extra-substituters = {} - extra-trusted-public-keys = {} - - And disable automatic cache configuration in `devenv.nix`: - - {{ - cachix.enable = false; - }} - ", whoami::username() - , caches.caches.pull.iter().map(|cache| format!("https://{}.cachix.org", cache)).collect::>().join(" ") - , caches.known_keys.values().cloned().collect::>().join(" ") - )); - } else { - self.logger.error(&indoc::formatdoc!( - "You're not a trusted user of the Nix store. You have the following options: - - a) Add yourself to the trusted-users list in /etc/nix/nix.conf by editing configuration.nix for devenv to manage caches for you. - - {{ - nix.extraOptions = '' - trusted-users = root {} - ''; - }} - - b) Add binary caches to /etc/nix/nix.conf yourself by editing configuration.nix: - {{ - nix.extraOptions = '' - extra-substituters = {}; - extra-trusted-public-keys = {}; - ''; - }} - - Lastly rebuild your system - - $ sudo nixos-rebuild switch - ", whoami::username() - , caches.caches.pull.iter().map(|cache| format!("https://{}.cachix.org", cache)).collect::>().join(" ") - , caches.known_keys.values().cloned().collect::>().join(" ") - )); - } - bail!("You're not a trusted user of the Nix store.") - } - } - - self.cachix_caches = Some(caches.clone()); - Ok(caches) - } - } - } -} - -/// Display a command as a pretty string. -fn display_command(cmd: &std::process::Command) -> String { - let command = cmd.get_program().to_string_lossy(); - let args = cmd - .get_args() - .map(|arg| arg.to_str().unwrap()) - .collect::>() - .join(" "); - format!("{command} {args}") -} - -#[derive(Deserialize, Clone)] -pub struct Cachix { - pull: Vec, - push: Option, -} - -#[derive(Deserialize, Clone)] -pub struct CachixCaches { - caches: Cachix, - known_keys: HashMap, -} - -#[derive(Deserialize, Clone)] -struct CachixResponse { - #[serde(rename = "publicSigningKeys")] - public_signing_keys: Vec, -} - -#[derive(Deserialize, Clone)] -struct StorePing { - trusted: Option, -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_trusted() { - let store_ping = r#"{"trusted":1,"url":"daemon","version":"2.18.1"}"#; - let store_ping: StorePing = serde_json::from_str(store_ping).unwrap(); - assert_eq!(store_ping.trusted, Some(1)); - } - - #[test] - fn test_no_trusted() { - let store_ping = r#"{"url":"daemon","version":"2.18.1"}"#; - let store_ping: StorePing = serde_json::from_str(store_ping).unwrap(); - assert_eq!(store_ping.trusted, None); - } - - #[test] - fn test_not_trusted() { - let store_ping = r#"{"trusted":0,"url":"daemon","version":"2.18.1"}"#; - let store_ping: StorePing = serde_json::from_str(store_ping).unwrap(); - assert_eq!(store_ping.trusted, Some(0)); - } -} diff --git a/devenv/src/devenv.rs b/devenv/src/devenv.rs index 5e8bbc2e4..6dfe94275 100644 --- a/devenv/src/devenv.rs +++ b/devenv/src/devenv.rs @@ -1,17 +1,16 @@ -use super::{cli, command, config, log}; +use super::{cli, cnix, config, log, tasks}; use clap::crate_version; use cli_table::Table; use cli_table::{print_stderr, WithTitle}; use include_dir::{include_dir, Dir}; use miette::{bail, IntoDiagnostic, Result, WrapErr}; +use nix::sys::signal; +use nix::unistd::Pid; use serde::Deserialize; use sha2::Digest; use std::collections::HashMap; use std::io::Write; -use std::os::unix::fs::symlink; use std::os::unix::{fs::PermissionsExt, process::CommandExt}; -use std::str::FromStr; -use std::time::{SystemTime, UNIX_EPOCH}; use std::{ fs, path::{Path, PathBuf}, @@ -41,19 +40,17 @@ pub struct Devenv { pub(crate) logger: log::Logger, pub(crate) log_progress: log::LogProgressCreator, + nix: cnix::Nix<'static>, + // All kinds of paths xdg_dirs: xdg::BaseDirectories, - devenv_root: PathBuf, + pub(crate) devenv_root: PathBuf, devenv_dotfile: PathBuf, devenv_dot_gc: PathBuf, devenv_home_gc: PathBuf, devenv_tmp: String, devenv_runtime: PathBuf, - // Caching - pub cachix_caches: Option, - pub cachix_trusted_keys: PathBuf, - pub(crate) assembled: bool, pub(crate) dirs_created: bool, pub(crate) has_processes: Option, @@ -65,6 +62,7 @@ impl Devenv { pub fn new(options: DevenvOptions) -> Self { let xdg_dirs = xdg::BaseDirectories::with_prefix("devenv").unwrap(); let devenv_home = xdg_dirs.get_data_home(); + let cachix_trusted_keys = devenv_home.join("cachix_trusted_keys.json"); let devenv_home_gc = devenv_home.join("gc"); let devenv_root = options @@ -88,7 +86,6 @@ impl Devenv { }; let devenv_runtime = Path::new(&devenv_tmp).join(format!("devenv-{}", &devenv_state_hash[..7])); - let cachix_trusted_keys = devenv_home.join("cachix_trusted_keys.json"); let global_options = options.global_options.unwrap_or_default(); @@ -105,6 +102,16 @@ impl Devenv { log::LogProgressCreator::Logging }; + let nix = cnix::Nix::new( + logger.clone(), + options.config.clone(), + global_options.clone(), + cachix_trusted_keys, + devenv_home_gc.clone(), + devenv_dot_gc.clone(), + devenv_root.clone(), + ); + Self { config: options.config, global_options, @@ -117,8 +124,7 @@ impl Devenv { devenv_home_gc, devenv_tmp, devenv_runtime, - cachix_caches: None, - cachix_trusted_keys, + nix, assembled: false, dirs_created: false, has_processes: None, @@ -126,20 +132,6 @@ impl Devenv { } } - pub fn devenv_root(&self) -> &Path { - self.devenv_root.as_ref() - } - - // TODO: refactor test to be able to remove this - pub fn update_devenv_dotfile

(&mut self, devenv_dotfile: P) - where - P: AsRef, - { - let devenv_dotfile = devenv_dotfile.as_ref(); - self.devenv_dotfile = devenv_dotfile.to_path_buf(); - self.devenv_dot_gc = devenv_dotfile.join("gc"); - } - pub fn processes_log(&self) -> PathBuf { self.devenv_dotfile.join("processes.log") } @@ -207,8 +199,8 @@ impl Devenv { Ok(()) } - pub fn print_dev_env(&mut self, json: bool) -> Result<()> { - let (env, _) = self.get_dev_environment(json, false)?; + pub async fn print_dev_env(&mut self, json: bool) -> Result<()> { + let (env, _) = self.get_dev_environment(json, false).await?; print!( "{}", String::from_utf8(env).expect("Failed to convert env to utf-8") @@ -216,31 +208,30 @@ impl Devenv { Ok(()) } - pub fn shell( + pub async fn shell( &mut self, cmd: &Option, args: &[String], replace_shell: bool, ) -> Result<()> { - let develop_args = self.prepare_shell(cmd, args)?; - - let options = command::Options { - replace_shell, - ..command::Options::default() - }; + let develop_args = self.prepare_develop_args(cmd, args).await?; let develop_args = develop_args .iter() .map(|s| s.as_str()) .collect::>(); - self.run_nix("nix", &develop_args, &options)?; + self.nix.develop(&develop_args, replace_shell).await?; Ok(()) } - pub fn prepare_shell(&mut self, cmd: &Option, args: &[String]) -> Result> { + pub async fn prepare_develop_args( + &mut self, + cmd: &Option, + args: &[String], + ) -> Result> { self.assemble(false)?; - let (_, gc_root) = self.get_dev_environment(false, true)?; + let (_, gc_root) = self.get_dev_environment(false, true).await?; let mut develop_args = vec![ "develop", @@ -294,22 +285,11 @@ impl Devenv { let _logprogress = self.log_progress.with_newline(&msg); self.assemble(false)?; - match input_name { - Some(input_name) => { - self.run_nix( - "nix", - &["flake", "lock", "--update-input", input_name], - &command::Options::default(), - )?; - } - None => { - self.run_nix("nix", &["flake", "update"], &command::Options::default())?; - } - } + self.nix.update(input_name)?; Ok(()) } - pub fn container_build(&mut self, name: &str) -> Result { + pub async fn container_build(&mut self, name: &str) -> Result { if cfg!(target_os = "macos") { bail!("Containers are not supported on macOS yet: https://github.com/cachix/devenv/issues/430"); } @@ -320,52 +300,35 @@ impl Devenv { self.assemble(false)?; - let container_store_path = self.run_nix( - "nix", - &[ - "build", - "--print-out-paths", - "--no-link", - &format!(".#devenv.containers.{name}.derivation"), - ], - &command::Options::default(), - )?; - - let container_store_path_string = String::from_utf8_lossy(&container_store_path.stdout) - .to_string() - .trim() - .to_string(); - println!("{}", container_store_path_string); - Ok(container_store_path_string) + let container_store_path = self + .nix + .build(&[&format!("devenv.containers.{name}.derivation")]) + .await?; + let container_store_path = container_store_path[0] + .to_str() + .expect("Failed to get container store path"); + println!("{}", &container_store_path); + Ok(container_store_path.to_string()) } - pub fn container_copy( + pub async fn container_copy( &mut self, name: &str, copy_args: &[String], registry: Option<&str>, ) -> Result<()> { - let spec = self.container_build(name)?; + let spec = self.container_build(name).await?; let _logprogress = self .log_progress .without_newline(&format!("Copying {name} container")); - let copy_script = self.run_nix( - "nix", - &[ - "build", - "--print-out-paths", - "--no-link", - &format!(".#devenv.containers.{name}.copyScript"), - ], - &command::Options::default(), - )?; - - let copy_script = String::from_utf8_lossy(©_script.stdout) - .to_string() - .trim() - .to_string(); + let copy_script = self + .nix + .build(&[&format!("devenv.containers.{name}.copyScript")]) + .await?; + let copy_script = ©_script[0]; + let copy_script_string = ©_script.to_string_lossy(); let copy_args = [ spec, @@ -373,8 +336,10 @@ impl Devenv { copy_args.join(" "), ]; - self.logger - .info(&format!("Running {copy_script} {}", copy_args.join(" "))); + self.logger.info(&format!( + "Running {copy_script_string} {}", + copy_args.join(" ") + )); let status = std::process::Command::new(copy_script) .args(copy_args) @@ -390,7 +355,7 @@ impl Devenv { } } - pub fn container_run( + pub async fn container_run( &mut self, name: &str, copy_args: &[String], @@ -400,29 +365,19 @@ impl Devenv { self.logger .warn("Ignoring --registry flag when running container"); }; - self.container_copy(name, copy_args, Some("docker-daemon:"))?; + self.container_copy(name, copy_args, Some("docker-daemon:")) + .await?; let _logprogress = self .log_progress .without_newline(&format!("Running {name} container")); - let run_script = self.run_nix( - "nix", - &[ - "build", - "--print-out-paths", - "--no-link", - &format!(".#devenv.containers.{name}.dockerRun"), - ], - &command::Options::default(), - )?; - - let run_script = String::from_utf8_lossy(&run_script.stdout) - .to_string() - .trim() - .to_string(); - - let status = std::process::Command::new(run_script) + let run_script = self + .nix + .build(&[&format!("devenv.containers.{name}.dockerRun")]) + .await?; + + let status = std::process::Command::new(&run_script[0]) .status() .expect("Failed to run container script"); @@ -435,10 +390,7 @@ impl Devenv { pub fn repl(&mut self) -> Result<()> { self.assemble(false)?; - - let mut cmd = self.prepare_command("nix", &["repl", "."])?; - cmd.exec(); - Ok(()) + self.nix.repl() } pub fn gc(&mut self) -> Result<()> { @@ -451,9 +403,10 @@ impl Devenv { )); cleanup_symlinks(&self.devenv_home_gc) }; + let to_gc_len = to_gc.len(); self.logger - .info(&format!("Found {} active environments.", to_gc.len())); + .info(&format!("Found {} active environments.", to_gc_len)); self.logger.info(&format!( "Deleted {} dangling environments (most likely due to previous GC).", removed_symlinks.len() @@ -464,17 +417,7 @@ impl Devenv { .log_progress .with_newline("Running garbage collection (this process will take some time) ..."); self.logger.warn("If you'd like this to run faster, leave a thumbs up at https://github.com/NixOS/nix/issues/7239"); - let paths: std::collections::HashSet<&str> = to_gc - .iter() - .filter_map(|path_buf| path_buf.to_str()) - .collect(); - for path in paths { - self.logger.info(&format!("Deleting {}...", path)); - let args: Vec<&str> = ["store", "delete", path].iter().copied().collect(); - let cmd = self.prepare_command("nix", &args); - // we ignore if this command fails, because root might be in use - let _ = cmd?.output(); - } + self.nix.gc(to_gc)?; } let (after_gc, _) = cleanup_symlinks(&self.devenv_home_gc); @@ -483,24 +426,17 @@ impl Devenv { eprintln!(); self.logger.info(&format!( "Done. Successfully removed {} symlinks in {}s.", - to_gc.len() - after_gc.len(), + to_gc_len - after_gc.len(), (end - start).as_secs_f32() )); Ok(()) } - pub fn search(&mut self, name: &str) -> Result<()> { + pub async fn search(&mut self, name: &str) -> Result<()> { self.assemble(false)?; - let options = self.run_nix( - "nix", - &["build", "--no-link", "--print-out-paths", ".#optionsJSON"], - &command::Options::default(), - )?; - - let options_str = std::str::from_utf8(&options.stdout).unwrap().trim(); - let options_path = PathBuf::from_str(options_str) - .expect("options store path should be a utf-8") + let options = self.nix.build(&["optionsJSON"]).await?; + let options_path = options[0] .join("share") .join("doc") .join("nixos") @@ -522,11 +458,7 @@ impl Devenv { .collect::>(); let results_options_count = options_results.len(); - let search = self.run_nix( - "nix", - &["search", "--json", "nixpkgs", name], - &command::Options::default(), - )?; + let search = self.nix.search(name).await?; let search_json: PackageResults = serde_json::from_slice(&search.stdout).expect("Failed to parse search results"); let search_results = search_json @@ -555,64 +487,79 @@ impl Devenv { Ok(()) } - pub fn has_processes(&mut self) -> Result { + pub async fn has_processes(&mut self) -> Result { if self.has_processes.is_none() { - let result = self.run_nix( - "nix", - &["eval", ".#devenv.processes", "--json"], - &command::Options::default(), - )?; - let processes = String::from_utf8_lossy(&result.stdout).to_string(); + let processes = self.nix.eval(&["devenv.processes"]).await?; self.has_processes = Some(processes.trim() != "{}"); } Ok(self.has_processes.unwrap()) } - pub fn test(&mut self) -> Result<()> { + pub async fn tasks_run(&mut self, roots: Vec) -> Result<()> { + if roots.is_empty() { + bail!("No tasks specified."); + } + let tasks_json_file = { + let _logprogress = self.log_progress.without_newline("Evaluating tasks"); + self.nix.build(&["devenv.task.config"]).await? + }; + // parse tasks config + let tasks_json = + std::fs::read_to_string(&tasks_json_file[0]).expect("Failed to read config file"); + let tasks: Vec = + serde_json::from_str(&tasks_json).expect("Failed to parse tasks config"); + // run tasks + let config = tasks::Config { roots, tasks }; + self.logger.debug(&format!( + "Tasks config: {}", + serde_json::to_string_pretty(&config).unwrap() + )); + let mut tui = tasks::TasksUi::new(config).await?; + let (tasks_status, outputs) = tui.run().await?; + + if tasks_status.failed > 0 || tasks_status.dependency_failed > 0 { + miette::bail!("Some tasks failed"); + } + + println!( + "{}", + serde_json::to_string(&outputs).expect("parsing of outputs failed") + ); + Ok(()) + } + + pub async fn test(&mut self) -> Result<()> { self.assemble(true)?; // collect tests let test_script = { let _logprogress = self.log_progress.with_newline("Building tests"); - self.run_nix( - "nix", - &["build", ".#devenv.test", "--no-link", "--print-out-paths"], - &command::Options::default(), - )? + self.nix.build(&["devenv.test"]).await? }; + let test_script = test_script[0].to_string_lossy().to_string(); - let test_script_string = String::from_utf8_lossy(&test_script.stdout) - .to_string() - .trim() - .to_string(); - if test_script_string.is_empty() { - self.logger.error("No tests found."); - bail!("No tests found"); - } - - if self.has_processes()? { - self.up(None, &true, &false)?; + if self.has_processes().await? { + self.up(None, &true, &false).await?; } let result = { let _logprogress = self.log_progress.with_newline("Running tests"); - self.logger - .debug(&format!("Running command: {test_script_string}")); - - let develop_args = self.prepare_shell(&Some(test_script_string), &[])?; - let develop_args = develop_args - .iter() - .map(|s| s.as_str()) - .collect::>(); - let mut cmd = self.prepare_command("nix", &develop_args)?; - cmd.stdin(std::process::Stdio::inherit()); - cmd.stderr(std::process::Stdio::inherit()); - cmd.stdout(std::process::Stdio::inherit()); - cmd.output().expect("Failed to run tests") + .debug(&format!("Running command: {test_script}")); + let develop_args = self.prepare_develop_args(&Some(test_script), &[]).await?; + // TODO: replace_shell? + self.nix + .develop( + &develop_args + .iter() + .map(|s| s.as_str()) + .collect::>(), + false, + ) + .await? }; - if self.has_processes()? { + if self.has_processes().await? { self.down()?; } @@ -627,37 +574,17 @@ impl Devenv { pub fn info(&mut self) -> Result<()> { self.assemble(false)?; - - // TODO: use --json - let metadata = self.run_nix("nix", &["flake", "metadata"], &command::Options::default())?; - - let re = regex::Regex::new(r"(Inputs:.+)$").unwrap(); - let metadata_str = String::from_utf8_lossy(&metadata.stdout); - let inputs = match re.captures(&metadata_str) { - Some(captures) => captures.get(1).unwrap().as_str(), - None => "", - }; - - let info_ = self.run_nix( - "nix", - &["eval", "--raw", ".#info"], - &command::Options::default(), - )?; - println!("{}\n{}", inputs, &String::from_utf8_lossy(&info_.stdout)); + let output = self.nix.metadata()?; + println!("{}", output); Ok(()) } - pub fn build(&mut self, attributes: &[String]) -> Result<()> { + pub async fn build(&mut self, attributes: &[String]) -> Result<()> { self.assemble(false)?; - - let build_attrs: Vec = if attributes.is_empty() { + let attributes: Vec = if attributes.is_empty() { // construct dotted names of all attributes that we need to build - let build_output = self.run_nix( - "nix", - &["eval", ".#build", "--json"], - &command::Options::default(), - )?; - serde_json::from_slice::(&build_output.stdout) + let build_output = self.nix.eval(&["build"]).await?; + serde_json::from_str::(&build_output) .map_err(|e| miette::miette!("Failed to parse build output: {}", e))? .as_object() .ok_or_else(|| miette::miette!("Build output is not an object"))? @@ -669,7 +596,7 @@ impl Devenv { .iter() .flat_map(|(k, v)| flatten_object(&format!("{}.{}", prefix, k), v)) .collect(), - _ => vec![format!(".#devenv.{}", prefix)], + _ => vec![format!("devenv.{}", prefix)], } } flatten_object(key, value) @@ -678,40 +605,27 @@ impl Devenv { } else { attributes .iter() - .map(|attr| format!(".#devenv.{}", attr)) + .map(|attr| format!("devenv.{}", attr)) .collect() }; - - let mut args = vec!["build", "--print-out-paths", "--no-link"]; - if !build_attrs.is_empty() { - args.extend(build_attrs.iter().map(|s| s.as_str())); - let output = self.run_nix("nix", &args, &command::Options::default())?; - println!("{}", String::from_utf8_lossy(&output.stdout)); + let paths = self + .nix + .build(&attributes.iter().map(AsRef::as_ref).collect::>()) + .await?; + for path in paths { + println!("{}", path.display()); } Ok(()) } - pub fn add_gc(&mut self, name: &str, path: &Path) -> Result<()> { - self.run_nix( - "nix-store", - &[ - "--add-root", - self.devenv_dot_gc.join(name).to_str().unwrap(), - "-r", - path.to_str().unwrap(), - ], - &command::Options::default(), - )?; - let link_path = self - .devenv_dot_gc - .join(format!("{}-{}", name, get_now_with_nanoseconds())); - symlink_force(&self.logger, path, &link_path); - Ok(()) - } - - pub fn up(&mut self, process: Option<&str>, detach: &bool, log_to_file: &bool) -> Result<()> { + pub async fn up( + &mut self, + process: Option<&str>, + detach: &bool, + log_to_file: &bool, + ) -> Result<()> { self.assemble(false)?; - if !self.has_processes()? { + if !self.has_processes().await? { self.logger .error("No 'processes' option defined: https://devenv.sh/processes/"); bail!("No processes defined"); @@ -720,25 +634,13 @@ impl Devenv { let proc_script_string: String; { let _logprogress = self.log_progress.with_newline("Building processes"); - - let proc_script = self.run_nix( - "nix", - &[ - "build", - "--no-link", - "--print-out-paths", - ".#procfileScript", - ], - &command::Options::default(), - )?; - - proc_script_string = String::from_utf8_lossy(&proc_script.stdout) - .to_string() - .trim() + let proc_script = self.nix.build(&["procfileScript"]).await?; + proc_script_string = proc_script[0] + .to_str() + .expect("Failed to get proc script path") .to_string(); - self.add_gc("procfilescript", Path::new(&proc_script_string))?; + self.nix.add_gc("procfilescript", &proc_script[0])?; } - { let _logprogress = self.log_progress.with_newline("Starting processes"); @@ -764,10 +666,21 @@ impl Devenv { std::fs::set_permissions(&processes_script, std::fs::Permissions::from_mode(0o755)) .expect("Failed to set permissions"); - let args = - self.prepare_shell(&Some(processes_script.to_str().unwrap().to_string()), &[])?; - let args = args.iter().map(|s| s.as_str()).collect::>(); - let mut cmd = self.prepare_command("nix", &args)?; + let develop_args = self + .prepare_develop_args(&Some(processes_script.to_str().unwrap().to_string()), &[]) + .await?; + + let mut cmd = self + .nix + .prepare_command_with_substituters( + "nix", + &develop_args + .iter() + .map(AsRef::as_ref) + .collect::>(), + &self.nix.options, + ) + .await?; if *detach { let log_file = std::fs::File::create(self.processes_log()) @@ -795,7 +708,8 @@ impl Devenv { } self.logger.info("Stop: $ devenv processes stop"); } else { - cmd.exec(); + let err = cmd.exec(); + bail!(err); } Ok(()) } @@ -815,8 +729,8 @@ impl Devenv { self.logger .info(&format!("Stopping process with PID {}", pid)); - let pid = nix::unistd::Pid::from_raw(pid); - match nix::sys::signal::kill(pid, nix::sys::signal::Signal::SIGTERM) { + let pid = Pid::from_raw(pid); + match signal::kill(pid, signal::Signal::SIGTERM) { Ok(_) => {} Err(_) => { self.logger @@ -923,7 +837,11 @@ impl Devenv { Ok(()) } - pub fn get_dev_environment(&mut self, json: bool, logging: bool) -> Result<(Vec, PathBuf)> { + pub async fn get_dev_environment( + &mut self, + json: bool, + logging: bool, + ) -> Result<(Vec, PathBuf)> { self.assemble(false)?; let _logprogress = if logging { Some(self.log_progress.with_newline("Building shell")) @@ -931,55 +849,11 @@ impl Devenv { None }; let gc_root = self.devenv_dot_gc.join("shell"); - let gc_root_str = gc_root.to_str().expect("gc root should be utf-8"); - - let mut args: Vec<&str> = vec!["print-dev-env", "--profile", gc_root_str]; - if json { - args.push("--json"); - } - - let env = self.run_nix("nix", &args, &command::Options::default())?; - - let options = command::Options { - logging: false, - ..command::Options::default() - }; - - let args: Vec<&str> = vec!["-p", gc_root_str, "--delete-generations", "old"]; - self.run_nix("nix-env", &args, &options)?; - let now_ns = get_now_with_nanoseconds(); - let target = format!("{}-shell", now_ns); - symlink_force( - &self.logger, - &fs::canonicalize(&gc_root).expect("to resolve gc_root"), - &self.devenv_home_gc.join(target), - ); - - Ok((env.stdout, gc_root)) + let env = self.nix.dev_env(json, &gc_root).await?; + Ok((env, gc_root)) } } -fn symlink_force(logger: &log::Logger, link_path: &Path, target: &Path) { - let _lock = dotlock::Dotlock::create(target.with_extension("lock")).unwrap(); - logger.debug(&format!( - "Creating symlink {} -> {}", - link_path.display(), - target.display() - )); - - if target.exists() { - fs::remove_file(target).unwrap_or_else(|_| panic!("Failed to remove {}", target.display())); - } - - symlink(link_path, target).unwrap_or_else(|_| { - panic!( - "Failed to create symlink: {} -> {}", - link_path.display(), - target.display() - ) - }); -} - #[derive(Deserialize)] struct PackageResults(HashMap); @@ -1022,14 +896,6 @@ struct DevenvPackageResult { description: String, } -fn get_now_with_nanoseconds() -> String { - let now = SystemTime::now(); - let duration = now.duration_since(UNIX_EPOCH).expect("Time went backwards"); - let secs = duration.as_secs(); - let nanos = duration.subsec_nanos(); - format!("{}.{}", secs, nanos) -} - fn cleanup_symlinks(root: &Path) -> (Vec, Vec) { let mut to_gc = Vec::new(); let mut removed_symlinks = Vec::new(); diff --git a/devenv/src/flake.tmpl.nix b/devenv/src/flake.tmpl.nix index 6f6b051ca..3da44bc29 100644 --- a/devenv/src/flake.tmpl.nix +++ b/devenv/src/flake.tmpl.nix @@ -100,6 +100,7 @@ v ); }; + build = options: config: lib.concatMapAttrs (name: option: diff --git a/devenv/src/lib.rs b/devenv/src/lib.rs index 1274afaa5..1c78f0c0c 100644 --- a/devenv/src/lib.rs +++ b/devenv/src/lib.rs @@ -1,8 +1,9 @@ mod cli; -pub mod command; +pub mod cnix; pub mod config; mod devenv; pub mod log; +pub mod tasks; pub use cli::{default_system, GlobalOptions}; pub use devenv::{Devenv, DevenvOptions}; diff --git a/devenv/src/log.rs b/devenv/src/log.rs index 15e2fc5fd..92108ebd8 100644 --- a/devenv/src/log.rs +++ b/devenv/src/log.rs @@ -75,7 +75,7 @@ pub enum Level { #[derive(Clone)] pub struct Logger { - level: Level, + pub level: Level, } impl Logger { diff --git a/devenv/src/main.rs b/devenv/src/main.rs index 483cedf9a..25d36634a 100644 --- a/devenv/src/main.rs +++ b/devenv/src/main.rs @@ -1,17 +1,28 @@ mod cli; -mod command; +mod cnix; mod config; mod devenv; mod log; +mod tasks; use clap::{crate_version, Parser}; -use cli::{Cli, Commands, ContainerCommand, InputsCommand, ProcessesCommand}; +use cli::{Cli, Commands, ContainerCommand, InputsCommand, ProcessesCommand, TasksCommand}; use devenv::Devenv; use miette::Result; -fn main() -> Result<()> { +#[tokio::main] +async fn main() -> Result<()> { let cli = Cli::parse(); + if let Commands::Version { .. } = cli.command { + println!( + "devenv {} ({})", + crate_version!(), + cli.global_options.system + ); + return Ok(()); + } + let level = if cli.global_options.verbose { log::Level::Debug } else if cli.global_options.quiet { @@ -27,40 +38,35 @@ fn main() -> Result<()> { config.add_input(&input[0].clone(), &input[1].clone(), &[]); } - let options = devenv::DevenvOptions { + let mut options = devenv::DevenvOptions { logger: Some(logger.clone()), global_options: Some(cli.global_options), config, ..Default::default() }; - let mut devenv = Devenv::new(options); - - if !matches!(cli.command, Commands::Version {} | Commands::Gc { .. }) { - devenv.create_directories()?; + if let Commands::Test { + dont_override_dotfile, + } = cli.command + { + let pwd = std::env::current_dir().expect("Failed to get current directory"); + let tmpdir = + tempdir::TempDir::new_in(pwd, ".devenv").expect("Failed to create temporary directory"); + if !dont_override_dotfile { + logger.info(&format!( + "Overriding .devenv to {}", + tmpdir.path().file_name().unwrap().to_str().unwrap() + )); + options.devenv_dotfile = Some(tmpdir.path().to_path_buf()); + } } + let mut devenv = Devenv::new(options); + devenv.create_directories()?; + match cli.command { - Commands::Shell { cmd, args } => devenv.shell(&cmd, &args, true), - Commands::Test { - dont_override_dotfile, - } => { - let tmpdir = tempdir::TempDir::new_in(devenv.devenv_root(), ".devenv") - .expect("Failed to create temporary directory"); - if !dont_override_dotfile { - logger.info(&format!( - "Overriding .devenv to {}", - tmpdir.path().file_name().unwrap().to_str().unwrap() - )); - devenv.update_devenv_dotfile(tmpdir.as_ref()); - } - devenv.test() - } - Commands::Version {} => Ok(println!( - "devenv {} ({})", - crate_version!(), - devenv.global_options.system - )), + Commands::Shell { cmd, args } => devenv.shell(&cmd, &args, true).await, + Commands::Test { .. } => devenv.test().await, Commands::Container { registry, copy, @@ -76,15 +82,19 @@ fn main() -> Result<()> { match c { ContainerCommand::Build { name } => { devenv.container_name = Some(name.clone()); - let _ = devenv.container_build(&name)?; + let _ = devenv.container_build(&name).await?; } ContainerCommand::Copy { name } => { devenv.container_name = Some(name.clone()); - devenv.container_copy(&name, ©_args, registry.as_deref())?; + devenv + .container_copy(&name, ©_args, registry.as_deref()) + .await?; } ContainerCommand::Run { name } => { devenv.container_name = Some(name.clone()); - devenv.container_run(&name, ©_args, registry.as_deref())?; + devenv + .container_run(&name, ©_args, registry.as_deref()) + .await?; } } } @@ -95,17 +105,21 @@ fn main() -> Result<()> { logger.warn( "--copy flag is deprecated, use `devenv container copy` instead", ); - devenv.container_copy(&name, ©_args, registry.as_deref())?; + devenv + .container_copy(&name, ©_args, registry.as_deref()) + .await?; } (_, true) => { logger.warn( "--docker-run flag is deprecated, use `devenv container run` instead", ); - devenv.container_run(&name, ©_args, registry.as_deref())?; + devenv + .container_run(&name, ©_args, registry.as_deref()) + .await?; } _ => { logger.warn("Calling without a subcommand is deprecated, use `devenv container build` instead"); - let _ = devenv.container_build(&name)?; + let _ = devenv.container_build(&name).await?; } }; } @@ -113,29 +127,33 @@ fn main() -> Result<()> { Ok(()) } Commands::Init { target } => devenv.init(&target), - Commands::Search { name } => devenv.search(&name), + Commands::Search { name } => devenv.search(&name).await, Commands::Gc {} => devenv.gc(), Commands::Info {} => devenv.info(), Commands::Repl {} => devenv.repl(), - Commands::Build { attributes } => devenv.build(&attributes), + Commands::Build { attributes } => devenv.build(&attributes).await, Commands::Update { name } => devenv.update(&name), - Commands::Up { process, detach } => devenv.up(process.as_deref(), &detach, &detach), + Commands::Up { process, detach } => devenv.up(process.as_deref(), &detach, &detach).await, Commands::Processes { command } => match command { ProcessesCommand::Up { process, detach } => { - devenv.up(process.as_deref(), &detach, &detach) + devenv.up(process.as_deref(), &detach, &detach).await } ProcessesCommand::Down {} => devenv.down(), }, + Commands::Tasks { command } => match command { + TasksCommand::Run { tasks } => devenv.tasks_run(tasks).await, + }, Commands::Inputs { command } => match command { InputsCommand::Add { name, url, follows } => devenv.inputs_add(&name, &url, &follows), }, // hidden Commands::Assemble => devenv.assemble(false), - Commands::PrintDevEnv { json } => devenv.print_dev_env(json), + Commands::PrintDevEnv { json } => devenv.print_dev_env(json).await, Commands::GenerateJSONSchema => { config::write_json_schema(); Ok(()) } + Commands::Version {} => unreachable!(), } } diff --git a/devenv/src/tasks.rs b/devenv/src/tasks.rs new file mode 100644 index 000000000..eb3bbb76c --- /dev/null +++ b/devenv/src/tasks.rs @@ -0,0 +1,1198 @@ +use crossterm::{ + cursor, execute, + style::{self, Stylize}, + terminal::{Clear, ClearType}, +}; +use miette::Diagnostic; +use petgraph::algo::toposort; +use petgraph::graph::{DiGraph, NodeIndex}; +use petgraph::visit::EdgeRef; +use serde::{Deserialize, Serialize}; +use std::collections::{HashMap, HashSet}; +use std::fmt::Display; +use std::io; +use std::process::Stdio; +use std::sync::Arc; +use thiserror::Error; +use tokio::process::Command; +use tokio::sync::RwLock; +use tokio::task::JoinSet; +use tokio::time::{Duration, Instant}; +use tokio::{ + io::{AsyncBufReadExt, BufReader}, + sync::Mutex, +}; +use tracing::{error, info, instrument}; + +#[derive(Error, Diagnostic, Debug)] +pub enum Error { + #[error(transparent)] + IoError(#[from] std::io::Error), + TaskNotFound(String), + MissingCommand(String), + TasksNotFound(Vec<(String, String)>), + InvalidTaskName(String), + // TODO: be more precies where the cycle happens + CycleDetected(String), +} + +impl Display for Error { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Error::IoError(e) => write!(f, "IO Error: {}", e), + Error::TasksNotFound(tasks) => write!( + f, + "Task dependencies not found: {}", + tasks + .iter() + .map(|(task, dep)| format!("{} is depending on non-existent {}", task, dep)) + .collect::>() + .join(", ") + ), + Error::TaskNotFound(task) => write!(f, "Task does not exist: {}", task), + Error::CycleDetected(task) => write!(f, "Cycle detected at task: {}", task), + Error::MissingCommand(task) => write!( + f, + "Task {} defined a status, but is missing a command", + task + ), + Error::InvalidTaskName(task) => write!( + f, + "Invalid task name: {}, expected [a-zA-Z-_]+:[a-zA-Z-_]+", + task + ), + } + } +} + +#[derive(Debug, Deserialize, Serialize)] +pub struct TaskConfig { + name: String, + #[serde(default)] + after: Vec, + #[serde(default)] + command: Option, + #[serde(default)] + status: Option, + #[serde(default)] + inputs: Option, +} + +#[derive(Deserialize, Serialize)] +pub struct Config { + pub tasks: Vec, + pub roots: Vec, +} + +#[derive(Serialize)] +pub struct Outputs(HashMap); +#[derive(Debug, Clone)] +pub struct Output(Option); + +impl TryFrom for Config { + type Error = serde_json::Error; + + fn try_from(json: serde_json::Value) -> Result { + serde_json::from_value(json) + } +} + +type LinesOutput = Vec<(std::time::Instant, String)>; + +impl std::ops::Deref for Outputs { + type Target = HashMap; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +#[derive(Debug, Clone)] +struct TaskFailure { + stdout: LinesOutput, + stderr: LinesOutput, + error: String, +} + +#[derive(Debug, Clone)] +enum Skipped { + Cached(Output), + NotImplemented, +} + +#[derive(Debug, Clone)] +enum TaskCompleted { + Success(Duration, Output), + Skipped(Skipped), + Failed(Duration, TaskFailure), + DependencyFailed, +} + +impl TaskCompleted { + fn has_failed(&self) -> bool { + matches!( + self, + TaskCompleted::Failed(_, _) | TaskCompleted::DependencyFailed + ) + } +} + +#[derive(Debug, Clone)] +enum TaskStatus { + Pending, + Running(Instant), + Completed(TaskCompleted), +} + +#[derive(Debug)] +struct TaskState { + task: TaskConfig, + status: TaskStatus, +} + +impl TaskState { + fn new(task: TaskConfig) -> Self { + Self { + task, + status: TaskStatus::Pending, + } + } + + fn prepare_command( + &self, + cmd: &str, + outputs: &HashMap, + ) -> (Command, tempfile::NamedTempFile) { + let mut command = Command::new(cmd); + command.stdout(Stdio::piped()).stderr(Stdio::piped()); + + // Set DEVENV_TASK_INPUTS + if let Some(inputs) = &self.task.inputs { + command.env("DEVENV_TASK_INPUT", serde_json::to_string(inputs).unwrap()); + } + + // Create a temporary file for DEVENV_TASK_OUTPUT_FILE + let outputs_file = tempfile::NamedTempFile::new().unwrap(); + command.env("DEVENV_TASK_OUTPUT_FILE", outputs_file.path()); + + // Set environment variables from task outputs + let mut devenv_env = String::new(); + for (_, value) in outputs.iter() { + if let Some(env) = value.get("devenv").and_then(|d| d.get("env")) { + if let Some(env_obj) = env.as_object() { + for (env_key, env_value) in env_obj { + if let Some(env_str) = env_value.as_str() { + command.env(env_key, env_str); + devenv_env.push_str(&format!("export {}={}\n", env_key, env_str)); + } + } + } + } + } + // Internal for now + command.env("DEVENV_TASK_ENV", devenv_env); + + // Set DEVENV_TASKS_OUTPUTS + let outputs_json = serde_json::to_string(outputs).unwrap(); + command.env("DEVENV_TASKS_OUTPUTS", outputs_json); + + (command, outputs_file) + } + + fn get_outputs(outputs_file: &tempfile::NamedTempFile) -> Output { + let output = match std::fs::File::open(outputs_file.path()) { + // TODO: report JSON parsing errors + Ok(file) => serde_json::from_reader(file).ok(), + Err(_) => None, + }; + Output(output) + } + + #[instrument(ret)] + async fn run( + &self, + now: Instant, + outputs: &HashMap, + ) -> TaskCompleted { + if let Some(cmd) = &self.task.status { + let (mut command, outputs_file) = self.prepare_command(cmd, outputs); + + let result = command.status().await; + match result { + Ok(status) => { + if status.success() { + return TaskCompleted::Skipped(Skipped::Cached(Self::get_outputs( + &outputs_file, + ))); + } + } + Err(e) => { + // TODO: stdout, stderr + return TaskCompleted::Failed( + now.elapsed(), + TaskFailure { + stdout: Vec::new(), + stderr: Vec::new(), + error: e.to_string(), + }, + ); + } + } + } + if let Some(cmd) = &self.task.command { + let (mut command, outputs_file) = self.prepare_command(cmd, outputs); + + let result = command.spawn(); + + let mut child = match result { + Ok(c) => c, + Err(e) => { + return TaskCompleted::Failed( + now.elapsed(), + TaskFailure { + stdout: Vec::new(), + stderr: Vec::new(), + error: e.to_string(), + }, + ); + } + }; + + let stdout = match child.stdout.take() { + Some(stdout) => stdout, + None => { + return TaskCompleted::Failed( + now.elapsed(), + TaskFailure { + stdout: Vec::new(), + stderr: Vec::new(), + error: "Failed to capture stdout".to_string(), + }, + ) + } + }; + let stderr = match child.stderr.take() { + Some(stderr) => stderr, + None => { + return TaskCompleted::Failed( + now.elapsed(), + TaskFailure { + stdout: Vec::new(), + stderr: Vec::new(), + error: "Failed to capture stderr".to_string(), + }, + ) + } + }; + + let mut stderr_reader = BufReader::new(stderr).lines(); + let mut stdout_reader = BufReader::new(stdout).lines(); + + let mut stdout_lines = Vec::new(); + let mut stderr_lines = Vec::new(); + + loop { + tokio::select! { + result = stdout_reader.next_line() => { + match result { + Ok(Some(line)) => { + info!(stdout = %line); + stdout_lines.push((std::time::Instant::now(), line)); + }, + Ok(None) => {}, + Err(e) => { + error!("Error reading stdout: {}", e); + stderr_lines.push((std::time::Instant::now(), e.to_string())); + }, + } + } + result = stderr_reader.next_line() => { + match result { + Ok(Some(line)) => { + stderr_lines.push((std::time::Instant::now(), line)); + }, + Ok(None) => {}, + Err(e) => { + stderr_lines.push((std::time::Instant::now(), e.to_string())); + }, + } + } + result = child.wait() => { + match result { + Ok(status) => { + if status.success() { + return TaskCompleted::Success(now.elapsed(), Self::get_outputs(&outputs_file)); + } else { + return TaskCompleted::Failed( + now.elapsed(), + TaskFailure { + stdout: stdout_lines, + stderr: stderr_lines, + error: format!("Task exited with status: {}", status), + }, + ); + } + }, + Err(e) => { + error!("Error waiting for command: {}", e); + return TaskCompleted::Failed( + now.elapsed(), + TaskFailure { + stdout: stdout_lines, + stderr: stderr_lines, + error: format!("Error waiting for command: {}", e), + }, + ); + } + } + } + } + } + } else { + return TaskCompleted::Skipped(Skipped::NotImplemented); + } + } +} + +#[derive(Debug)] +struct Tasks { + roots: Vec, + // Stored for reporting + root_names: Vec, + longest_task_name: usize, + graph: DiGraph>, ()>, + tasks_order: Vec, +} + +impl Tasks { + async fn new(config: Config) -> Result { + let mut graph = DiGraph::new(); + let mut task_indices = HashMap::new(); + let mut longest_task_name = 0; + for task in config.tasks { + let name = task.name.clone(); + longest_task_name = longest_task_name.max(name.len()); + if !task.name.contains(':') + || task.name.split(':').count() < 2 + || task.name.starts_with(':') + || task.name.ends_with(':') + || !task + .name + .chars() + .all(|c| c.is_ascii_alphanumeric() || c == ':' || c == '_' || c == '-') + { + return Err(Error::InvalidTaskName(name)); + } + if task.status.is_some() && task.command.is_none() { + return Err(Error::MissingCommand(name)); + } + let index = graph.add_node(Arc::new(RwLock::new(TaskState::new(task)))); + task_indices.insert(name, index); + } + let mut roots = Vec::new(); + for name in config.roots.clone() { + if let Some(index) = task_indices.get(&name) { + roots.push(*index); + } else { + return Err(Error::TaskNotFound(name)); + } + } + let mut tasks = Self { + roots, + root_names: config.roots, + longest_task_name, + graph, + tasks_order: vec![], + }; + tasks.resolve_dependencies(task_indices).await?; + tasks.tasks_order = tasks.schedule().await?; + Ok(tasks) + } + + async fn resolve_dependencies( + &mut self, + task_indices: HashMap, + ) -> Result<(), Error> { + let mut unresolved = HashSet::new(); + let mut edges_to_add = Vec::new(); + + for index in self.graph.node_indices() { + let task_state = &self.graph[index].read().await; + + for dep_name in &task_state.task.after { + if let Some(dep_idx) = task_indices.get(dep_name) { + edges_to_add.push((*dep_idx, index)); + } else { + unresolved.insert((task_state.task.name.clone(), dep_name.clone())); + } + } + } + + for (dep_idx, idx) in edges_to_add { + self.graph.add_edge(dep_idx, idx, ()); + } + + if unresolved.is_empty() { + Ok(()) + } else { + Err(Error::TasksNotFound(unresolved.into_iter().collect())) + } + } + + #[instrument(skip(self), fields(graph, subgraph), ret)] + async fn schedule(&mut self) -> Result, Error> { + let mut subgraph = DiGraph::new(); + let mut node_map = HashMap::new(); + let mut visited = HashSet::new(); + let mut to_visit = Vec::new(); + + // Start with root nodes + for &root_index in &self.roots { + to_visit.push(root_index); + } + + // Depth-first search including dependencies + while let Some(node) = to_visit.pop() { + if visited.insert(node) { + let new_node = subgraph.add_node(self.graph[node].clone()); + node_map.insert(node, new_node); + + // Add dependencies to visit + for neighbor in self + .graph + .neighbors_directed(node, petgraph::Direction::Incoming) + { + to_visit.push(neighbor); + } + } + } + + // Add edges to subgraph + for (&old_node, &new_node) in &node_map { + for edge in self.graph.edges(old_node) { + let target = edge.target(); + if let Some(&new_target) = node_map.get(&target) { + subgraph.add_edge(new_node, new_target, ()); + } + } + } + + self.graph = subgraph; + + // Run topological sort on the subgraph + match toposort(&self.graph, None) { + Ok(indexes) => Ok(indexes), + Err(cycle) => Err(Error::CycleDetected( + self.graph[cycle.node_id()].read().await.task.name.clone(), + )), + } + } + + #[instrument(skip(self))] + async fn run(&self) -> Outputs { + let mut running_tasks = JoinSet::new(); + let outputs = Arc::new(Mutex::new(HashMap::new())); + + for index in &self.tasks_order { + let task_state = &self.graph[*index]; + + let mut dependency_failed = false; + + 'dependency_check: loop { + let mut dependencies_completed = true; + for dep_index in self + .graph + .neighbors_directed(*index, petgraph::Direction::Incoming) + { + match &self.graph[dep_index].read().await.status { + TaskStatus::Completed(completed) => { + if completed.has_failed() { + dependency_failed = true; + break 'dependency_check; + } + } + TaskStatus::Pending => { + dependencies_completed = false; + break; + } + TaskStatus::Running(_) => { + dependencies_completed = false; + break; + } + } + } + + if dependencies_completed { + break; + } + + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + } + + if dependency_failed { + let mut task_state = task_state.write().await; + task_state.status = TaskStatus::Completed(TaskCompleted::DependencyFailed); + } else { + let now = Instant::now(); + + // hold write lock only to update the status + { + let mut task_state = task_state.write().await; + task_state.status = TaskStatus::Running(now); + } + + let task_state_clone = Arc::clone(task_state); + let outputs_clone = Arc::clone(&outputs); + running_tasks.spawn(async move { + let completed = { + let outputs = outputs_clone.lock().await.clone(); + task_state_clone.read().await.run(now, &outputs).await + }; + { + let mut task_state = task_state_clone.write().await; + match &completed { + TaskCompleted::Success(_, Output(Some(output))) => { + outputs_clone + .lock() + .await + .insert(task_state.task.name.clone(), output.clone()); + } + TaskCompleted::Skipped(Skipped::Cached(Output(Some(output)))) => { + outputs_clone + .lock() + .await + .insert(task_state.task.name.clone(), output.clone()); + } + _ => {} + } + task_state.status = TaskStatus::Completed(completed); + } + }); + } + } + + while let Some(res) = running_tasks.join_next().await { + match res { + Ok(_) => (), + Err(e) => eprintln!("Task crashed: {}", e), + } + } + + Outputs(Arc::try_unwrap(outputs).unwrap().into_inner()) + } +} + +pub struct TasksStatus { + lines: Vec, + pub pending: usize, + pub running: usize, + pub succeeded: usize, + pub failed: usize, + pub skipped: usize, + pub dependency_failed: usize, +} + +impl TasksStatus { + fn new() -> Self { + Self { + lines: vec![], + pending: 0, + running: 0, + succeeded: 0, + failed: 0, + skipped: 0, + dependency_failed: 0, + } + } +} + +pub struct TasksUi { + tasks: Arc, +} + +impl TasksUi { + pub async fn new(config: Config) -> Result { + let tasks = Tasks::new(config).await?; + Ok(Self { + tasks: Arc::new(tasks), + }) + } + + async fn get_tasks_status(&self) -> TasksStatus { + let mut tasks_status = TasksStatus::new(); + + for index in &self.tasks.tasks_order { + let (task_status, task_name) = { + let task_state = self.tasks.graph[*index].read().await; + (task_state.status.clone(), task_state.task.name.clone()) + }; + let (status_text, duration) = match task_status { + TaskStatus::Pending => { + tasks_status.pending += 1; + continue; + } + TaskStatus::Running(started) => { + tasks_status.running += 1; + ( + format!("{:17}", "Running").blue().bold(), + Some(started.elapsed()), + ) + } + TaskStatus::Completed(TaskCompleted::Skipped(skipped)) => { + tasks_status.skipped += 1; + let status = match skipped { + Skipped::Cached(_) => "Cached", + Skipped::NotImplemented => "Not implemented", + }; + (format!("{:17}", status).blue().bold(), None) + } + TaskStatus::Completed(TaskCompleted::Success(duration, _)) => { + tasks_status.succeeded += 1; + (format!("{:17}", "Succeeded").green().bold(), Some(duration)) + } + TaskStatus::Completed(TaskCompleted::Failed(duration, _)) => { + tasks_status.failed += 1; + (format!("{:17}", "Failed").red().bold(), Some(duration)) + } + TaskStatus::Completed(TaskCompleted::DependencyFailed) => { + tasks_status.dependency_failed += 1; + (format!("{:17}", "Dependency failed").magenta().bold(), None) + } + }; + + let duration = match duration { + Some(d) => d.as_millis().to_string() + "ms", + None => "".to_string(), + }; + tasks_status.lines.push(format!( + "{} {} {}", + status_text, + format!("{:width$}", task_name, width = self.tasks.longest_task_name).bold(), + duration, + )); + } + + tasks_status + } + + pub async fn run(&mut self) -> Result<(TasksStatus, Outputs), Error> { + let names = self.tasks.root_names.join(", ").bold(); + eprint!("{:17} {}", "Running tasks", names); + + // start processing tasks + let started = std::time::Instant::now(); + let tasks_clone = Arc::clone(&self.tasks); + let handle = tokio::spawn(async move { tasks_clone.run().await }); + + // start TUI + let mut last_list_height: u16 = 0; + let mut stderr = io::stderr(); + + loop { + let mut finished = false; + if handle.is_finished() { + finished = true; + } + + let tasks_status = self.get_tasks_status().await; + + let output = style::PrintStyledContent( + format!( + "{}\n{:width$} {}\n", + tasks_status.lines.join("\n"), + [ + if tasks_status.pending > 0 { + format!("{} {}", tasks_status.pending, "Pending".blue().bold()) + } else { + String::new() + }, + if tasks_status.running > 0 { + format!("{} {}", tasks_status.running, "Running".blue().bold()) + } else { + String::new() + }, + if tasks_status.skipped > 0 { + format!("{} {}", tasks_status.skipped, "Skipped".blue().bold()) + } else { + String::new() + }, + if tasks_status.succeeded > 0 { + format!("{} {}", tasks_status.succeeded, "Succeeded".green().bold()) + } else { + String::new() + }, + if tasks_status.failed > 0 { + format!("{} {}", tasks_status.failed, "Failed".red().bold()) + } else { + String::new() + }, + if tasks_status.dependency_failed > 0 { + format!( + "{} {}", + tasks_status.dependency_failed, + "Dependency Failed".red().bold() + ) + } else { + String::new() + }, + ] + .into_iter() + .filter(|s| !s.is_empty()) + .collect::>() + .join(", "), + format!("{:.2?}", started.elapsed()), + width = self.tasks.longest_task_name + 36 + ) + .stylize(), + ); + + if last_list_height > 0 { + execute!( + stderr, + cursor::MoveUp(last_list_height), + Clear(ClearType::FromCursorDown), + output + )?; + } else { + execute!(stderr, output)?; + } + + if finished { + break; + } + + last_list_height = tasks_status.lines.len() as u16 + 1; + + // Sleep briefly to avoid excessive redraws + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + } + + let errors = { + let mut errors = String::new(); + for index in &self.tasks.tasks_order { + let task_state = self.tasks.graph[*index].read().await; + if let TaskStatus::Completed(TaskCompleted::Failed(_, failure)) = &task_state.status + { + errors.push_str(&format!( + "\n--- {} failed with error: {}\n", + task_state.task.name, failure.error + )); + errors.push_str(&format!("--- {} stdout:\n", task_state.task.name)); + for (time, line) in &failure.stdout { + errors.push_str(&format!( + "{:07.2}: {}\n", + time.elapsed().as_secs_f32(), + line + )); + } + errors.push_str(&format!("--- {} stderr:\n", task_state.task.name)); + for (time, line) in &failure.stderr { + errors.push_str(&format!( + "{:07.2}: {}\n", + time.elapsed().as_secs_f32(), + line + )); + } + errors.push_str("---\n") + } + } + errors.stylize() + }; + execute!(stderr, style::PrintStyledContent(errors))?; + + let tasks_status = self.get_tasks_status().await; + Ok((tasks_status, handle.await.unwrap())) + } +} + +#[cfg(test)] +mod test { + use super::*; + + use pretty_assertions::assert_matches; + use serde_json::json; + use std::fs; + use std::io::Write; + use std::os::unix::fs::PermissionsExt; + + #[tokio::test] + async fn test_task_name() -> Result<(), Error> { + let invalid_names = vec![ + "invalid:name!", + "invalid name", + "invalid@name", + ":invalid", + "invalid:", + "invalid", + ]; + for task in invalid_names { + assert_matches!( + Config::try_from(json!({ + "roots": [], + "tasks": [{ + "name": task.to_string() + }] + })) + .map(Tasks::new) + .unwrap() + .await, + Err(Error::InvalidTaskName(_)) + ); + } + let valid_names = vec![ + "devenv:enterShell", + "devenv:enter-shell", + "devenv:enter_shell", + "devenv:python:virtualenv", + ]; + for task in valid_names { + assert_matches!( + Config::try_from(serde_json::json!({ + "roots": [], + "tasks": [{ + "name": task.to_string() + }] + })) + .map(Tasks::new) + .unwrap() + .await, + Ok(_) + ); + } + Ok(()) + } + + #[tokio::test] + async fn test_basic_tasks() -> Result<(), Error> { + let script1 = create_script( + "#!/bin/sh\necho 'Task 1 is running' && sleep 0.5 && echo 'Task 1 completed'", + )?; + let script2 = create_script( + "#!/bin/sh\necho 'Task 2 is running' && sleep 0.5 && echo 'Task 2 completed'", + )?; + let script3 = create_script( + "#!/bin/sh\necho 'Task 3 is running' && sleep 0.5 && echo 'Task 3 completed'", + )?; + let script4 = + create_script("#!/bin/sh\necho 'Task 4 is running' && echo 'Task 4 completed'")?; + + let tasks = Tasks::new( + Config::try_from(json!({ + "roots": ["myapp:task_1", "myapp:task_4"], + "tasks": [ + { + "name": "myapp:task_1", + "command": script1.to_str().unwrap() + }, + { + "name": "myapp:task_2", + "command": script2.to_str().unwrap() + }, + { + "name": "myapp:task_3", + "after": ["myapp:task_1"], + "command": script3.to_str().unwrap() + }, + { + "name": "myapp:task_4", + "after": ["myapp:task_3"], + "command": script4.to_str().unwrap() + } + ] + })) + .unwrap(), + ) + .await?; + tasks.run().await; + + let task_statuses = inspect_tasks(&tasks).await; + let task_statuses = task_statuses.as_slice(); + assert_matches!( + task_statuses, + [ + (name1, TaskStatus::Completed(TaskCompleted::Success(_, _))), + (name2, TaskStatus::Completed(TaskCompleted::Success(_, _))), + (name3, TaskStatus::Completed(TaskCompleted::Success(_, _))) + ] if name1 == "myapp:task_1" && name2 == "myapp:task_3" && name3 == "myapp:task_4" + ); + Ok(()) + } + + #[tokio::test] + async fn test_tasks_cycle() -> Result<(), Error> { + let result = Tasks::new( + Config::try_from(json!({ + "roots": ["myapp:task_1"], + "tasks": [ + { + "name": "myapp:task_1", + "after": ["myapp:task_2"], + "command": "echo 'Task 1 is running' && echo 'Task 1 completed'" + }, + { + "name": "myapp:task_2", + "after": ["myapp:task_1"], + "command": "echo 'Task 2 is running' && echo 'Task 2 completed'" + } + ] + })) + .unwrap(), + ) + .await; + if let Err(Error::CycleDetected(task)) = result { + assert_eq!(task, "myapp:task_2".to_string()); + } else { + panic!("Expected Error::CycleDetected, got {:?}", result); + } + Ok(()) + } + + #[tokio::test] + async fn test_status() -> Result<(), Error> { + let command_script1 = + create_script("#!/bin/sh\necho 'Task 1 is running' && echo 'Task 1 completed'")?; + let status_script1 = create_script("#!/bin/sh\nexit 0")?; + let command_script2 = + create_script("#!/bin/sh\necho 'Task 2 is running' && echo 'Task 2 completed'")?; + let status_script2 = create_script("#!/bin/sh\nexit 1")?; + + let command1 = command_script1.to_str().unwrap(); + let status1 = status_script1.to_str().unwrap(); + let command2 = command_script2.to_str().unwrap(); + let status2 = status_script2.to_str().unwrap(); + + let create_tasks = |root: &'static str| async move { + Tasks::new( + Config::try_from(json!({ + "roots": [root], + "tasks": [ + { + "name": "myapp:task_1", + "command": command1, + "status": status1 + }, + { + "name": "myapp:task_2", + "command": command2, + "status": status2 + } + ] + })) + .unwrap(), + ) + .await + }; + + let tasks = create_tasks("myapp:task_1").await.unwrap(); + tasks.run().await; + assert_eq!(tasks.tasks_order.len(), 1); + assert_matches!( + tasks.graph[tasks.tasks_order[0]].read().await.status, + TaskStatus::Completed(TaskCompleted::Skipped(Skipped::Cached(_))) + ); + + let tasks = create_tasks("myapp:task_2").await.unwrap(); + tasks.run().await; + assert_eq!(tasks.tasks_order.len(), 1); + assert_matches!( + tasks.graph[tasks.tasks_order[0]].read().await.status, + TaskStatus::Completed(TaskCompleted::Success(_, _)) + ); + + Ok(()) + } + + #[tokio::test] + async fn test_nonexistent_script() -> Result<(), Error> { + let tasks = Tasks::new( + Config::try_from(json!({ + "roots": ["myapp:task_1"], + "tasks": [ + { + "name": "myapp:task_1", + "command": "/path/to/nonexistent/script.sh" + } + ] + })) + .unwrap(), + ) + .await?; + + tasks.run().await; + + let task_statuses = inspect_tasks(&tasks).await; + let task_statuses = task_statuses.as_slice(); + assert_matches!( + &task_statuses, + [( + task_1, + TaskStatus::Completed(TaskCompleted::Failed( + _, + TaskFailure { + stdout: _, + stderr: _, + error + } + )) + )] if error == "No such file or directory (os error 2)" && task_1 == "myapp:task_1" + ); + + Ok(()) + } + + #[tokio::test] + async fn test_status_without_command() -> Result<(), Error> { + let status_script = create_script("#!/bin/sh\nexit 0")?; + + let result = Tasks::new( + Config::try_from(json!({ + "roots": ["myapp:task_1"], + "tasks": [ + { + "name": "myapp:task_1", + "status": status_script.to_str().unwrap() + } + ] + })) + .unwrap(), + ) + .await; + + assert!(matches!(result, Err(Error::MissingCommand(_)))); + Ok(()) + } + + #[tokio::test] + async fn test_dependency_failure() -> Result<(), Error> { + let failing_script = create_script("#!/bin/sh\necho 'Failing task' && exit 1")?; + let dependent_script = create_script("#!/bin/sh\necho 'Dependent task' && exit 0")?; + + let tasks = Tasks::new( + Config::try_from(json!({ + "roots": ["myapp:task_2"], + "tasks": [ + { + "name": "myapp:task_1", + "command": failing_script.to_str().unwrap() + }, + { + "name": "myapp:task_2", + "after": ["myapp:task_1"], + "command": dependent_script.to_str().unwrap() + } + ] + })) + .unwrap(), + ) + .await?; + + tasks.run().await; + + let task_statuses = inspect_tasks(&tasks).await; + let task_statuses_slice = &task_statuses.as_slice(); + assert_matches!( + *task_statuses_slice, + [ + (task_1, TaskStatus::Completed(TaskCompleted::Failed(_, _))), + ( + task_2, + TaskStatus::Completed(TaskCompleted::DependencyFailed) + ) + ] if task_1 == "myapp:task_1" && task_2 == "myapp:task_2" + ); + + Ok(()) + } + + #[tokio::test] + async fn test_inputs_outputs() -> Result<(), Error> { + let input_script = create_script( + r#"#!/bin/sh +echo "{\"key\": \"value\"}" > $DEVENV_TASK_OUTPUT_FILE +if [ "$DEVENV_TASK_INPUT" != '{"test":"input"}' ]; then + echo "Error: Input does not match expected value" >&2 + echo "Expected: $expected" >&2 + echo "Actual: $input" >&2 + exit 1 +fi +"#, + )?; + + let output_script = create_script( + r#"#!/bin/sh + if [ "$DEVENV_TASKS_OUTPUTS" != '{"myapp:task_1":{"key":"value"}}' ]; then + echo "Error: Outputs do not match expected value" >&2 + echo "Expected: {\"myapp:task_1\":{\"key\":\"value\"}}" >&2 + echo "Actual: $DEVENV_TASKS_OUTPUTS" >&2 + exit 1 + fi + echo "{\"result\": \"success\"}" > $DEVENV_TASK_OUTPUT_FILE +"#, + )?; + + let tasks = Tasks::new( + Config::try_from(json!({ + "roots": ["myapp:task_1", "myapp:task_2"], + "tasks": [ + { + "name": "myapp:task_1", + "command": input_script.to_str().unwrap(), + "inputs": {"test": "input"} + }, + { + "name": "myapp:task_2", + "command": output_script.to_str().unwrap(), + "after": ["myapp:task_1"] + } + ] + })) + .unwrap(), + ) + .await?; + + let outputs = tasks.run().await; + let task_statuses = inspect_tasks(&tasks).await; + let task_statuses = task_statuses.as_slice(); + assert_matches!( + task_statuses, + [ + (name1, TaskStatus::Completed(TaskCompleted::Success(_, _))), + (name2, TaskStatus::Completed(TaskCompleted::Success(_, _))) + ] if name1 == "myapp:task_1" && name2 == "myapp:task_2" + ); + + assert_eq!( + outputs.get("myapp:task_1").unwrap(), + &json!({"key": "value"}) + ); + assert_eq!( + outputs.get("myapp:task_2").unwrap(), + &json!({"result": "success"}) + ); + + Ok(()) + } + + #[cfg(test)] + async fn inspect_tasks(tasks: &Tasks) -> Vec<(String, TaskStatus)> { + let mut result = Vec::new(); + for index in &tasks.tasks_order { + let task_state = tasks.graph[*index].read().await; + result.push((task_state.task.name.clone(), task_state.status.clone())); + } + result + } + + #[cfg(test)] + fn create_script(script: &str) -> std::io::Result { + let mut temp_file = tempfile::Builder::new() + .prefix("script") + .suffix(".sh") + .tempfile()?; + temp_file.write_all(script.as_bytes())?; + temp_file + .as_file_mut() + .set_permissions(fs::Permissions::from_mode(0o755))?; + Ok(temp_file.into_temp_path()) + } +} diff --git a/docs/.pages b/docs/.pages index 003f29a09..131be05d3 100644 --- a/docs/.pages +++ b/docs/.pages @@ -7,13 +7,14 @@ nav: - Basics: basics.md - Packages: packages.md - Scripts: scripts.md - - Languages: + - Tasks: tasks.md + - Languages: - Overview: languages.md - Supported Languages: supported-languages - - Processes: + - Processes: - Overview: processes.md - Supported Process Managers: supported-process-managers - - Services: + - Services: - Overview: services.md - Supported Services: supported-services - Containers: containers.md @@ -21,6 +22,7 @@ nav: - Pre-Commit Hooks: pre-commit-hooks.md - Outputs: outputs.md - Tests: tests.md + - Outputs: outputs.md - Common Patterns: common-patterns.md - Writing devenv.yaml: - Inputs: inputs.md diff --git a/docs/assets/images/tasks.gif b/docs/assets/images/tasks.gif new file mode 100644 index 000000000..801add92b Binary files /dev/null and b/docs/assets/images/tasks.gif differ diff --git a/docs/blog/posts/devenv-v1.2-tasks.md b/docs/blog/posts/devenv-v1.2-tasks.md new file mode 100644 index 000000000..cdeb17379 --- /dev/null +++ b/docs/blog/posts/devenv-v1.2-tasks.md @@ -0,0 +1,95 @@ +--- +draft: false +date: 2024-09-24 +authors: + - domenkozar +--- + +# devenv 1.2: Tasks for convergent configuration with Nix + +For devenv, our mission is to make Nix the ultimate tool for managing developer environments. Nix +excels at [congruent configuration](https://constructolution.wordpress.com/2012/07/08/divergent-convergent-and-congruent-infrastructures/), +where the system state is fully described by declarative code. + +However, the real world often throws curveballs. Side-effects like database migrations, one-off +tasks such as data imports, or external API calls don't always fit neatly into this paradigm. +In these cases, we often resort to [convergent configuration](https://constructolution.wordpress.com/2012/07/08/divergent-convergent-and-congruent-infrastructures/), +where we define the desired end-state and let the system figure out how to get there. + +To bridge this gap and make Nix more versatile, we're introducing tasks. These allow you to +handle those pesky real-world scenarios while still leveraging Nix's powerful ecosystem. + +![Tasks interactive example](/assets/images/tasks.gif) + +## Usage + +For example if you'd like to execute python code after virtualenv has been created: + +```nix title="devenv.nix" +{ pkgs, lib, config, ... }: { + languages.python.enable = true; + languages.python.venv.enable = true; + + tasks = { + "python:setup" = { + exec = "python ${pkgs.writeText "setup.py" '' + print("hello world") + ''}"; + after = [ "devenv:python:virtualenv" ]; + }; + "devenv:enterShell".after = [ "python:setup" ]; + }; +} +``` + +`python:setup` task executes before `devenv:enterShell` but after `python:virtualenv` task: + +For all supported use cases see [tasks documentation](/tasks/). + + +## Task Server Protocol for SDKs + +We've talked to many teams that **dropped Nix** after a while and they usually fit into two categories: + +* 1) Maintaining **Nix was too complex** and the team didn't fully onboard, **creating friction inside the teams**. +* 2) Went **all-in Nix** and it took **a big toll on the team productivity**. + +While devenv already addresses (1), bridging **the gap between Nix provided developer environments +and existing devops tooling written in your favorite language is still an unsolved problem until now**. +
+ +We've designed [Task Server Protocol](https://github.com/cachix/devenv/issues/1457) so that you can write tasks +using your existing automation by providing an executable that exposes the tasks to devenv: +
+ +```nix title="devenv.nix" +{ pkgs, ... }: +let + myexecutable = pkgs.rustPlatform.buildRustPackage rec { + pname = "foo-bar"; + version = "0.1"; + cargoLock.lockFile = ./myexecutable/Cargo.lock; + src = pkgs.lib.cleanSource ./myexecutable; + } +in { + task.serverProtocol = [ "${myexecutable}/bin/myexecutable" ]; +} +``` + +In a few weeks we're planning to provide [Rust TSP SDK](https://github.com/cachix/devenv/issues/1457) +with a **full test suite** so you can implement your own abstraction in your language of choice. +
+ +You can now use your **preferred language for automation**, running tasks with a simple `devenv tasks run ` command. This +**flexibility** allows for more **intuitive and maintainable scripts**, tailored to your team's familiarity. + +For devenv itself, we'll slowly **transition from bash to Rust for +internal glue code**, enhancing performance and reliability. This change will make devenv more +**robust and easier to extend**, ultimately providing you with a **smoother development experience**. + +## Upgrading + +If you run `devenv update` on your existing repository you should already be using tasks, +without needing to upgrade to devenv 1.2. + +Domen diff --git a/docs/overrides/home.html b/docs/overrides/home.html index 15c96b52e..3cbde9ca8 100644 --- a/docs/overrides/home.html +++ b/docs/overrides/home.html @@ -33,7 +33,8 @@

Fast, Decla Develop natively • Deploy containers • 100.000+ packages - • Write scripts + • Write scripts and + tasks • 50+ supported languages • Define processes • Reuse services @@ -113,7 +114,7 @@

Fast, Decla 
{ pkgs, config, ... }: {
   env.GREET = "determinism";
 
-  packages = [ 
+  packages = [
     pkgs.ncdu
   ];
 
@@ -154,9 +155,16 @@ 
Fast, Decla
                 
               
 
{ pkgs, ... }: {
-  scripts.build.exec = "parcel build";
+  packages = [ pkgs.yarn ];
 
-  # Runs on git commit and CI
+  scripts.build.exec = "yarn build";
+
+  tasks = {
+    "myapp:build".exec = "build";
+    "devenv:enterShell".after = [ "myapp:build" ];
+  };
+
+  # Runs on `git commit` and `devenv test`
   pre-commit.hooks = {
     black.enable = true;
     # Your custom hooks
@@ -172,8 +180,14 @@ 
Fast, Decla
           
 
           

-
devenv shell build
+
devenv shell
 ...
+Running tasks     devenv:enterShell
+Succeeded         devenv:pre-commit:install 15ms
+Succeeded         myapp:build               23ms
+Succeeded         devenv:enterShell         23ms
+3 Succeeded                                 50.14ms
+$
 
 

           

@@ -182,9 +196,9 @@ 
Fast, Decla
 
       

         

-          
Scripts and Git hooks

+          
Scripts and Tasks

           

-            Define scripts and
+            Define scripts,  tasks and
             git hooks
             to automate your development workflow.
           

@@ -199,20 +213,20 @@ 
Fast, Decla
             

             

               

-                git hooks.
+                Tasks.
               

               

-                Pick from builtin and language specific linters and formatters using git-hooks.nix.
+                Form dependencies between automation code, executed in parallel and written in your favorite language.
+              

             

             

               

-                Invoke commands inside the environment.
+                Git hooks.
               

               

-                Particularly useful in CI/CD and scripting.
-              

+                Pick from builtin and language specific linters and formatters using git-hooks.nix.
             

           
         

@@ -291,7 +305,7 @@ 
Fast, Decla
                 Examples
               
               

-                Checkout examples collection 
+                Checkout examples collection
                 to get started.
             
           
@@ -363,7 +377,7 @@ 
Fast, Decla
                 
               
 
{ pkgs, ... }: {
-  packages = [ 
+  packages = [
     pkgs.mkdocs
     pkgs.watchexec
   ];
@@ -516,7 +530,7 @@ 
Fast, Decla
                 
               
 
{ pkgs, ... }: {
-  packages = [ 
+  packages = [
     pkgs.mkdocs
     pkgs.curl
   ];
@@ -624,7 +638,7 @@ 
Fast, Decla
                 
               
 
{ pkgs, ... }: {
-  packages = [ 
+  packages = [
     pkgs.mkdocs
     pkgs.curl
   ];
diff --git a/docs/reference/options.md b/docs/reference/options.md
index ec05f8bec..a31dac9f2 100644
--- a/docs/reference/options.md
+++ b/docs/reference/options.md
@@ -41484,6 +41484,190 @@ package
 
 
 
+## tasks
+
+
+
+This option has no description.
+
+
+
+*Type:*
+attribute set of (submodule)
+
+*Declared by:*
+ - [https://github.com/cachix/devenv/blob/main/src/modules/tasks.nix](https://github.com/cachix/devenv/blob/main/src/modules/tasks.nix)
+
+
+
+## tasks.\.package
+
+
+
+Package to install for this task.
+
+
+
+*Type:*
+package
+
+
+
+*Default:*
+`  `
+
+*Declared by:*
+ - [https://github.com/cachix/devenv/blob/main/src/modules/tasks.nix](https://github.com/cachix/devenv/blob/main/src/modules/tasks.nix)
+
+
+
+## tasks.\.after
+
+
+
+List of tasks to run before this task.
+
+
+
+*Type:*
+list of string
+
+
+
+*Default:*
+` [ ] `
+
+*Declared by:*
+ - [https://github.com/cachix/devenv/blob/main/src/modules/tasks.nix](https://github.com/cachix/devenv/blob/main/src/modules/tasks.nix)
+
+
+
+## tasks.\.binary
+
+
+
+Override the binary name if it doesn’t match package name
+
+
+
+*Type:*
+string
+
+
+
+*Default:*
+` "bash" `
+
+*Declared by:*
+ - [https://github.com/cachix/devenv/blob/main/src/modules/tasks.nix](https://github.com/cachix/devenv/blob/main/src/modules/tasks.nix)
+
+
+
+## tasks.\.description
+
+
+
+Description of the task.
+
+
+
+*Type:*
+string
+
+
+
+*Default:*
+` "" `
+
+*Declared by:*
+ - [https://github.com/cachix/devenv/blob/main/src/modules/tasks.nix](https://github.com/cachix/devenv/blob/main/src/modules/tasks.nix)
+
+
+
+## tasks.\.exec
+
+
+
+Command to execute the task.
+
+
+
+*Type:*
+null or string
+
+
+
+*Default:*
+` null `
+
+*Declared by:*
+ - [https://github.com/cachix/devenv/blob/main/src/modules/tasks.nix](https://github.com/cachix/devenv/blob/main/src/modules/tasks.nix)
+
+
+
+## tasks.\.exports
+
+
+
+List of environment variables to export.
+
+
+
+*Type:*
+list of string
+
+
+
+*Default:*
+` [ ] `
+
+*Declared by:*
+ - [https://github.com/cachix/devenv/blob/main/src/modules/tasks.nix](https://github.com/cachix/devenv/blob/main/src/modules/tasks.nix)
+
+
+
+## tasks.\.input
+
+
+
+Input values for the task, encoded as JSON.
+
+
+
+*Type:*
+attribute set of anything
+
+
+
+*Default:*
+` { } `
+
+*Declared by:*
+ - [https://github.com/cachix/devenv/blob/main/src/modules/tasks.nix](https://github.com/cachix/devenv/blob/main/src/modules/tasks.nix)
+
+
+
+## tasks.\.status
+
+
+
+Check if the command should be ran
+
+
+
+*Type:*
+null or string
+
+
+
+*Default:*
+` null `
+
+*Declared by:*
+ - [https://github.com/cachix/devenv/blob/main/src/modules/tasks.nix](https://github.com/cachix/devenv/blob/main/src/modules/tasks.nix)
+
+
+
 ## unsetEnvVars
 
 
diff --git a/docs/tasks.md b/docs/tasks.md
new file mode 100644
index 000000000..bbb5e1e60
--- /dev/null
+++ b/docs/tasks.md
@@ -0,0 +1,117 @@
+# Tasks
+
+!!! info "New in version 1.2"
+
+Tasks allow you to form dependencies between code, executed in parallel.
+
+## Defining tasks
+
+```nix title="devenv.nix"
+{ pkgs, ... }:
+
+{
+  tasks."myapp:hello" = {
+    exec = ''echo "Hello, world!"'';
+  };
+}
+```
+
+```shell-session
+$ devenv tasks run myapp:hello
+Running tasks     myapp:hello
+Succeeded         myapp:hello         9ms
+1 Succeeded                           50.14ms
+```
+
+## enterShell / enterTest
+
+If you'd like the tasks to run as part of the `enterShell` or `enterTest`:
+
+```nix title="devenv.nix"
+{ pkgs, lib, config, ... }:
+
+{
+  tasks = {
+    "bash:hello".exec = "echo 'Hello world from bash!'";
+    "devenv:enterShell".after = [ "bash:hello" ];
+    "devenv:enterTest".after = [ "bash:hello" ];
+  };
+}
+```
+
+```shell-session
+$ devenv shell
+...
+Running tasks     devenv:enterShell
+Succeeded         devenv:pre-commit:install 25ms
+Succeeded         bash:hello                 9ms
+Succeeded         devenv:enterShell         23ms
+3 Succeeded                                 103.14ms
+```
+
+## Using your favourite language
+
+Tasks can also use another package for execution, for example when entering the shell:
+
+```nix title="devenv.nix"
+{ pkgs, lib, config, ... }:
+
+{
+  tasks = {
+    "python:hello"" = {
+      exec = ''
+        print("Hello world from Python!")
+      '';
+      package = config.languages.python.package;
+    };
+  };
+}
+```
+
+## Avoiding running expensive `exec` via `status` check
+
+If you define a `status` command, it will be executed first and if it returns `0`, `exec` will be skipped.
+
+```nix title="devenv.nix"
+{ pkgs, lib, config, ... }:
+
+{
+  tasks = {
+    "myapp:migrations" = {
+      exec = "db-migrate";
+      status = "db-needs-migrations";
+    };
+  };
+}
+```
+
+## Inputs / Outputs
+
+Tasks support passing inputs and produce outputs, both as JSON objects:
+
+- `$DEVENV_TASK_INPUT`: JSON object of  `tasks."myapp:mytask".input`.
+- `$DEVENV_TASKS_OUTPUTS`: JSON object with dependent tasks as keys and their outputs as values.
+- `$DEVENV_TASK_OUTPUT_FILE`: a writable file with tasks' outputs in JSON.
+
+```nix title="devenv.nix"
+{ pkgs, lib, config, ... }:
+
+{
+  tasks = {
+    "myapp:mytask" = {
+      exec = ''
+        echo $DEVENV_TASK_INPUTS> $DEVENV_ROOT/input.json
+        echo '{ "output" = 1; }' > $DEVENV_TASK_OUTPUT_FILE
+        echo $DEVENV_TASKS_OUTPUTS > $DEVENV_ROOT/outputs.json
+      '';
+      input = {
+        value = 1;
+      };
+    };
+  };
+}
+```
+
+## SDK using Task Server Protocol
+
+See [Task Server Protocol](https://github.com/cachix/devenv/issues/1457) for a proposal how defining tasks in your favorite language would look like.
diff --git a/examples/mongodb/devenv.yaml b/examples/mongodb/devenv.yaml
index 89a8475be..09bce897b 100644
--- a/examples/mongodb/devenv.yaml
+++ b/examples/mongodb/devenv.yaml
@@ -1,4 +1 @@
 allowUnfree: true
-inputs:
-  nixpkgs:
-    url: github:NixOS/nixpkgs/nixpkgs-unstable
diff --git a/package.nix b/package.nix
index ec4116a7e..e3a17069c 100644
--- a/package.nix
+++ b/package.nix
@@ -1,4 +1,4 @@
-{ pkgs, inputs }:
+{ pkgs, inputs, build_tasks ? false }:
 
 pkgs.rustPlatform.buildRustPackage {
   pname = "devenv";
@@ -9,11 +9,12 @@ pkgs.rustPlatform.buildRustPackage {
     "Cargo\.toml"
     "Cargo\.lock"
     "devenv(/.*)?"
+    "tasks(/.*)?"
     "devenv-run-tests(/.*)?"
     "xtask(/.*)?"
   ];
 
-  cargoBuildFlags = [ "-p devenv -p devenv-run-tests" ];
+  cargoBuildFlags = if build_tasks then [ "-p tasks" ] else [ "-p devenv -p devenv-run-tests" ];
 
   cargoLock = {
     lockFile = ./Cargo.lock;
@@ -29,7 +30,7 @@ pkgs.rustPlatform.buildRustPackage {
     pkgs.darwin.apple_sdk.frameworks.SystemConfiguration
   ];
 
-  postInstall = ''
+  postInstall = pkgs.lib.optionalString (!build_tasks) ''
     wrapProgram $out/bin/devenv \
       --set DEVENV_NIX ${inputs.nix.packages.${pkgs.stdenv.system}.nix} \
       --prefix PATH ":" "$out/bin:${inputs.cachix.packages.${pkgs.stdenv.system}.cachix}/bin"
diff --git a/requirements.txt b/requirements.txt
index 2ade804fa..58f44d96b 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,7 +1,7 @@
 mkdocs
 mkdocs-material[imaging]
 # https://github.com/Guts/mkdocs-rss-plugin/issues/257#issuecomment-2170940396
-mkdocs-rss-plugin==1.13.1
+mkdocs-rss-plugin==1.15.0
 mkdocs-include-markdown-plugin
 mkdocs-markdownextradata-plugin
 mkdocs-awesome-pages-plugin
diff --git a/src/modules/integrations/pre-commit.nix b/src/modules/integrations/pre-commit.nix
index 5e27432ec..bfec2b34d 100644
--- a/src/modules/integrations/pre-commit.nix
+++ b/src/modules/integrations/pre-commit.nix
@@ -20,11 +20,14 @@
 
   config = lib.mkIf ((lib.filterAttrs (id: value: value.enable) config.pre-commit.hooks) != { }) {
     ci = [ config.pre-commit.run ];
-    enterTest = ''
-      pre-commit run -a
-    '';
     # Add the packages for any enabled hooks at the end to avoid overriding the language-defined packages.
     packages = lib.mkAfter ([ config.pre-commit.package ] ++ (config.pre-commit.enabledPackages or [ ]));
-    enterShell = config.pre-commit.installationScript;
+    tasks = {
+      # TODO: split installation script into status + exec
+      "devenv:pre-commit:install".exec = config.pre-commit.installationScript;
+      "devenv:pre-commit:run".exec = "pre-commit run -a";
+      "devenv:enterShell".after = [ "devenv:pre-commit:install" ];
+      "devenv:enterTest".after = [ "devenv:pre-commit:run" ];
+    };
   };
 }
diff --git a/src/modules/languages/nix.nix b/src/modules/languages/nix.nix
index e547914a1..204904197 100644
--- a/src/modules/languages/nix.nix
+++ b/src/modules/languages/nix.nix
@@ -2,7 +2,7 @@
 
 let
   cfg = config.languages.nix;
-  cachix = "${lib.getBin config.cachix.package}";
+  cachix = lib.getBin config.cachix.package;
 
   # a bit of indirection to prevent mkShell from overriding the installed Nix
   vulnix = pkgs.buildEnv {
@@ -27,8 +27,7 @@ in
       statix
       deadnix
       cfg.lsp.package
-    ] ++ (lib.optional config.cachix.enable cachix) ++ [
       vulnix
-    ];
+    ] ++ (lib.optional config.cachix.enable cachix);
   };
 }
diff --git a/src/modules/languages/python.nix b/src/modules/languages/python.nix
index ca8ed346c..9cfb18551 100644
--- a/src/modules/languages/python.nix
+++ b/src/modules/languages/python.nix
@@ -43,7 +43,7 @@ let
     let
       USE_UV_SYNC = cfg.uv.sync.enable && builtins.compareVersions cfg.uv.package.version "0.4.4" >= 0;
     in
-    pkgs.writeShellScript "init-venv.sh" ''
+    ''
       pushd "${cfg.directory}"
 
       # Make sure any tools are not attempting to use the Python interpreter from any
@@ -107,7 +107,7 @@ let
       popd
     '';
 
-  initUvScript = pkgs.writeShellScript "init-uv.sh" ''
+  initUvScript = ''
     pushd "${cfg.directory}"
 
     VENV_PATH="${config.env.DEVENV_STATE}/venv"
@@ -173,7 +173,7 @@ let
     popd
   '';
 
-  initPoetryScript = pkgs.writeShellScript "init-poetry.sh" ''
+  initPoetryScript = ''
     pushd "${cfg.directory}"
 
     function _devenv_init_poetry_venv
@@ -454,19 +454,32 @@ in
       }
     ];
 
-    enterShell = lib.concatStringsSep "\n" ([
-      ''
-        export PYTHONPATH="$DEVENV_PROFILE/${package.sitePackages}''${PYTHONPATH:+:$PYTHONPATH}"
-      ''
-    ] ++
-    (lib.optional cfg.venv.enable ''
-      source ${initVenvScript}
-    '') ++
-    (lib.optional cfg.poetry.install.enable ''
-      source ${initPoetryScript}
-    '') ++
-    (lib.optional cfg.uv.sync.enable ''
-      source ${initUvScript}
-    ''));
+    tasks = {
+      "devenv:python:virtualenv" = lib.mkIf cfg.venv.enable {
+        description = "Initialize Python virtual environment";
+        exec = initVenvScript;
+        exports = [ "PATH" "VIRTUAL_ENV" ];
+      };
+
+      "devenv:python:poetry" = lib.mkIf cfg.poetry.install.enable {
+        description = "Initialize Poetry";
+        exec = initPoetryScript;
+        exports = [ "PATH" "VIRTUAL_ENV" ];
+      };
+
+      "devenv:python:uv" = lib.mkIf cfg.uv.sync.enable {
+        description = "Initialize uv sync";
+        exec = initUvScript;
+        exports = [ "PATH" "VIRTUAL_ENV" ];
+      };
+
+      "devenv:enterShell".after = lib.optional cfg.venv.enable "devenv:python:virtualenv"
+        ++ lib.optional cfg.poetry.install.enable "devenv:python:poetry"
+        ++ lib.optional cfg.uv.sync.enable "devenv:python:uv";
+    };
+
+    enterShell = ''
+      export PYTHONPATH="$DEVENV_PROFILE/${package.sitePackages}''${PYTHONPATH:+:$PYTHONPATH}"
+    '';
   };
 }
diff --git a/src/modules/scripts.nix b/src/modules/scripts.nix
index c74ece408..de69288ec 100644
--- a/src/modules/scripts.nix
+++ b/src/modules/scripts.nix
@@ -46,10 +46,7 @@ let
   );
 
   renderInfoSection = name: script:
-    ''
-      ${name}${lib.optionalString (script.description != "") ": ${script.description}"}
-        ${script.scriptPackage}
-    '';
+    "${name}${lib.optionalString (script.description != "") ": ${script.description}"} ${script.scriptPackage}";
 in
 {
   options = {
diff --git a/src/modules/tasks.nix b/src/modules/tasks.nix
new file mode 100644
index 000000000..7b1401d71
--- /dev/null
+++ b/src/modules/tasks.nix
@@ -0,0 +1,137 @@
+{ pkgs, lib, config, ... }@inputs:
+let
+  types = lib.types;
+  devenv = import ./../../package.nix { inherit pkgs inputs; build_tasks = true; };
+  taskType = types.submodule
+    ({ name, config, ... }:
+      let
+        mkCommand = command:
+          if builtins.isNull command
+          then null
+          else
+            pkgs.writeScript name ''
+              #!${pkgs.lib.getBin config.package}/bin/${config.binary}
+              ${command}
+              ${lib.optionalString (config.exports != []) "${devenv}/bin/tasks export ${lib.concatStringsSep " " config.exports}"}
+            '';
+      in
+      {
+        options = {
+          exec = lib.mkOption {
+            type = types.nullOr types.str;
+            default = null;
+            description = "Command to execute the task.";
+          };
+          binary = lib.mkOption {
+            type = types.str;
+            description = "Override the binary name if it doesn't match package name";
+            default = config.package.pname;
+          };
+          package = lib.mkOption {
+            type = types.package;
+            default = pkgs.bash;
+            description = "Package to install for this task.";
+          };
+          command = lib.mkOption {
+            type = types.nullOr types.package;
+            internal = true;
+            default = mkCommand config.exec;
+            description = "Path to the script to run.";
+          };
+          status = lib.mkOption {
+            type = types.nullOr types.str;
+            default = null;
+            description = "Check if the command should be ran";
+          };
+          statusCommand = lib.mkOption {
+            type = types.nullOr types.package;
+            internal = true;
+            default = mkCommand config.status;
+            description = "Path to the script to run.";
+          };
+          config = lib.mkOption {
+            type = types.attrsOf types.anything;
+            internal = true;
+            default = {
+              name = name;
+              description = config.description;
+              status = config.statusCommand;
+              after = config.after;
+              command = config.command;
+              input = config.input;
+            };
+          };
+          exports = lib.mkOption {
+            type = types.listOf types.str;
+            default = [ ];
+            description = "List of environment variables to export.";
+          };
+          description = lib.mkOption {
+            type = types.str;
+            default = "";
+            description = "Description of the task.";
+          };
+          after = lib.mkOption {
+            type = types.listOf types.str;
+            description = "List of tasks to run before this task.";
+            default = [ ];
+          };
+          input = lib.mkOption {
+            type = types.attrsOf types.anything;
+            default = { };
+            description = "Input values for the task, encoded as JSON.";
+          };
+        };
+      });
+  tasksJSON = (lib.mapAttrsToList (name: value: { inherit name; } // value.config) config.tasks);
+in
+{
+  options.tasks = lib.mkOption {
+    type = types.attrsOf taskType;
+  };
+
+  options.task.config = lib.mkOption {
+    type = types.package;
+    internal = true;
+  };
+
+  config = {
+    env.DEVENV_TASKS = builtins.toJSON tasksJSON;
+
+    assertions = [
+      {
+        assertion = lib.all (task: task.binary == "bash" || task.export == [ ]) (lib.attrValues config.tasks);
+        message = "The 'export' option can only be set when 'binary' is set to 'bash'.";
+      }
+    ];
+
+    infoSections."tasks" =
+      lib.mapAttrsToList
+        (name: task: "${name}: ${task.description} (${if task.command == null then "no command" else task.command})")
+        config.tasks;
+
+    task.config = (pkgs.formats.json { }).generate "tasks.json" tasksJSON;
+
+    tasks = {
+      "devenv:enterShell" = {
+        description = "Runs when entering the shell";
+        exec = ''
+          echo "$DEVENV_TASK_ENV" > "$DEVENV_DOTFILE/load-exports"
+          chmod +x "$DEVENV_DOTFILE/load-exports"
+        '';
+      };
+      "devenv:enterTest" = {
+        description = "Runs when entering the test environment";
+      };
+    };
+    enterShell = ''
+      ${devenv}/bin/tasks run devenv:enterShell
+      if [ -f "$DEVENV_DOTFILE/load-exports" ]; then
+        source "$DEVENV_DOTFILE/load-exports"
+      fi
+    '';
+    enterTest = ''
+      ${devenv}/bin/tasks run devenv:enterTest
+    '';
+  };
+}
diff --git a/src/modules/top-level.nix b/src/modules/top-level.nix
index fb7aa2272..9f9d254e3 100644
--- a/src/modules/top-level.nix
+++ b/src/modules/top-level.nix
@@ -221,6 +221,7 @@ in
     ./info.nix
     ./outputs.nix
     ./processes.nix
+    ./outputs.nix
     ./scripts.nix
     ./update-check.nix
     ./containers.nix
@@ -228,6 +229,7 @@ in
     ./lib.nix
     ./tests.nix
     ./cachix.nix
+    ./tasks.nix
   ]
   ++ (listEntries ./languages)
   ++ (listEntries ./services)
diff --git a/tasks/Cargo.toml b/tasks/Cargo.toml
new file mode 100644
index 000000000..69472260a
--- /dev/null
+++ b/tasks/Cargo.toml
@@ -0,0 +1,11 @@
+[package]
+name = "tasks"
+version = "0.1.0"
+edition.workspace = true
+license.workspace = true
+
+[dependencies]
+clap.workspace = true
+devenv = { path = "../devenv" }
+serde_json.workspace = true
+tokio.workspace = true
diff --git a/tasks/src/main.rs b/tasks/src/main.rs
new file mode 100644
index 000000000..4d70776e5
--- /dev/null
+++ b/tasks/src/main.rs
@@ -0,0 +1,72 @@
+use clap::{Parser, Subcommand};
+use devenv::tasks::{Config, TaskConfig, TasksUi};
+use std::env;
+
+#[derive(Parser)]
+#[clap(author, version, about)]
+struct Args {
+    #[clap(subcommand)]
+    command: Command,
+}
+
+#[derive(Subcommand)]
+enum Command {
+    Run {
+        #[clap()]
+        roots: Vec,
+    },
+    Export {
+        #[clap()]
+        strings: Vec,
+    },
+}
+
+#[tokio::main]
+async fn main() -> Result<(), Box> {
+    let args = Args::parse();
+
+    match args.command {
+        Command::Run { roots } => {
+            let tasks_json = env::var("DEVENV_TASKS")?;
+            let tasks: Vec = serde_json::from_str(&tasks_json)?;
+
+            let config = Config { tasks, roots };
+
+            let mut tasks_ui = TasksUi::new(config).await?;
+            tasks_ui.run().await?;
+        }
+        Command::Export { strings } => {
+            let output_file =
+                env::var("DEVENV_TASK_OUTPUT_FILE").expect("DEVENV_TASK_OUTPUT_FILE not set");
+            let mut output: serde_json::Value = std::fs::read_to_string(&output_file)
+                .map(|content| serde_json::from_str(&content).unwrap_or(serde_json::json!({})))
+                .unwrap_or(serde_json::json!({}));
+
+            let mut exported_vars = serde_json::Map::new();
+            for var in strings {
+                if let Ok(value) = env::var(&var) {
+                    exported_vars.insert(var, serde_json::Value::String(value));
+                }
+            }
+
+            if !output.as_object().unwrap().contains_key("devenv") {
+                output["devenv"] = serde_json::json!({});
+            }
+            if !output["devenv"].as_object().unwrap().contains_key("env") {
+                output["devenv"]["env"] = serde_json::json!({});
+            }
+            output["devenv"]["env"] = serde_json::Value::Object(
+                output["devenv"]["env"]
+                    .as_object()
+                    .cloned()
+                    .unwrap_or_default()
+                    .into_iter()
+                    .chain(exported_vars)
+                    .collect(),
+            );
+            std::fs::write(output_file, serde_json::to_string_pretty(&output)?)?;
+        }
+    }
+
+    Ok(())
+}
diff --git a/tests/tasks/.gitignore b/tests/tasks/.gitignore
new file mode 100644
index 000000000..c75670085
--- /dev/null
+++ b/tests/tasks/.gitignore
@@ -0,0 +1,2 @@
+shell
+test
diff --git a/tests/tasks/devenv.nix b/tests/tasks/devenv.nix
new file mode 100644
index 000000000..6674c8cba
--- /dev/null
+++ b/tests/tasks/devenv.nix
@@ -0,0 +1,22 @@
+{
+  tasks = {
+    "myapp:shell".exec = "touch shell";
+    "devenv:enterShell".after = [ "myapp:shell" ];
+    "myapp:test".exec = "touch test";
+    "devenv:enterTest".after = [ "myapp:test" ];
+  };
+
+  enterTest = ''
+    if [ ! -f shell ]; then
+      echo "shell does not exist"
+      exit 1
+    fi
+    rm -f shell
+    rm -f test
+    devenv tasks run myapp:test >/dev/null
+    if [ ! -f test ]; then
+      echo "test does not exist"
+      exit 1
+    fi
+  '';
+}