Towards automated migrations

src/eval.rs

@@ -561,6 +561,30 @@ fn handle_non_by_name_attribute(
                         NonApplicable
                     }
                     _ => {
+                        // TODO: Insert check to make sure that all files under the referenced path can be
+                        // moved without breakage
+                        //
+                        // foo = callPackage ../applications/foo { };         # already there before
+                        //
+                        // foo_2 = callPackage ../applications/foo/2.nix { }; # new
+                        //
+                        //
+                        // # foo/common.nix exists
+                        // # foo/default.nix and foo/2.nix reference foo/common.nix
+                        //
+                        // Currently this gives an error, saying that `foo_2` should be migrated.
+                        //
+                        // To do:
+                        // - Collect all files transitively referenced by the entry-point file
+                        // - Check that they all are within the directory of the entry-point file
+                        // - Check that they all have a movable ancestor within the directory of the entry-point file
+                        //   - Ignore the one reference being considered in all-packages.nix
+                        //
+                        // Logic in
+                        // https://github.com/nixpkgs-architecture/nix-spp/blob/2a6ff6cb2a74f55032aa48531eac5a14dc4fc2bb/src/main.rs#L22C1-L22C6
+                        // is really good
+
+
                         // Otherwise, the path is outside `pkgs/by-name`, which means it can be
                         // migrated.
                         Loose((syntactic_call_package, location.file))

src/files.rs

@@ -1,19 +1,20 @@
-use relative_path::RelativePath;
 use relative_path::RelativePathBuf;
 use std::collections::BTreeMap;
+use std::collections::HashSet;
 use std::path::Path;
 
 use crate::nix_file::NixFileStore;
 use crate::validation::ResultIteratorExt;
 use crate::validation::Validation::Success;
-use crate::{nix_file, ratchet, structure, validation};
+use crate::{nix_file, ratchet, validation};
 
 /// Runs check on all Nix files, returning a ratchet result for each
 pub fn check_files(
     nixpkgs_path: &Path,
+    paths: &HashSet<RelativePathBuf>,
     nix_file_store: &mut NixFileStore,
 ) -> validation::Result<BTreeMap<RelativePathBuf, ratchet::File>> {
-    process_nix_files(nixpkgs_path, nix_file_store, |_nix_file| {
+    process_nix_files(nixpkgs_path, paths, nix_file_store, |_nix_file| {
         // Noop for now, only boilerplate to make it easier to add future file-based checks
         Ok(Success(ratchet::File {}))
     })
@@ -23,23 +24,18 @@ pub fn check_files(
 /// results into a mapping from each file to a ratchet value.
 fn process_nix_files(
     nixpkgs_path: &Path,
+    paths: &HashSet<RelativePathBuf>,
     nix_file_store: &mut NixFileStore,
     f: impl Fn(&nix_file::NixFile) -> validation::Result<ratchet::File>,
 ) -> validation::Result<BTreeMap<RelativePathBuf, ratchet::File>> {
-    // Get all Nix files
-    let files = {
-        let mut files = vec![];
-        collect_nix_files(nixpkgs_path, &RelativePathBuf::new(), &mut files)?;
-        files
-    };
-
-    let results = files
-        .into_iter()
+    let results = paths
+        .iter()
+        .filter(|path| path.extension() == Some(".nix") && path.to_path(nixpkgs_path).is_file())
         .map(|path| {
             // Get the (optionally-cached) parsed Nix file
             let nix_file = nix_file_store.get(&path.to_path(nixpkgs_path))?;
             let result = f(nix_file)?;
-            let val = result.map(|ratchet| (path, ratchet));
+            let val = result.map(|ratchet| (path.clone(), ratchet));
             Ok::<_, anyhow::Error>(val)
         })
         .collect_vec()?;
@@ -49,29 +45,3 @@ fn process_nix_files(
         entries.into_iter().collect()
     }))
 }
-
-/// Recursively collects all Nix files in the relative `dir` within `base`
-/// into the `files` `Vec`.
-fn collect_nix_files(
-    base: &Path,
-    dir: &RelativePath,
-    files: &mut Vec<RelativePathBuf>,
-) -> anyhow::Result<()> {
-    for entry in structure::read_dir_sorted(&dir.to_path(base))? {
-        let mut relative_path = dir.to_relative_path_buf();
-        relative_path.push(entry.file_name().to_string_lossy().into_owned());
-
-        let absolute_path = entry.path();
-
-        // We'll get to every file based on directory recursion, no need to follow symlinks.
-        if absolute_path.is_symlink() {
-            continue;
-        }
-        if absolute_path.is_dir() {
-            collect_nix_files(base, &relative_path, files)?
-        } else if absolute_path.extension().is_some_and(|x| x == "nix") {
-            files.push(relative_path)
-        }
-    }
-    Ok(())
-}

src/index.rs

@@ -0,0 +1,162 @@
+use crate::nix_file;
+use relative_path::Component;
+use relative_path::RelativePath;
+use relative_path::RelativePathBuf;
+use rowan::ast::AstNode;
+use std::collections::HashMap;
+use std::collections::HashSet;
+use std::path::Path;
+use std::process::Command;
+use std::str;
+
+#[derive(Debug, PartialEq, Eq, Hash, Clone)]
+pub struct Reference {
+    pub line: usize,
+
+    // The longest ancestor of the referenced path that can be moved
+    // around without breaking the reference
+    // E.g. if the reference is `./foo`, then this is `./.`, since we can move the current
+    // directory without breaking this reference. It can't be `./foo` because moving `./foo` around
+    // would break the reference
+    // Another example: If the reference is `../bar`, then movable_ancestor is `..`. It's not `./.`
+    // because if we moved the current directory around we could break this reference.
+    pub movable_ancestor: RelativePathBuf,
+
+    pub rel_to_root: RelativePathBuf,
+
+    pub text: String,
+}
+
+#[derive(Debug, Clone)]
+pub struct PathIndex {
+    pub references: Vec<Reference>,
+    pub referenced_by: Vec<(RelativePathBuf, usize)>,
+}
+
+impl PathIndex {
+    fn new() -> PathIndex {
+        PathIndex {
+            references: Vec::new(),
+            referenced_by: Vec::new(),
+        }
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct GlobalIndex {
+    // For each Nix file, what paths it references
+    pub path_indices: HashMap<RelativePathBuf, PathIndex>,
+}
+
+impl GlobalIndex {
+    pub fn new(
+        nixpkgs_path: impl AsRef<Path>,
+        paths: &HashSet<RelativePathBuf>,
+        nix_file_store: &mut nix_file::NixFileStore,
+    ) -> GlobalIndex {
+        // TODO: Remove the unwrap's return Result
+
+        let mut path_indices: HashMap<RelativePathBuf, PathIndex> = paths
+            .iter()
+            .map(|p| (p.clone(), PathIndex::new()))
+            .collect();
+
+        //eprintln!("{:#?}", path_indices);
+
+        paths
+            .iter()
+            .filter(|p| !p.to_path(nixpkgs_path.as_ref()).is_dir() && p.extension() == Some("nix"))
+            .for_each(|subpath| {
+                //eprintln!("Processing {subpath}");
+
+                let abs_path = subpath.to_path(nixpkgs_path.as_ref());
+
+                if abs_path.is_dir() || subpath.extension() != Some("nix") {
+                    return;
+                }
+
+                // TODO: Handle error
+                let file = nix_file_store.get(&abs_path).unwrap();
+
+                'nodes: for node in file.syntax_root.syntax().descendants() {
+                    let text = node.text().to_string();
+                    let line = file.line_index.line(node.text_range().start().into());
+
+                    let Some(ast_path) = rnix::ast::Path::cast(node) else {
+                        continue 'nodes;
+                    };
+
+                    //eprintln!("Processing reference {text} on line {line}");
+
+                    // TODO: Error reporting
+                    let nix_file::ResolvedPath::Within(mut rel_to_root, movable_ancestor) =
+                        file.static_resolve_path(&ast_path, nixpkgs_path.as_ref())
+                    else {
+                        continue 'nodes;
+                    };
+
+                    let mut rel_to_source = RelativePathBuf::from(&text);
+
+                    let abs = rel_to_root.to_path(&nixpkgs_path);
+
+                    // FIXME: This should not be necessary, it's something `import` specific
+                    if abs.is_dir() && abs.join("default.nix").exists() {
+                        rel_to_root = rel_to_root.join("default.nix");
+                        rel_to_source = rel_to_source.join("default.nix");
+                    }
+
+                    let reference = Reference {
+                        line,
+                        movable_ancestor,
+                        rel_to_root,
+                        text: text.clone(),
+                    };
+
+                    let path_index = path_indices.get_mut(&*subpath).unwrap();
+                    let current_length = path_index.references.len();
+                    let pointer = (subpath.clone(), current_length);
+
+                    // Insert the reference
+                    path_index.references.push(reference);
+                    // We can't move the file that contains the reference itself without breaking the
+                    // reference contained in it
+                    path_index.referenced_by.push(pointer.clone());
+
+                    let mut focused_dir = subpath.parent().unwrap().to_relative_path_buf();
+                    //eprintln!("Focused dir is: {focused_dir}");
+                    // The directory of the file is referenced by the file
+                    path_indices
+                        .get_mut(&focused_dir)
+                        .unwrap()
+                        .referenced_by
+                        .push(pointer.clone());
+
+                    for component in rel_to_source.components() {
+                        match component {
+                            Component::CurDir => {}
+                            Component::ParentDir => {
+                                path_indices
+                                    .get_mut(&focused_dir)
+                                    .unwrap()
+                                    .referenced_by
+                                    .push(pointer.clone());
+                                focused_dir = focused_dir.parent().unwrap().to_relative_path_buf();
+                                //eprintln!("Focused dir is: {focused_dir}");
+                            }
+                            Component::Normal(osstr) => {
+                                focused_dir = focused_dir.join(osstr).to_relative_path_buf();
+                                //eprintln!("Focused dir is: {focused_dir}");
+                                path_indices
+                                    .get_mut(&focused_dir)
+                                    .unwrap()
+                                    .referenced_by
+                                    .push(pointer.clone());
+                            }
+                        }
+                    }
+                }
+            });
+
+        GlobalIndex { path_indices }
+    }
+}

src/main.rs

@@ -9,8 +9,14 @@
 // #![allow(clippy::use_self)]
 // #![allow(clippy::missing_const_for_fn)]
 
+use ignore::DirEntry;
+use relative_path::RelativePathBuf;
+use std::collections::HashSet;
+use std::io::Write;
+use std::process::Command;
 mod eval;
 mod files;
+mod index;
 mod location;
 mod nix_file;
 mod problem;
@@ -22,6 +28,7 @@ mod validation;
 
 use anyhow::Context as _;
 use clap::Parser;
+use ignore::Walk;
 use std::collections::BTreeMap;
 use std::path::{Path, PathBuf};
 use std::process::ExitCode;
@@ -117,6 +124,35 @@ fn check_nixpkgs(nixpkgs_path: &Path) -> validation::Result<ratchet::Nixpkgs> {
 
     let mut nix_file_store = NixFileStore::default();
 
+    let all_paths = {
+        let output = Command::new("git")
+            .arg("-C")
+            .arg(nixpkgs_path.as_os_str())
+            .arg("ls-files")
+            .arg("-z")
+            .output()
+            .expect("failed to execute `git ls-files`");
+
+        let mut subpaths: HashSet<RelativePathBuf> = HashSet::new();
+
+        for file in output.stdout.rsplit(|i| *i == 0).skip(1) {
+            let f = std::str::from_utf8(file).unwrap();
+            let mut x = RelativePathBuf::from(f);
+            subpaths.insert(x.clone());
+            while x.pop() {
+                subpaths.insert(x.clone());
+            }
+        }
+        subpaths
+    };
+
+    //let index = index::GlobalIndex::new(&nixpkgs_path, &all_paths, &mut nix_file_store);
+
+    eprintln!("Index calculated");
+    //let mut file = std::fs::File::create("index")?;
+    //writeln!(file, "{:#?}", index)?;
+    //eprintln!("Done writing");
+
     let package_result = {
         if !nixpkgs_path.join(structure::BASE_SUBPATH).exists() {
             // No pkgs/by-name directory, always valid
@@ -131,7 +167,7 @@ fn check_nixpkgs(nixpkgs_path: &Path) -> validation::Result<ratchet::Nixpkgs> {
         }
     };
 
-    let file_result = files::check_files(&nixpkgs_path, &mut nix_file_store)?;
+    let file_result = files::check_files(&nixpkgs_path, &all_paths, &mut nix_file_store)?;
 
     Ok(
         package_result.and(file_result, |packages, files| ratchet::Nixpkgs {