workspace glob exclusions

docs/install/workspaces.md

@@ -35,12 +35,30 @@ In the root `package.json`, the `"workspaces"` key is used to indicate which sub
     "example-package-in-monorepo": "workspace:*"
   }
 }
+
+```
+
+You can exclude specific directories from your workspaces using the standard glob negation syntax. In the example below, `!packages/excluded-pkg` excludes a specific package, while `!packages/test-*` excludes all packages that start with `test-`. This is useful when you have directories within your monorepo that shouldn't be treated as workspaces (i.e. they should not be importable by other packages in the workspace), such as scripts and utilites or testing packages.
+
+```json
+{
+  "name": "my-project",
+  "version": "1.0.0",
+  "workspaces": [
+    "packages/*",
+    "!packages/excluded-pkg",
+    "!packages/test-*"
+  ]
+}
 ```
 
+
 {% callout %}
-**Glob support** — Bun supports full glob syntax in `"workspaces"` (see [here](https://bun.sh/docs/api/glob#supported-glob-patterns) for a comprehensive list of supported syntax), _except_ for exclusions (e.g. `!**/excluded/**`), which are not implemented yet.
+**Glob support** — Bun supports full glob syntax in `"workspaces"` (see [here](https://bun.sh/docs/api/glob#supported-glob-patterns) for a comprehensive list of supported syntax) for both included and excluded packages.
 {% /callout %}
 
+
+
 Each workspace has it's own `package.json`. When referencing other packages in the monorepo, semver or workspace protocols (e.g. `workspace:*`) can be used as the version field in your `package.json`.
 
 ```json

src/install/lockfile.zig

@@ -4916,6 +4916,206 @@ pub const Package = extern struct {
         }
     };
 
+    const WorkspaceExclusions = struct {
+        abs_root_path: []const u8, // absolute path of workspace root - not owned
+        arena: std.heap.ArenaAllocator,
+        exclusions: Exclusions,
+
+        const we_debug = Output.scoped(.WorkspaceExclusions, true);
+
+        const Exclusions = bun.StringArrayHashMap(void);
+
+        pub fn init(allocator: std.mem.Allocator, root_path: []const u8) WorkspaceExclusions {
+            if (comptime Environment.allow_assert) {
+                assert(std.fs.path.isAbsolute(root_path));
+            }
+            const arena = std.heap.ArenaAllocator.init(allocator);
+            return .{
+                .abs_root_path = root_path,
+                .arena = arena,
+                .exclusions = Exclusions.init(allocator),
+            };
+        }
+
+        pub fn deinit(self: *WorkspaceExclusions) void {
+            self.arena.deinit();
+
+            for (self.exclusions.keys()) |key| {
+                self.exclusions.allocator.free(key);
+            }
+            self.exclusions.deinit();
+        }
+
+        pub fn insert(self: *WorkspaceExclusions, log: *logger.Log, source: *const logger.Source, loc: logger.Loc, glob: []const u8) bun.OOM!void {
+            if (comptime Environment.allow_assert) {
+                assert(isExclusion(glob));
+            }
+
+            const non_negated_glob = brk: {
+                var negation_count: u32 = 0;
+                while (negation_count < glob.len and glob[negation_count] == '!') : (negation_count += 1) {}
+                if (comptime Environment.allow_assert) {
+                    assert(negation_count >= 1); // should be negated
+                    assert(negation_count % 2 == 1); // negation_count should be odd to be negated
+                }
+                if (glob.len == negation_count) {
+                    // TODO: is empty negated glob an error or deserving of a warning?
+                    return;
+                }
+                break :brk glob[negation_count..];
+            };
+
+            we_debug("found exclusion: {s}\n", .{glob});
+
+            defer _ = self.arena.reset(.retain_capacity);
+
+            const arena_alloc = self.arena.allocator();
+
+            const filepath_bufOS = arena_alloc.create(bun.PathBuffer) catch unreachable;
+            const filepath_buf = std.mem.asBytes(filepath_bufOS);
+
+            // TODO: see warning about windows paths on detectGlobSyntax
+            if (!Glob.Ascii.detectGlobSyntax(non_negated_glob)) {
+                const key_path = self.prepareRelPathForHash(filepath_buf, non_negated_glob);
+
+                const entry = try self.exclusions.getOrPut(key_path);
+
+                if (!entry.found_existing) {
+                    entry.key_ptr.* = try self.exclusions.allocator.dupe(u8, key_path);
+                }
+
+                we_debug("path: {s} - saved as exclusion (early)\n", .{key_path});
+                return;
+            }
+
+            const glob_pattern = brk: {
+                const parts = [_][]const u8{ non_negated_glob, "package.json" };
+                break :brk arena_alloc.dupe(u8, bun.path.join(parts, .auto)) catch bun.outOfMemory();
+            };
+
+            var walker: GlobWalker = .{};
+
+            if ((try walker.initWithCwd(&self.arena, glob_pattern, self.abs_root_path, false, false, false, false, true)).asErr()) |e| {
+                log.addWarningFmt(
+                    source,
+                    loc,
+                    self.exclusions.allocator,
+                    "Failed to run workspace exclusion pattern <b>{s}<r> due to error <b>{s}<r>",
+                    .{ glob, @tagName(e.getErrno()) },
+                ) catch {};
+
+                // invalid path in exclusion is a reasonable error to allow
+                return;
+            }
+
+            defer walker.deinit(false);
+
+            var iter: GlobWalker.Iterator = .{
+                .walker = &walker,
+            };
+            defer iter.deinit();
+
+            if ((try iter.init()).asErr()) |e| {
+                log.addWarningFmt(
+                    source,
+                    loc,
+                    self.exclusions.allocator,
+                    "Failed to run workspace exclusion pattern <b>{s}<r> due to error <b>{s}<r>",
+                    .{ glob, @tagName(e.getErrno()) },
+                ) catch {};
+
+                // invalid path in exclusion is a reasonable error to allow
+                return;
+            }
+
+            while (true) {
+                const excluded_package_json_path = switch (try iter.next()) {
+                    .result => |result| result orelse break,
+                    .err => |e| {
+                        log.addWarningFmt(
+                            source,
+                            loc,
+                            self.exclusions.allocator,
+                            "Failed to run workspace exclusion pattern <b>{s}<r> due to error <b>{s}<r>",
+                            .{ glob, @tagName(e.getErrno()) },
+                        ) catch {};
+
+                        // invalid path in exclusion is a reasonable error to allow
+                        // continue instead of return in case some paths described by glob are valid unlike this one
+                        continue;
+                    },
+                };
+                const excluded_package_json_dir: []const u8 = Path.dirname(excluded_package_json_path, .auto);
+
+                const key_path = self.prepareRelPathForHash(filepath_buf, excluded_package_json_dir);
+
+                const entry = try self.exclusions.getOrPut(key_path);
+
+                if (!entry.found_existing) {
+                    entry.key_ptr.* = try self.exclusions.allocator.dupe(u8, key_path);
+                }
+
+                we_debug("path: {s} - saved as exclusion\n", .{key_path});
+            }
+        }
+
+        fn prepareRelPathForHash(self: WorkspaceExclusions, path_buf: []u8, rel_path: []const u8) []const u8 {
+            const abs_path = Path.joinAbsStringBuf(
+                self.abs_root_path,
+                path_buf,
+                &.{rel_path},
+                .auto,
+            );
+
+            return self.prepareAbsPathForHash(abs_path);
+        }
+
+        fn prepareAbsPathForHash(self: WorkspaceExclusions, abs_path: []const u8) []const u8 {
+            // strip trailing sep
+            const abs_path_no_trailing_sep = if (abs_path.len > 0 and abs_path[abs_path.len - 1] == std.fs.path.sep)
+                abs_path[0 .. abs_path.len - 1]
+            else
+                abs_path;
+
+            // make relative
+            assert(abs_path_no_trailing_sep.len >= self.abs_root_path.len);
+            var res_path = abs_path_no_trailing_sep[self.abs_root_path.len..];
+            if (res_path.len > 0 and res_path[0] == std.fs.path.sep) {
+                res_path = res_path[1..];
+            }
+
+            return res_path;
+        }
+
+        pub fn isExcludedPath(self: WorkspaceExclusions, abs_path: []const u8) bool {
+            if (self.exclusions.count() == 0) return false;
+
+            if (comptime Environment.allow_assert) {
+                assert(std.fs.path.isAbsolute(abs_path));
+                assert(!std.mem.endsWith(u8, abs_path, "package.json"));
+                assert(std.mem.startsWith(u8, abs_path, self.abs_root_path));
+                // NOTE: expects path to be normalized as well
+            }
+
+            const key = self.prepareAbsPathForHash(abs_path);
+
+            // if path matches an excluded path and the excluded path was
+            // entered after the path in the package.json/workspaces array
+            // then it should be excluded
+            const is_excluded = self.exclusions.contains(key);
+
+            we_debug("path: {s} - {s}excluded\n", .{ key, if (is_excluded) "" else "NOT " });
+
+            return is_excluded;
+        }
+
+        pub fn isExclusion(glob: []const u8) bool {
+            var negation_count: u32 = 0;
+            while (negation_count < glob.len and glob[negation_count] == '!') : (negation_count += 1) {}
+            return negation_count % 2 == 1;
+        }
+    };
+
     const WorkspaceEntry = struct {
         name: []const u8 = "",
         name_loc: logger.Loc = logger.Loc.Empty,
@@ -4966,12 +5166,21 @@ pub const Package = extern struct {
 
         var workspace_globs = std.ArrayList(string).init(allocator);
         defer workspace_globs.deinit();
+        var workspace_exclusions = WorkspaceExclusions.init(allocator, source.path.name.dir);
+        defer workspace_exclusions.deinit();
         const filepath_bufOS = allocator.create(bun.PathBuffer) catch unreachable;
         const filepath_buf = std.mem.asBytes(filepath_bufOS);
         defer allocator.destroy(filepath_bufOS);
 
-        for (arr.slice()) |item| {
-            // TODO: when does this get deallocated?
+        const arr_slice = arr.slice();
+        // ?PERF: use ArrayList instead with initial capacity and remove
+        //  ?hard to predict branch? while iterating input_paths and
+        //  checking if .len == 0
+        const input_paths = allocator.alloc(string, arr_slice.len) catch bun.outOfMemory();
+        defer allocator.free(input_paths);
+
+        for (arr_slice, 0..) |item, i| {
+            // TODO: when does this get deallocated if it's not an exclusion?
             const input_path = try item.asStringZ(allocator) orelse {
                 log.addErrorFmt(source, item.loc, allocator,
                     \\Workspaces expects an array of strings, like:
@@ -4982,22 +5191,43 @@ pub const Package = extern struct {
                 return error.InvalidPackageJSON;
             };
 
-            if (input_path.len == 0 or input_path.len == 1 and input_path[0] == '.') continue;
+            if (input_path.len == 0 or input_path.len == 1 and input_path[0] == '.') {
+                input_paths[i].len = 0; // signal no path
+                allocator.free(input_path);
+            }
+
+            if (WorkspaceExclusions.isExclusion(input_path)) {
+                workspace_exclusions.insert(log, source, loc, input_path) catch bun.outOfMemory();
+                input_paths[i].len = 0; // signal no path
+                allocator.free(input_path);
+                continue;
+            }
 
             if (Glob.Ascii.detectGlobSyntax(input_path)) {
                 workspace_globs.append(input_path) catch bun.outOfMemory();
+                input_paths[i].len = 0; // signal no path
                 continue;
             }
 
+            input_paths[i] = input_path;
+        }
+
+        for (input_paths, arr_slice) |input_path, item| {
+            if (input_path.len == 0) continue;
+
             const abs_package_json_path: stringZ = Path.joinAbsStringBufZ(
                 source.path.name.dir,
                 filepath_buf,
                 &.{ input_path, "package.json" },
                 .auto,
             );
 
+            const abs_workspace_dir_path: string = Path.dirname(abs_package_json_path, .auto);
+
             // skip root package.json
-            if (strings.eqlLong(bun.path.dirname(abs_package_json_path, .auto), source.path.name.dir, true)) continue;
+            if (strings.eqlLong(abs_workspace_dir_path, source.path.name.dir, true)) continue;
+
+            if (workspace_exclusions.isExcludedPath(abs_workspace_dir_path)) continue;
 
             const workspace_entry = processWorkspaceName(
                 allocator,
@@ -5133,7 +5363,9 @@ pub const Package = extern struct {
                         &.{ entry_dir, "package.json" },
                         .auto,
                     );
-                    const abs_workspace_dir_path: string = strings.withoutSuffixComptime(abs_package_json_path, "package.json");
+                    const abs_workspace_dir_path = Path.dirname(abs_package_json_path, .auto);
+
+                    if (workspace_exclusions.isExcludedPath(abs_workspace_dir_path)) continue;
 
                     const workspace_entry = processWorkspaceName(
                         allocator,