fix(s3) Support Virtual Hosted-Style endpoints

docs/api/s3.md

@@ -181,8 +181,8 @@ const download = s3.presign("my-file.txt"); // GET, text/plain, expires in 24 ho
 
 const upload = s3.presign("my-file", {
   expiresIn: 3600, // 1 hour
-  method: 'PUT',
-  type: 'application/json', // No extension for inferring, so we can specify the content type to be JSON
+  method: "PUT",
+  type: "application/json", // No extension for inferring, so we can specify the content type to be JSON
 });
 
 // You can call .presign() if on a file reference, but avoid doing so
@@ -361,6 +361,56 @@ const minio = new S3Client({
 });
 ```
 
+### Using Bun's S3Client with supabase
+
+To use Bun's S3 client with [supabase](https://supabase.com/), set `endpoint` to the supabase endpoint in the `S3Client` constructor. The supabase endpoint includes your account ID and /storage/v1/s3 path. Make sure to set Enable connection via S3 protocol on in the supabase dashboard in https://supabase.com/dashboard/project/<account-id>/settings/storage and to set the region informed in the same section.
+
+```ts
+import { S3Client } from "bun";
+
+const supabase = new S3Client({
+  accessKeyId: "access-key",
+  secretAccessKey: "secret-key",
+  bucket: "my-bucket",
+  region: "us-west-1",
+  endpoint: "https://<account-id>.supabase.co/storage/v1/s3/storage",
+});
+```
+
+### Using Bun's S3Client with S3 Virtual Hosted-Style endpoints
+
+When using a S3 Virtual Hosted-Style endpoint, you need to set the `virtualHostedStyle` option to `true` and if no endpoint is provided, Bun will use region and bucket to infer the endpoint to AWS S3, if no region is provided it will use `us-east-1`. If you provide a the endpoint, there are no need to provide the bucket name.
+
+```ts
+import { S3Client } from "bun";
+
+// AWS S3 endpoint inferred from region and bucket
+const s3 = new S3Client({
+  accessKeyId: "access-key",
+  secretAccessKey: "secret-key",
+  bucket: "my-bucket",
+  virtualHostedStyle: true,
+  // endpoint: "https://my-bucket.s3.us-east-1.amazonaws.com",
+  // region: "us-east-1",
+});
+
+// AWS S3
+const s3WithEndpoint = new S3Client({
+  accessKeyId: "access-key",
+  secretAccessKey: "secret-key",
+  endpoint: "https://<bucket-name>.s3.<region>.amazonaws.com",
+  virtualHostedStyle: true,
+});
+
+// Cloudflare R2
+const r2WithEndpoint = new S3Client({
+  accessKeyId: "access-key",
+  secretAccessKey: "secret-key",
+  endpoint: "https://<bucket-name>.<account-id>.r2.cloudflarestorage.com",
+  virtualHostedStyle: true,
+});
+```
+
 ## Credentials
 
 Credentials are one of the hardest parts of using S3, and we've tried to make it as easy as possible. By default, Bun reads the following environment variables for credentials.

packages/bun-types/bun.d.ts

@@ -1382,6 +1382,18 @@ declare module "bun" {
      */
     endpoint?: string;
 
+    /**
+     * Use virtual hosted style endpoint. default to false, when true if `endpoint` is informed it will ignore the `bucket`
+     *
+     * @example
+     *     // Using virtual hosted style
+     *     const file = s3("my-file.txt", {
+     *       virtualHostedStyle: true,
+     *       endpoint: "https://my-bucket.s3.us-east-1.amazonaws.com"
+     *     });
+     */
+    virtualHostedStyle?: boolean;
+
     /**
      * The size of each part in multipart uploads (in bytes).
      * - Minimum: 5 MiB

src/bun.js/webcore/S3Client.zig

@@ -17,7 +17,7 @@ pub fn writeFormatCredentials(credentials: *S3Credentials, options: bun.S3.Multi
         formatter.indent += 1;
         defer formatter.indent -|= 1;
 
-        const endpoint = if (credentials.endpoint.len > 0) credentials.endpoint else "https://s3.<region>.amazonaws.com";
+        const endpoint = if (credentials.endpoint.len > 0) credentials.endpoint else (if (credentials.virtual_hosted_style) "https://<bucket>.s3.<region>.amazonaws.com" else "https://s3.<region>.amazonaws.com");
 
         try formatter.writeIndent(Writer, writer);
         try writer.writeAll(comptime bun.Output.prettyFmt("<r>endpoint<d>:<r> \"", enable_ansi_colors));
@@ -112,11 +112,13 @@ pub const S3Client = struct {
 
     pub fn writeFormat(this: *@This(), comptime Formatter: type, formatter: *Formatter, writer: anytype, comptime enable_ansi_colors: bool) !void {
         try writer.writeAll(comptime bun.Output.prettyFmt("<r>S3Client<r>", enable_ansi_colors));
-        if (this.credentials.bucket.len > 0) {
+        // detect virtual host style bucket name
+        const bucket_name = if (this.credentials.virtual_hosted_style and this.credentials.endpoint.len > 0) S3Credentials.guessBucket(this.credentials.endpoint) orelse this.credentials.bucket else this.credentials.bucket;
+        if (bucket_name.len > 0) {
             try writer.print(
                 comptime bun.Output.prettyFmt(" (<green>\"{s}\"<r>)<r> {{", enable_ansi_colors),
                 .{
-                    this.credentials.bucket,
+                    bucket_name,
                 },
             );
         } else {

src/bun.js/webcore/S3File.zig

@@ -11,15 +11,17 @@ const Output = bun.Output;
 const S3Client = @import("./S3Client.zig");
 const S3 = bun.S3;
 const S3Stat = @import("./S3Stat.zig").S3Stat;
-pub fn writeFormat(s3: *Blob.S3Store, comptime Formatter: type, formatter: *Formatter, writer: anytype, comptime enable_ansi_colors: bool) !void {
+pub fn writeFormat(s3: *Blob.S3Store, comptime Formatter: type, formatter: *Formatter, writer: anytype, comptime enable_ansi_colors: bool, content_type: []const u8, offset: usize) !void {
     try writer.writeAll(comptime Output.prettyFmt("<r>S3Ref<r>", enable_ansi_colors));
     const credentials = s3.getCredentials();
+    // detect virtual host style bucket name
+    const bucket_name = if (credentials.virtual_hosted_style and credentials.endpoint.len > 0) S3.S3Credentials.guessBucket(credentials.endpoint) orelse credentials.bucket else credentials.bucket;
 
-    if (credentials.bucket.len > 0) {
+    if (bucket_name.len > 0) {
         try writer.print(
             comptime Output.prettyFmt(" (<green>\"{s}/{s}\"<r>)<r> {{", enable_ansi_colors),
             .{
-                credentials.bucket,
+                bucket_name,
                 s3.path(),
             },
         );
@@ -32,6 +34,40 @@ pub fn writeFormat(s3: *Blob.S3Store, comptime Formatter: type, formatter: *Form
         );
     }
 
+    if (content_type.len > 0) {
+        try writer.writeAll("\n");
+        formatter.indent += 1;
+        defer formatter.indent -|= 1;
+
+        try formatter.writeIndent(@TypeOf(writer), writer);
+        try writer.print(
+            comptime Output.prettyFmt("type<d>:<r> <green>\"{s}\"<r>", enable_ansi_colors),
+            .{
+                content_type,
+            },
+        );
+
+        try formatter.printComma(@TypeOf(writer), writer, enable_ansi_colors);
+        if (offset > 0) {
+            try writer.writeAll("\n");
+        }
+    }
+
+    if (offset > 0) {
+        formatter.indent += 1;
+        defer formatter.indent -|= 1;
+
+        try formatter.writeIndent(@TypeOf(writer), writer);
+
+        try writer.print(
+            comptime Output.prettyFmt("offset<d>:<r> <yellow>{d}<r>", enable_ansi_colors),
+            .{
+                offset,
+            },
+        );
+
+        try formatter.printComma(@TypeOf(writer), writer, enable_ansi_colors);
+    }
     try S3Client.writeFormatCredentials(credentials, s3.options, s3.acl, Formatter, formatter, writer, enable_ansi_colors);
     try formatter.writeIndent(@TypeOf(writer), writer);
     try writer.writeAll("}");

src/bun.js/webcore/blob.zig

@@ -705,7 +705,7 @@ pub const Blob = struct {
             const store = this.store.?;
             switch (store.data) {
                 .s3 => |*s3| {
-                    try S3File.writeFormat(s3, Formatter, formatter, writer, enable_ansi_colors);
+                    try S3File.writeFormat(s3, Formatter, formatter, writer, enable_ansi_colors, this.content_type, this.offset);
                 },
                 .file => |file| {
                     try writer.writeAll(comptime Output.prettyFmt("<r>FileRef<r>", enable_ansi_colors));
@@ -752,7 +752,7 @@ pub const Blob = struct {
         }
 
         const show_name = (this.is_jsdom_file and this.getNameString() != null) or (!this.name.isEmpty() and this.store != null and this.store.?.data == .bytes);
-        if (this.content_type.len > 0 or this.offset > 0 or show_name or this.last_modified != 0.0) {
+        if (!this.isS3() and (this.content_type.len > 0 or this.offset > 0 or show_name or this.last_modified != 0.0)) {
             try writer.writeAll(" {\n");
             {
                 formatter.indent += 1;

src/env_loader.zig

@@ -144,9 +144,9 @@ pub const Loader = struct {
             region = region_;
         }
         if (this.get("S3_ENDPOINT")) |endpoint_| {
-            endpoint = bun.URL.parse(endpoint_).host;
+            endpoint = bun.URL.parse(endpoint_).hostWithPath();
         } else if (this.get("AWS_ENDPOINT")) |endpoint_| {
-            endpoint = bun.URL.parse(endpoint_).host;
+            endpoint = bun.URL.parse(endpoint_).hostWithPath();
         }
         if (this.get("S3_BUCKET")) |bucket_| {
             bucket = bucket_;

src/s3/credentials.zig

@@ -21,7 +21,8 @@ pub const S3Credentials = struct {
     storage_class: ?StorageClass = null,
     /// Important for MinIO support.
     insecure_http: bool = false,
-
+    /// indicates if the endpoint is a virtual hosted style bucket
+    virtual_hosted_style: bool = false,
     ref_count: u32 = 1,
     pub usingnamespace bun.NewRefCounted(@This(), deinit, null);
 
@@ -113,7 +114,7 @@ pub const S3Credentials = struct {
                                 new_credentials._endpointSlice = str.toUTF8(bun.default_allocator);
                                 const endpoint = new_credentials._endpointSlice.?.slice();
                                 const url = bun.URL.parse(endpoint);
-                                const normalized_endpoint = url.host;
+                                const normalized_endpoint = url.hostWithPath();
                                 if (normalized_endpoint.len > 0) {
                                     new_credentials.credentials.endpoint = normalized_endpoint;
 
@@ -148,6 +149,11 @@ pub const S3Credentials = struct {
                     }
                 }
 
+                if (try opts.getBooleanStrict(globalObject, "virtualHostedStyle")) |virtual_hosted_style| {
+                    new_credentials.credentials.virtual_hosted_style = virtual_hosted_style;
+                    new_credentials.changed_credentials = true;
+                }
+
                 if (try opts.getTruthyComptime(globalObject, "sessionToken")) |js_value| {
                     if (!js_value.isEmptyOrUndefinedOrNull()) {
                         if (js_value.isString()) {
@@ -242,6 +248,7 @@ pub const S3Credentials = struct {
                 "",
 
             .insecure_http = this.insecure_http,
+            .virtual_hosted_style = this.virtual_hosted_style,
         });
     }
     pub fn deinit(this: *@This()) void {
@@ -387,7 +394,33 @@ pub const S3Credentials = struct {
         acl: ?ACL = null,
         storage_class: ?StorageClass = null,
     };
-
+    /// This is not used for signing but for console.log output, is just nice to have
+    pub fn guessBucket(endpoint: []const u8) ?[]const u8 {
+        // check if is amazonaws.com
+        if (strings.indexOf(endpoint, ".amazonaws.com")) |_| {
+            // check if is .s3. virtual host style
+            if (strings.indexOf(endpoint, ".s3.")) |end| {
+                // its https://bucket-name.s3.region-code.amazonaws.com/key-name
+                const start = strings.indexOf(endpoint, "/") orelse {
+                    return endpoint[0..end];
+                };
+                return endpoint[start + 1 .. end];
+            }
+        } else if (strings.indexOf(endpoint, ".r2.cloudflarestorage.com")) |r2_start| {
+            // check if is <BUCKET>.<ACCOUNT_ID>.r2.cloudflarestorage.com
+            const end = strings.indexOf(endpoint, ".") orelse return null; // actually unreachable
+            if (end > 0 and r2_start == end) {
+                // its https://<ACCOUNT_ID>.r2.cloudflarestorage.com
+                return null;
+            }
+            // ok its virtual host style
+            const start = strings.indexOf(endpoint, "/") orelse {
+                return endpoint[0..end];
+            };
+            return endpoint[start + 1 .. end];
+        }
+        return null;
+    }
     pub fn guessRegion(endpoint: []const u8) []const u8 {
         if (endpoint.len > 0) {
             if (strings.endsWith(endpoint, ".r2.cloudflarestorage.com")) return "auto";
@@ -485,24 +518,24 @@ pub const S3Credentials = struct {
         var path: []const u8 = full_path;
         var bucket: []const u8 = this.bucket;
 
-        if (bucket.len == 0) {
-            //TODO: r2 supports bucket in the endpoint
-
-            // guess bucket using path
-            if (strings.indexOf(full_path, "/")) |end| {
-                if (strings.indexOf(full_path, "\\")) |backslash_index| {
-                    if (backslash_index < end) {
-                        bucket = full_path[0..backslash_index];
-                        path = full_path[backslash_index + 1 ..];
+        if (!this.virtual_hosted_style) {
+            if (bucket.len == 0) {
+                // guess bucket using path
+                if (strings.indexOf(full_path, "/")) |end| {
+                    if (strings.indexOf(full_path, "\\")) |backslash_index| {
+                        if (backslash_index < end) {
+                            bucket = full_path[0..backslash_index];
+                            path = full_path[backslash_index + 1 ..];
+                        }
                     }
+                    bucket = full_path[0..end];
+                    path = full_path[end + 1 ..];
+                } else if (strings.indexOf(full_path, "\\")) |backslash_index| {
+                    bucket = full_path[0..backslash_index];
+                    path = full_path[backslash_index + 1 ..];
+                } else {
+                    return error.InvalidPath;
                 }
-                bucket = full_path[0..end];
-                path = full_path[end + 1 ..];
-            } else if (strings.indexOf(full_path, "\\")) |backslash_index| {
-                bucket = full_path[0..backslash_index];
-                path = full_path[backslash_index + 1 ..];
-            } else {
-                return error.InvalidPath;
             }
         }
         if (strings.endsWith(path, "/")) {
@@ -524,7 +557,44 @@ pub const S3Credentials = struct {
         var bucket_buffer: [63]u8 = undefined;
         bucket = encodeURIComponent(bucket, &bucket_buffer, false) catch return error.InvalidPath;
         path = encodeURIComponent(path, &path_buffer, false) catch return error.InvalidPath;
-        const normalizedPath = std.fmt.bufPrint(&normalized_path_buffer, "/{s}/{s}", .{ bucket, path }) catch return error.InvalidPath;
+        // Default to https. Only use http if they explicit pass "http://" as the endpoint.
+        const protocol = if (this.insecure_http) "http" else "https";
+
+        // detect service name and host from region or endpoint
+        var endpoint = this.endpoint;
+        var extra_path: []const u8 = "";
+        const host = brk_host: {
+            if (this.endpoint.len > 0) {
+                if (this.endpoint.len >= 2048) return error.InvalidEndpoint;
+                var host = this.endpoint;
+                if (bun.strings.indexOf(this.endpoint, "/")) |index| {
+                    host = this.endpoint[0..index];
+                    extra_path = this.endpoint[index..];
+                }
+                // only the host part is needed here
+                break :brk_host try bun.default_allocator.dupe(u8, host);
+            } else {
+                if (this.virtual_hosted_style) {
+                    // virtual hosted style requires a bucket name if an endpoint is not provided
+                    if (bucket.len == 0) {
+                        return error.InvalidEndpoint;
+                    }
+                    // default to https://<BUCKET_NAME>.s3.<REGION>.amazonaws.com/
+                    endpoint = try std.fmt.allocPrint(bun.default_allocator, "{s}.s3.{s}.amazonaws.com", .{ bucket, region });
+                    break :brk_host endpoint;
+                }
+                endpoint = try std.fmt.allocPrint(bun.default_allocator, "s3.{s}.amazonaws.com", .{region});
+                break :brk_host endpoint;
+            }
+        };
+        errdefer bun.default_allocator.free(host);
+        const normalizedPath = brk: {
+            if (this.virtual_hosted_style) {
+                break :brk std.fmt.bufPrint(&normalized_path_buffer, "{s}/{s}", .{ extra_path, path }) catch return error.InvalidPath;
+            } else {
+                break :brk std.fmt.bufPrint(&normalized_path_buffer, "{s}/{s}/{s}", .{ extra_path, bucket, path }) catch return error.InvalidPath;
+            }
+        };
 
         const date_result = getAMZDate(bun.default_allocator);
         const amz_date = date_result.date;
@@ -595,22 +665,8 @@ pub const S3Credentials = struct {
             }
         };
 
-        // Default to https. Only use http if they explicit pass "http://" as the endpoint.
-        const protocol = if (this.insecure_http) "http" else "https";
-
-        // detect service name and host from region or endpoint
-        const host = brk_host: {
-            if (this.endpoint.len > 0) {
-                if (this.endpoint.len >= 512) return error.InvalidEndpoint;
-                break :brk_host try bun.default_allocator.dupe(u8, this.endpoint);
-            } else {
-                break :brk_host try std.fmt.allocPrint(bun.default_allocator, "s3.{s}.amazonaws.com", .{region});
-            }
-        };
         const service_name = "s3";
 
-        errdefer bun.default_allocator.free(host);
-
         const aws_content_hash = if (content_hash) |hash| hash else ("UNSIGNED-PAYLOAD");
         var tmp_buffer: [4096]u8 = undefined;
 
@@ -890,7 +946,8 @@ pub const S3CredentialsWithOptions = struct {
     storage_class: ?StorageClass = null,
     /// indicates if the credentials have changed
     changed_credentials: bool = false,
-
+    /// indicates if the virtual hosted style is used
+    virtual_hosted_style: bool = false,
     _accessKeyIdSlice: ?JSC.ZigString.Slice = null,
     _secretAccessKeySlice: ?JSC.ZigString.Slice = null,
     _regionSlice: ?JSC.ZigString.Slice = null,