Add endpoint for REST version and capabilities. (#5429)

This adds the REST capabilities endpoint to `RestClientRemote` and
serialization for the `RestCapabilities` response. The endpoint is
currently not in use internally and there is no way to make the request
using public APIs.

The `RestCapabilities` response currently provides the TileDB core
version deployed to REST, and the minimum TileDB core client version
supported by the REST server. The response may be updated in the future
to add more information as needed - [discussion in
shortcut](https://app.shortcut.com/tiledb-inc/story/61403/rest-route-to-check-server-side-libtiledb-version-and-capabilities#activity-62215)
and [design
document](https://www.notion.so/Design-REST-route-to-check-server-side-libtiledb-version-181326501175800ea184eb6909b65d20?pvs=4)

---
TYPE: FEATURE
DESC: Add endpoint for REST version and capabilities.

---------

Co-authored-by: Theodore Tsirpanis <[email protected]>

tiledb/CMakeLists.txt

@@ -296,6 +296,7 @@ set(TILEDB_CORE_SOURCES
   ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/serialization/query_plan.cc
   ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/serialization/consolidation.cc
   ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/serialization/vacuum.cc
+  ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/serialization/rest_capabilities.cc
   ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/stats/global_stats.cc
   ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/stats/stats.cc
   ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/storage_manager/cancellation_source.cc

tiledb/sm/rest/curl.cc

@@ -162,7 +162,9 @@ size_t write_header_callback(
   auto* const header_buffer = static_cast<char*>(res_data);
   auto* const pmHeader = static_cast<HeaderCbData*>(userdata);
 
-  if (pmHeader->uri->empty()) {
+  // If we have enabled caching of the redirect URI ensure it's not empty.
+  // If disabled for this request, do not treat an empty asset URI as an error.
+  if (pmHeader->should_cache_redirect && pmHeader->uri->empty()) {
     LOG_ERROR("Rest components as array_ns and array_uri cannot be empty");
     return 0;
   }
@@ -540,6 +542,7 @@ Status Curl::make_curl_request_common(
         data->get_offset();
   }
 
+  stats->add_counter("rest_http_requests", 1);
   // <= because the 0ths retry is actually the initial request
   for (uint8_t i = 0; i <= retry_count_; i++) {
     WriteCbState write_cb_state;

tiledb/sm/rest/rest_capabilities.h

@@ -0,0 +1,82 @@
+/**
+ * @file   rest_capabilities.h
+ *
+ * @section LICENSE
+ *
+ * The MIT License
+ *
+ * @copyright Copyright (c) 2025 TileDB, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ *
+ * @section DESCRIPTION
+ *
+ * Helper struct to encapsulate REST supported versions and capabilities.
+ */
+
+namespace tiledb::sm {
+
+struct RestCapabilities {
+  struct TileDBVersion {
+    TileDBVersion() = default;
+
+    TileDBVersion(int major, int minor, int patch)
+        : major_(major)
+        , minor_(minor)
+        , patch_(patch) {
+    }
+
+    bool operator==(const TileDBVersion& rhs) const = default;
+
+    uint16_t major_, minor_, patch_;
+  };
+
+  /**
+   * Default constructor allows the class to be constructed without submitting
+   * a REST request to initialize member variables.
+   */
+  RestCapabilities() = default;
+
+  ~RestCapabilities() = default;
+
+  /**
+   * Fully initialized constructor contains all REST version and capabilities
+   * information required for handling edge cases between client & server
+   * releases.
+   */
+  RestCapabilities(
+      TileDBVersion rest_version, TileDBVersion rest_minimum_version)
+      : detected_(true)
+      , rest_tiledb_version_(rest_version)
+      , rest_minimum_supported_version_(rest_minimum_version) {
+  }
+
+  bool operator==(const RestCapabilities& rhs) const = default;
+
+  /// Whether or not the REST capabilities have been initialized.
+  bool detected_ = false;
+
+  /// The currently deployed TileDB version available on the REST server.
+  TileDBVersion rest_tiledb_version_{};
+
+  /// The minimum TileDB version supported by the REST server.
+  TileDBVersion rest_minimum_supported_version_{};
+};
+
+}  // namespace tiledb::sm

tiledb/sm/rest/rest_client.h

@@ -175,6 +175,7 @@
 #include "tiledb/common/status.h"
 #include "tiledb/common/thread_pool/thread_pool.h"
 #include "tiledb/sm/group/group.h"
+#include "tiledb/sm/rest/rest_capabilities.h"
 #include "tiledb/sm/serialization/query.h"
 #include "tiledb/sm/stats/stats.h"
 
@@ -259,6 +260,8 @@ class RestClient {
   std::string rest_server_;
 
  public:
+  using TileDBVersion = RestCapabilities::TileDBVersion;
+
   RestClient(const Config& config);
 
   virtual ~RestClient() = default;
@@ -294,6 +297,22 @@ class RestClient {
     return rest_server_;
   }
 
+  /// Operation disabled in base class.
+  inline virtual const TileDBVersion& rest_tiledb_version() const {
+    throw RestClientDisabledException();
+  }
+
+  /// Operation disabled in base class.
+  inline virtual const TileDBVersion& rest_minimum_supported_tiledb_version()
+      const {
+    throw RestClientDisabledException();
+  }
+
+  /// Operation disabled in base class.
+  inline virtual bool rest_capabilities_detected() const {
+    throw RestClientDisabledException();
+  }
+
   //-------------------------------------------------------
   // Rest client operations
   //-------------------------------------------------------
@@ -473,6 +492,12 @@ class RestClient {
   post_consolidation_plan_from_rest(const URI&, const Config&, uint64_t) {
     throw RestClientDisabledException();
   }
+
+  /// Operation disabled in base class.
+  inline virtual const tiledb::sm::RestCapabilities&
+  get_capabilities_from_rest() const {
+    throw RestClientDisabledException();
+  }
 };
 
 // Forward declaration.

tiledb/sm/rest/rest_client_remote.cc

@@ -1516,4 +1516,24 @@ RestClientRemote::post_consolidation_plan_from_rest(
       serialization_type_, returned_data);
 }
 
+const RestCapabilities& RestClientRemote::get_capabilities_from_rest() const {
+  // Return early if already detected REST capabilities for this REST client.
+  if (rest_capabilities_.detected_) {
+    return rest_capabilities_;
+  }
+
+  // Init curl and form the URL
+  Curl curlc(logger_);
+  std::unordered_map<std::string, std::string> redirect_meta;
+  throw_if_not_ok(curlc.init(
+      config_, extra_headers_, &redirect_meta, &redirect_mtx_, false));
+  const std::string url = rest_server_ + "/v4/capabilities";
+
+  Buffer data;
+  throw_if_not_ok(curlc.get_data(stats_, url, serialization_type_, &data, {}));
+  rest_capabilities_ =
+      serialization::rest_capabilities_deserialize(serialization_type_, data);
+  return rest_capabilities_;
+}
+
 }  // namespace tiledb::sm

tiledb/sm/rest/rest_client_remote.h

@@ -42,6 +42,7 @@
 // clang-format off
 #include "tiledb/sm/serialization/capnp_utils.h"
 #include "tiledb/sm/serialization/array.h"
+#include "tiledb/sm/serialization/array_schema.h"
 #include "tiledb/sm/serialization/array_schema_evolution.h"
 #include "tiledb/sm/serialization/config.h"
 #include "tiledb/sm/serialization/consolidation.h"
@@ -53,6 +54,7 @@
 #include "tiledb/sm/serialization/query_plan.h"
 #include "tiledb/sm/serialization/tiledb-rest.capnp.h"
 #include "tiledb/sm/serialization/vacuum.h"
+#include "tiledb/sm/serialization/rest_capabilities.h"
 #include "tiledb/sm/rest/curl.h" // must be included last to avoid Windows.h
 // clang-format on
 
@@ -73,7 +75,6 @@
 #include "tiledb/sm/query/query_buffer.h"
 #include "tiledb/sm/query/query_remote_buffer_storage.h"
 #include "tiledb/sm/rest/rest_client.h"
-#include "tiledb/sm/serialization/array_schema.h"
 #include "tiledb/type/apply_with_type.h"
 
 using namespace tiledb::common;
@@ -117,6 +118,32 @@ class RestClientRemote : public RestClient {
    * */
   static bool use_refactored_query(const Config& config);
 
+  /**
+   * @return TileDB core version currently deployed to the REST server.
+   */
+  inline const TileDBVersion& rest_tiledb_version() const override {
+    return get_capabilities_from_rest().rest_tiledb_version_;
+  }
+
+  /**
+   * @return Minimum TileDB core version currently supported by the REST server.
+   */
+  inline const TileDBVersion& rest_minimum_supported_tiledb_version()
+      const override {
+    return get_capabilities_from_rest().rest_minimum_supported_version_;
+  }
+
+  /**
+   * Check if REST capabilities are currently known to the RestClient. This
+   * will not attempt to initialize them if they are currently unknown.
+   *
+   * @return True if RestCapabilities member has been initialized by a previous
+   * REST capabilities endpoint request, else False.
+   */
+  inline bool rest_capabilities_detected() const override {
+    return rest_capabilities_.detected_;
+  }
+
   /**
    * Check if an array exists by making a REST call. To start with this fetches
    * the schema but ignores the body returned if non-error
@@ -444,6 +471,13 @@ class RestClientRemote : public RestClient {
   std::vector<std::vector<std::string>> post_consolidation_plan_from_rest(
       const URI& uri, const Config& config, uint64_t fragment_size) override;
 
+  /**
+   * Get REST capabilities from the REST server.
+   *
+   * @return RestCapabilities object initialized with context from REST server.
+   */
+  const RestCapabilities& get_capabilities_from_rest() const override;
+
  private:
   /* ********************************* */
   /*        PRIVATE ATTRIBUTES         */
@@ -487,6 +521,9 @@ class RestClientRemote : public RestClient {
   /** The class MemoryTracker. */
   shared_ptr<MemoryTracker> memory_tracker_;
 
+  /** REST supported TileDB versions and capabilities. */
+  mutable RestCapabilities rest_capabilities_;
+
   /* ********************************* */
   /*         PRIVATE METHODS           */
   /* ********************************* */