Merge pull request #5113 from gangj/private/gangj/CP-43916

CP-43916: Merge only required and clean commits from feature/stream-updates into master

ocaml/idl/datamodel_common.ml

@@ -10,7 +10,7 @@ open Datamodel_roles
               to leave a gap for potential hotfixes needing to increment the schema version.*)
 let schema_major_vsn = 5
 
-let schema_minor_vsn = 763
+let schema_minor_vsn = 764
 
 (* Historical schema versions just in case this is useful later *)
 let rio_schema_major_vsn = 5

ocaml/idl/datamodel_errors.ml

@@ -1931,8 +1931,6 @@ let _ =
       "The hash of updateinfo doesn't match with current one. There may be \
        newer available updates."
     () ;
-  error Api_errors.updates_require_sync []
-    ~doc:"A call to pool.sync_updates is required before this operation." () ;
   error Api_errors.cannot_restart_device_model ["ref"]
     ~doc:"Cannot restart device models of paused VMs residing on the host." () ;
   error Api_errors.invalid_repository_proxy_url ["url"]
@@ -1941,6 +1939,8 @@ let _ =
     ~doc:"The repository proxy username/password is invalid." () ;
   error Api_errors.apply_livepatch_failed ["livepatch"]
     ~doc:"Failed to apply a livepatch." () ;
+  error Api_errors.updates_require_recommended_guidance ["recommended_guidance"]
+    ~doc:"Requires recommended guidance after applying updates." () ;
   error Api_errors.update_guidance_changed ["guidance"]
     ~doc:"Guidance for the update has changed" () ;
 
@@ -1954,6 +1954,9 @@ let _ =
        now."
     () ;
 
+  error Api_errors.invalid_update_sync_day ["day"]
+    ~doc:"Invalid day of the week chosen for weekly update sync." () ;
+
   message
     (fst Api_messages.ha_pool_overcommitted)
     ~doc:

ocaml/idl/datamodel_host.ml

@@ -1709,7 +1709,10 @@ let apply_updates =
         )
       ]
     ~result:
-      (Set (Set String), "The list of warnings happened in applying updates")
+      ( Set (Set String)
+      , "The list of results after applying updates, including livepatch apply \
+         failures and recommended guidances"
+      )
     ~allowed_roles:(_R_POOL_OP ++ _R_CLIENT_CERT)
     ()
 
@@ -1743,6 +1746,39 @@ let set_https_only =
       ]
     ~allowed_roles:_R_POOL_OP ()
 
+let apply_recommended_guidances =
+  call ~name:"apply_recommended_guidances"
+    ~doc:
+      "apply all recommended guidances both on the host and on all HVM VMs on \
+       the host after updates are applied on the host"
+    ~lifecycle:[]
+    ~params:
+      [
+        ( Ref _host
+        , "self"
+        , "The host whose recommended guidances will be applied"
+        )
+      ]
+    ~allowed_roles:_R_POOL_OP ()
+
+let latest_synced_updates_applied_state =
+  Enum
+    ( "latest_synced_updates_applied_state"
+    , [
+        ( "yes"
+        , "The host is up to date with the latest updates synced from remote \
+           CDN"
+        )
+      ; ( "no"
+        , "The host is outdated with the latest updates synced from remote CDN"
+        )
+      ; ( "unknown"
+        , "If the host is up to date with the latest updates synced from \
+           remote CDN is unknown"
+        )
+      ]
+    )
+
 (** Hosts *)
 let t =
   create_obj ~in_db:true ~in_product_since:rel_rio ~in_oss_since:oss_since_303
@@ -1877,6 +1913,7 @@ let t =
       ; apply_updates
       ; copy_primary_host_certs
       ; set_https_only
+      ; apply_recommended_guidances
       ]
     ~contents:
       ([
@@ -2098,6 +2135,15 @@ let t =
         ; field ~qualifier:DynamicRO ~lifecycle:[] ~ty:Bool
             ~default_value:(Some (VBool false)) "https_only"
             "Reflects whether port 80 is open (false) or not (true)"
+        ; field ~qualifier:DynamicRO ~lifecycle:[] ~ty:(Set update_guidances)
+            "recommended_guidances" ~default_value:(Some (VSet []))
+            "The set of recommended guidances after applying updates"
+        ; field ~qualifier:DynamicRO ~lifecycle:[]
+            ~ty:latest_synced_updates_applied_state
+            "latest_synced_updates_applied"
+            ~default_value:(Some (VEnum "unknown"))
+            "Default as 'unknown', 'yes' if the host is up to date with \
+             updates synced from remote CDN, otherwise 'no'"
         ]
       )
     ()

ocaml/idl/datamodel_lifecycle.ml

@@ -29,12 +29,26 @@ let prototyped_of_field = function
       Some "22.26.0"
   | "VTPM", "persistence_backend" ->
       Some "22.26.0"
+  | "host", "latest_synced_updates_applied" ->
+      Some "23.16.2-next"
+  | "host", "recommended_guidances" ->
+      Some "23.16.2-next"
   | "host", "https_only" ->
       Some "22.27.0"
   | "host", "last_software_update" ->
       Some "22.20.0"
+  | "VM", "recommended_guidances" ->
+      Some "23.16.2-next"
   | "VM", "actions__after_softreboot" ->
       Some "23.1.0"
+  | "pool", "update_sync_enabled" ->
+      Some "23.16.2-next"
+  | "pool", "update_sync_day" ->
+      Some "23.16.2-next"
+  | "pool", "update_sync_frequency" ->
+      Some "23.16.2-next"
+  | "pool", "last_update_sync" ->
+      Some "23.16.2-next"
   | "pool", "telemetry_next_collection" ->
       Some "23.9.0"
   | "pool", "telemetry_frequency" ->
@@ -77,8 +91,14 @@ let prototyped_of_message = function
       Some "22.26.0"
   | "VTPM", "create" ->
       Some "22.26.0"
+  | "host", "apply_recommended_guidances" ->
+      Some "23.16.2-next"
   | "host", "set_https_only" ->
       Some "22.27.0"
+  | "pool", "set_update_sync_enabled" ->
+      Some "23.16.2-next"
+  | "pool", "configure_update_sync" ->
+      Some "23.16.2-next"
   | "pool", "reset_telemetry_uuid" ->
       Some "23.9.0"
   | "pool", "set_telemetry_next_collection" ->

ocaml/idl/datamodel_pool.ml

@@ -1055,6 +1055,54 @@ let reset_telemetry_uuid =
     ~params:[(Ref _pool, "self", "The pool")]
     ~allowed_roles:_R_POOL_ADMIN ()
 
+let update_sync_frequency =
+  Enum
+    ( "update_sync_frequency"
+    , [
+        ("daily", "The update synchronizations happen every day")
+      ; ( "weekly"
+        , "The update synchronizations happen every week on the chosen day"
+        )
+      ]
+    )
+
+let configure_update_sync =
+  call ~name:"configure_update_sync"
+    ~doc:
+      "Configure periodic update synchronization to sync updates from a remote \
+       CDN"
+    ~lifecycle:[]
+    ~params:
+      [
+        (Ref _pool, "self", "The pool")
+      ; ( update_sync_frequency
+        , "update_sync_frequency"
+        , "The frequency at which updates are synchronized from a remote CDN: \
+           daily or weekly."
+        )
+      ; ( Int
+        , "update_sync_day"
+        , "The day of the week the update synchronization will happen, based \
+           on pool's local timezone. Valid values are 0 to 6, 0 being Sunday. \
+           For 'daily' schedule, the value is ignored."
+        )
+      ]
+    ~allowed_roles:_R_POOL_OP ()
+
+let set_update_sync_enabled =
+  call ~name:"set_update_sync_enabled" ~lifecycle:[]
+    ~doc:
+      "enable or disable periodic update synchronization depending on the value"
+    ~params:
+      [
+        (Ref _pool, "self", "The pool")
+      ; ( Bool
+        , "value"
+        , "true - enable periodic update synchronization, false - disable it"
+        )
+      ]
+    ~allowed_roles:_R_POOL_OP ()
+
 (** A pool class *)
 let t =
   create_obj ~in_db:true ~in_product_since:rel_rio ~in_oss_since:None
@@ -1139,6 +1187,8 @@ let t =
       ; set_https_only
       ; set_telemetry_next_collection
       ; reset_telemetry_uuid
+      ; configure_update_sync
+      ; set_update_sync_enabled
       ]
     ~contents:
       ([uid ~in_oss_since:None _pool]
@@ -1380,6 +1430,21 @@ let t =
             "telemetry_next_collection"
             "The earliest timestamp (in UTC) when the next round of telemetry \
              collection can be carried out"
+        ; field ~qualifier:DynamicRO ~lifecycle:[] ~ty:DateTime
+            ~default_value:(Some (VDateTime Date.epoch)) "last_update_sync"
+            "time of the last update sychronization"
+        ; field ~qualifier:DynamicRO ~lifecycle:[] ~ty:update_sync_frequency
+            ~default_value:(Some (VEnum "weekly")) "update_sync_frequency"
+            "The frequency at which updates are synchronized from a remote \
+             CDN: daily or weekly."
+        ; field ~qualifier:DynamicRO ~lifecycle:[] ~ty:Int "update_sync_day"
+            ~default_value:(Some (VInt 0L))
+            "The day of the week the update synchronizations will be \
+             scheduled, based on pool's local timezone. Ignored when \
+             update_sync_frequency is daily"
+        ; field ~qualifier:DynamicRO ~lifecycle:[] ~ty:Bool
+            ~default_value:(Some (VBool false)) "update_sync_enabled"
+            "Whether periodic update synchronization is enabled or not"
         ]
       )
     ()

ocaml/idl/datamodel_repository.ml

@@ -173,7 +173,15 @@ let t =
           "SHA256 checksum of latest updateinfo.xml.gz in this repository if \
            its 'update' is true"
       ; field ~qualifier:DynamicRO
-          ~lifecycle:[(Published, "1.301.0", "")]
+          ~lifecycle:
+            [
+              (Published, "1.301.0", "")
+            ; (Deprecated, "23.12.0-next", "Dummy transition")
+            ; ( Removed
+              , "23.12.0-next"
+              , "The up_to_date field of repository was removed"
+              )
+            ]
           ~ty:Bool ~default_value:(Some (VBool false)) "up_to_date"
           "True if all hosts in pool is up to date with this repository"
       ; field ~qualifier:StaticRO ~lifecycle:[] ~ty:String

ocaml/idl/datamodel_vm.ml

@@ -2141,6 +2141,9 @@ let t =
             ~ty:(Set update_guidances) "pending_guidances"
             ~default_value:(Some (VSet []))
             "The set of pending guidances after applying updates"
+        ; field ~qualifier:DynamicRO ~lifecycle:[] ~ty:(Set update_guidances)
+            "recommended_guidances" ~default_value:(Some (VSet []))
+            "The set of recommended guidances after applying updates"
         ]
       )
     ()

ocaml/idl/schematest.ml

@@ -2,7 +2,7 @@ let hash x = Digest.string x |> Digest.to_hex
 
 (* BEWARE: if this changes, check that schema has been bumped accordingly in
    ocaml/idl/datamodel_common.ml, usually schema_minor_vsn *)
-let last_known_schema_hash = "6e38b31df3f16f18608bf5eaf74ae5d2"
+let last_known_schema_hash = "3efd34e77e3d098653f4f0e1c89bae1d"
 
 let current_schema_hash : string =
   let open Datamodel_types in

ocaml/tests/common/test_common.ml

@@ -209,7 +209,8 @@ let make_host2 ~__context ?(ref = Ref.make ()) ?(uuid = make_uuid ())
     ~control_domain:Ref.null ~updates_requiring_reboot:[] ~iscsi_iqn:""
     ~multipathing:false ~uefi_certificates:"" ~editions:[] ~pending_guidances:[]
     ~tls_verification_enabled
-    ~last_software_update:(Xapi_host.get_servertime ~__context ~host:ref) ;
+    ~last_software_update:(Xapi_host.get_servertime ~__context ~host:ref)
+    ~recommended_guidances:[] ~latest_synced_updates_applied:`unknown ;
   ref
 
 let make_pif ~__context ~network ~host ?(device = "eth0")
@@ -291,7 +292,9 @@ let make_pool ~__context ~master ?(name_label = "") ?(name_description = "")
     ?(repository_proxy_username = "") ?(repository_proxy_password = Ref.null)
     ?(migration_compression = false) ?(coordinator_bias = true)
     ?(telemetry_uuid = Ref.null) ?(telemetry_frequency = `weekly)
-    ?(telemetry_next_collection = API.Date.never) () =
+    ?(telemetry_next_collection = API.Date.never)
+    ?(last_update_sync = API.Date.epoch) ?(update_sync_frequency = `daily)
+    ?(update_sync_day = 0L) ?(update_sync_enabled = false) () =
   let pool_ref = Ref.make () in
   Db.Pool.create ~__context ~ref:pool_ref ~uuid:(make_uuid ()) ~name_label
     ~name_description ~master ~default_SR ~suspend_image_SR ~crash_dump_SR
@@ -307,7 +310,8 @@ let make_pool ~__context ~master ?(name_label = "") ?(name_description = "")
     ~client_certificate_auth_enabled ~client_certificate_auth_name
     ~repository_proxy_url ~repository_proxy_username ~repository_proxy_password
     ~migration_compression ~coordinator_bias ~telemetry_uuid
-    ~telemetry_frequency ~telemetry_next_collection ;
+    ~telemetry_frequency ~telemetry_next_collection ~last_update_sync
+    ~update_sync_frequency ~update_sync_day ~update_sync_enabled ;
   pool_ref
 
 let default_sm_features =

ocaml/tests/dune

@@ -7,7 +7,8 @@
       test_cluster_host test_cluster test_pusb test_network_sriov
       test_vm_placement test_vm_helpers test_repository test_repository_helpers
       test_ref
-      test_livepatch test_rpm test_updateinfo test_storage_smapiv1_wrapper test_storage_quicktest test_observer))
+      test_livepatch test_rpm test_updateinfo test_storage_smapiv1_wrapper test_storage_quicktest test_observer
+      test_pool_periodic_update_sync))
   (libraries
     alcotest
     angstrom
@@ -59,13 +60,13 @@
 (tests
   (names test_vm_helpers test_vm_placement test_network_sriov test_vdi_cbt
     test_clustering test_pusb test_daemon_manager test_repository test_repository_helpers
-    test_livepatch test_rpm test_updateinfo)
+    test_livepatch test_rpm test_updateinfo test_pool_periodic_update_sync)
   (package xapi)
   (modes exe)
   (modules test_vm_helpers test_vm_placement test_network_sriov test_vdi_cbt
     test_event test_clustering test_cluster_host test_cluster test_pusb
     test_daemon_manager test_repository test_repository_helpers test_livepatch test_rpm
-    test_updateinfo)
+    test_updateinfo test_pool_periodic_update_sync)
   (libraries
     alcotest
     fmt