Merge master into feature/configure-ssh-phase2 (#6409)

Author: gangj

.github/workflows/docs.yml

@@ -31,15 +31,23 @@ jobs:
       - name: Update Ubuntu repositories
         run: sudo apt-get update
 
+      # We set DUNE_CACHE_STORAGE_MODE, it is required for dune cache to work inside opam for now,
+      # otherwise it gets EXDEV and considers it a cache miss
       - name: Use ocaml
-        uses: ocaml/setup-ocaml@v2
+        uses: ocaml/setup-ocaml@v3
         with:
           ocaml-compiler: ${{ steps.dotenv.outputs.ocaml_version_full }}
           opam-repositories: |
             xs-opam: ${{ steps.dotenv.outputs.repository }}
+          dune-cache: true
+          opam-pin: false
+          cache-prefix: v3-${{ steps.system-info.outputs.name }}-${{ steps.system-info.outputs.release }}
+        env:
+          DUNE_CACHE_STORAGE_MODE: copy
 
       - name: Install dependencies
-        run: opam pin list --short | xargs opam install --deps-only -v
+        shell: bash
+        run: opam install . --deps-only -v
 
       - name: Generate xapi-storage docs
         run: |

dune-project

@@ -3,6 +3,7 @@
 (formatting (enabled_for ocaml))
 (using menhir 2.0)
 (using directory-targets 0.1)
+(opam_file_location inside_opam_directory)
 
 (cram enable)
 (implicit_transitive_deps false)

ocaml/libs/xapi-rrd/lib/rrd.ml

@@ -379,121 +379,124 @@ let process_ds_value ds value interval new_rrd =
     rate
 
 let ds_update rrd timestamp valuesandtransforms new_rrd =
-  (* Interval is the time between this and the last update
-
-     Currently ds_update is called with datasources that belong to a single
-     plugin, correspondingly they all have the same timestamp.
-     Further refactoring is needed if timestamps per measurement are to be
-     introduced. *)
-  let first_ds_index, _ = valuesandtransforms.(0) in
-  let last_updated = rrd.rrd_dss.(first_ds_index).ds_last_updated in
-  let interval = timestamp -. last_updated in
-  (* Work around the clock going backwards *)
-  let interval = if interval < 0. then 5. else interval in
-
-  (* start time (st) and age of the last processed pdp and the currently occupied one *)
-  let proc_pdp_st, _proc_pdp_age = get_times last_updated rrd.timestep in
-  let occu_pdp_st, occu_pdp_age = get_times timestamp rrd.timestep in
-
-  (* The number of pdps that should result from this update *)
-  let elapsed_pdp_st =
-    Int64.to_int ((occu_pdp_st --- proc_pdp_st) /// rrd.timestep)
-  in
-
-  (* if we're due one or more PDPs, pre_int is the amount of the
-     current update interval that will be used in calculating them, and
-     post_int is the amount left over
-     this step. If a PDP isn't post is what's left over *)
-  let pre_int, post_int =
-    if elapsed_pdp_st > 0 then
-      let pre = interval -. occu_pdp_age in
-      (pre, occu_pdp_age)
-    else
-      (interval, 0.0)
-  in
-
-  (* We're now done with the last_updated value, so update it *)
-  rrd.last_updated <- timestamp ;
+  (* CA-408841 - don't update the rrd at all if list of datasources is empty *)
+  if valuesandtransforms <> [||] then (
+    (* Interval is the time between this and the last update
+
+       Currently ds_update is called with datasources that belong to a single
+       plugin, correspondingly they all have the same timestamp.
+       Further refactoring is needed if timestamps per measurement are to be
+       introduced. *)
+    let first_ds_index, _ = valuesandtransforms.(0) in
+    let last_updated = rrd.rrd_dss.(first_ds_index).ds_last_updated in
+    let interval = timestamp -. last_updated in
+    (* Work around the clock going backwards *)
+    let interval = if interval < 0. then 5. else interval in
+
+    (* start time (st) and age of the last processed pdp and the currently occupied one *)
+    let proc_pdp_st, _proc_pdp_age = get_times last_updated rrd.timestep in
+    let occu_pdp_st, occu_pdp_age = get_times timestamp rrd.timestep in
+
+    (* The number of pdps that should result from this update *)
+    let elapsed_pdp_st =
+      Int64.to_int ((occu_pdp_st --- proc_pdp_st) /// rrd.timestep)
+    in
 
-  (* Calculate the values we're going to store based on the input data and the type of the DS *)
-  let v2s =
-    Array.map
-      (fun (i, {value; _}) ->
-        let v = process_ds_value rrd.rrd_dss.(i) value interval new_rrd in
-        rrd.rrd_dss.(i).ds_last_updated <- timestamp ;
-        (i, v)
-      )
-      valuesandtransforms
-  in
-  (* Update the PDP accumulators up until the most recent PDP *)
-  Array.iter
-    (fun (i, value) ->
-      let ds = rrd.rrd_dss.(i) in
-      if Utils.isnan value then
-        ds.ds_unknown_sec <- pre_int
+    (* if we're due one or more PDPs, pre_int is the amount of the
+       current update interval that will be used in calculating them, and
+       post_int is the amount left over
+       this step. If a PDP isn't post is what's left over *)
+    let pre_int, post_int =
+      if elapsed_pdp_st > 0 then
+        let pre = interval -. occu_pdp_age in
+        (pre, occu_pdp_age)
       else
-        (* CA-404597 - Gauge and Absolute values should be passed as-is,
-           without being involved in time-based calculations at all.
-           This applies to calculations below as well *)
-        match ds.ds_ty with
-        | Gauge | Absolute ->
-            ds.ds_value <- value
-        | Derive ->
-            ds.ds_value <- ds.ds_value +. (pre_int *. value /. interval)
-    )
-    v2s ;
+        (interval, 0.0)
+    in
+
+    (* We're now done with the last_updated value, so update it *)
+    rrd.last_updated <- timestamp ;
 
-  (* If we've passed a PDP point, we need to update the RRAs *)
-  if elapsed_pdp_st > 0 then (
-    (* Calculate the PDPs for each DS *)
-    let pdps =
+    (* Calculate the values we're going to store based on the input data and the type of the DS *)
+    let v2s =
       Array.map
-        (fun (i, {transform; _}) ->
-          let ds = rrd.rrd_dss.(i) in
-          if interval > ds.ds_mrhb then
-            (i, nan)
-          else
-            let raw =
-              let proc_pdp_st = get_float_time last_updated rrd.timestep in
-              let occu_pdp_st = get_float_time timestamp rrd.timestep in
-
-              match ds.ds_ty with
-              | Gauge | Absolute ->
-                  ds.ds_value
-              | Derive ->
-                  ds.ds_value
-                  /. (occu_pdp_st -. proc_pdp_st -. ds.ds_unknown_sec)
-            in
-            (* Apply the transform after the raw value has been calculated *)
-            let raw = apply_transform_function transform raw in
-            (* Make sure the values are not out of bounds after all the processing *)
-            if raw < ds.ds_min || raw > ds.ds_max then
-              (i, nan)
-            else
-              (i, raw)
+        (fun (i, {value; _}) ->
+          let v = process_ds_value rrd.rrd_dss.(i) value interval new_rrd in
+          rrd.rrd_dss.(i).ds_last_updated <- timestamp ;
+          (i, v)
         )
         valuesandtransforms
     in
-
-    rra_update rrd proc_pdp_st elapsed_pdp_st pdps ;
-
-    (* Reset the PDP accumulators *)
+    (* Update the PDP accumulators up until the most recent PDP *)
     Array.iter
       (fun (i, value) ->
         let ds = rrd.rrd_dss.(i) in
-        if Utils.isnan value then (
-          ds.ds_value <- 0.0 ;
-          ds.ds_unknown_sec <- post_int
-        ) else (
-          ds.ds_unknown_sec <- 0.0 ;
+        if Utils.isnan value then
+          ds.ds_unknown_sec <- pre_int
+        else
+          (* CA-404597 - Gauge and Absolute values should be passed as-is,
+             without being involved in time-based calculations at all.
+             This applies to calculations below as well *)
           match ds.ds_ty with
           | Gauge | Absolute ->
               ds.ds_value <- value
           | Derive ->
-              ds.ds_value <- post_int *. value /. interval
-        )
+              ds.ds_value <- ds.ds_value +. (pre_int *. value /. interval)
       )
-      v2s
+      v2s ;
+
+    (* If we've passed a PDP point, we need to update the RRAs *)
+    if elapsed_pdp_st > 0 then (
+      (* Calculate the PDPs for each DS *)
+      let pdps =
+        Array.map
+          (fun (i, {transform; _}) ->
+            let ds = rrd.rrd_dss.(i) in
+            if interval > ds.ds_mrhb then
+              (i, nan)
+            else
+              let raw =
+                let proc_pdp_st = get_float_time last_updated rrd.timestep in
+                let occu_pdp_st = get_float_time timestamp rrd.timestep in
+
+                match ds.ds_ty with
+                | Gauge | Absolute ->
+                    ds.ds_value
+                | Derive ->
+                    ds.ds_value
+                    /. (occu_pdp_st -. proc_pdp_st -. ds.ds_unknown_sec)
+              in
+              (* Apply the transform after the raw value has been calculated *)
+              let raw = apply_transform_function transform raw in
+              (* Make sure the values are not out of bounds after all the processing *)
+              if raw < ds.ds_min || raw > ds.ds_max then
+                (i, nan)
+              else
+                (i, raw)
+          )
+          valuesandtransforms
+      in
+
+      rra_update rrd proc_pdp_st elapsed_pdp_st pdps ;
+
+      (* Reset the PDP accumulators *)
+      Array.iter
+        (fun (i, value) ->
+          let ds = rrd.rrd_dss.(i) in
+          if Utils.isnan value then (
+            ds.ds_value <- 0.0 ;
+            ds.ds_unknown_sec <- post_int
+          ) else (
+            ds.ds_unknown_sec <- 0.0 ;
+            match ds.ds_ty with
+            | Gauge | Absolute ->
+                ds.ds_value <- value
+            | Derive ->
+                ds.ds_value <- post_int *. value /. interval
+          )
+        )
+        v2s
+    )
   )
 
 (** Update the rrd with named values rather than just an ordered array

ocaml/tests/test_sm_features.ml

@@ -249,6 +249,7 @@ module CreateSMObject = Generic.MakeStateful (struct
       ; features
       ; configuration= []
       ; required_cluster_stack= []
+      ; smapi_version= SMAPIv2
       }
 
   let extract_output __context _ =

ocaml/tests/test_storage_migrate_state.ml

@@ -17,11 +17,11 @@ open Test_highlevel
 module StorageMigrateState = struct
   type state_t = unit
 
-  let create_default_state () = Storage_migrate.State.clear ()
+  let create_default_state () = Storage_migrate_helper.State.clear ()
 end
 
 let sample_send_state =
-  Storage_migrate.State.Send_state.
+  Storage_migrate_helper.State.Send_state.
     {
       url= "url"
     ; dest_sr= Storage_interface.Sr.of_string "dest_sr"
@@ -45,7 +45,7 @@ let sample_send_state =
 
 let sample_receive_state =
   let open Storage_interface in
-  Storage_migrate.State.Receive_state.
+  Storage_migrate_helper.State.Receive_state.
     {
       sr= Sr.of_string "my_sr"
     ; dummy_vdi= Vdi.of_string "dummy_vdi"
@@ -57,7 +57,7 @@ let sample_receive_state =
     }
 
 let sample_copy_state =
-  Storage_migrate.State.Copy_state.
+  Storage_migrate_helper.State.Copy_state.
     {
       base_dp= "base_dp"
     ; leaf_dp= "leaf_dp"
@@ -70,7 +70,7 @@ let sample_copy_state =
 
 module MapOf = Generic.MakeStateful (struct
   module Io = struct
-    open Storage_migrate.State
+    open Storage_migrate_helper.State
 
     type input_t =
       (string * osend operation) option
@@ -88,7 +88,7 @@ module MapOf = Generic.MakeStateful (struct
   end
 
   module State = StorageMigrateState
-  open Storage_migrate.State
+  open Storage_migrate_helper.State
 
   let load_input () (send, recv, copy) =
     Option.iter (fun (id, send) -> add id send) send ;
@@ -116,7 +116,7 @@ module MapOf = Generic.MakeStateful (struct
 end)
 
 let test_clear () =
-  let open Storage_migrate.State in
+  let open Storage_migrate_helper.State in
   clear () ;
   add "foo" (Send_op sample_send_state) ;
   add "bar" (Recv_op sample_receive_state) ;
@@ -130,5 +130,5 @@ let test_clear () =
 let test = [("clear", `Quick, test_clear)]
 
 let tests =
-  Storage_migrate.State.persist_root := Test_common.working_area ;
+  Storage_migrate_helper.State.persist_root := Test_common.working_area ;
   [("storage_migrate_state_map_of", MapOf.tests)]

ocaml/tests/test_vdi_cbt.ml

@@ -30,10 +30,11 @@ let register_smapiv2_server (module S : Storage_interface.Server_impl) sr_ref =
       ; features= []
       ; configuration= []
       ; required_cluster_stack= []
+      ; smapi_version= SMAPIv2
       }
   in
 
-  Storage_mux.register sr_ref rpc "" dummy_query_result
+  Storage_mux_reg.register sr_ref rpc "" dummy_query_result
 
 let make_smapiv2_storage_server ?vdi_enable_cbt ?vdi_disable_cbt
     ?vdi_list_changed_blocks ?vdi_data_destroy ?vdi_snapshot ?vdi_clone:_ () =