Re-apply eval-variables on upgrades

eval-variables are in opamrc from the 2.x de-OCaml-ing of opam, but
they're conceptually in the wrong place (they should be declared in the
repository where they can be synchronised and upgraded).

This is a problem for the new variables added in 2.1, because they don't
get added to the config of existing roots.

This alters the format upgrade to update pre-existing variables and add
any new ones.

Obviously, if the user had initialised with a different opamrc then
we're overwriting that, but that is the conceptual weakness of the field
being in opamrc.

master_changes.md

@@ -78,6 +78,7 @@ users)
  * Add support for Wolfi OS, treat it like Apline family as it uses apk too [#5878 @xnox]
 
 ## Format upgrade
+  * Handle init OCaml `sys-ocaml-*` eval variables during format upgrade from 2.0 -> 2.1 -> 2.2 [#5829 @dra27]
 
 ## Sandbox
   * Mark the user temporary directory (as returned by `getconf DARWIN_USER_TEMP_DIR`) as writable when TMPDIR is not defined on macOS [#5780 @ElectreAAS]

src/state/opamFormatUpgrade.ml

@@ -1080,6 +1080,36 @@ let from_2_1_alpha2_to_2_1_rc ~on_the_fly:_ root conf =
 
 let from_2_1_rc_to_2_1 ~on_the_fly:_ _ conf = conf, gtc_none
 
+let apply_eval_variables conf old_vars new_vars =
+  let current_eval_variables = OpamFile.Config.eval_variables conf in
+  let add map (name, cmd, docstring) = OpamVariable.Map.add name (cmd, docstring) map in
+  let old_vars = List.fold_left add OpamVariable.Map.empty old_vars in
+  let new_vars = List.fold_left add OpamVariable.Map.empty new_vars in
+  let update new_vars ((name, cmd, _) as var) =
+    match OpamVariable.Map.find_opt name old_vars with
+    | Some (cmd', _) when cmd = cmd' ->
+        let cmd, docstring = OpamVariable.Map.find name new_vars in
+        (OpamVariable.Map.remove name new_vars, (name, cmd, docstring))
+    | _ -> (new_vars, var) in
+  let (missing, eval_variables) = OpamStd.List.fold_left_map update new_vars current_eval_variables in
+  let eval_variables =
+    let add name (cmd, docstring) eval_variables =
+      (name, cmd, docstring)::eval_variables in
+    OpamVariable.Map.fold add missing eval_variables in
+  OpamFile.Config.with_eval_variables eval_variables conf
+
+let sys_config_variables_unix_2_1 = [
+  OpamVariable.of_string "sys-ocaml-arch",
+  ["sh"; "-c"; "ocamlc -config 2>/dev/null | tr -d '\\r' | grep '^architecture: ' | sed -e 's/.*: //' -e 's/i386/i686/' -e 's/amd64/x86_64/'"],
+  "Target architecture of the OCaml compiler present on your system";
+  OpamVariable.of_string "sys-ocaml-cc",
+  ["sh"; "-c"; "ocamlc -config 2>/dev/null | tr -d '\\r' | grep '^ccomp_type: ' | sed -e 's/.*: //'"],
+  "Host C Compiler type of the OCaml compiler present on your system";
+  OpamVariable.of_string "sys-ocaml-libc",
+  ["sh"; "-c"; "ocamlc -config 2>/dev/null | tr -d '\\r' | grep '^os_type: ' | sed -e 's/.*: //' -e 's/Win32/msvc/' -e '/^msvc$/!s/.*/libc/'"],
+  "Host C Runtime Library type of the OCaml compiler present on your system";
+]
+
 let from_2_0_to_2_1 ~on_the_fly _ conf =
   (* In opam < 2.1 "jobs" was set during initialisation
      This creates problems when upgrading from opam 2.0 as it
@@ -1100,11 +1130,14 @@ let from_2_0_to_2_1 ~on_the_fly _ conf =
    | Some prev_jobs when prev_jobs = max 1 (OpamSysPoll.cores () - 1) -> ()
    | Some prev_jobs -> info_jobs_changed ~prev_jobs
    | None -> info_jobs_changed ~prev_jobs:1);
+  let conf = apply_eval_variables conf [] sys_config_variables_unix_2_1 in
   OpamFile.Config.with_jobs_opt None conf, gtc_none
 
 let v2_2_alpha = OpamVersion.of_string "2.2~alpha"
 
-let from_2_1_to_2_2_alpha ~on_the_fly:_ _ conf = conf, gtc_none
+let from_2_1_to_2_2_alpha ~on_the_fly:_ _ conf =
+  apply_eval_variables conf sys_config_variables_unix_2_1
+    OpamEnv.sys_ocaml_eval_variables, gtc_none
 
 (* To add an upgrade layer
    * If it is a light upgrade, returns as second element if the repo or switch

tests/reftests/init-ocaml-eval-variables.unix.test

@@ -65,7 +65,7 @@ Continue? [y/n] y
 Format upgrade done.
 Set to '4' the field jobs in global configuration
 ### opam-cat $OPAMROOT/config | grep eval-variables
-eval-variables: [sys-ocaml-version ["ocamlc" "-vnum"] "OCaml version present on your system independently of opam, if any"]
+eval-variables: [[sys-ocaml-libc ["sh" "-c" "ocamlc -config 2>/dev/null | tr -d '/r' | sed -n -e 's/^os_type: Win32/msvc/p;s/^os_type: .*/libc/p'"] "Host C Runtime Library type of the OCaml compiler present on your system"] [sys-ocaml-cc ["sh" "-c" "ocamlc -config 2>/dev/null | tr -d '/r' | sed -n -e 's/^ccomp_type: //p'"] "Host C Compiler type of the OCaml compiler present on your system"] [sys-ocaml-arch ["sh" "-c" "ocamlc -config 2>/dev/null | tr -d '/r' | sed -n -e 's/i386/i686/;s/amd64/x86_64/;s/^architecture: //p'"] "Target architecture of the OCaml compiler present on your system"] [sys-ocaml-version ["ocamlc" "-vnum"] "OCaml version present on your system independently of opam, if any"]]
 ### rm "$OPAMROOT/config"
 ### :::::::::::::::::::::
 ### :III: From 2.1 root :
@@ -84,4 +84,4 @@ Continue? [y/n] y
 Format upgrade done.
 Set to '4' the field jobs in global configuration
 ### opam-cat $OPAMROOT/config | grep eval-variables
-eval-variables: [[sys-ocaml-version ["ocamlc" "-vnum"] "OCaml version present on your system independently of opam, if any"] [sys-ocaml-arch ["sh" "-c" "ocamlc -config 2>/dev/null | tr -d '/r' | grep '^architecture: ' | sed -e 's/.*: //' -e 's/i386/i686/' -e 's/amd64/x86_64/'"] "Target architecture of the OCaml compiler present on your system"] [sys-ocaml-cc ["sh" "-c" "ocamlc -config 2>/dev/null | tr -d '/r' | grep '^ccomp_type: ' | sed -e 's/.*: //'"] "Host C Compiler type of the OCaml compiler present on your system"] [sys-ocaml-libc ["sh" "-c" "ocamlc -config 2>/dev/null | tr -d '/r' | grep '^os_type: ' | sed -e 's/.*: //' -e 's/Win32/msvc/' -e '/^msvc$/!s/.*/libc/'"] "Host C Runtime Library type of the OCaml compiler present on your system"]]
+eval-variables: [[sys-ocaml-version ["ocamlc" "-vnum"] "OCaml version present on your system independently of opam, if any"] [sys-ocaml-arch ["sh" "-c" "ocamlc -config 2>/dev/null | tr -d '/r' | sed -n -e 's/i386/i686/;s/amd64/x86_64/;s/^architecture: //p'"] "Target architecture of the OCaml compiler present on your system"] [sys-ocaml-cc ["sh" "-c" "ocamlc -config 2>/dev/null | tr -d '/r' | sed -n -e 's/^ccomp_type: //p'"] "Host C Compiler type of the OCaml compiler present on your system"] [sys-ocaml-libc ["sh" "-c" "ocamlc -config 2>/dev/null | tr -d '/r' | sed -n -e 's/^os_type: Win32/msvc/p;s/^os_type: .*/libc/p'"] "Host C Runtime Library type of the OCaml compiler present on your system"]]

tests/reftests/init-ocaml-eval-variables.win32.test

@@ -65,7 +65,7 @@ Continue? [y/n] y
 Format upgrade done.
 Set to '4' the field jobs in global configuration
 ### opam-cat $OPAMROOT/config | grep eval-variables
-eval-variables: [sys-ocaml-version ["ocamlc" "-vnum"] "OCaml version present on your system independently of opam, if any"]
+eval-variables: [[sys-ocaml-libc ["cmd" "/d" "/c" "for /f %f in ('ocamlc -config-var os_type 2^>nul') do @if '%f' equ 'Win32' (echo msvc) else (echo libc)"] "Host C Runtime Library type of the OCaml compiler present on your system"] [sys-ocaml-cc ["cmd" "/d" "/c" "ocamlc -config-var ccomp_type 2>nul"] "Host C Compiler type of the OCaml compiler present on your system"] [sys-ocaml-arch ["cmd" "/d" "/c" "for /f %f in ('ocamlc -config-var architecture 2^>nul') do @if '%f' equ 'i386' (echo i686) else if '%f' equ 'amd64' (echo x86_64) else (echo %f)"] "Target architecture of the OCaml compiler present on your system"] [sys-ocaml-version ["ocamlc" "-vnum"] "OCaml version present on your system independently of opam, if any"]]
 ### rm "$OPAMROOT/config"
 ### :::::::::::::::::::::
 ### :III: From 2.1 root :
@@ -84,4 +84,4 @@ Continue? [y/n] y
 Format upgrade done.
 Set to '4' the field jobs in global configuration
 ### opam-cat $OPAMROOT/config | grep eval-variables
-eval-variables: [[sys-ocaml-version ["ocamlc" "-vnum"] "OCaml version present on your system independently of opam, if any"] [sys-ocaml-arch ["sh" "-c" "ocamlc -config 2>/dev/null | tr -d '/r' | grep '^architecture: ' | sed -e 's/.*: //' -e 's/i386/i686/' -e 's/amd64/x86_64/'"] "Target architecture of the OCaml compiler present on your system"] [sys-ocaml-cc ["sh" "-c" "ocamlc -config 2>/dev/null | tr -d '/r' | grep '^ccomp_type: ' | sed -e 's/.*: //'"] "Host C Compiler type of the OCaml compiler present on your system"] [sys-ocaml-libc ["sh" "-c" "ocamlc -config 2>/dev/null | tr -d '/r' | grep '^os_type: ' | sed -e 's/.*: //' -e 's/Win32/msvc/' -e '/^msvc$/!s/.*/libc/'"] "Host C Runtime Library type of the OCaml compiler present on your system"]]
+eval-variables: [[sys-ocaml-version ["ocamlc" "-vnum"] "OCaml version present on your system independently of opam, if any"] [sys-ocaml-arch ["cmd" "/d" "/c" "for /f %f in ('ocamlc -config-var architecture 2^>nul') do @if '%f' equ 'i386' (echo i686) else if '%f' equ 'amd64' (echo x86_64) else (echo %f)"] "Target architecture of the OCaml compiler present on your system"] [sys-ocaml-cc ["cmd" "/d" "/c" "ocamlc -config-var ccomp_type 2>nul"] "Host C Compiler type of the OCaml compiler present on your system"] [sys-ocaml-libc ["cmd" "/d" "/c" "for /f %f in ('ocamlc -config-var os_type 2^>nul') do @if '%f' equ 'Win32' (echo msvc) else (echo libc)"] "Host C Runtime Library type of the OCaml compiler present on your system"]]

tests/reftests/init.test

@@ -106,14 +106,13 @@ No configuration file found, using built-in defaults.
 
 <><> Fetching repository information ><><><><><><><><><><><><><><><><><><><><><>
 [default] Initialised
-### opam-cat $OPAMROOT/config | 'opam-root-version: "${OPAMROOTVERSION}"' -> 'opam-root-version: current' | grep -v sys-pkg-manager-cmd | grep -v global-variables
+### opam-cat $OPAMROOT/config | 'opam-root-version: "${OPAMROOTVERSION}"' -> 'opam-root-version: current' | grep -v sys-pkg-manager-cmd | grep -v global-variables | grep -v eval-variables:
 default-compiler: ["ocaml-system" "ocaml-base-compiler"]
 default-invariant: ["ocaml" {>= "4.05.0"}]
 depext: true
 depext-cannot-install: false
 depext-run-installs: true
 download-jobs: 3
-eval-variables: [[sys-ocaml-version ["ocamlc" "-vnum"] "OCaml version present on your system independently of opam, if any"] [sys-ocaml-arch ["sh" "-c" "ocamlc -config 2>/dev/null | tr -d '/r' | grep '^architecture: ' | sed -e 's/.*: //' -e 's/i386/i686/' -e 's/amd64/x86_64/'"] "Target architecture of the OCaml compiler present on your system"] [sys-ocaml-cc ["sh" "-c" "ocamlc -config 2>/dev/null | tr -d '/r' | grep '^ccomp_type: ' | sed -e 's/.*: //'"] "Host C Compiler type of the OCaml compiler present on your system"] [sys-ocaml-libc ["sh" "-c" "ocamlc -config 2>/dev/null | tr -d '/r' | grep '^os_type: ' | sed -e 's/.*: //' -e 's/Win32/msvc/' -e '/^msvc$/!s/.*/libc/'"] "Host C Runtime Library type of the OCaml compiler present on your system"]]
 opam-root-version: current
 opam-version: "2.0"
 repositories: "default"