Generate source jobs using the new index (#1036)

Signed-off-by: Jose Luis Rivero <[email protected]>

jenkins-scripts/dsl/gazebo_libs.dsl

@@ -55,12 +55,11 @@ boolean is_testing_enabled(lib_name, ci_config)
   return ! ci_config.tests_disabled?.contains(lib_name)
 }
 
-
 /*
  * Generate an index that facilitates the operations with the yaml values,
  * avoiding to parse them several times.
  *
- * ci_configs_by_lib index structure:
+ * # ci_configs_by_lib index structure:
  *   lib_name : [ ci_config_name : [ .branch .collection ] ]
  *
  *   The index main keys are the lib names (i.e: gz-cmake) and associated them
@@ -71,16 +70,33 @@ boolean is_testing_enabled(lib_name, ci_config)
  *
  *   index[gz-cmake][jammy] -> [ branch: gz-cmake3, collection: garden ,
  *                               branch: gz-cmake3, collection: harmonic]
+ *
+ * # pkgconf_per_src_inde index structure:
+ *   pkg_src_name : [ packaging_config_name : [ .lib_name .collection ] ]
+ *
+ *   The index main keys are package source names (i.e gz-cmake3 or gz-harmonic and associated them
+ *   another map of packaging configuration names supported as keys (i.e: jammy) with the
+ *   list of associated items composed by a map: lib_name (canonical name for the source package)
+ *   (and collection)
+ *
+ *   index[gz-cmake3][jammy] -> [ lib_name: gz-cmake, collection: harmonic ]
  */
-void generate_ciconfigs_by_lib(config, configs_per_lib_index)
+void generate_ciconfigs_by_lib(config, ciconf_per_lib_index, pkgconf_per_src_index)
 {
   config.collections.each { collection ->
     collection.libs.each { lib ->
       def libName = lib.name
       def branch = lib.repo.current_branch
       collection.ci.configs.each { config_name ->
-        configs_per_lib_index["$libName"]["${config_name}"] = configs_per_lib_index["$libName"]["${config_name}"]?: []
-        configs_per_lib_index["$libName"]["${config_name}"].contains(branch) ?: configs_per_lib_index["$libName"]["${config_name}"] << [branch: branch, collection: collection.name]
+        ciconf_per_lib_index[libName][config_name] = ciconf_per_lib_index[libName][config_name]?: []
+        ciconf_per_lib_index[libName][config_name].contains(branch) ?: ciconf_per_lib_index[libName][config_name] << [branch: branch, collection: collection.name]
+      }
+      def pkg_name = lib.name + lib.major_version
+      if (collection.packaging.linux?.ignore_major_version?.contains(libName))
+        pkg_name = lib.name
+      collection.packaging.configs?.each { config_name ->
+        pkgconf_per_src_index[pkg_name][config_name] = pkgconf_per_src_index[pkg_name][config_name]?: []
+        pkgconf_per_src_index[pkg_name][config_name] << [ lib_name: libName, collection: collection.name ]
       }
     }
   }
@@ -122,11 +138,12 @@ void generate_ci_job(gz_ci_job, lib_name, branch, ci_config,
   }
 }
 
-def configs_per_lib_index = [:].withDefault { [:] }
-generate_ciconfigs_by_lib(gz_collections_yaml, configs_per_lib_index)
+def ciconf_per_lib_index = [:].withDefault { [:] }
+def pkgconf_per_src_index = [:].withDefault { [:] }
+generate_ciconfigs_by_lib(gz_collections_yaml, ciconf_per_lib_index, pkgconf_per_src_index)
 
 // Generate PR jobs: 1 per ci configuration on each lib
-configs_per_lib_index.each { lib_name, lib_configs ->
+ciconf_per_lib_index.each { lib_name, lib_configs ->
   lib_configs.each { ci_configs ->
     def config_name = ci_configs.getKey()
     def ci_config = gz_collections_yaml.ci_configs.find{ it.name == config_name }
@@ -141,6 +158,7 @@ configs_per_lib_index.each { lib_name, lib_configs ->
     if (ci_config.exclude.contains(lib_name))
       return
 
+
     // Main PR jobs (-ci-pr_any-) (pulling check every 5 minutes)
     // --------------------------------------------------------------
     def distro = ci_config.system.version
@@ -180,7 +198,6 @@ configs_per_lib_index.each { lib_name, lib_configs ->
       // TODO: remove after testing
       if (branch_and_collection.collection != 'harmonic')
         return
-
       branch_name = branch_and_collection.branch
       def gz_ci_job = job("${gz_job_name_prefix}-ci-${branch_name}-${distro}-${arch}")
       generate_ci_job(gz_ci_job, lib_name, branch_name, ci_config)
@@ -199,6 +216,26 @@ configs_per_lib_index.each { lib_name, lib_configs ->
   } //en of lib_configs
 } // end of lib
 
+pkgconf_per_src_index.each { pkg_src, pkg_src_configs ->
+  pkg_src_configs.each { pkg_src_config ->
+    def config_name = pkg_src_config.getKey()
+    def pkg_config = gz_collections_yaml.packaging_configs.find{ it.name == config_name }
+    // lib_names are the same in all the entries
+    def canonical_lib_name = pkg_src_config.getValue()[0].lib_name
+
+    if (pkg_config.exclude?.contains(canonical_lib_name))
+      return
+
+    def gz_source_job = job("${pkg_src}-source")
+    OSRFSourceCreation.create(gz_source_job, [
+      PACKAGE: pkg_src ,
+      SOURCE_REPO_URI: "https://github.com/gazebosim/${canonical_lib_name}.git"])
+    OSRFSourceCreation.call_uploader_and_releasepy(gz_source_job,
+      'repository_uploader_packages',
+      '_releasepy')
+  }
+}
+
 if (WRITE_JOB_LOG) {
   File log_file = new File("jobs.txt")
   logging_list.each { log_type, items ->

jenkins-scripts/dsl/gz-collections.yaml

@@ -96,8 +96,9 @@ collections:
       configs:
         - bionic
     packaging:
+      configs:
+        - bionic
       linux:
-        package_name:
           ignore_major_version:
             - ign-citadel
   - name: 'fortress'
@@ -174,10 +175,11 @@ collections:
       configs:
         - focal
     packaging:
+      configs:
+        - focal
       linux:
-        package_name:
-          ignore_major_version:
-            - gz-fortress
+        ignore_major_version:
+          - gz-fortress
   - name: 'garden'
     libs:
       - name: gz-cmake
@@ -252,10 +254,11 @@ collections:
       configs:
         - focal
     packaging:
+      configs:
+        - focal
       linux:
-        package_name:
-          ignore_major_version:
-            - gz-garden
+        ignore_major_version:
+          - gz-garden
   - name: 'harmonic'
     libs:
       - name: gz-cmake
@@ -330,10 +333,11 @@ collections:
       configs:
         - jammy
     packaging:
+      configs:
+        - jammy
       linux:
-        package_name:
-          ignore_major_version:
-            - gz-harmonic
+        ignore_major_version:
+          - gz-harmonic     
   - name: 'ionic'
     libs:
       - name: gz-cmake
@@ -405,12 +409,11 @@ collections:
         - jammy
     packaging:
       linux:
-        package_name:
-          ignore_major_version:
-            - gz-ionic
+        ignore_major_version:
+          - gz-ionic
         nightly:
-            distros:
-              - jammy
+          distros:
+            - jammy
   - name: '__upcoming__'
     libs:
       - name: gz-cmake
@@ -481,6 +484,8 @@ collections:
       configs:
         - jammy
     packaging:
+      configs:
+        - jammy
       exclude:
         - __upcoming__
 ci_configs:
@@ -546,3 +551,30 @@ ci_configs:
       gz-physics:
         - "export MAKE_JOBS=1"
     tests_disabled:
+packaging_configs:
+  - name: bionic
+    system:
+      so: linux
+      distribution: ubuntu
+      version: bionic
+      arch:
+        - i386
+        - amd64
+        - arm64
+  - name: focal
+    system:
+      so: linux
+      distribution: ubuntu
+      version: focal
+      arch:
+        - amd64
+        - arm64
+  - name: jammy
+    system:
+      so: linux
+      distribution: ubuntu
+      version: jammy
+      arch:
+        - amd64
+        - arm64
+        - armhf

jenkins-scripts/dsl/ignition_collection.dsl

@@ -17,7 +17,7 @@ String get_debbuilder_name(parsed_yaml_lib, parsed_yaml_packaging)
 {
   major_version = parsed_yaml_lib.major_version
 
-  ignore_major_version = parsed_yaml_packaging.linux?.package_name?.ignore_major_version
+  ignore_major_version = parsed_yaml_packaging.linux?.ignore_major_version
   if (ignore_major_version && ignore_major_version.contains(parsed_yaml_lib.name))
     major_version = ""