cms-YYYY-luminosity: update scripts for HI type data taking

Author: katilp

.github/workflows/cms-luminosity-tables.yaml

@@ -2,8 +2,16 @@ name: Build listings and records
 on:
   workflow_dispatch
 env:
+  RECID: 1058
   YEAR: 2015
-  ERAS: Run2015D
+  ERAS: Run2015E
+  TYPE: pphiref
+  # Inputs:
+  # - 2013 pPb: 1056 2013 HIRun2013 pPb
+  # - 2013 ppref: 1057 2013 Run2013A pphiref
+  # - 2015 ppref: 1058 2015 Run2015E pphiref
+  # - 2015 pp: 1055 2015 Run2015D pp (not used to build the current record, but tested to produce the same files)
+  # - 2016 pp: 1059 2016 Run2016G Run2016H pp
 defaults:
   run: 
     working-directory: cms-YYYY-luminosity
@@ -12,27 +20,35 @@ jobs:
     runs-on: ubuntu-latest
     name: create lumi tables
     steps:
-    - uses: actions/checkout@v2    
+    - uses: actions/checkout@v3    
     - name: Prepare
       run: |
         mkdir outputs
         echo ls `ls`
         echo pwd `pwd`
+        RECID=${{env.RECID}}
+        YEAR=${{env.YEAR}}
+        TYPE=${{env.TYPE}}
     # Get the part of the normtag file corresponding to validated runs (muons cert file as it has a wider range)
     - name: Normtag file
       run: |
-        echo "Get the normtag file for $YEAR"
-        curl http://api-server-cms-release-info.app.cern.ch/years/val_json?year=$YEAR > myjs.json
-        muonurl=$(jq -r '.[0][1].url' myjs.json)
-        wget $muonurl -O cert.txt
-        wget https://raw.githubusercontent.com/CMS-LUMI-POG/Normtags/master/normtag_PHYSICS.json -O normtag_PHYSICS.json
-        source normtag.sh > outputs/normtag_PHYSICS_$YEAR.json
-        cp outputs/normtag_PHYSICS_$YEAR.json normtag_PHYSICS.json
+        if (( $YEAR > 2013 ))
+        then
+          echo "Get the normtag file for $TYPE of $YEAR"
+          curl 'http://api-server-cms-release-info.app.cern.ch/years/val_json?year='"$YEAR"'&type='"$TYPE"'' > myjs.json
+          ls -l myjs.json
+          muonurl=$(jq -r '.[0][1].url' myjs.json)
+          echo $muonurl
+          wget $muonurl -O cert.txt
+          wget https://raw.githubusercontent.com/CMS-LUMI-POG/Normtags/master/normtag_PHYSICS.json -O normtag_PHYSICS.json
+          source normtag.sh > outputs/normtag_PHYSICS_$TYPE'_'$YEAR.json 
+          cp outputs/normtag_PHYSICS_$TYPE'_'$YEAR.json normtag_PHYSICS.json
+        fi
     # Get the parameters for lumi listings
     - name: Get validated runs file
       run: |
-        echo "Get validated runs for $YEAR"
-        curl http://api-server-cms-release-info.app.cern.ch/years/val_json?year=$YEAR > myjs.json
+        echo "Get validated runs for $TYPE of $YEAR"
+        curl 'http://api-server-cms-release-info.app.cern.ch/years/val_json?year='"$YEAR"'&type='"$TYPE"'' > myjs.json
         url=$(jq -r '.[0][0].url' myjs.json)
         wget $url -O cert.txt
     # run the container with the script in the repository
@@ -41,36 +57,40 @@ jobs:
       run: | 
         # Summary txt table per year
         echo "Create summary table for $YEAR"
-        docker run -v $(pwd):/mnt/vol -w /home/brilws gitlab-registry.cern.ch/cms-cloud/brilws-docker /bin/bash /mnt/vol/commands.sh cert.txt tab summ > outputs/$YEAR'lumi.txt'
+        # Use normtag file for Run-2 values 
+        # and hfoc (the only available option) for Run-1 HI and pp ref
+        if (( $YEAR > 2014 )); then option="normtag"; else option="hfoc"; fi;
+        echo "From workflow, type is $TYPE, value of option is $option"
+        [ $TYPE == "pp" ] && docker run -v $(pwd):/mnt/vol -w /home/brilws gitlab-registry.cern.ch/cms-cloud/brilws-docker /bin/bash /mnt/vol/commands.sh cert.txt tab $option summ > outputs/$YEAR'lumi.txt'
         # Prescale listing for each run in a single csv file
         echo "Get prescale listing for all run numbers"
-        docker run -v $(pwd):/mnt/vol -w /home/brilws gitlab-registry.cern.ch/cms-cloud/brilws-docker /bin/bash /mnt/vol/commands.sh cert.txt csv summ | grep -v "#" | awk '{ print $1 }' FS=':' > run_numbers.txt
-        docker run -v $(pwd):/mnt/vol -w /home/brilws gitlab-registry.cern.ch/cms-cloud/brilws-docker /bin/bash /mnt/vol/prescale.sh > outputs/'prescale'$YEAR'.csv'
+        docker run -v $(pwd):/mnt/vol -w /home/brilws gitlab-registry.cern.ch/cms-cloud/brilws-docker /bin/bash /mnt/vol/commands.sh cert.txt csv $option summ | grep -v "#" |  grep  "/" | awk '{ print $1 }' FS=':' > run_numbers.txt
+        docker run -v $(pwd):/mnt/vol -w /home/brilws gitlab-registry.cern.ch/cms-cloud/brilws-docker /bin/bash /mnt/vol/prescale.sh > outputs/'prescale_'$TYPE$YEAR'.csv'
         # Summary txt table per era, for all listed eras
+        ERAS=${{env.ERAS}}
         echo "Create summary tables for $ERAS"
         for era in ${ERAS[*]} 
         do 
           echo "doing " $era
-          curl http://api-server-cms-release-info.app.cern.ch/runeras?run_era=$era > myjs.json
+          curl 'http://api-server-cms-release-info.app.cern.ch/runeras?run_era='"$era"'' > myjs.json
           runmin=$(cat myjs.json | jq -r '.[0].run_min')
           runmax=$(cat myjs.json | jq -r '.[0].run_max')
-          docker run -v $(pwd):/mnt/vol -w /home/brilws gitlab-registry.cern.ch/cms-cloud/brilws-docker /bin/bash /mnt/vol/commands.sh cert.txt tab summ $runmin $runmax > outputs/$era'lumi.txt'
+          docker run -v $(pwd):/mnt/vol -w /home/brilws gitlab-registry.cern.ch/cms-cloud/brilws-docker /bin/bash /mnt/vol/commands.sh cert.txt tab $option summ $runmin $runmax > outputs/$era'lumi.txt'
         done        
         # Detailed csv per year
         echo "Create full csv for $YEAR"
-        docker run -v $(pwd):/mnt/vol -w /home/brilws gitlab-registry.cern.ch/cms-cloud/brilws-docker /bin/bash /mnt/vol/commands.sh cert.txt csv byls > outputs/$YEAR'lumibyls.csv'
+        docker run -v $(pwd):/mnt/vol -w /home/brilws gitlab-registry.cern.ch/cms-cloud/brilws-docker /bin/bash /mnt/vol/commands.sh cert.txt csv $option byls > outputs/$TYPE'_'$YEAR'lumibyls.csv'
         echo ls -l outputs `ls -l outputs`
         echo pwd `pwd`
     # Build the luminosity record
     - name: Build the luminosity records
       id: record
       run: |
-        python code/lumi_records.py > outputs/lumi_records.json
+        python code/lumi_records.py $RECID $YEAR ${ERAS[0]} $TYPE > outputs/lumi_records.json
         #python code/vali_records.py > outputs/vali_records.json
     # upload output file NB bug: https://github.com/actions/upload-artifact/issues/294
     - name: Upload artifact
-      uses: actions/upload-artifact@v2
+      uses: actions/upload-artifact@v3
       with:
         name: output
-        path: cms-YYYY-luminosity/outputs/
-   
+        path: cms-YYYY-luminosity/outputs/

cms-YYYY-luminosity/README.md

@@ -4,4 +4,12 @@ This directory contains helper scripts to prepare CMS luminosity information rec
 
 - subdirectory `code` contains the script to prepare the luminosity information record
 - shell scripts `commands.sh` and `prescale.sh` are used in workflow `cms-luminosity-tables.yaml` in `.github/workflows` to build the luminosity information tables to be attached in the luminosity record 
-- subdirectory `inputs` contains the "normtag" file needed for the luminosity calculation.
+- subdirectory `inputs` contains the "normtag" file needed for the luminosity calculation.
+
+The inputs are given as GitHub workflow variables and are
+
+- 2013 pPb: 1056 2013 HIRun2013 pPb
+- 2013 ppref: 1057 2013 Run2013A pphiref
+- 2015 ppref: 1058 2015 Run2015E pphiref
+- 2015 pp: 1055 2015 Run2015D pp (not used to build the current record, but tested to produce the same files)
+- 2016 pp: 1059 2016 Run2016G Run2016H pp

cms-YYYY-luminosity/code/lumi_records.py

@@ -9,36 +9,76 @@
 
 """
 Create a luminosity record.
-"""
-
-RECID_START = 1055
-YEAR_RELEASED = 2015
-
+Input arguments
+- recid (May 2023, the last used recid was 1055 for 2015 pp)
+- year to be released
+- run era (for pp, give one of the released eras, it is needed to get the collision energy, stored per era)
+- type (pp, pPb, pphiref, PbPb)
+
+Inputs:
+- 2013 pPb: 1056 2013 HIRun2013 pPb
+- 2013 ppref: 1057 2013 Run2013A pphiref
+- 2015 ppref: 1058 2015 Run2015E pphiref
+- 2015 pp: 1055 2015 Run2015D pp (not used to build the current record, but tested to produce the same files)
 
+"""
 
-def create_record(recid, year, uncertainty, lumi_ref, val_recid):
+def create_record(recid, year, era, runtype, uncertainty, lumi_ref, val_recid):
     """Create record for the given year."""
 
     rec = {}
 
-    year = str(year)
     year_created = year
     year_published = datetime.date.today().strftime("%Y")
+    runtype = str(runtype)
+    if "pphiref" in runtype :
+        display_runtype = 'pp'
+    else:
+        display_runtype = runtype
+        
+    # Get the energy
+    # Using the run_era, for pp it is needed only here
+    # Could be done differently but this is good enough
+    url = 'http://api-server-cms-release-info.app.cern.ch/runeras/?run_era='+era
+    this_json=json.loads(requests.get(url).text.strip())
+    energy=this_json[0]["energy"]
+
     # NB the reference needs to be to cds for this to work:
     url = lumi_ref+'/?of=tm&ot=245__a'
     lumi_ref_title = requests.get(url).text.strip()
 
+    pp_text=''
+    if "pphiref" in runtype:
+        collision_text = energy+' proton-proton collision data, needed as reference data for heavy-ion data analysis,'
+    elif "PbPb" in runtype:
+        collision_text = energy+' PbPb heavy-ion collision data'
+    elif "pPb" in runtype:
+        collision_text = energy+' proton-Pb heavy-ion collision data'
+    elif "pp" in runtype:
+        collision_text = energy+' proton-proton collision data'
+        run_range_input = year
+        pp_text='(The integrated luminosity for validated runs and luminosity sections of all '+year+' p-p data taking is available in '+year+'lumi.txt.)'
+    else:
+        print('Runtype unknown!')
+
+# normtag file only after Run-1
+    normtag_text=''
+    if int(year) > 2014:
+        normtag_text='The luminometer giving the best value for each luminosity section is recorded in a <strong>normtag</strong> file <a href=\"/record/'+str(recid)+'/files/normtag_PHYSICS_'+runtype+'_'+year+'.json\">normtag_PHYSICS_'+runtype+'_'+year+'.json</a> that is used in the luminosity calculation.'
+
     rec["abstract"] = {}
 
-    url = 'http://api-server-cms-release-info.app.cern.ch/runeras/run_era?year='+year+'&released=yes'
+    url = 'http://api-server-cms-release-info.app.cern.ch/runeras/run_era?year='+year+'&type='+runtype+'-phys&released=yes'
     od_runs = json.loads(requests.get(url).text.strip())
 
+# The use of variable in string with two different notations could be fixed in the following...
+# NB in the  +var+ notation, var needs to be a string
     rec["abstract"]["description"] = (
-            "<p>CMS measures the luminosity using different luminometers (luminosity detectors) and algorithms. The luminometer giving the best value for each luminosity section is recorded in a 'normtag' file <a href=\"/record/%s/files/normtag_PHYSICS_%s.json\">normtag_PHYSICS_%s.json</a> that is used in the luminosity calculation.</p>" % (recid, year, year)
-            + "<p>The integrated luminosity for validated runs and luminosity sections of the %s public data (%s) is available in %slumi.txt. (The integrated luminosity for validated runs and luminosity sections of all %s p-p data taking is available in %slumi.txt.)</p>" % (year, ",".join(od_runs), ",".join(od_runs), year, year)
-            + "<p> For luminosity calculation, a detailed list of luminosity by lumi section is provided in <a href=\"/record/%s/files/%slumibyls.csv\">%slumibyls.csv</a> for the <a href=\"/record/%s\">list of validated runs</a> and lumi sections.</p>" % (recid, year, year, val_recid)
+            "<p>CMS measures the luminosity using different luminometers (luminosity detectors) and algorithms. "+normtag_text+"</p>"
+            "<p>The integrated luminosity for validated runs and luminosity sections of the %s taken in %s (%s) is available in %slumi.txt. %s</p>" % (collision_text, year, ",".join(od_runs), ",".join(od_runs), pp_text)
+            + "<p> For luminosity calculation, a detailed list of luminosity by lumi section is provided in <a href=\"/record/%s/files/%s_%slumibyls.csv\">%s_%slumibyls.csv</a> for the <a href=\"/record/%s\">list of validated runs</a> and lumi sections.</p>" % (recid, runtype, year, runtype, year, val_recid)
             + "<p>The uncertainty in the luminosity measurement of %s data should be considered as %s%% (reference <a href=\"%s\">%s</a>).</p>" % (year, uncertainty, lumi_ref, lumi_ref_title)
-            + "<p>In your estimate for the integrated luminosity, check for which runs the trigger you have selected is active and sum the values for those runs. If you are using prescaled triggers, you can find the trigger prescale factors as shown in <a href=\"/record/5004\">the trigger examples</a>. The change of prescales (run, lumi section, index of prescales) is recorded in <a href=\"/record/%s/files/prescale%s.csv\">prescale%s.csv</a></p>" % (recid, year, year)
+            + "<p>In your estimate for the integrated luminosity, check for which runs the trigger you have selected is active and sum the values for those runs. For prescaled triggers, the change of prescales (run, lumi section, index of prescales referring to the PrescaleService module in the High-Level Trigger configuration files) is recorded in <a href=\"/record/%s/files/prescale_%s%s.csv\">prescale_%s%s.csv</a>.</p>" % (recid, runtype, year, runtype, year)
             + "<p>Additional information on how to extract luminosity values using the <strong>brilcalc tool</strong> can be found in the <a href=\"/docs/cms-guide-luminosity-calculation\"> luminosity calculation guide</a>.</p>"
         )
 
@@ -52,8 +92,8 @@ def create_record(recid, year, uncertainty, lumi_ref, val_recid):
     ]
 
     rec["collision_information"] = {}
-    rec["collision_information"]["energy"] = "13TeV"
-    rec["collision_information"]["type"] = "pp"
+    rec["collision_information"]["energy"] = energy
+    rec["collision_information"]["type"] = display_runtype
 
     rec["date_created"] = [
         year_created,
@@ -85,13 +125,12 @@ def create_record(recid, year, uncertainty, lumi_ref, val_recid):
         }
     ]
 
-    url = 'http://api-server-cms-release-info.app.cern.ch/runeras/run_era?year='+year+'&type=pp-phys'
+    url = 'http://api-server-cms-release-info.app.cern.ch/runeras/run_era?year='+year+'&type='+runtype+'-phys'
     #rec["run_period"] = read_run_periods(year, 'pp-phys')
     rec["run_period"] = json.loads(requests.get(url).text.strip())
 
     rec["title"] = (
-        "CMS luminosity information, for %s CMS open data"
-        % year
+        "CMS luminosity information for "+collision_text+" taken in "+year
         )
 
     rec["type"] = {}
@@ -107,8 +146,10 @@ def main():
     "Do the job."
 
     records = []
-    recid = RECID_START
-    year = str(YEAR_RELEASED)
+    recid = sys.argv[1]
+    year = sys.argv[2]
+    era = sys.argv[3]
+    runtype = sys.argv[4]
 
     # this would read from the local json file
     # with open('./inputs/cms_release_info.json') as f:
@@ -118,14 +159,16 @@ def main():
     # all_years = json.loads(data)    
     # this_year = all_years[year]
 
-    # this gets json from the api server
-    url = 'http://api-server-cms-release-info.app.cern.ch/years?year='+year+'&output=plain'
-    this_year = json.loads(requests.get(url).text.strip())
+    # this gets json from the api server 
+    url = 'http://api-server-cms-release-info.app.cern.ch/years?year='+year+'&type='+runtype+'&output=plain'
+    this_year = json.loads(requests.get(url).text.strip()) 
 
     records.append(
         create_record(
             recid,
-            this_year["year"],
+            year,
+            era,
+            runtype,            
             this_year["lumi_uncertainty"],
             this_year["luminosity_reference"],
             this_year["val_json"][0]["recid"]) # This requires the json files to be in a specific order, with "golden" first

cms-YYYY-luminosity/commands.sh

@@ -2,24 +2,40 @@
 # parameters: 
 #   $1 certified run json
 #   $2 output style csv or tab
-#   $3 summary or byls (if not byls, no option taken)
-#   $4 run range first
-#   $5 run range last
+#   $3 use normtag file or pxl
+#   $4 summary or byls (if not byls, no option taken)
+#   $5 run range first
+#   $6 run range last
 # exit on error
 set -e
 
 cert=$1
 style=$2
-if [ "$3" == "byls" ]; then mode="--"$3; fi;
 
-if [  -z "$4" ]
+#echo "# cert file:"
+#cat /mnt/vol/$cert
+if [ "$3" == "normtag" ]
+then 
+  option="--normtag /mnt/vol/normtag_PHYSICS.json"
+  #echo "# normtag file:"
+  #cat /mnt/vol/normtag_PHYSICS.json
+else
+  option="--type $3"
+fi
+#echo "#from commands: value of options is $option, value of input is $3" 
+
+if [ "$4" == "byls" ]; then mode="--"$4; fi;
+
+if [  -z "$5" ]
 then
-  brilcalc lumi -c web $mode -i /mnt/vol/$cert -u /fb --normtag /mnt/vol/normtag_PHYSICS.json --output-style $style
-elif [ -z "$5" ]
+  echo "#Brilcalc command:  brilcalc lumi -c web $mode -i /mnt/vol/$cert -u /fb $option --output-style $style"
+  brilcalc lumi -c web $mode -i /mnt/vol/$cert -u /fb $option --output-style $style
+elif [ -z "$6" ]
 then
   echo "Give maximum range" 
 else
-  runmin=$4
-  runmax=$5
-  brilcalc lumi -c web $mode --begin $runmin --end $runmax -i /mnt/vol/$cert -u /fb --normtag /mnt/vol/normtag_PHYSICS.json  --output-style $style
+  runmin=$5
+  runmax=$6
+  echo "#Brilcalc command:  brilcalc lumi -c web $mode --begin $runmin --end $runmax -i /mnt/vol/$cert -u /fb $option  --output-style $style"
+  brilcalc lumi -c web $mode --begin $runmin --end $runmax -i /mnt/vol/$cert -u /fb $option  --output-style $style
 fi