feat: QADB for RG-A Fa18 (#254)

bin/error-print.sh

@@ -6,6 +6,7 @@
 cmd() {
   grep -HE '.*' /farm_out/$LOGNAME/clas12-timeline--*.err |\
     grep -vE '\[DataSourceDump\] --> opened file with events #' |\
+    grep -vE 'Picked up _JAVA_OPTIONS:' |\
     grep --color -E '^.*\.err:'
 }
 

bin/run-physics-timelines.sh

@@ -86,11 +86,16 @@ mkdir -p $logDir
 logFile=$logDir/physics.err
 logTmp=$logFile.tmp
 > $logFile
-function exe { 
+function exe {
   echo $sep
   echo "EXECUTE: $*"
   echo $sep
   $* 2> >(tee $logTmp >&2)
+  mv $logTmp{,.bak}
+  cat $logTmp.bak |\
+    { grep -v '^Picked up _JAVA_OPTIONS:' || test $? = 1; } \
+    > $logTmp
+  rm $logTmp.bak
   if [ -s $logTmp ]; then
     echo "stderr from command:  $*" >> $logFile
     cat $logTmp >> $logFile

qa-physics/QA/modifyQaTree.groovy

@@ -2,6 +2,7 @@ import groovy.json.JsonSlurper
 import groovy.json.JsonOutput
 import java.util.Date
 import org.jlab.clas.timeline.util.Tools
+import org.rcdb.*
 Tools T = new Tools()
 
 
@@ -13,6 +14,7 @@ usage["delbit"]     = "delete specified bit from defectBit(s)"
 usage["sectorloss"] = "specify a sector loss"
 usage["lossft"]     = "specify a FT loss"
 usage["nobeam"]     = "add 'PossiblyNoBeam' bit"
+usage["misc"]       = "add the Misc bit, with default comment from shift expert"
 usage["setcomment"] = "change or delete the comment"
 usage["addcomment"] = "append a comment"
 usage["custom"]     = "do a custom action (see code)"
@@ -176,8 +178,7 @@ else if(cmd=="sectorloss") {
       -$helpStr
       - set [lastBin] to -1 to denote last time bin of run
       - use \"all\" in place of [list_of_sectors] to apply to all sectors
-      - this will set the SectorLoss bit for specified time bins and sectors;
-        it will unset any other relevant bits
+      - this will add the SectorLoss bit for specified time bins and sectors
       - you will be prompted to enter a comment
     """)
     System.exit(101)
@@ -236,9 +237,73 @@ else if(cmd=="nobeam") {
   }
 }
 
+else if(cmd=="misc") {
+  def rnum,bnumL,bnumR
+  def secList = []
+  if(args.length>1) {
+    rnum = args[1].toInteger()
+    bnumL = args.length < 3 ?  0 : args[2].toInteger()
+    bnumR = args.length < 4 ? -1 : args[3].toInteger()
+    if(args.length<5 || args[4]=="all") secList = (1..6).collect{it}
+    else (4..<args.length).each{ secList<<args[it].toInteger() }
+
+    def db = RCDB.createProvider("mysql://rcdb@clasdb/rcdb")
+    def shift_expert_comment = null
+    try {
+      db.connect()
+      shift_expert_comment = db.getCondition(Long.valueOf(rnum), 'user_comment').toString()
+    }
+    catch(Exception e) {
+      System.err.println("Unable to connect to RCDB provider")
+      System.exit(100)
+    }
+    if(shift_expert_comment == null) {
+      System.err.println("Failed to get shift expert's comment from RCDB")
+      System.exit(100)
+    }
+
+    println("run $rnum bins ${bnumL}-"+(bnumR==-1 ? "END" : bnumR) +
+      " sectors ${secList}: add Misc bit with shift expert's comment")
+    println("\n---------------- Shift Expert's Comment ----------------\n${shift_expert_comment}\n--------------------------------------------------------\n")
+    println("Enter a different comment, if you want, otherwise press return to use this one")
+    print("> ")
+    def cmt = System.in.newReader().readLine()
+    if(cmt == "") {
+      cmt = shift_expert_comment
+    }
+
+    qaTree["$rnum"].each { k,v ->
+      def qaFnum = k.toInteger()
+      if( qaFnum>=bnumL && ( bnumR==-1 || qaFnum<=bnumR ) ) {
+        secList.each{
+          qaTree["$rnum"]["$qaFnum"]["sectorDefects"]["$it"] += T.bit("Misc")
+        }
+        recomputeDefMask(rnum,qaFnum)
+        if(cmt.length()>0) {
+          qaTree["$rnum"]["$qaFnum"]["comment"] = cmt
+        }
+      }
+    }
+
+  }
+  else {
+    def helpStr = usage["$cmd"]
+    System.err.println(
+    """
+    SYNTAX: ${exe} ${cmd} [run] [firstBin (default=0)] [lastBin (default=-1)] [list_of_sectors (default=all)]
+      -$helpStr
+      - set [lastBin] to -1 to denote last time bin of run
+      - use \"all\" in place of [list_of_sectors] to apply to all sectors
+      - you will be prompted to enter a different comment, if you do not
+        want to use the shift expert comment
+    """)
+    System.exit(101)
+  }
+}
+
 else if(cmd=="lossft") {
   def rnum,bnumL,bnumR
-  if(args.length>4) {
+  if(args.length>3) {
     rnum = args[1].toInteger()
     bnumL = args[2].toInteger()
     bnumR = args[3].toInteger()
@@ -272,8 +337,7 @@ else if(cmd=="lossft") {
     SYNTAX: ${exe} ${cmd} [run] [firstBin] [lastBin]
       -$helpStr
       - set [lastBin] to -1 to denote last time bin of run
-      - this will set the LossFT bit for specified time bins;
-        it will unset any other relevant bits
+      - this will add the LossFT bit for specified time bins
       - you will be prompted to enter a comment
     """)
     System.exit(101)
@@ -357,9 +421,17 @@ else if( cmd=="custom") {
     def cmt = "setup period; possible beam modulation issues"
     */
 
-    ///* // add misc bit to sector 6 only
+    /* // add misc bit to sector 6 only
     qaTree["$rnum"]["$bnum"]["sectorDefects"]["6"] += T.bit("Misc")
     def cmt = "FADC failure in ECAL sector 6; see https://logbooks.jlab.org/entry/3678262"
+    */
+
+    ///* // low helicity fraction
+    def secList = (1..6).collect{it}
+    secList.each{
+      qaTree["$rnum"]["$bnum"]["sectorDefects"]["$it"] += T.bit("Misc")
+    }
+    def cmt = "fraction of events with defined helicity is low"
     //*/
 
     if(!qaTree["$rnum"]["$bnum"].containsKey("comment")) {

qa-physics/README.md

@@ -144,10 +144,15 @@ First step is to read DST or Skim files, producing HIPO files and data tables
       * `grN*`: N vs. time bin
       * `grT*`: livetime vs. time bin
 
-### Automated QA of Normalized Electron Yield
+### Automatic QA
 This section will run the automated QA of the FC-charge normalized electron yield (N/F); it will ultimately
 generate QA timelines, and a `json` file which is used for the manual followup QA
 
+* Decide cut definitions
+  * add the file `cutdefs/${dataset}.json`; you may copy one of the existing ones, most likely
+    the default one
+  * after generating the automatic QA, check to make sure the results look reasonable; if they
+    do not, tune the settings in this file (see `qaCut.groovy` to see how the numbers are used)
 * Determine epoch lines
   * At the moment, this step is manual, but could be automated in a future release
   * You need to generate `epochs/epochs.${dataset}.txt`, which is a list epoch boundary lines

qa-physics/cutdefs/default.json

@@ -0,0 +1,19 @@
+{
+  "OutlierFD": {
+    "IQR_cut_factor": 4.0
+  },
+  "OutlierFT": {
+    "IQR_cut_factor": 4.0
+  },
+  "LowLiveTime": {
+    "min_live_time": 0.9
+  },
+  "ChargeHigh": {
+    "IQR_cut_factor": 4.0
+  },
+  "PossiblyNoBeam": {
+    "max_num_events": 40000,
+    "max_num_electrons": 100,
+    "max_FC_charge": 20
+  }
+}

qa-physics/cutdefs/rga_fa18_inbending_nSidis.json

@@ -0,0 +1,19 @@
+{
+  "OutlierFD": {
+    "IQR_cut_factor": 4.0
+  },
+  "OutlierFT": {
+    "IQR_cut_factor": 4.0
+  },
+  "LowLiveTime": {
+    "min_live_time": 0.9
+  },
+  "ChargeHigh": {
+    "IQR_cut_factor": 4.0
+  },
+  "PossiblyNoBeam": {
+    "max_num_events": 125000,
+    "max_num_electrons": 100,
+    "max_FC_charge": 150
+  }
+}

qa-physics/cutdefs/rga_fa18_outbending_nSidis.json

@@ -0,0 +1,19 @@
+{
+  "OutlierFD": {
+    "IQR_cut_factor": 4.0
+  },
+  "OutlierFT": {
+    "IQR_cut_factor": 4.0
+  },
+  "LowLiveTime": {
+    "min_live_time": 0.9
+  },
+  "ChargeHigh": {
+    "IQR_cut_factor": 4.0
+  },
+  "PossiblyNoBeam": {
+    "max_num_events": 125000,
+    "max_num_electrons": 100,
+    "max_FC_charge": 150
+  }
+}

qa-physics/cutdefs/rga_sp19.json

@@ -0,0 +1,19 @@
+{
+  "OutlierFD": {
+    "IQR_cut_factor": 4.0
+  },
+  "OutlierFT": {
+    "IQR_cut_factor": 4.0
+  },
+  "LowLiveTime": {
+    "min_live_time": 0.9
+  },
+  "ChargeHigh": {
+    "IQR_cut_factor": 4.0
+  },
+  "PossiblyNoBeam": {
+    "max_num_events": 40000,
+    "max_num_electrons": 100,
+    "max_FC_charge": 20
+  }
+}

qa-physics/cutdefs/rga_sp19_nSidis.json

@@ -0,0 +1 @@
+rga_sp19.json

qa-physics/cutdefs/rga_sp19_prescaled.json

@@ -0,0 +1 @@
+rga_sp19.json

qa-physics/cutdefs/rgc_su22.json

@@ -0,0 +1,19 @@
+{
+  "OutlierFD": {
+    "IQR_cut_factor": 4.0
+  },
+  "OutlierFT": {
+    "IQR_cut_factor": 4.0
+  },
+  "LowLiveTime": {
+    "min_live_time": 0.9
+  },
+  "ChargeHigh": {
+    "IQR_cut_factor": 4.0
+  },
+  "PossiblyNoBeam": {
+    "max_num_events": 40000,
+    "max_num_electrons": 100,
+    "max_FC_charge": 20
+  }
+}

qa-physics/cutdefs/rgc_su22_prescaled.json

@@ -0,0 +1 @@
+rgc_su22.json

qa-physics/cutdefs/rgc_su22_sidisdvcs.json

@@ -0,0 +1 @@
+rgc_su22.json

qa-physics/notes/rga_fa18.md

@@ -0,0 +1,44 @@
+# Run Group A, Fall 2019, Pass 2
+
+## Run monitoring
+
+> [!IMPORTANT]
+> Check any run-dependent settings in `qa-physics/monitorRead.groovy`, such as beam energy.
+
+We will use the `nSidis` train.
+
+First make sure all skim files are cached:
+```bash
+bin/run-monitoring.sh -d rga_fa18_inbending_nSidis  --check-cache --flatdir --focus-physics /cache/clas12/rg-a/production/recon/fall2018/torus-1/pass2/main/train/nSidis
+bin/run-monitoring.sh -d rga_fa18_outbending_nSidis --check-cache --flatdir --focus-physics /cache/clas12/rg-a/production/recon/fall2018/torus+1/pass2/train/nSidis
+```
+then run monitoring
+```bash
+bin/run-monitoring.sh -d rga_fa18_inbending_nSidis  --submit --flatdir --focus-physics /cache/clas12/rg-a/production/recon/fall2018/torus-1/pass2/main/train/nSidis
+bin/run-monitoring.sh -d rga_fa18_outbending_nSidis --submit --flatdir --focus-physics /cache/clas12/rg-a/production/recon/fall2018/torus+1/pass2/train/nSidis
+```
+
+## Double check that we have all the runs
+
+> [!IMPORTANT]
+> In case any runs disappeared from `/cache` while running monitoring, be sure to cross check the output
+> runs with those from `/mss`
+
+## Make timelines
+
+Make the timelines:
+```bash
+bin/run-physics-timelines.sh -d rga_fa18_inbending_nSidis
+bin/run-physics-timelines.sh -d rga_fa18_outbending_nSidis
+```
+
+Deploy either to your area or the common area (remove the `-D` option once you confirm this is the correct directory):
+```bash
+# your area, for testing
+bin/deploy-timelines.sh -d rga_fa18_inbending_nSidis  -t $LOGNAME -D
+bin/deploy-timelines.sh -d rga_fa18_outbending_nSidis -t $LOGNAME -D
+
+# common area
+bin/deploy-timelines.sh -d rga_fa18_inbending_nSidis  -t rga/pass2/fa18/qa -D
+bin/deploy-timelines.sh -d rga_fa18_outbending_nSidis -t rga/pass2/fa18/qa -D
+```