[added] align and merge FC h2

analysis/cc_setup.sh

-ccenv python 3.9.1
+module add Programming_Languages/python/3.9.1
+module add Analysis/gnuplot
 python -m venv `hostname`_env
 source `hostname`_env/bin/activate
 python -m pip install --upgrade pip
-python -m pip install -r requirements.txt
+python -m pip install -r requirements.txt
\ No newline at end of file

analysis/etc/FC_align.yaml

-GAMMAFLASH_DETECTION_THRESHOLD_FRACTION = 1.0 / 3.0
-FASTER_SLOPE = 1.0
\ No newline at end of file

analysis/etc/FC_time_align.yaml

+GAMMAFLASH_DETECTION_THRESHOLD_FRACTION: 0.333
\ No newline at end of file

analysis/etc/fc_dof.yaml

+FC_DOF: 
+    U3O8: 27.3981
+    UF4: 27.285
+u_FC_DOF: 
+    U3O8: 0.005
+    UF4: 0.005
+    
+SPEED_OF_LIGHT: 0.299792458 # meters per nanoseconds
+TNT_TS_STEP: 10. # nanoseconds per timestamp
\ No newline at end of file

analysis/run_on_cc.sh

 #!/bin/bash
 
 #SBATCH --ntasks 4
-# #SBATCH --mem 10000
+#SBATCH --mem 10000
 #SBATCH --time 5-00:00:00             
 # #SBATCH --mail-user=ghenning@iphc.cnrs.fr 
 # #SBATCH --mail-type=END,FAIL
@@ -13,8 +13,8 @@ echo "# [STARTING]"
 date
 
 echo "# Setting up the environnement"
-ccenv python 3.9.1
-ccenv gnuplot
+module add Programming_Languages/python/3.9.1
+module add Analysis/gnuplot
 python -m venv `hostname`_env
 source `hostname`_env/bin/activate
 python -m pip install --upgrade pip
@@ -33,6 +33,8 @@ pwd && cd src/fill_into_h2/ && make binaries && cp bin/fill_into_h2.`hostname` .
 echo "# Running tasks"
 # echo "#... raw2histo"
 #./run_task tasks/do_raw2histo.py
+echo "#... FC time align"
+./run_task tasks/do_fc_timealign.py
 # echo "#... ge_time_align"
 #./run_task tasks/do_ge_timealign.py
 #echo "#... ge_calibrate"
@@ -41,11 +43,14 @@ echo "# Running tasks"
 #./run_task tasks/do_ge_calibrate_2D.py
 #echo "#... cut the bidim to the interesting region"
 #./run_task tasks/do_ge_inel.py
-echo "#... getting the transitions yields"
-./run_task tasks/do_extract_transitions_gnum.py
+#echo "#... getting the transitions yields"
+#./run_task tasks/do_extract_transitions_gnum.py
 
 echo "# Removing job venv"
 rm -rf `hostname`_env
 
 echo "... done"
 date 
+echo "..."
+echo " .."
+echo "  ."

analysis/scripts/calibrate1D.py

@@ -32,7 +32,7 @@ def calibrate1D(h: histo,
         b.lowedge = b.lowedge * slope + offset
         b.upedge = b.upedge * slope + offset
         b.upedge, b.lowedge = max(b.upedge, b.lowedge), min(b.upedge, b.lowedge)
-    new_h.frombins(new_h.bins)
+    new_h._frombins(new_h.bins)
     return new_h
 
 if __name__=="__main__":

analysis/tasks/do_alignrawh.py

-'''
-This file contain all the necessary steps for a raw .evt to histogram conversion
-'''
-
-from doit.tools import create_folder
-import yaml
-import pathlib
-from itertools import product, chain
-from random import uniform as rand_uniform
-
-from pyhisto import Histogram
-
-import evt_tools as tools
-
-
-the_mapping = yaml.load(open('etc/mapping.yaml'),
-                        Loader=yaml.Loader)
-detectors = the_mapping['detectors']
-FC_detectors = list(filter(lambda x:x.startswith('U'), detectors))
-Ge_detectors = list(filter(lambda x: not x.startswith('U'), detectors))
-
-validated_data_weeks = yaml.load(open('data/validated_data_list.yaml'),
-                               Loader=yaml.Loader)['data_weeks']
-
-GAMMAFLASH_DETECTION_THRESHOLD_FRACTION = 1.0 / 3.0
-
-
-
-def task_FC_simple_time_projection():
-    '''do a simple time projection to get the gamma flash'''
-    for FC_det, flavor, week in product(FC_detectors, 
-                                        ['all', 'clean'], 
-                                        validated_data_weeks):
-        
-        for FC_file in pathlib.Path().glob(f'output/raw/{week}/{FC_det}_*.h2_{flavor}.txt'):
-            number = str(FC_file).split('.')[0][-4:]
-            target_dir = f'output/fc_simple_t_proj/{week}'
-            target_file = target_dir + f'/{FC_det}_{number}_{flavor}.time_proj.h1.txt'
-            Ecut = yaml.load(open(f'etc/{FC_det}_Ecut4timecalib.yaml'),
-                                  Loader=yaml.Loader)['energy_cut']
-            yield {
-                'name': FC_file,
-                'actions': [(create_folder, (target_dir,)),
-                            f'./env.run scripts/project2D.py --axis=x --min={Ecut} {str(FC_file)} > {target_file}',
-                            f'./env.run scripts/Draw1Dh.py {target_file}',
-                           ],
-                'targets': [target_file],
-                'file_dep': [FC_file, f'etc/{FC_det}_Ecut4timecalib.yaml'],
-            }
-            
-def task_FC_find_γflash():
-    '''find gamma flash position'''
-    for FC_simpletimeproj in pathlib.Path().glob(f'output/fc_simple_t_proj/*/*.time_proj.h1.txt'):
-        target_file = str(FC_simpletimeproj).replace("fc_simple_t_proj", "gflash_position").replace(".time_projh1.txt", ".txt")
-        yield {
-            'name': FC_simpletimeproj,
-            'file_dep': [FC_simpletimeproj],
-            'targets': [target_file],
-            'actions': [(create_folder, (str(FC_simpletimeproj.parent).replace("fc_simple_t_proj", "gflash_position"),)),
-                        f'./env.run scripts/find_first_bin_above_threshold.py --behaviour="rel" --threshold={GAMMAFLASH_DETECTION_THRESHOLD_FRACTION} {FC_simpletimeproj} > {target_file}']
-        }
-        
-def task_check_γflash():
-    '''check gamma flash position'''
-    for FC_det, flavor in product(FC_detectors, ['all', 'clean']):
-        position_files = map(str, pathlib.Path().glob(f'output/gflash_position/*/*txt'))
-        position_files_str = " ".join(position_files)
-        target_file = f"output/gflash_position/{FC_det}_{flavor}.txt"
-        yield {
-            'name': target_file,
-            'file_dep': list(position_files),
-            'targets': [target_file],
-            'actions': [
-                f"cat {position_files_str} > {target_file}",
-                f"gnuplot -e 'set term png; set output \"{target_file}.png\"; plot \"{target_file}\" u 0:1 w p t\"\"'"
-            ],
-            'verbosity': 2,
-        }
-            
-def notask_FC_get_γflash():
-    '''get gamma flash position for FC detectors'''
-    def time_calib_this(targets, dependencies, *w, **kw):
-        '''Time calibrate one detector'''
-        print(f"# Making {targets} from {dependencies}")
-        the_FC = 'U3O8' if 'U3O8' in targets[0] else 'UF4'
-        # print(f"# Getting gamma flash positoin and time calibration from {dependencies[0]}")
-        this_histo = Histogram(fromfile=dependencies[0])
-        max_bin = max(this_histo)
-        fout = open(targets[0], 'w')
-        for b in this_histo:
-            if b.count > max_bin * GAMMAFLASH_DETECTION_THRESHOLD_FRACTION :
-                γflash_position = b.center()
-                fout.write(f"# found gamma flash at {γflash_position} \n")
-                fout.write(f"# {b} vs. {max_bin} \n")
-                the_offset = -γflash_position
-                fout.write('\n'.join([f"offset = {the_offset}"]))
-                return True
-            
-    for week, FC_det in product(validated_data_weeks, FC_detectors):
-        for FC_file in pathlib.Path().glob(f'output/raw/{week}/{FC_det}*h1_time.txt'):
-            num = str(FC_file).split(FC_det)[1][1:].split(".")[0]
-            yield { 
-                'name': str(FC_file),
-                'file_dep': [FC_file],
-                'targets': [f"output/alignh/{week}/{FC_det}_{num}.timecalib"],
-                'actions': [(time_calib_this,)],
-            }            
-
-
-def notask_create_directories():
-    '''create directories'''
-    _dir_to_create = validated_data_weeks
-    for _dir in _dir_to_create:
-        yield {
-            'name': _dir,
-            'actions': [
-                (create_folder, (f"output/alignh/{_dir}",)),
-                ],
-            'targets': [f"output/alignh/{_dir}"],
-            'file_dep': ['data/validated_data_list.yaml']
-            }
-
-        
-        
-        
-
-            
-            
-def task_FC_merge_h2():
-    '''merging time histograms'''
-    for FC_det, flavor in product(FC_detectors, ('clean', 'all')):
-        source_h2 = []
-        # Getting the file list with alignement values
-        for FC_h2 in pathlib.Path().glob(f'output/raw/*/{FC_det}*h2_{flavor}.txt'): 
-            #skipp non validated weeks
-            if not any( [week in str(FC_h2) for week in validated_data_weeks]):
-                #print(f"# skipping {FC_h2}")
-                continue
-            # find the alignement value
-            align_file = str(FC_h2).replace("raw", 'alignh').replace(f'h2_{flavor}.txt', 'timecalib')
-            align_offset = float(next(filter(lambda s:s.startswith('offset'), 
-                                         open(align_file).readlines())).split('=')[-1])
-            source_h2.append(f"{FC_h2}:{align_offset}")
-        #print(source_h2)
-        command_ = " ".join(["./env.run scripts/fill_into_h2.py",
-                             "--nxbins=8000 --xmin=-1000.0 --xmax=7000.",
-                             "--nybins=16384 --ymin=0.0 --ymax=23768.0",
-                             " ".join(source_h2),
-                             f"> output/alignh/{FC_det}.h2_{flavor}.merged.txt"])
-        #print(command_)
-        #end for files
-        yield {
-            'name': f"{FC_det}_{flavor}",
-            'file_dep': list(chain(pathlib.Path().glob(f'output/raw/*/{FC_det}*h2_{flavor}.txt'),
-                                  pathlib.Path().glob(f'output/alignh/*/{FC_det}*.timecalib'))),
-            'targets': [f"output/alignh/{FC_det}.h2_{flavor}.merged.txt"],
-            'actions': [command_,
-                        f'./env.run scripts/Draw2Dh.py --grid output/alignh/{FC_det}.h2_{flavor}.merged.txt',
-                        f'./env.run scripts/project2D.py --axis=x output/alignh/{FC_det}.h2_{flavor}.merged.txt > output/alignh/{FC_det}.h2_{flavor}.merged_px.txt',
-                        f'./env.run scripts/project2D.py --axis=y output/alignh/{FC_det}.h2_{flavor}.merged.txt > output/alignh/{FC_det}.h2_{flavor}.merged_py.txt',
-                        f'./env.run scripts/Draw1Dh.py output/alignh/{FC_det}.h2_{flavor}.merged_px.txt output/alignh/{FC_det}.h2_{flavor}.merged_py.txt'
-                       ],
-            #'verbosity': 2
-        }
-# EOF

analysis/tasks/do_fc_timealign.py

+'''
+This file contain all the necessary steps for a raw .evt to histogram conversion
+'''
+
+from doit.tools import create_folder
+import yaml
+import pathlib
+from itertools import product, chain
+from random import uniform as rand_uniform
+
+from pyhisto import Histogram
+
+import evt_tools as tools
+
+
+the_mapping = yaml.load(open('etc/mapping.yaml'),
+                        Loader=yaml.Loader)
+detectors = the_mapping['detectors']
+FC_detectors = list(filter(lambda x:x.startswith('U'), detectors))
+Ge_detectors = list(filter(lambda x: not x.startswith('U'), detectors))
+
+validated_data_weeks = yaml.load(open('data/validated_data_list.yaml'),
+                               Loader=yaml.Loader)['data_weeks']
+
+
+
+
+
+def task_FC_simple_time_projection():
+    '''do a simple time projection to get the gamma flash'''
+    for FC_det, flavor, week in product(FC_detectors, 
+                                        ['all', 'clean'], 
+                                        validated_data_weeks):
+        
+        for FC_file in pathlib.Path().glob(f'output/raw/{week}/{FC_det}_*.h2_{flavor}.txt'):
+            number = str(FC_file).split('.')[0][-4:]
+            target_dir = f'output/fc_simple_t_proj/{week}'
+            target_file = target_dir + f'/{FC_det}_{number}_{flavor}.time_proj.h1.txt'
+            Ecut = yaml.load(open(f'etc/{FC_det}_Ecut4timecalib.yaml'),
+                                  Loader=yaml.Loader)['energy_cut']
+            yield {
+                'name': FC_file,
+                'actions': [(create_folder, (target_dir,)),
+                            f'./env.run scripts/project2D.py --axis=x --min={Ecut} {str(FC_file)} > {target_file}',
+                            f'./env.run scripts/Draw1Dh.py {target_file}',
+                           ],
+                'targets': [target_file],
+                'file_dep': [FC_file, f'etc/{FC_det}_Ecut4timecalib.yaml'],
+            }
+            
+def task_FC_find_γflash():
+    '''find gamma flash position'''
+    for FC_simpletimeproj in pathlib.Path().glob(f'output/fc_simple_t_proj/*/*.time_proj.h1.txt'):
+        target_file = str(FC_simpletimeproj).replace("fc_simple_t_proj", "gflash_position").replace(".time_proj.h1.txt", ".txt")
+        GAMMAFLASH_DETECTION_THRESHOLD_FRACTION: float =  yaml.load(open('etc/FC_time_align.yaml'),
+                                                          Loader=yaml.Loader)['GAMMAFLASH_DETECTION_THRESHOLD_FRACTION']
+        yield {
+            'name': FC_simpletimeproj,
+            'file_dep': [FC_simpletimeproj, 'etc/FC_time_align.yaml'],
+            'targets': [target_file],
+            'actions': [(create_folder, (str(FC_simpletimeproj.parent).replace("fc_simple_t_proj", "gflash_position"),)),
+                        f'./env.run scripts/find_first_bin_above_threshold.py --behaviour="rel" --threshold={GAMMAFLASH_DETECTION_THRESHOLD_FRACTION} {FC_simpletimeproj} > {target_file}']
+        }
+        
+def task_check_γflash():
+    '''check gamma flash position'''
+    for FC_det, flavor in product(FC_detectors, ['all', 'clean']):
+        position_files = map(str, pathlib.Path().glob(f'output/gflash_position/*/{FC_det}_*_{flavor}.txt'))
+        position_files_str = " ".join(position_files)
+        target_file = f"output/gflash_position/{FC_det}_{flavor}.txt"
+        yield {
+            'name': target_file,
+            'file_dep': list(pathlib.Path().glob(f'output/gflash_position/*/{FC_det}_*_{flavor}.txt')), #list(position_files),
+            'targets': [target_file],
+            'actions': [
+                f"cat {position_files_str} > {target_file}",
+                f"gnuplot -e 'set term png; set output \"{target_file}.png\"; plot \"{target_file}\" u 0:1 w p t\"\"'"
+            ],
+        }
+            
+
+def task_get_time_calibration():
+    '''Compute the time shift value for time calibration'''
+    def FC_timecalib(FC_det: str,
+                     runfile: str,
+                     ymlfile: str,
+                     targetfile: str,):
+        '''inner function that does the job'''
+        yml_data = yaml.load(open(ymlfile), Loader=yaml.Loader)
+        fc_dof = yml_data['FC_DOF'][FC_det]
+        gflash_tof = fc_dof / yml_data['SPEED_OF_LIGHT']
+        run_gflash = float(open(runfile, 'r').readline().split()[0])
+        gf_shift = gflash_tof - yml_data['TNT_TS_STEP'] * run_gflash
+        open(targetfile, 'w').write(f'''slope: {yml_data["TNT_TS_STEP"]} \noffset: {gf_shift} \n''')
+        return
+    
+    for FC_det, flavor in product(FC_detectors, ['all', 'clean']):
+        for position_file in pathlib.Path().glob(f'output/gflash_position/*/{FC_det}_*_{flavor}.txt'):
+            target_file = str(position_file).replace("gflash_position", "fc_time_calib")
+            yield {
+                'name': target_file,
+                'file_dep': [position_file, 'etc/fc_dof.yaml'],
+                'targets': [target_file],
+                'actions': [
+                    (create_folder, (str(position_file.parent).replace("gflash_position", "fc_time_calib"),)),
+                    (FC_timecalib, None, {'FC_det': FC_det,
+                                          'runfile': position_file,
+                                          'ymlfile': 'etc/fc_dof.yaml',
+                                          'targetfile': target_file}),
+                ]
+            }
+        
+        
+def task_FC_calibrate_1D_time():
+    '''calibrate the clean and all simple time projection spectra for Ge'''
+    
+    for FC_det, week in product(FC_detectors,
+                                validated_data_weeks):
+        for run_file in pathlib.Path().glob(f'output/fc_simple_t_proj/{week}/{FC_det}_*.time_proj.h1.txt'):
+            calib_file = str(run_file).replace('fc_simple_t_proj', 'fc_time_calib').replace('.time_proj.h1.txt', '.txt')
+            target_file = str(run_file).replace('fc_simple_t_proj', 'fc_time_aligned')
+            # if calib_file is missing...
+            yield {
+                'name': target_file,
+                'file_dep': [run_file, calib_file],
+                'targets': [target_file],
+                'actions': [
+                    (create_folder, (str(run_file.parent).replace("fc_simple_t_proj", "fc_time_aligned"),)),
+                    f'./env.run scripts/calibrate1D.py --offset=`grep offset {calib_file} | cut -d ":" -f 2| xargs` --slope=`grep slope {calib_file} | cut -d ":" -f 2| xargs` {run_file} > {target_file}',
+                    f'./env.run scripts/Draw1Dh.py {target_file}',
+                ]
+            }
+
+def task_FC_merge_time_aligned():
+    '''merge time algined spectra'''
+    for FC_det, flavor in product(FC_detectors, ['all', 'clean']):
+        target_file = f"output/fc_time_aligned/{FC_det}_{flavor}.time_proj.merged.h1.txt"
+        source_h1 = []
+        for time_spect in pathlib.Path().glob(f'output/fc_time_aligned/*/{FC_det}*{flavor}*.time_proj.h1.txt'): 
+            if not any( [week in str(time_spect) for week in validated_data_weeks]):
+                continue
+            source_h1.append(str(time_spect))
+        command_ = " ".join(["./env.run scripts/fill_into_histogram.py",
+                             "--nbins=7000 --xmin=-200.0 --xmax=6800.",
+                             " ".join(source_h1),
+                             f"> {target_file}"])
+        yield {
+            'name': f"{FC_det}_{flavor}",
+            'file_dep': list(pathlib.Path().glob(f'output/fc_time_aligned/*/{FC_det}*{flavor}*.time_proj.h1.txt')),
+            'targets': [target_file],
+            'actions': [command_,
+                        f'./env.run scripts/Draw1Dh.py --yscale=log {target_file}'
+                       ],
+            #'verbosity': 2
+        }
+            
+def task_calibrate_FC_2D():
+    '''performs time and energy calibration on Ge bidims'''
+    for FC_det in FC_detectors:
+        for flavor in ('all', 'clean'):
+            for week in validated_data_weeks:
+                for h2_file in pathlib.Path().glob(f'./output/raw/{week}/{FC_det}_*.h2_{flavor}.txt'):
+                    zh2_file = str(h2_file)
+                    #print(zh2_file)
+                    file_number = zh2_file.split('/')[-1].split('_')[1].split('.')[0]
+                    time_params =  str(list(pathlib.Path().glob(f'./output/fc_time_calib/{week}/{FC_det}_{file_number}_{flavor}.txt'))[0])
+                    time_offset = yaml.load(open(time_params), Loader=yaml.Loader)['offset']
+                    time_slope = yaml.load(open(time_params), Loader=yaml.Loader)['slope']
+                    energy_slope, energy_offset = 1.0, 0.0
+                    target_file = zh2_file.replace("raw", "calibrated_2D")
+                    yield {
+                        'name': f"{FC_det} {flavor} {week} {file_number}",
+                        'file_dep': [zh2_file, time_params],
+                        'targets': [target_file, 
+                                   pathlib.Path( f'{target_file}_time.txt'), pathlib.Path(f'{target_file}_energy.txt'),
+                                   pathlib.Path(f'{target_file}.png')],
+                        'actions': [
+                            (create_folder, (str(h2_file.parent).replace("raw", "calibrated_2D"),)),
+                            f"""./env.run ./scripts/calibrate2D.py {zh2_file} --xoffset={time_offset} --xslope={time_slope} --yoffset={energy_offset} --yslope={energy_slope} > {target_file}""",
+                            f'./env.run scripts/project2D.py --axis=x {target_file} > {target_file}_time.txt',
+                            f'./env.run scripts/project2D.py --axis=y {target_file} > {target_file}_energy.txt',
+                            f'./env.run scripts/Draw1Dh.py {target_file}_energy.txt',
+                            f'./env.run scripts/Draw1Dh.py --yscale=log {target_file}_time.txt',
+                            f"gnuplot -e \"set term pngcairo; set output '{target_file}.png';set xrange[0:7000]; set yrange[0:16000]; set zrange[0:20000]; set view map; splot '{target_file}' t '' w d\"",
+                        ],
+                        'verbosity': 2
+                    }
+                    
+def task_merge_calibrated_bidim():
+    '''Merge the claibrated H2 into one file'''
+    for FC_det in FC_detectors:
+        for flavor in ('all', 'clean'):
+            # define target params
+            target_dir = './output/calibrated_2D'
+            target_file = f'{target_dir}/{FC_det}_merged.h2_{flavor}.txt'
+            files_list = []
+            for week in validated_data_weeks:
+                for h2_file in pathlib.Path().glob(f'./output/calibrated_2D/{week}/{FC_det}_*.h2_{flavor}.txt'):
+                    files_list.append(str(h2_file))
+            command_ = " ".join(["./bin/fill_into_h2.`hostname`",
+                             "--xnbins=7000 --xmin=-200.0 --xmax=6800.",
+                             "--ynbins=10000 --ymin=0.0 --ymax=20000.0",
+                             " ".join(files_list),
+                             f"> {target_file}"])
+            
+            yield {
+                'name': f"{FC_det}, {flavor}",
+                'file_dep': files_list,
+                'targets': [target_file],
+                'actions': [
+                    (create_folder, (target_dir,)),
+                    command_,
+                ]
+            }
+            
+def task_check_merged_h2():
+    '''prooject and draw the h2'''
+    for FC_det in FC_detectors:
+        for flavor in ('all', 'clean'):
+            source_file = f"./output/calibrated_2D/{FC_det}_merged.h2_{flavor}.txt"
+            yield {
+                'name': source_file,
+                'file_dep': [source_file],
+                'actions': [
+                    f'./env.run scripts/project2D.py --axis=x {source_file} > {source_file}_px.txt',
+                    f'./env.run scripts/project2D.py --axis=y {source_file} > {source_file}_py.txt',
+                    f'./env.run scripts/Draw1Dh.py --yscale=log {source_file}_py.txt',
+                    f'./env.run scripts/Draw1Dh.py {source_file}_px.txt',
+                    f"gnuplot -e \"set term pngcairo; set output '{source_file}.png';set xrange[0:10000]; set yrange[0:10000]; set zrange[0:20000]; set view map; splot '{source_file}' t '' w d\"",
+                ],
+                'targets': [
+                    pathlib.Path(f'{source_file}_px.txt'), pathlib.Path(f'{source_file}_px.txt.png'), 
+                    pathlib.Path(f'{source_file}_py.txt'), pathlib.Path(f'{source_file}_py.txt.png'),
+                    pathlib.Path(f'{source_file}.txt.png')
+                ]
+            }
+             
+                
\ No newline at end of file