Skip to content
Snippets Groups Projects
Commit c32c3fcc authored by Carlo Ferrigno's avatar Carlo Ferrigno
Browse files

Removes xlim and re-order options

parent b2a6f9b1
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Generic Revolution LC.ipynb
+ 17
− 18
View file @ c32c3fcc
%% Cell type:code id:f42afc40 tags:parameters
``` python
import os,sys
rev_num = 0
use_isgri = True
use_jemx1 = True
use_jemx2 = True
E1_keV = "28.0"
E2_keV = "40.0"
J_E1_keV = "3.0"
J_E2_keV = "20.0"
osa_version = 'OSA11.2'
detection_threshold = 7
host_type = 'staging'
isgri_systematic_fraction = 0.015
jemx_systematic_fraction = 0.05
lc_time_bin = 3000
data_version = 'CONS'
output_yaml_dir = 'rev' # A folder to store the yaml file
token=''
batch_run = False
notebooks_folder = os.getcwd()
```
%% Cell type:code id:f2f901c0 tags:
``` python
import sys
import oda_integral_wrapper.wrapper
from oda_api.plot_tools import OdaLightCurve
```
%% Cell type:code id:d2508e02 tags:
``` python
import oda_integral_wrapper.itime as itime
if rev_num <= 0:
now = itime.now()
rev_num = '%04d' % (int(now.REVNUM) + rev_num)
data_version = 'NRT'
use_jemx2 = False
print("We force NRT data and no use of JEM-X2")
else:
rev_num = '%04d' % int(rev_num)
print("Revolution ", rev_num)
```
%% Cell type:code id:97950e40 tags:
``` python
import logging
#logging.getLogger().setLevel(logging.WARNING)
logging.getLogger().setLevel(logging.INFO) #for more verbose logging
logging.getLogger('').addHandler(logging.StreamHandler())
```
%% Cell type:code id:0ba01826 tags:
``` python
import oda_api.token
if token == '':
token = oda_api.token.discover_token()
oda_api.token.decode_oda_token(token)
```
%% Cell type:code id:c2c1b193 tags:
``` python
import yaml
wrap=oda_integral_wrapper.wrapper.INTEGRALwrapper(token=token, integral_data_rights='all-private',
host_type=host_type)
#print(wrap.disp.url)
observation_title="rev. "+rev_num
yaml_file_path = os.path.join('rev', 'rev_%s.yaml' % rev_num)
#print(observation_title)
try:
output_get = wrap.disp.get_yaml_files_observation_with_title(observation_title=observation_title, token=token)
observations = yaml.safe_load(output_get['file_content'])
if output_get['file_content'] == '':
try:
with open(yaml_file_path) as file:
observations = yaml.load(file, Loader=yaml.FullLoader)
except:
raise Exception('The file rev_%s.yaml must exist to process the LCs' % rev_num)
except:
try:
with open(yaml_file_path) as file:
observations = yaml.load(file, Loader=yaml.FullLoader)
except:
raise Exception('The file rev_%s.yaml must exist to process the LCs' % rev_num)
```
%% Cell type:code id:47014b40 tags:
``` python
for source in observations:
tt = source['scw_list']
pp = ["%016.3f" %(float(x)) for x in tt]
source['scw_list'] = pp
print(source['title'], pp)
#print(observations)
```
%% Cell type:code id:f02e1c5b tags:
``` python
import astropy.io.fits as fits
import numpy as np
from astroquery.simbad import Simbad
from astropy import units as u
from astropy.coordinates import SkyCoord
import copy
from importlib import reload
import json
```
%% Cell type:code id:8b5d064b tags:
``` python
## It does not work from papermill
# import ipynbname
# nb_fname = ipynbname.name()
nb_fname="Generic Revolution LC.ipynb"
from git import Repo
try:
repo = Repo(notebooks_folder)
repo_name = repo.remotes.origin.url.split('.git')[0]
origin_notebook = repo_name.replace(':','/').replace('git@', 'https://') + \
'/-/blob/master/' + nb_fname
except:
origin_notebook = 'https://gitlab.astro.unige.ch/oda/product-gallery/gallery-notebooks/-/blob/master/' + nb_fname
print(origin_notebook)
```
%% Cell type:code id:bc873b69 tags:
``` python
reload(oda_integral_wrapper.wrapper)
wrap=oda_integral_wrapper.wrapper.INTEGRALwrapper(token=token, integral_data_rights='all-private', )
```
%% Cell type:code id:055d4bf6 tags:
``` python
looping = True
import time
# This looping allows us to submit all jobs at once !
n_loops=0
max_loops=200
while looping:
for source in observations:
if 'processed_mosaics' in source and source['processed_mosaics']:
scw_list=source['scw_list']
#print(len(scw_list))
if use_isgri:
if source.get('isgri_lc', None) is None:
if source.get('isgri_source_catalog', None) is None or \
(len(json.loads(source['isgri_source_catalog'])['cat_column_list'][1]) == 0):
print('ISGRI LCs for %s are not extracted because no sources are present' % source['title'])
source.update({'isgri_lc' : 'Error'})
else:
par_dict = {
'instrument': 'isgri',
'product': 'isgri_lc',
'E1_keV' : E1_keV,
'E2_keV' : E2_keV,
'osa_version' : osa_version,
'product_type' : 'Real',
'src_name' : source['title'],
'RA':source['RA'],
'DEC' : source['Dec'],
'selected_catalog': source['isgri_source_catalog']
}
if 'T1' in source.keys():
pardict.update({
'T1' : source['tstart'],
'T2' : source['tstop']
})
try:
data=wrap.long_scw_list_call(scw_list, s_max=500, sleep_time=120,
save_partial_products=False,
wait=False, **par_dict)
source.update({'isgri_lc' : data})
except:
print('ISGRI lc for %s failed' % source['title'])
source.update({'isgri_lc' : 'Error'})
if use_jemx1:
if source.get('jemx1_lc', None) is None:
#print("Catlog length ", len(json.loads(source['jemx1_source_catalog'])['cat_column_list'][1]))
if source.get('jemx1_source_catalog',None) is None or \
(len(json.loads(source['jemx1_source_catalog'])['cat_column_list'][1]) == 0):
print('JEMX1 LCs for %s are not extracted because no sources are present' % source['title'])
source.update({'jemx1_lc' : 'Error'})
else:
par_dict_j = { 'instrument' : 'jemx',
'product' : 'jemx_lc',
'jemx_num' : 1,
'E1_keV' : J_E1_keV,
'E2_keV' : J_E2_keV,
'osa_version' : osa_version,
'product_type' : 'Real',
'src_name' : source['title'],
'RA' : source['RA'],
'DEC' : source['Dec'],
'selected_catalog': source['jemx1_source_catalog']
}
if 'tstart' in source.keys():
par_dict_j.update({ 'T1' : source['tstart'],
'T2' : source['tstop'] })
try:
data=wrap.long_scw_list_call(scw_list, s_max=500, sleep_time=120, save_partial_products=False,
wait=False, **par_dict_j )
source.update({'jemx1_lc' : data})
except:
print('JEM-X1 lc for %s failed' % source['title'])
source.update({'jemx1_lc' : 'Error'})
if use_jemx2:
if source.get('jemx2_lc', None) is None:
if source.get('jemx2_source_catalog', None) is None or \
(len(json.loads(source['jemx2_source_catalog'])['cat_column_list'][1]) == 0):
print('JEMX2 LCs for %s are not extracted because no sources are present' % source['title'])
source.update({'jemx2_lc' : 'Error'})
else:
par_dict_j = { 'instrument' : 'jemx',
'product' : 'jemx_lc',
'jemx_num' : 2,
'E1_keV' : J_E1_keV,
'E2_keV' : J_E2_keV,
'osa_version' : osa_version,
'product_type' : 'Real',
'src_name' : source['title'],
'RA' : source['RA'],
'DEC' : source['Dec'],
'time_bin' : lc_time_bin,
'selected_catalog': source['jemx2_source_catalog']
}
if 'tstart' in source.keys():
par_dict_j.update({ 'T1' : source['tstart'],
'T2' : source['tstop'] })
try:
data=wrap.long_scw_list_call(scw_list, s_max=500, sleep_time=120,
save_partial_products=False,
wait=False, **par_dict_j )
source.update({'jemx2_lc' : data})
except:
print('JEM-X2 lc for %s failed' % source['title'])
source.update({'jemx2_lc' : 'Error'})
else:
source.update({'isgri_lc' : 'Error'})
source.update({'jemx1_lc' : 'Error'})
source.update({'jemx2_lc' : 'Error'})
#Checks if I need to ask again
looping=False
for ii in observations:
if use_isgri:
if ii.get('isgri_lc',None) is None:
print('Need to loop again on isgri for %s ' % ( ii['title']))
looping = True
if use_jemx1:
if ii.get('jemx1_lc',None) is None:
print('Need to loop again on jemx1 for %s ' % ( ii['title']))
looping = True
if use_jemx2:
if ii.get('jemx2_lc',None) is None:
print('Need to loop again on jemx2 for %s ' % ( ii['title']))
looping = True
if looping and n_loops < max_loops:
time.sleep(120)
n_loops+=1
else:
looping=False
```
%% Cell type:code id:783a1be1 tags:
``` python
reload(oda_integral_wrapper.wrapper)
wrap=oda_integral_wrapper.wrapper.INTEGRALwrapper(token=token, integral_data_rights='all-private', )
for source in observations:
sanitized_source_title = oda_integral_wrapper.wrapper.INTEGRALwrapper.clean_source_title(source['title'])
pattern = sanitized_source_title + '_' + str(source['expid'])
if use_isgri:
lc_isgri=source.get('isgri_lc', None)
if lc_isgri is not None and type(lc_isgri) is not str:
isgri_sources, isgri_lcs, isgri_tstarts, isgri_tstops, isgri_exposures = \
wrap.write_all_lc_fits_files(lc_isgri, pattern)
source.update({'isgri_files' : {'fname': isgri_lcs, 'tstart':isgri_tstarts,
'tstop': isgri_tstops,
'exposure': isgri_exposures},
'isgri_sources': isgri_sources})
if use_jemx1:
lc_jemx1 = source.get('jemx1_lc', None)
#print(type(lc_jemx1))
if lc_jemx1 is not None and type(lc_jemx1) is not str:
jemx1_sources, jemx1_lcs, jemx1_tstarts, jemx1_tstops, jemx1_exposures = \
wrap.write_all_lc_fits_files(lc_jemx1, pattern)
source.update({'jemx1_files' : {'fname': jemx1_lcs, 'tstart':jemx1_tstarts,
'tstop': jemx1_tstops,
'exposure': jemx1_exposures},
'jemx1_sources': jemx1_sources})
if use_jemx2:
lc_jemx2 = source.get('jemx2_lc', None)
#print(type(lc_jemx1))
if lc_jemx2 is not None and type(lc_jemx2) is not str:
jemx2_sources, jemx2_lcs, jemx2_tstarts, jemx2_tstops, jemx2_exposures = \
wrap.write_all_lc_fits_files(lc_jemx2, pattern)
source.update({'jemx2_files' : {'fname': jemx2_lcs, 'tstart':jemx2_tstarts,
'tstop': jemx2_tstops,
'exposure': jemx2_exposures},
'jemx2_sources': jemx2_sources})
```
%% Cell type:code id:3a980fda tags:
``` python
# Upload LC to Gallery
import oda_api.plot_tools
import re
reload(oda_api.plot_tools)
reload(oda_integral_wrapper.wrapper)
wrap2=oda_integral_wrapper.wrapper.INTEGRALwrapper(token=token, host_type=host_type)
instruments = []
if use_isgri:
instruments.append('isgri')
if use_jemx1:
instruments.append('jemx1')
if use_jemx2:
instruments.append('jemx2')
if batch_run:
additional_information = _i2
else:
additional_information = _i1
# remove any token from the additional_information
token_pattern = r"token\s*=\s*[\'\"].*?[\'\"]"
additional_information = re.sub(token_pattern, 'token = \"<Insert yout token here>\"', additional_information, flags=re.DOTALL)
host_type_pattern = r"host_type\s*=\s*[\'\"].*?[\'\"]"
additional_information = re.sub(host_type_pattern, 'host_type = \"production\"', additional_information, flags=re.DOTALL)
notebooks_folder_pattern = r"notebooks_folder\s*=\s*[\'\"].*?[\'\"]\n"
additional_information = re.sub(notebooks_folder_pattern, '', additional_information, flags=re.DOTALL)
for obs in observations:
print(obs)
for ins in instruments:
lc = obs.get('%s_lc' % ins, None)
#Sets upload variables, defaults ISGRI
E1_keV = 28
E2_keV = 40
systematic_fraction = isgri_systematic_fraction
k1 = 'E1_isgri'
k2 = 'E2_isgri'
if 'jemx' in ins:
systematic_fraction = jemx_systematic_fraction
E1_keV = 3
E2_keV = 20
k1 = 'E1_jemx'
k2 = 'E2_jemx'
if lc is not None and type(lc) is not str:
sources = obs['%s_sources' % ins]
image_product = oda_api.plot_tools.OdaLightCurve(lc)
#print(lc.as_list())
systematic_fraction = isgri_systematic_fraction
xlim = [20,200]
if 'jemx' in ins:
systematic_fraction = jemx_systematic_fraction
xlim = [2,30]
E1_keV = 1
E2_keV = 100
k1 = 'E1_isgri'
k2 = 'E2_isgri'
if 'jemx' in ins:
k1 = 'E1_jemx'
k2 = 'E2_jemx'
E1_keV = obs.get(k1, E1_keV)
E2_keV = obs.get(k2, E2_keV)
print(f"E1_kev: {E1_keV}")
print(f"E2_keV: {E2_keV}")
print(f"k1: {k1}")
print(f"k2: {k2}")
for i,src in enumerate(sources):
#print(src)
if 'MULTIPLE ID' in src:
continue
print(src)
if not os.path.isdir('out'):
os.mkdir('out')
img_fn = image_product.get_image_for_gallery( in_source_name=src,
systematic_fraction=systematic_fraction,
output_folder='out')
par_dict_product_id = {
'source_name': src,
# 't1': obs['tstart'],
# 't2': obs['tstop'],
# 'e1_kev' : E1_keV,
# 'e2_kev' : E2_keV,
'obsid': obs['obsid'],
'instrument' : '%s' % ins,
'product_type' : '%s_lc' % ins,
"rev_num" : rev_num
}
product_id = oda_api.api.DispatcherAPI.calculate_param_dict_id(par_dict_product_id)
lc = obs['%s_files' % ins]['fname'][i]
nrt_string = ''
if data_version.upper() == 'NRT':
nrt_string = ' (NRT)'
par_dict={
'token': token,
#if observation is incomplete (NRT) it would create new products at a later run
'T1': re.sub('\.\d{3}', '', obs['tstart']),
'T2': re.sub('\.\d{3}', '', obs['tstop']),
'e1_kev' : E1_keV,
'e2_kev' : E2_keV,
'product_title' : src + ' %s light curve' % ins + nrt_string,
'gallery_image_path' : img_fn,
'fits_file_path' : [lc],
'src_name' : src,
'instrument' : ins,
'insert_new_source' : True,
'force_insert_not_valid_new_source' : False,
'validate_source' : True,
'apply_fields_source_resolution': True,
'product_type' : '%s_lc' % ins ,
'product_id' : product_id,
#input parameters assuming they are in cell #1
'additional_information' : additional_information,
'html_image': image_product.get_html_image(src, systematic_fraction),
'produced_by' : origin_notebook
}
n_max_tries = 3
n_tries_left = n_max_tries
#print(par_dict)
while True:
try:
d = wrap2.disp.post_data_product_to_gallery(**par_dict)
source.update({
"%s_gallery_object" % ins : d.copy(),
})
if 'error_message' not in d:
source.update({
'processed_lc': True
})
break
else:
n_tries_left -= 1
if n_tries_left == 0:
break
else:
print(f"Exception while posting a product on the gallery, will re-attempt to post {n_tries_left} times")
except Exception as e:
print(f"Exception while posting a product on the gallery, will re-attemp to post:\n{e}")
n_tries_left -= 1
if n_tries_left == 0:
break
else:
print(f"Exception while posting a product on the gallery, will re-attempt to post {n_tries_left} times")
print(obs)
```
%% Cell type:code id:308d2096 tags:
``` python
wrap2.disp.url
```
%% Cell type:code id:9c547dc3 tags:
``` python
oda_integral_wrapper.wrapper.dump_yaml(observations, yaml_file_path)
```
%% Cell type:code id:6d140f74 tags:
``` python
reload(oda_integral_wrapper.wrapper)
if not os.path.isdir(output_yaml_dir):
os.mkdir(output_yaml_dir)
yaml_file_path = os.path.join(output_yaml_dir, 'rev_%s.yaml' % rev_num)
observations = oda_integral_wrapper.wrapper.INTEGRALwrapper.clean_and_update_observations(
observations, dictionary_to_update={'E1_isgri' : E1_keV,
'E2_isgri' : E2_keV,
'E1_jemx' : J_E1_keV,
'E2_jemx' : J_E2_keV})
oda_integral_wrapper.wrapper.dump_yaml(observations, yaml_file_path)
with open(yaml_file_path) as file:
observations = yaml.load(file, Loader=yaml.FullLoader)
print(observations)
```
%% Cell type:code id:ab8fde01 tags:
``` python
time_dict = oda_integral_wrapper.wrapper.INTEGRALwrapper.converttime('REVNUM', rev_num, 'ANY')
utc_start_rev = oda_integral_wrapper.wrapper.INTEGRALwrapper.converttime('IJD', time_dict['IJD'].split()[1],
'UTC')
utc_start_rev = re.sub('\.\d{3}', '', utc_start_rev)
utc_end_rev = oda_integral_wrapper.wrapper.INTEGRALwrapper.converttime('IJD', time_dict['IJD'].split()[2],
'UTC')
utc_end_rev = re.sub('\.\d{3}', '', utc_end_rev)
```
%% Cell type:code id:efca9980 tags:
``` python
reload(oda_integral_wrapper.wrapper)
obsids = [oo['obsid'] for oo in observations]
par_dict_gallery = {
'observation_title' : "rev. %s" % rev_num,
'T1' : utc_start_rev,
'T2' : utc_end_rev,
'yaml_file_path': yaml_file_path,
'obsid' : obsids,
'token' : token,
'create_new': True
}
wrap2=oda_integral_wrapper.wrapper.INTEGRALwrapper(token=token, host_type=host_type)
d = wrap2.disp.update_observation_with_title(**par_dict_gallery)
```
%% Cell type:code id:5e8d334b tags:output
``` python
observations
```
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment