Skip to content
Snippets Groups Projects
Commit 4fab744f authored by Carlo Ferrigno's avatar Carlo Ferrigno
Browse files

Add use_jemx2 optional parameter

parent 7a19073e
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
ProcessNRTRevolutions.ipynb
+ 2
− 1
View file @ 4fab744f
%% Cell type:code id:cd05caa6 tags:parameters %% Cell type:code id:cd05caa6 tags:parameters
``` python ``` python
import os,sys import os,sys
token='' token=''
# This defines the range of revlutions to process, default is the last 10 # This defines the range of revlutions to process, default is the last 10
rev_num_indicator = 0 rev_num_indicator = 0
force_reprocessing = True force_reprocessing = True
data_processing = 'NRT' data_processing = 'NRT'
host_type = 'staging' host_type = 'staging'
use_jemx2 = False
notebooks_folder = os.getcwd() notebooks_folder = os.getcwd()
``` ```
%% Cell type:code id:0924c9db tags: %% Cell type:code id:0924c9db tags:
``` python ``` python
import pandas as pd import pandas as pd
import requests import requests
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
import numpy as np import numpy as np
import os,sys import os,sys
import oda_integral_wrapper.wrapper import oda_integral_wrapper.wrapper
import yaml import yaml
import json import json
import papermill as pm import papermill as pm
from datetime import datetime from datetime import datetime
from importlib import reload from importlib import reload
``` ```
%% Cell type:code id:374f2a15 tags: %% Cell type:code id:374f2a15 tags:
``` python ``` python
import logging import logging
#logging.getLogger().setLevel(logging.WARNING) #logging.getLogger().setLevel(logging.WARNING)
logging.getLogger().setLevel(logging.INFO) #for more verbose logging logging.getLogger().setLevel(logging.INFO) #for more verbose logging
logger = logging.getLogger('').addHandler(logging.StreamHandler()) logger = logging.getLogger('').addHandler(logging.StreamHandler())
``` ```
%% Cell type:code id:d1b6bb71 tags: %% Cell type:code id:d1b6bb71 tags:
``` python ``` python
import oda_api.token import oda_api.token
print(token) print(token)
if token is None or token == '': if token is None or token == '':
token = oda_api.token.discover_token() token = oda_api.token.discover_token()
# extend token lifetime # extend token lifetime
from oda_api.api import DispatcherAPI from oda_api.api import DispatcherAPI
disp = DispatcherAPI(url='https://www.astro.unige.ch/mmoda/dispatch-data') disp = DispatcherAPI(url='https://www.astro.unige.ch/mmoda/dispatch-data')
token = disp.refresh_token(token, write_token=True) token = disp.refresh_token(token, write_token=True)
oda_api.token.decode_oda_token(token) oda_api.token.decode_oda_token(token)
``` ```
%% Cell type:code id:c626a24b tags: %% Cell type:code id:c626a24b tags:
``` python ``` python
import oda_integral_wrapper.itime as itime import oda_integral_wrapper.itime as itime
if rev_num_indicator <= 0: if rev_num_indicator <= 0:
now = itime.now() now = itime.now()
rev_num = int(now.REVNUM) + rev_num_indicator rev_num = int(now.REVNUM) + rev_num_indicator
else: else:
rev_num = int(rev_num_indicator) rev_num = int(rev_num_indicator)
``` ```
%% Cell type:markdown id:6ba77738 tags: %% Cell type:markdown id:6ba77738 tags:
## Actual processing using papermill ## Actual processing using papermill
%% Cell type:code id:b021fee7 tags: %% Cell type:code id:b021fee7 tags:
``` python ``` python
failed = False failed = False
now = datetime.now().strftime("%Y_%m_%d_%H_%M_%S") now = datetime.now().strftime("%Y_%m_%d_%H_%M_%S")
par_dict = { par_dict = {
'rev_num' : rev_num, 'rev_num' : rev_num,
'host_type' : host_type, 'host_type' : host_type,
'token': token, 'token': token,
'data_version': data_processing, 'data_version': data_processing,
'use_jemx2': False, 'use_jemx2': use_jemx2,
'notebooks_folder': notebooks_folder, 'notebooks_folder': notebooks_folder,
'batch_run' : True 'batch_run' : True
} }
print(f"Running ProcessNRTRevolutions notebook for revolution number: {rev_num}") print(f"Running ProcessNRTRevolutions notebook for revolution number: {rev_num}")
try: try:
print(f"Processing Mosaics for revolution number: {rev_num}") print(f"Processing Mosaics for revolution number: {rev_num}")
mosaics_notebook_path = os.path.join(notebooks_folder,"Generic Revolution Mosaics.ipynb") mosaics_notebook_path = os.path.join(notebooks_folder,"Generic Revolution Mosaics.ipynb")
print(f"Mosaics notebook path: {mosaics_notebook_path}") print(f"Mosaics notebook path: {mosaics_notebook_path}")
output_mosaics_notebook_path = os.path.join(notebooks_folder, f"out/Generic Revolution Mosaics_output_{rev_num}_{now}.ipynb") output_mosaics_notebook_path = os.path.join(notebooks_folder, f"out/Generic Revolution Mosaics_output_{rev_num}_{now}.ipynb")
print(f"Output Mosaics notebook path: {output_mosaics_notebook_path}") print(f"Output Mosaics notebook path: {output_mosaics_notebook_path}")
nb_mosaics_execution = pm.execute_notebook( nb_mosaics_execution = pm.execute_notebook(
mosaics_notebook_path, mosaics_notebook_path,
output_mosaics_notebook_path, output_mosaics_notebook_path,
parameters=par_dict, parameters=par_dict,
log_output=True log_output=True
) )
print("Succesfully processed Mosaics for revolution number: %s" %(rev_num)) print("Succesfully processed Mosaics for revolution number: %s" %(rev_num))
except: except:
failed = True failed = True
print(f'Mosaics failed for revolution number: {rev_num}') print(f'Mosaics failed for revolution number: {rev_num}')
print('Check ' + f"out/Generic Revolution Mosaics_output_{rev_num}_{now}.ipynb") print('Check ' + f"out/Generic Revolution Mosaics_output_{rev_num}_{now}.ipynb")
if not failed: if not failed:
try: try:
print(f"Processing Spectra for revolution number: {rev_num}") print(f"Processing Spectra for revolution number: {rev_num}")
spectra_notebook_path = os.path.join(notebooks_folder,"Generic Revolution Spectra.ipynb") spectra_notebook_path = os.path.join(notebooks_folder,"Generic Revolution Spectra.ipynb")
print(f"Spectra notebook path: {spectra_notebook_path}") print(f"Spectra notebook path: {spectra_notebook_path}")
output_spectra_notebook_path = os.path.join(notebooks_folder, f"out/Generic Revolution Spectra_output_{rev_num}_{now}.ipynb") output_spectra_notebook_path = os.path.join(notebooks_folder, f"out/Generic Revolution Spectra_output_{rev_num}_{now}.ipynb")
print(f"Output Spectra notebook path: {output_spectra_notebook_path}") print(f"Output Spectra notebook path: {output_spectra_notebook_path}")
nb_spectra_execution = pm.execute_notebook( nb_spectra_execution = pm.execute_notebook(
spectra_notebook_path, spectra_notebook_path,
output_spectra_notebook_path, output_spectra_notebook_path,
parameters=par_dict, parameters=par_dict,
log_output=True log_output=True
) )
print(f"Succesfully processed Spectra for revolution number: {rev_num}") print(f"Succesfully processed Spectra for revolution number: {rev_num}")
except: except:
failed = True failed = True
print(f'Spectra failed for revolution number: {rev_num}') print(f'Spectra failed for revolution number: {rev_num}')
print('Check ' + f"out/Generic Revolution Spectra_output_{rev_num}_{now}.ipynb") print('Check ' + f"out/Generic Revolution Spectra_output_{rev_num}_{now}.ipynb")
try: try:
print(f"Processing LC for revolution number: {rev_num}") print(f"Processing LC for revolution number: {rev_num}")
lc_notebook_path = os.path.join(notebooks_folder, "Generic Revolution LC.ipynb") lc_notebook_path = os.path.join(notebooks_folder, "Generic Revolution LC.ipynb")
print(f"LC notebook path: {lc_notebook_path}") print(f"LC notebook path: {lc_notebook_path}")
output_lc_notebook_path = os.path.join(notebooks_folder, f"out/Generic Revolution LC_output_{rev_num}_{now}.ipynb") output_lc_notebook_path = os.path.join(notebooks_folder, f"out/Generic Revolution LC_output_{rev_num}_{now}.ipynb")
print(f"Output LC notebook path: {output_lc_notebook_path}") print(f"Output LC notebook path: {output_lc_notebook_path}")
nb_lc_execution = pm.execute_notebook( nb_lc_execution = pm.execute_notebook(
lc_notebook_path, lc_notebook_path,
output_lc_notebook_path, output_lc_notebook_path,
parameters=par_dict, parameters=par_dict,
log_output=True log_output=True
) )
print(f"Succesfully processed LC for revolution number: {rev_num}") print(f"Succesfully processed LC for revolution number: {rev_num}")
except: except:
failed = True failed = True
print(f'Light Curve failed for revolution number: {rev_num}') print(f'Light Curve failed for revolution number: {rev_num}')
print('Check ' + f"out/Generic Revolution LC_output_{rev_num}_{now}.ipynb") print('Check ' + f"out/Generic Revolution LC_output_{rev_num}_{now}.ipynb")
``` ```
%% Cell type:code id:a7435cc1 tags: %% Cell type:code id:a7435cc1 tags:
``` python ``` python
``` ```
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment