Skip to content

Commit

Permalink
Merge pull request #468 from oda-hub/update-astro-entity
Browse files Browse the repository at this point in the history 
Update astro entity
  • Loading branch information
@burnout87
burnout87 authored Oct 7, 2022
2 parents 6bd163a + d579c31 commit 74436e3
Show file tree
Hide file tree
Showing 3 changed files with 310 additions and 41 deletions.
166 changes: 137 additions & 29 deletions cdci_data_analysis/analysis/drupal_helper.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
from dateutil import parser, tz
from datetime import datetime
from enum import Enum, auto
from astropy.coordinates import SkyCoord
from astropy.coordinates import SkyCoord, Angle
from astropy import units as u

from cdci_data_analysis.analysis import tokenHelper
Expand Down Expand Up @@ -466,6 +466,7 @@ def post_content_to_gallery(decoded_token,
sentry_dsn=sentry_dsn,
**par_dic)
elif content_type == content_type.OBSERVATION:
# TODO build the body to send to the gallery in more automated fashion (like done for the data-product)
t1 = kwargs.pop('T1', None)
t2 = kwargs.pop('T2', None)
revnum_1 = kwargs.pop('revnum_1', None)
Expand Down Expand Up @@ -493,6 +494,81 @@ def post_content_to_gallery(decoded_token,
output_content_post = {}
logger.info(f"no observation has been created or updated")

elif content_type == content_type.ASTROPHYSICAL_ENTITY:
# TODO build the body to send to the gallery in more automated fashion (like done for the data-product)
update_astro_entity = kwargs.pop('update_astro_entity', 'False') == 'True'
src_name = kwargs.pop('src_name', None)
source_entity_id = None
source_ra = None
source_dec = None
src_portal_link = None
object_type = None
object_ids = None
if update_astro_entity:
auto_update = kwargs.pop('auto_update', 'False') == 'True'
if auto_update is True:
name_resolver_url = disp_conf.name_resolver_url
entities_portal_url = disp_conf.entities_portal_url
resolved_obj = resolve_name(name_resolver_url=name_resolver_url,
entities_portal_url=entities_portal_url,
name=src_name)
if resolved_obj is not None:
msg = ''
if 'message' in resolved_obj:
if 'could not be resolved' in resolved_obj['message']:
msg = f'\nSource {src_name} could not be validated'
elif 'successfully resolved' in resolved_obj['message']:
msg = f'\nSource {src_name} was successfully validated'
msg += '\n'
logger.info(msg)
if 'RA' in resolved_obj:
source_ra = Angle(resolved_obj["RA"], unit='degree').deg
if 'DEC' in resolved_obj:
source_dec = Angle(resolved_obj["DEC"], unit='degree').deg
if 'entity_portal_link' in resolved_obj:
src_portal_link = resolved_obj['entity_portal_link']
if 'object_type' in resolved_obj:
object_type = resolved_obj['object_type']
if 'object_ids' in resolved_obj:
object_ids = resolved_obj['object_ids']
else:
src_portal_link = kwargs.pop('src_portal_link', None)
source_ra = kwargs.pop('source_ra', None)
source_dec = kwargs.pop('source_dec', None)
object_type = kwargs.pop('object_type', None)
object_ids = kwargs.pop('object_ids', None)

source_entity_id = get_source_astrophysical_entity_id_by_source_name(product_gallery_url,
gallery_jwt_token,
source_name=src_name,
sentry_dsn=sentry_dsn)
if update_astro_entity and source_entity_id is None:
logger.warning(f'an update of an astrophysical entity could not be performed since the correspondent one '
f'could not be found, please check the provided name')
raise RequestNotUnderstood(message="Request data not found",
payload={'drupal_helper_error_message': 'error while updating astrophysical and '
'entity product: no correspondent entity '
'could be found with the provided name'})

output_content_post = post_astro_entity(product_gallery_url=product_gallery_url,
gallery_jwt_token=gallery_jwt_token,
astro_entity_name=src_name.strip(),
astro_entity_portal_link=src_portal_link,
source_ra=source_ra,
source_dec=source_dec,
object_type=object_type,
object_ids=object_ids,
sentry_dsn=sentry_dsn,
update_astro_entity=update_astro_entity,
astro_entity_id=source_entity_id)
if output_content_post is not None:
# extract the id of the observation
astrophysical_entity_drupal_id = output_content_post['nid'][0]['value']
logger.info(f"Astrophysical entity with id {astrophysical_entity_drupal_id} has been successfully posted")
else:
output_content_post = {}
logger.info(f"no astrophysical entity has been created or updated")

return output_content_post


Expand Down Expand Up @@ -526,7 +602,9 @@ def post_astro_entity(product_gallery_url, gallery_jwt_token, astro_entity_name,
source_dec=None,
object_type=None,
object_ids=None,
sentry_dsn=None):
sentry_dsn=None,
astro_entity_id=None,
update_astro_entity=False):
# post new observation with or without a specific time range
body_gallery_astro_entity_node = copy.deepcopy(body_article_product_gallery.body_node)
astro_entity_name = astro_entity_name.strip()
Expand All @@ -539,9 +617,10 @@ def post_astro_entity(product_gallery_url, gallery_jwt_token, astro_entity_name,
body_gallery_astro_entity_node["field_source_name"] = [{
"value": astro_entity_name
}]
body_gallery_astro_entity_node["field_link"] = [{
"value": astro_entity_portal_link
}]
if astro_entity_portal_link is not None:
body_gallery_astro_entity_node["field_link"] = [{
"value": astro_entity_portal_link
}]
if object_ids is not None:
body_gallery_astro_entity_node["field_alternative_names_long_str"] = [{
"value": ','.join(object_ids)
Expand All @@ -564,18 +643,22 @@ def post_astro_entity(product_gallery_url, gallery_jwt_token, astro_entity_name,

headers = get_drupal_request_headers(gallery_jwt_token)

log_res = execute_drupal_request(f"{product_gallery_url}/node",
method='post',
data=json.dumps(body_gallery_astro_entity_node),
headers=headers,
sentry_dsn=sentry_dsn)
if update_astro_entity:
log_res = execute_drupal_request(os.path.join(product_gallery_url, 'node', astro_entity_id),
method='patch',
data=json.dumps(body_gallery_astro_entity_node),
headers=headers,
sentry_dsn=sentry_dsn)
else:
log_res = execute_drupal_request(f"{product_gallery_url}/node",
method='post',
data=json.dumps(body_gallery_astro_entity_node),
headers=headers,
sentry_dsn=sentry_dsn)

output_post = analyze_drupal_output(log_res, operation_performed="posting a new astrophysical entity")

# extract the id of the observation
astro_entity_drupal_id = output_post['nid'][0]['value']

return astro_entity_drupal_id
return output_post


def build_gallery_observation_node(product_gallery_url,
Expand Down Expand Up @@ -748,10 +831,22 @@ def get_instrument_product_type_id(product_gallery_url, gallery_jwt_token, produ
return output_dict


def get_all_source_astrophysical_entities(product_gallery_url, gallery_jwt_token, sentry_dsn=None) -> Optional[list]:
entities = []
headers = get_drupal_request_headers(gallery_jwt_token)
log_res = execute_drupal_request(f"{product_gallery_url}/astro_entities/source/all",
headers=headers,
sentry_dsn=sentry_dsn)
output_get = analyze_drupal_output(log_res, operation_performed="retrieving the astrophysical entity information")
if isinstance(output_get, list):
entities = list(obj['title'] for obj in output_get)

return entities


def get_source_astrophysical_entity_id_by_source_name(product_gallery_url, gallery_jwt_token, source_name=None, sentry_dsn=None) \
-> Optional[str]:
entities_id = None
# get from the drupal the relative id
headers = get_drupal_request_headers(gallery_jwt_token)

# the URL-reserved characters should be quoted eg GX 1+4 -> GX%201%2B4
Expand Down Expand Up @@ -1053,12 +1148,9 @@ def post_data_product_to_gallery(product_gallery_url, gallery_jwt_token, convert
arg_source_coord_dec = arg_source_coord.get('source_dec', None)
if source_entity_coord_ra is not None and source_entity_coord_dec is not None and \
arg_source_coord_ra is not None and arg_source_coord_dec is not None:
drupal_source_sky_coord = SkyCoord(source_entity_coord_ra, source_entity_coord_dec, unit=(u.hourangle, u.deg))
arg_source_sky_coord = SkyCoord(arg_source_coord_ra, arg_source_coord_dec, unit=(u.hourangle, u.deg), frame="fk5")
separation = drupal_source_sky_coord.separation(arg_source_sky_coord).deg
tolerance = 1. / 60.
ind = np.logical_or(source_entity_title == src_name, separation <= tolerance)
if np.count_nonzero(ind) > 0:
matching_coords = check_matching_coords(source_entity_title, source_entity_coord_ra, source_entity_coord_dec,
src_name, arg_source_coord_ra, arg_source_coord_dec)
if matching_coords:
source_entity_id = source_entity['nid']
break

Expand All @@ -1073,14 +1165,17 @@ def post_data_product_to_gallery(product_gallery_url, gallery_jwt_token, convert
object_type = None
if object_type_list is not None and object_type_list[src_name_idx] != '':
object_type = object_type_list[src_name_idx]
source_entity_id = post_astro_entity(product_gallery_url, gallery_jwt_token,
astro_entity_name=src_name.strip(),
astro_entity_portal_link=src_portal_link,
source_ra=arg_source_coord.get('source_ra', None),
source_dec=arg_source_coord.get('source_dec', None),
object_type=object_type,
object_ids=object_ids,
sentry_dsn=sentry_dsn)
output_post = post_astro_entity(product_gallery_url, gallery_jwt_token,
astro_entity_name=src_name.strip(),
astro_entity_portal_link=src_portal_link,
source_ra=arg_source_coord.get('source_ra', None),
source_dec=arg_source_coord.get('source_dec', None),
object_type=object_type,
object_ids=object_ids,
sentry_dsn=sentry_dsn)

# extract the id of the observation
source_entity_id = output_post['nid'][0]['value']

if source_entity_id is not None:
if 'field_describes_astro_entity' not in body_gallery_article_node:
Expand Down Expand Up @@ -1172,6 +1267,19 @@ def post_data_product_to_gallery(product_gallery_url, gallery_jwt_token, convert
return output_post


def check_matching_coords(source_1_name, source_1_coord_ra, source_1_coord_dec,
source_2_name, source_2_coord_ra, source_2_coord_dec,
tolerance=1. / 60):
drupal_source_sky_coord = SkyCoord(source_1_coord_ra, source_1_coord_dec, unit=(u.hourangle, u.deg))
arg_source_sky_coord = SkyCoord(source_2_coord_ra, source_2_coord_dec, unit=(u.hourangle, u.deg), frame="fk5")
separation = drupal_source_sky_coord.separation(arg_source_sky_coord).deg
ind = np.logical_or(source_1_name == source_2_name, separation <= tolerance)
if np.count_nonzero(ind) > 0:
return True

return False


def resolve_name(name_resolver_url: str, entities_portal_url: str = None, name: str = None):
resolved_obj = {}
if name is not None:
Expand Down
78 changes: 78 additions & 0 deletions cdci_data_analysis/flask_app/app.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -657,6 +657,84 @@ def get_observation_attachments():
return output_get


@app.route('/get_all_astro_entities', methods=['GET'])
def get_all_astro_entities():
logger.info("request.args: %s ", request.args)
logger.info("request.files: %s ", request.files)

token = request.args.get('token', None)
app_config = app.config.get('conf')
secret_key = app_config.secret_key

output, output_code = tokenHelper.validate_token_from_request(token=token, secret_key=secret_key,
required_roles=['gallery contributor'],
action="post on the product gallery")

if output_code is not None:
return make_response(output, output_code)
decoded_token = output

par_dic = request.values.to_dict()
par_dic.pop('token')

sentry_dsn = getattr(app_config, 'sentry_url', None)
if sentry_dsn is not None:
sentry_sdk.init(
dsn=sentry_dsn,
# Set traces_sample_rate to 1.0 to capture 100%
# of transactions for performance monitoring.
# We recommend adjusting this value in production.
traces_sample_rate=1.0,
debug=True,
max_breadcrumbs=50,
)

gallery_secret_key = app_config.product_gallery_secret_key
product_gallery_url = app_config.product_gallery_url
user_email = tokenHelper.get_token_user_email_address(decoded_token)
user_id_product_creator = drupal_helper.get_user_id(product_gallery_url=product_gallery_url,
user_email=user_email,
sentry_dsn=sentry_dsn)
# update the token
gallery_jwt_token = drupal_helper.generate_gallery_jwt_token(gallery_secret_key, user_id=user_id_product_creator)

output_get = drupal_helper.get_all_source_astrophysical_entities(product_gallery_url=product_gallery_url,
gallery_jwt_token=gallery_jwt_token,
sentry_dsn=sentry_dsn)
output_list = json.dumps(output_get)

return output_list


@app.route('/post_astro_entity_to_gallery', methods=['POST'])
def post_astro_entity_to_gallery():
logger.info("request.args: %s ", request.args)
logger.info("request.files: %s ", request.files)

token = request.args.get('token', None)
app_config = app.config.get('conf')
secret_key = app_config.secret_key

output, output_code = tokenHelper.validate_token_from_request(token=token, secret_key=secret_key,
required_roles=['gallery contributor'],
action="post on the product gallery")

if output_code is not None:
return make_response(output, output_code)
decoded_token = output

par_dic = request.values.to_dict()
par_dic.pop('token')

output_post = drupal_helper.post_content_to_gallery(decoded_token=decoded_token,
content_type="astrophysical_entity",
disp_conf=app_config,
files=request.files,
**par_dic)

return output_post


@app.route('/post_observation_to_gallery', methods=['POST'])
def post_observation_to_gallery():
logger.info("request.args: %s ", request.args)
Expand Down
Loading

0 comments on commit 74436e3

Please sign in to comment.