Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use main id source search #469

Draft
wants to merge 8 commits into
base: master
Choose a base branch
from
Draft

Use main id source search #469

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
4ba4a46
update url gallery drupal, using main_id to search for the source in …
burnout87 Oct 18, 2022
b228f26
extract main_id resolve_name,
burnout87 Oct 19, 2022
3bf30dc
not using main_id to resolve name
burnout87 Oct 19, 2022
5534944
better log message
burnout87 Oct 19, 2022
8d893cd
using development version of the gallery for testing
burnout87 Oct 25, 2022
202f8f0
some refactoring of the post astro entity method
burnout87 Oct 25, 2022
3f3b5d0
header argument for content type
burnout87 Oct 26, 2022
219bc66
some code refactoring to pass args to post astro entity
burnout87 Oct 26, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
120 changes: 66 additions & 54 deletions cdci_data_analysis/analysis/drupal_helper.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -269,9 +269,9 @@ def execute_drupal_request(url,
payload={'drupal_helper_error_message': str(e)})


def get_drupal_request_headers(gallery_jwt_token=None):
def get_drupal_request_headers(gallery_jwt_token=None, content_type='application/hal+json'):
headers = {
'Content-type': 'application/hal+json'
'Content-type': content_type
}
if gallery_jwt_token is not None:
headers['Authorization'] = 'Bearer ' + gallery_jwt_token
Expand Down Expand Up @@ -498,12 +498,8 @@ def post_content_to_gallery(decoded_token,
# TODO build the body to send to the gallery in more automated fashion (like done for the data-product)
update_astro_entity = kwargs.pop('update_astro_entity', 'False') == 'True'
src_name = kwargs.pop('src_name', None)
# main_id = None
source_entity_id = None
source_ra = None
source_dec = None
src_portal_link = None
object_type = None
object_ids = None
if update_astro_entity:
auto_update = kwargs.pop('auto_update', 'False') == 'True'
if auto_update is True:
Expand All @@ -522,45 +518,33 @@ def post_content_to_gallery(decoded_token,
msg += '\n'
logger.info(msg)
if 'RA' in resolved_obj:
source_ra = Angle(resolved_obj["RA"], unit='degree').deg
kwargs['source_ra'] = Angle(resolved_obj["RA"], unit='degree').deg
if 'DEC' in resolved_obj:
source_dec = Angle(resolved_obj["DEC"], unit='degree').deg
kwargs['source_dec'] = Angle(resolved_obj["DEC"], unit='degree').deg
if 'entity_portal_link' in resolved_obj:
src_portal_link = resolved_obj['entity_portal_link']
kwargs['src_portal_link'] = resolved_obj['entity_portal_link']
if 'object_type' in resolved_obj:
object_type = resolved_obj['object_type']
kwargs['object_type'] = resolved_obj['object_type']
if 'object_ids' in resolved_obj:
object_ids = resolved_obj['object_ids']
else:
src_portal_link = kwargs.pop('src_portal_link', None)
source_ra = kwargs.pop('source_ra', None)
source_dec = kwargs.pop('source_dec', None)
object_type = kwargs.pop('object_type', None)
object_ids = kwargs.pop('object_ids', None)
kwargs['object_ids'] = resolved_obj['object_ids']
# if 'main_id' in resolved_obj:
# main_id = resolved_obj['main_id']

# src_name_search = src_name.strip()
# if main_id is not None:
# src_name_search = main_id.strip()
source_entity_id = get_source_astrophysical_entity_id_by_source_name(product_gallery_url,
gallery_jwt_token,
source_name=src_name,
sentry_dsn=sentry_dsn)
if update_astro_entity and source_entity_id is None:
logger.warning(f'an update of an astrophysical entity could not be performed since the correspondent one '
f'could not be found, please check the provided name')
raise RequestNotUnderstood(message="Request data not found",
payload={'drupal_helper_error_message': 'error while updating astrophysical and '
'entity product: no correspondent entity '
'could be found with the provided name'})

output_content_post = post_astro_entity(product_gallery_url=product_gallery_url,
gallery_jwt_token=gallery_jwt_token,
astro_entity_name=src_name.strip(),
astro_entity_portal_link=src_portal_link,
source_ra=source_ra,
source_dec=source_dec,
object_type=object_type,
object_ids=object_ids,
sentry_dsn=sentry_dsn,
update_astro_entity=update_astro_entity,
astro_entity_id=source_entity_id)
astro_entity_id=source_entity_id,
sentry_dsn=sentry_dsn,
**kwargs)
if output_content_post is not None:
# extract the id of the observation
astrophysical_entity_drupal_id = output_content_post['nid'][0]['value']
Expand Down Expand Up @@ -597,14 +581,19 @@ def get_observations_for_time_range(product_gallery_url, gallery_jwt_token, t1=N


def post_astro_entity(product_gallery_url, gallery_jwt_token, astro_entity_name,
astro_entity_portal_link=None,
source_ra=None,
source_dec=None,
object_type=None,
object_ids=None,
sentry_dsn=None,
astro_entity_id=None,
update_astro_entity=False):
sentry_dsn=None,
update_astro_entity=False,
**kwargs):

if update_astro_entity and astro_entity_id is None:
logger.warning(f'an update of an astrophysical entity could not be performed since the correspondent one '
f'could not be found in the gallery, please check the provided name')
raise RequestNotUnderstood(message="Request data not found",
payload={'drupal_helper_error_message': 'error while updating astrophysical and '
'entity product: no correspondent entity '
'could be found with the provided name'})

# post new observation with or without a specific time range
body_gallery_astro_entity_node = copy.deepcopy(body_article_product_gallery.body_node)
astro_entity_name = astro_entity_name.strip()
Expand All @@ -617,25 +606,31 @@ def post_astro_entity(product_gallery_url, gallery_jwt_token, astro_entity_name,
body_gallery_astro_entity_node["field_source_name"] = [{
"value": astro_entity_name
}]
astro_entity_portal_link = kwargs.pop('src_portal_link', None)
if astro_entity_portal_link is not None:
body_gallery_astro_entity_node["field_link"] = [{
"value": astro_entity_portal_link
}]

object_ids = kwargs.pop('object_ids', None)
if object_ids is not None:
body_gallery_astro_entity_node["field_alternative_names_long_str"] = [{
"value": ','.join(object_ids)
}]

source_ra = kwargs.pop('source_ra', None)
if source_ra is not None:
body_gallery_astro_entity_node["field_source_ra"] = [{
"value": source_ra
}]

source_dec = kwargs.pop('source_dec', None)
if source_dec is not None:
body_gallery_astro_entity_node["field_source_dec"] = [{
"value": source_dec
}]

object_type = kwargs.pop('object_type', None)
if object_type is not None:
body_gallery_astro_entity_node["field_object_type"] = [{
"value": object_type
Expand Down Expand Up @@ -1094,14 +1089,22 @@ def post_data_product_to_gallery(product_gallery_url, gallery_jwt_token, convert
# set the source astrophysical entity if available
src_name_concat = None
src_name_arg = kwargs.pop('src_name', None)
# src_obj_arg = kwargs.pop('src_obj', None)
# src_obj = json.loads(src_obj_arg)
main_id_arg = kwargs.pop('main_id', None)
src_portal_link_arg = kwargs.pop('entity_portal_link_list', None)
object_ids_arg = kwargs.pop('object_ids_list', None)
source_coord_arg = kwargs.pop('source_coord_list', None)
object_type_arg = kwargs.pop('object_type_list', None)
if src_name_arg is not None:
# src_obj = json.loads(src_name_arg)
src_name_list = src_name_arg.split(',')
src_name_concat = "_".join(src_name_list)

main_id_list = None
if main_id_arg is not None:
main_id_list = json.loads(main_id_arg)

src_portal_link_list = None
if src_portal_link_arg is not None:
# TODO consider using json.loads
Expand All @@ -1122,11 +1125,22 @@ def post_data_product_to_gallery(product_gallery_url, gallery_jwt_token, convert
for src_name in src_name_list:
src_name_idx = src_name_list.index(src_name)
arg_source_coord = {}
main_id = None
if source_coord_obj_list is not None and source_coord_obj_list[src_name_idx] != {}:
arg_source_coord = source_coord_obj_list[src_name_idx]
source_entity_list = get_source_astrophysical_entity_id_by_source_and_alternative_name(product_gallery_url, gallery_jwt_token,
source_name=src_name,
sentry_dsn=sentry_dsn)
if main_id_list is not None and source_coord_obj_list[src_name_idx] != '':
main_id = main_id_list[src_name_idx]

if main_id is not None:
source_entity_list = get_source_astrophysical_entity_id_by_source_and_alternative_name(product_gallery_url,
gallery_jwt_token,
source_name=main_id,
sentry_dsn=sentry_dsn)
else:
source_entity_list = get_source_astrophysical_entity_id_by_source_and_alternative_name(product_gallery_url,
gallery_jwt_token,
source_name=src_name,
sentry_dsn=sentry_dsn)
source_entity_id = None
if len(source_entity_list) == 1:
source_entity_id = source_entity_list[0]['nid']
Expand Down Expand Up @@ -1156,23 +1170,19 @@ def post_data_product_to_gallery(product_gallery_url, gallery_jwt_token, convert

# create a new source ? yes if the user wants it
if source_entity_id is None and insert_new_source:
src_portal_link = None
source_kwargs = dict()
if src_portal_link_list is not None and src_portal_link_list[src_name_idx] != '':
src_portal_link = src_portal_link_list[src_name_idx].strip()
object_ids = None
source_kwargs['src_portal_link'] = src_portal_link_list[src_name_idx].strip()
if object_ids_lists is not None and object_ids_lists[src_name_idx] != []:
object_ids = object_ids_lists[src_name_idx]
object_type = None
source_kwargs['object_ids'] = object_ids_lists[src_name_idx]
if object_type_list is not None and object_type_list[src_name_idx] != '':
object_type = object_type_list[src_name_idx]
source_kwargs['object_type'] = object_type_list[src_name_idx]
source_kwargs['source_ra'] = arg_source_coord.get('source_ra', None)
source_kwargs['source_dec'] = arg_source_coord.get('source_dec', None)
output_post = post_astro_entity(product_gallery_url, gallery_jwt_token,
astro_entity_name=src_name.strip(),
astro_entity_portal_link=src_portal_link,
source_ra=arg_source_coord.get('source_ra', None),
source_dec=arg_source_coord.get('source_dec', None),
object_type=object_type,
object_ids=object_ids,
sentry_dsn=sentry_dsn)
sentry_dsn=sentry_dsn,
**source_kwargs)

# extract the id of the observation
source_entity_id = output_post['nid'][0]['value']
Expand Down Expand Up @@ -1299,6 +1309,8 @@ def resolve_name(name_resolver_url: str, entities_portal_url: str = None, name:
resolved_obj['object_ids'] = returned_resolved_obj['object_ids']
if 'object_type' in returned_resolved_obj:
resolved_obj['object_type'] = returned_resolved_obj['object_type']
if 'main_id' in returned_resolved_obj:
resolved_obj['main_id'] = returned_resolved_obj['main_id']
resolved_obj['entity_portal_link'] = entities_portal_url.format(quoted_name)
resolved_obj['message'] = f'{name} successfully resolved'
elif not returned_resolved_obj['success']:
Expand Down
4 changes: 2 additions & 2 deletions cdci_data_analysis/pytest_fixtures.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -417,7 +417,7 @@ def dispatcher_test_conf_with_gallery_fn(dispatcher_test_conf_fn):
f.write(f_default.read())

f.write('\n product_gallery_options:'
'\n product_gallery_url: "http://cdciweb02.isdc.unige.ch/mmoda/galleryd9"'
'\n product_gallery_url: "http://cdciweb02.isdc.unige.ch/mmoda/galleryd"'
f'\n product_gallery_secret_key: "{os.getenv("DISPATCHER_PRODUCT_GALLERY_SECRET_KEY", "secret_key")}"'
'\n product_gallery_timezone: "Europe/Zurich"'
'\n name_resolver_url: "https://resolver-prod.obsuks1.unige.ch/api/v1.1/byname/{}"'
Expand All @@ -436,7 +436,7 @@ def dispatcher_test_conf_with_gallery_no_resolver_fn(dispatcher_test_conf_fn):
f.write(f_default.read())

f.write('\n product_gallery_options:'
'\n product_gallery_url: "http://cdciweb02.isdc.unige.ch/mmoda/galleryd9"'
'\n product_gallery_url: "http://cdciweb02.isdc.unige.ch/mmoda/galleryd"'
'\n product_gallery_timezone: "Europe/Zurich"'
f'\n product_gallery_secret_key: "{os.getenv("DISPATCHER_PRODUCT_GALLERY_SECRET_KEY", "secret_key")}"')

Expand Down