diff --git a/plugins/lookup/json2xml.py b/plugins/lookup/json2xml.py new file mode 100644 index 0000000..89dcea1 --- /dev/null +++ b/plugins/lookup/json2xml.py @@ -0,0 +1,122 @@ +# (c) 2020 Red Hat, Inc. +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see . +from __future__ import absolute_import, division, print_function + +__metaclass__ = type + +DOCUMENTATION = """ +lookup: json2xml +author: Ganesh Nalawade (@ganeshrn) +short_description: Validates json configuration against yang data model and convert it to xml. +description: + - This plugin lookups the input json configuration, validates it against the respective yang data + model which is also given as input to this plugin and coverts it to xml format which can be used + as payload within Netconf rpc. +options: + _terms: + description: + - Input json configuration file path that adheres to a particular yang model. + required: True + type: path + doctype: + description: + - Specifies the target root node of the generated xml. The default value is C(config) + default: config + yang_file: + description: + - Path to yang model file against which the json configuration is validated and + converted to xml. + required: True + type: path + search_path: + description: + - This option is a colon C(:) separated list of directories to search for imported yang modules + in the yang file mentioned in C(path) option. If the value is not given it will search in + the current directory. + required: false + keep_tmp_files: + description: + - This is a boolean flag to indicate if the intermediate files generated while validation json + configuration should be kept or deleted. If the value is C(true) the files will not be deleted else by + default all the intermediate files will be deleted irrespective of whether task run is + successful or not. The intermediate files are stored in path C(~/.ansible/tmp/json2xml), this + option is mainly used for debugging purpose. + default: False + type: bool +""" + +EXAMPLES = """ +- name: translate json to xml + debug: msg="{{ lookup('yang_json2xml', config_json, + yang_file='openconfig/public/release/models/interfaces/openconfig-interfaces.yang', + search_path='openconfig/public/release/models:pyang/modules/') }}" +""" + +RETURN = """ +_raw: + description: The translated xml string from json +""" + +import os +import json + +from ansible.plugins.lookup import LookupBase +from ansible.module_utils._text import to_text +from ansible.errors import AnsibleError + +from ansible_collections.community.yang.plugins.module_utils.translator import ( + Translator, +) + +try: + import pyang # noqa +except ImportError: + raise AnsibleError("pyang is not installed") + +try: + from __main__ import display +except ImportError: + from ansible.utils.display import Display + + display = Display() + + +class LookupModule(LookupBase): + def run(self, terms, variables, **kwargs): + + res = [] + try: + json_config = terms[0] + except IndexError: + raise AnsibleError("path to json file must be specified") + + try: + yang_file = kwargs["yang_file"] + except KeyError: + raise AnsibleError("value of 'yang_file' must be specified") + + search_path = kwargs.pop("search_path", "") + keep_tmp_files = kwargs.pop("keep_tmp_files", False) + + json_config = os.path.realpath(os.path.expanduser(json_config)) + try: + # validate json + with open(json_config) as fp: + json.load(fp) + except Exception as exc: + raise AnsibleError( + "Failed to load json configuration: %s" + % (to_text(exc, errors="surrogate_or_strict")) + ) + + doctype = kwargs.get("doctype", "config") + + tl = Translator(yang_file, search_path, doctype, keep_tmp_files) + + xml_data = tl.json_to_xml(json_config) + res.append(xml_data) + + return res diff --git a/plugins/module_utils/files/yang/ietf-yang-metadata.yang b/plugins/module_utils/files/yang/ietf-yang-metadata.yang new file mode 100644 index 0000000..4369887 --- /dev/null +++ b/plugins/module_utils/files/yang/ietf-yang-metadata.yang @@ -0,0 +1,84 @@ + module ietf-yang-metadata { + + namespace "urn:ietf:params:xml:ns:yang:ietf-yang-metadata"; + + prefix "md"; + + organization + "IETF NETMOD (NETCONF Data Modeling Language) Working Group"; + + contact + "WG Web: + + WG List: + + WG Chair: Lou Berger + + + WG Chair: Kent Watsen + + + Editor: Ladislav Lhotka + "; + + description + "This YANG module defines an 'extension' statement that allows + for defining metadata annotations. + + Copyright (c) 2016 IETF Trust and the persons identified as + authors of the code. All rights reserved. + + Redistribution and use in source and binary forms, with or + without modification, is permitted pursuant to, and subject to + the license terms contained in, the Simplified BSD License set + forth in Section 4.c of the IETF Trust's Legal Provisions + Relating to IETF Documents + (http://trustee.ietf.org/license-info). + + This version of this YANG module is part of RFC 7952 + (http://www.rfc-editor.org/info/rfc7952); see the RFC itself + for full legal notices."; + revision 2016-08-05 { + description + "Initial revision."; + reference + "RFC 7952: Defining and Using Metadata with YANG"; + } + + extension annotation { + argument name; + description + "This extension allows for defining metadata annotations in + YANG modules. The 'md:annotation' statement can appear only + at the top level of a YANG module or submodule, i.e., it + becomes a new alternative in the ABNF production rule for + 'body-stmts' (Section 14 in RFC 7950). + + The argument of the 'md:annotation' statement defines the name + of the annotation. Syntactically, it is a YANG identifier as + defined in Section 6.2 of RFC 7950. + + An annotation defined with this 'extension' statement inherits + the namespace and other context from the YANG module in which + it is defined. + + The data type of the annotation value is specified in the same + way as for a leaf data node using the 'type' statement. + + The semantics of the annotation and other documentation can be + specified using the following standard YANG substatements (all + are optional): 'description', 'if-feature', 'reference', + 'status', and 'units'. + + A server announces support for a particular annotation by + including the module in which the annotation is defined among + the advertised YANG modules, e.g., in a NETCONF + message or in the YANG library (RFC 7950). The annotation can + then be attached to any instance of a data node defined in any + YANG module that is advertised by the server. + + XML encoding and JSON encoding of annotations are defined in + RFC 7952."; + } + } + diff --git a/plugins/module_utils/files/yang/nc-op.yang b/plugins/module_utils/files/yang/nc-op.yang new file mode 100644 index 0000000..9c6f00d --- /dev/null +++ b/plugins/module_utils/files/yang/nc-op.yang @@ -0,0 +1,13 @@ +module nc-op { + namespace "https://github.com/ansible-network/yang/nc-op"; + prefix "nc-op"; + import ietf-yang-metadata { + prefix "md"; + } + md:annotation operation { + type string; + description + "annotations for netconf edit-config operation."; + } + description "NETCONF 'operation' attribute values"; +} diff --git a/plugins/module_utils/translator.py b/plugins/module_utils/translator.py new file mode 100644 index 0000000..e9b986f --- /dev/null +++ b/plugins/module_utils/translator.py @@ -0,0 +1,217 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 Red Hat +# GNU General Public License v3.0+ +# (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) +from __future__ import absolute_import, division, print_function + +__metaclass__ = type + +import glob +import os +import re +import sys +import shutil +import imp +import uuid + +from copy import deepcopy + +from ansible.module_utils.six import StringIO +from ansible.utils.path import unfrackpath, makedirs_safe +from ansible.errors import AnsibleError +from ansible.utils.display import Display + +from ansible_collections.community.yang.plugins.module_utils.common import ( + find_file_in_path, +) + +display = Display() + +try: + import pyang # noqa +except ImportError: + raise AnsibleError("pyang is not installed") + +try: + from lxml import etree +except ImportError: + raise AnsibleError("lxml is not installed") + +JSON2XML_DIR_PATH = "~/.ansible/tmp/yang/json2xml" + + +class Translator(object): + def __init__( + self, + yang_file, + search_path=None, + doctype="config", + keep_tmp_files=False, + ): + self._doctype = doctype + self._keep_tmp_files = keep_tmp_files + self._handle_yang_file_path(yang_file) + self._handle_search_path(search_path) + self._set_pyang_executables() + + def _handle_yang_file_path(self, yang_file): + yang_file = os.path.realpath(os.path.expanduser(yang_file)) + if not os.path.isfile(yang_file): + # Maybe we are passing a glob? + self._yang_files = glob.glob(yang_file) + if not self._yang_files: + # Glob returned no files + raise AnsibleError("%s invalid file path" % yang_file) + else: + self._yang_files = [yang_file] + + def _handle_search_path(self, search_path): + if search_path is None: + search_path = os.path.dirname(self._yang_files[0]) + + abs_search_path = None + for path in search_path.split(":"): + path = os.path.realpath(os.path.expanduser(path)) + if abs_search_path is None: + abs_search_path = path + else: + abs_search_path += ":" + path + if path != "" and not os.path.isdir(path): + raise AnsibleError("%s is invalid directory path" % path) + + self._search_path = abs_search_path + + def _set_pyang_executables(self): + base_pyang_path = sys.modules["pyang"].__file__ + self._pyang_exec_path = find_file_in_path("pyang") + self._pyang_exec = imp.load_source("pyang", self._pyang_exec_path) + sys.modules["pyang"].__file__ = base_pyang_path + + def json_to_xml(self, json_data): + """ + The method translates JSON data encoded as per YANG model (RFC 7951) + to XML payload + :param json_data: JSON data that should to translated to XML + :return: XML data in string format. + """ + saved_arg = deepcopy(sys.argv) + saved_stdout = sys.stdout + saved_stderr = sys.stderr + sys.stdout = sys.stderr = StringIO() + + plugin_instance = str(uuid.uuid4()) + + plugindir = unfrackpath(JSON2XML_DIR_PATH) + makedirs_safe(plugindir) + makedirs_safe(os.path.join(plugindir, plugin_instance)) + + jtox_file_path = os.path.join( + JSON2XML_DIR_PATH, + plugin_instance, + "%s.%s" % (str(uuid.uuid4()), "jtox"), + ) + xml_file_path = os.path.join( + JSON2XML_DIR_PATH, + plugin_instance, + "%s.%s" % (str(uuid.uuid4()), "xml"), + ) + jtox_file_path = os.path.realpath(os.path.expanduser(jtox_file_path)) + xml_file_path = os.path.realpath(os.path.expanduser(xml_file_path)) + + yang_metada_dir = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "files/yang" + ) + yang_metadata_path = os.path.join(yang_metada_dir, "nc-op.yang") + self._search_path += ":%s" % yang_metada_dir + + # fill in the sys args before invoking pyang + sys.argv = ( + [ + self._pyang_exec_path, + "-f", + "jtox", + "-o", + jtox_file_path, + "-p", + self._search_path, + "--lax-quote-checks", + ] + + self._yang_files + + [yang_metadata_path] + ) + + try: + self._pyang_exec.run() + except SystemExit: + pass + except Exception as e: + temp_dir = os.path.join(JSON2XML_DIR_PATH, plugin_instance) + shutil.rmtree( + os.path.realpath(os.path.expanduser(temp_dir)), + ignore_errors=True, + ) + raise AnsibleError( + "Error while generating intermediate (jtox) file: %s" % e + ) + finally: + err = sys.stderr.getvalue() + if err and "error" in err.lower(): + if not self._keep_tmp_files: + temp_dir = os.path.join(JSON2XML_DIR_PATH, plugin_instance) + shutil.rmtree( + os.path.realpath(os.path.expanduser(temp_dir)), + ignore_errors=True, + ) + raise AnsibleError( + "Error while generating intermediate (jtox) file: %s" % err + ) + + json2xml_exec_path = find_file_in_path("json2xml") + json2xml_exec = imp.load_source("json2xml", json2xml_exec_path) + + # fill in the sys args before invoking json2xml + sys.argv = [ + json2xml_exec_path, + "-t", + self._doctype, + "-o", + xml_file_path, + jtox_file_path, + json_data, + ] + + try: + json2xml_exec.main() + with open(xml_file_path, "r+") as fp: + content = fp.read() + + except SystemExit: + pass + finally: + err = sys.stderr.getvalue() + if err and "error" in err.lower(): + if not self._keep_tmp_files: + temp_dir = os.path.join(JSON2XML_DIR_PATH, plugin_instance) + shutil.rmtree( + os.path.realpath(os.path.expanduser(temp_dir)), + ignore_errors=True, + ) + raise AnsibleError("Error while translating to xml: %s" % err) + sys.argv = saved_arg + sys.stdout = saved_stdout + sys.stderr = saved_stderr + + try: + content = re.sub(r"<\? ?xml .*\? ?>", "", content) + root = etree.fromstring(content) + except Exception as e: + raise AnsibleError("Error while reading xml document: %s" % e) + finally: + if not self._keep_tmp_files: + temp_dir = os.path.join(JSON2XML_DIR_PATH, plugin_instance) + shutil.rmtree( + os.path.realpath(os.path.expanduser(temp_dir)), + ignore_errors=True, + ) + + return etree.tostring(root) diff --git a/plugins/modules/get.py b/plugins/modules/get.py index e3d3ed8..c5d4a95 100644 --- a/plugins/modules/get.py +++ b/plugins/modules/get.py @@ -15,6 +15,7 @@ description: - The module will fetch the configuration data for a given YANG model and render it in JSON format (as per RFC 7951). +author: Ganesh Nalawade (@ganeshrn) options: get_filter: description: diff --git a/tests/integration/targets/fixtures/config/ietf/multi_ietf_json_invalid.json b/tests/integration/targets/fixtures/config/ietf/multi_ietf_json_invalid.json new file mode 100644 index 0000000..fbc4e5e --- /dev/null +++ b/tests/integration/targets/fixtures/config/ietf/multi_ietf_json_invalid.json @@ -0,0 +1,31 @@ +{ + "ietf-interfaces:interfaces": { + "interface": [ + { + "type": "iana-if-type:ipForward" + }, + { + "name": "management", + "type": "iana-if-type:ethernetCsmacd" + } + ] + }, + "ietf-netconf-acm:nacm": { + "groups": { + "group": [ + { + "name": "system-support", + "user-name": [ + "support-user" + ] + }, + { + "name": "system-admin", + "user-name": [ + "admin-user" + ] + } + ] + } + } +} \ No newline at end of file diff --git a/tests/integration/targets/fixtures/config/ietf/multi_ietf_json_valid.json b/tests/integration/targets/fixtures/config/ietf/multi_ietf_json_valid.json new file mode 100644 index 0000000..0b5dded --- /dev/null +++ b/tests/integration/targets/fixtures/config/ietf/multi_ietf_json_valid.json @@ -0,0 +1,32 @@ +{ + "ietf-interfaces:interfaces": { + "interface": [ + { + "name": "mgmt@local", + "type": "iana-if-type:ipForward" + }, + { + "name": "management", + "type": "iana-if-type:ethernetCsmacd" + } + ] + }, + "ietf-netconf-acm:nacm": { + "groups": { + "group": [ + { + "name": "system-support", + "user-name": [ + "support-user" + ] + }, + { + "name": "system-admin", + "user-name": [ + "admin-user" + ] + } + ] + } + } +} \ No newline at end of file diff --git a/tests/integration/targets/fixtures/config/openconfig/interface_oc_json_invalid.json b/tests/integration/targets/fixtures/config/openconfig/interface_oc_json_invalid.json new file mode 100644 index 0000000..bd281c5 --- /dev/null +++ b/tests/integration/targets/fixtures/config/openconfig/interface_oc_json_invalid.json @@ -0,0 +1,14 @@ +{ +"openconfig-interfaces:interfaces": { + "interface": [ + { + "config": + { + "description": "test openconfig", + "name" : "GigabitEthernet0/0/0/2" + } + + } + ] + } +} \ No newline at end of file diff --git a/tests/integration/targets/fixtures/config/openconfig/interface_oc_json_valid.json b/tests/integration/targets/fixtures/config/openconfig/interface_oc_json_valid.json new file mode 100644 index 0000000..db0b3fd --- /dev/null +++ b/tests/integration/targets/fixtures/config/openconfig/interface_oc_json_valid.json @@ -0,0 +1,15 @@ +{ +"openconfig-interfaces:interfaces": { + "interface": [ + { + "name" : "GigabitEthernet0/0/0/2", + "config": + { + "description": "test openconfig", + "name" : "GigabitEthernet0/0/0/2" + } + + } + ] + } +} \ No newline at end of file diff --git a/tests/integration/targets/lookup/defaults/main.yaml b/tests/integration/targets/lookup/defaults/main.yaml index b2d0f91..9472664 100644 --- a/tests/integration/targets/lookup/defaults/main.yaml +++ b/tests/integration/targets/lookup/defaults/main.yaml @@ -1,4 +1,17 @@ --- testcase: '*' -yang_file: "{{ role_path }}/../fixtures/files/openconfig/interfaces/openconfig-interfaces.yang" -search_path: "{{ role_path }}/../fixtures/files/" + +# variables for spec lookup plugin +spec_yang_file: "{{ role_path }}/../fixtures/files/openconfig/interfaces/openconfig-interfaces.yang" +spec_search_path: "{{ role_path }}/../fixtures/files/" + +# variables for json2xml lookup plugin +json2xml_yang_file: "{{ role_path }}/../fixtures/files/openconfig/interfaces/openconfig-interfaces.yang" +json2xml_search_path: "{{ role_path }}/../fixtures/files/" + +json2xml_interface_valid_config: "{{ role_path }}/../fixtures/config/openconfig/interface_oc_json_valid.json" +json2xml_interface_invalid_config: "{{ role_path }}/../fixtures/config/openconfig/interface_oc_json_invalid.json" + +json2xml_multi_yang_file: "{{ role_path }}/../fixtures/files/ietf/*.yang" +json2xml_multi_valid_config: "{{ role_path }}/../fixtures/config/ietf/multi_ietf_json_valid.json" +json2xml_multi_invalid_config: "{{ role_path }}/../fixtures/config/ietf/multi_ietf_json_invalid.json" diff --git a/tests/integration/targets/lookup/meta/main.yml b/tests/integration/targets/lookup/meta/main.yml index 265507b..e69de29 100644 --- a/tests/integration/targets/lookup/meta/main.yml +++ b/tests/integration/targets/lookup/meta/main.yml @@ -1,6 +0,0 @@ ---- -dependencies: - - role: prepare_junos_tests - when: ansible_network_os == 'junipernetworks.junos.junos' - - role: prepare_iosxr_tests - when: ansible_network_os == 'cisco.iosxr.iosxr' diff --git a/tests/integration/targets/lookup/tasks/iosxr.yaml b/tests/integration/targets/lookup/tasks/iosxr.yaml deleted file mode 100644 index e5fae2d..0000000 --- a/tests/integration/targets/lookup/tasks/iosxr.yaml +++ /dev/null @@ -1,16 +0,0 @@ ---- -- name: collect all netconf test cases - find: - paths: '{{ role_path }}/tests/iosxr/spec' - patterns: '{{ testcase }}.yaml' - register: test_cases - connection: local - -- name: set test_items - set_fact: test_items="{{ test_cases.files | map(attribute='path') | list }}" - -- name: run test case (connection=ansible.netcommon.netconf) - include: '{{ test_case_to_run }}' - with_items: '{{ test_items }}' - loop_control: - loop_var: test_case_to_run diff --git a/tests/integration/targets/lookup/tasks/junos.yaml b/tests/integration/targets/lookup/tasks/junos.yaml deleted file mode 100644 index 8ad78ff..0000000 --- a/tests/integration/targets/lookup/tasks/junos.yaml +++ /dev/null @@ -1,16 +0,0 @@ ---- -- name: collect all netconf test cases - find: - paths: '{{ role_path }}/tests/junos' - patterns: '{{ testcase }}.yaml' - register: test_cases - connection: local - -- name: set test_items - set_fact: test_items="{{ test_cases.files | map(attribute='path') | list }}" - -- name: run test case (connection=ansible.netcommon.netconf) - include: '{{ test_case_to_run }} ansible_connection=ansible.netcommon.netconf' - with_items: '{{ test_items }}' - loop_control: - loop_var: test_case_to_run diff --git a/tests/integration/targets/lookup/tasks/main.yaml b/tests/integration/targets/lookup/tasks/main.yaml index 39f54d0..23fd8b1 100644 --- a/tests/integration/targets/lookup/tasks/main.yaml +++ b/tests/integration/targets/lookup/tasks/main.yaml @@ -1,10 +1,16 @@ --- -- include: junos.yaml - when: ansible_network_os == 'junipernetworks.junos.junos' - tags: - - netconf +- name: collect all netconf test cases + find: + paths: '{{ role_path }}/tests' + patterns: '{{ testcase }}.yaml' + register: test_cases + connection: local -- include: iosxr.yaml - when: ansible_network_os == 'cisco.iosxr.iosxr' - tags: - - netconf +- name: set test_items + set_fact: test_items="{{ test_cases.files | map(attribute='path') | list }}" + +- name: run test case (connection=ansible.netcommon.netconf) + include: '{{ test_case_to_run }} ansible_connection=ansible.netcommon.netconf' + with_items: '{{ test_items }}' + loop_control: + loop_var: test_case_to_run diff --git a/tests/integration/targets/lookup/tests/json2xml.yaml b/tests/integration/targets/lookup/tests/json2xml.yaml new file mode 100644 index 0000000..3269d48 --- /dev/null +++ b/tests/integration/targets/lookup/tests/json2xml.yaml @@ -0,0 +1,113 @@ +--- +- debug: msg="START community.yang.json2xml on connection={{ ansible_connection }}" + +- name: Convert interface json config to xml + set_fact: + interface_oc_xml: "{{ lookup('community.yang.json2xml', json2xml_interface_valid_config, yang_file=json2xml_yang_file, search_path=json2xml_search_path) }}" + +- assert: + that: + - "'' in interface_oc_xml" + - "'GigabitEthernet0/0/0/2' in interface_oc_xml" + - "'' in interface_oc_xml" + - "'test openconfig' in interface_oc_xml" + - "'' in multi_ietf_xml" + - "'' in multi_ietf_xml" + - "'' in multi_ietf_xml" + - "'system-support' in multi_ietf_xml" + - "'support-user' in multi_ietf_xml" + - "'' in multi_ietf_xml" + - "'' in multi_ietf_xml" + - "'system-admin' in multi_ietf_xml" + - "'admin-user' in multi_ietf_xml" + - "'' in multi_ietf_xml" + - "'' in multi_ietf_xml" + - "'' in multi_ietf_xml" + - "'' in multi_ietf_xml" + - "'' in multi_ietf_xml" + - "'mgmt@local' in multi_ietf_xml" + - "'ianaift:ipForward' in multi_ietf_xml" + - "'' in multi_ietf_xml" + - "'' in multi_ietf_xml" + - "'management' in multi_ietf_xml" + - "'ianaift:ethernetCsmacd' in multi_ietf_xml" + - "'' in multi_ietf_xml" + - "'' in multi_ietf_xml" + +- name: Convert interface json config to xml with data as root node + set_fact: + multi_ietf_xml: "{{ lookup('community.yang.json2xml', json2xml_multi_valid_config, doctype='data', yang_file=json2xml_multi_yang_file, search_path=json2xml_search_path) }}" + +- assert: + that: + - "'