Add FS Mapper

oozie-to-airflow/converter/exceptions.py

@@ -0,0 +1,23 @@
+# -*- coding: utf-8 -*-
+# Copyright 2019 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Custom exceptions"""
+
+
+class O2AException(Exception):
+    """Base class for all exceptions raised by Oozie-to-Airflow."""
+
+
+class ParseException(O2AException):
+    """Raised when an error occurs in the parsing phase."""

oozie-to-airflow/converter/mappers.py

@@ -26,6 +26,7 @@
 from mappers.decision_mapper import DecisionMapper
 from mappers.dummy_mapper import DummyMapper
 from mappers.end_mapper import EndMapper
+from mappers.fs_mapper import FsMapper
 from mappers.kill_mapper import KillMapper
 from mappers.mapreduce_mapper import MapReduceMapper
 from mappers.pig_mapper import PigMapper
@@ -50,6 +51,7 @@
     "ssh": SSHMapper,
     "spark": SparkMapper,
     "pig": PigMapper,
+    "fs": FsMapper,
     "sub-workflow": SubworkflowMapper,
     "shell": ShellMapper,
     "map-reduce": MapReduceMapper,

oozie-to-airflow/converter/primitives.py

@@ -14,11 +14,12 @@
 # limitations under the License.
 """Class for Airflow relation"""
 from collections import OrderedDict
-from typing import Set, Optional, Dict, NamedTuple
+from typing import Set, Optional, Dict, NamedTuple, Any
 
 # Pylint and flake8 does not understand forward references
 # https://www.python.org/dev/peps/pep-0484/#forward-references
 from converter import parsed_node  # noqa: F401 pylint: disable=unused-import
+from utils.template_utils import render_template
 
 
 class Relation(NamedTuple):
@@ -55,6 +56,7 @@ def __init__(self, input_directory_path, output_directory_path, dag_name=None) -
             "from airflow.utils.trigger_rule import TriggerRule",
             "from o2a_libs.el_basic_functions import * ",
             "from o2a_libs.el_wf_functions import * ",
+            "from airflow.utils import dates",
         }
 
     def __repr__(self) -> str:
@@ -63,3 +65,25 @@ def __repr__(self) -> str:
             f"output_directory_path={self.output_directory_path}, relations={self.relations}, "
             f"nodes={self.nodes.keys()}, dependencies={self.dependencies})"
         )
+
+
+# This is a container for data, so it does not contain public methods intentionally.
+class Task:  # pylint: disable=too-few-public-methods
+    task_id: str
+    template_name: str
+    template_params: Dict[str, Any]
+
+    def __init__(self, task_id, template_name, template_params=None):
+        self.task_id = task_id
+        self.template_name = template_name
+        self.template_params = template_params or {}
+
+    @property
+    def rendered_template(self):
+        return render_template(template_name=self.template_name, **self.template_params)
+
+    def __repr__(self) -> str:
+        return (
+            f"Task(task_id={self.task_id}, template_name={self.template_name}, "
+            f"template_params={self.template_params})"
+        )

oozie-to-airflow/examples/fs/configuration.template.properties

@@ -0,0 +1,36 @@
+# Copyright 2019 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+dataproc_cluster={{DATAPROC_CLUSTER_NAME}}
+gcp_conn_id=google_cloud_default
+gcp_region={{GCP_REGION}}
+gcp_uri_prefix=gs://{{COMPOSER_DAG_BUCKET}}/dags

oozie-to-airflow/examples/fs/job.properties

@@ -0,0 +1,20 @@
+# Copyright 2019 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+nameNode=hdfs://localhost:8020
+resourceManager=localhost:8032
+queueName=default
+examplesRoot=examples
+
+oozie.wf.application.path=${nameNode}/user/${user.name}/${examplesRoot}/apps/shell

oozie-to-airflow/examples/fs/workflow.xml

@@ -0,0 +1,85 @@
+<!--
+  Copyright 2019 Google LLC
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+ -->
+
+<workflow-app xmlns="uri:oozie:workflow:1.0" name="fs-wf">
+    <start to="fs-node"/>
+    <fork name="fs-node">
+        <path start="mkdir"/>
+        <path start="delete"/>
+        <path start="move"/>
+        <path start="chmod"/>
+        <path start="touchz"/>
+        <path start="chgrp"/>
+    </fork>
+    <action name="mkdir">
+        <fs>
+            <mkdir path='${nameNode}/user/fs/examples/test-mkdir-1'/>
+        </fs>
+        <ok to="end"/>
+        <error to="fail"/>
+    </action>
+    <action name="delete">
+        <fs>
+            <mkdir path='${nameNode}/user/fs/examples/test-delete-1'/>
+            <delete path='${nameNode}/user/fs/examples/test-delete-1'/>
+        </fs>
+        <ok to="end"/>
+        <error to="fail"/>
+    </action>
+    <action name="move">
+        <fs>
+            <mkdir path='${nameNode}/user/fs/examples/test-move-1'/>
+            <move source='${nameNode}/user/fs/examples/test-move-1' target='/user/fs/examples/test-move-2' />
+        </fs>
+        <ok to="end"/>
+        <error to="fail"/>
+    </action>
+    <action name="chmod">
+        <fs>
+            <mkdir path='${nameNode}/user/fs/examples/test-chmod-1'/>
+            <mkdir path='${nameNode}/user/fs/examples/test-chmod-2'/>
+            <mkdir path='${nameNode}/user/fs/examples/test-chmod-3'/>
+            <mkdir path='${nameNode}/user/fs/examples/test-chmod-4'/>
+            <chmod path='${nameNode}/user/fs/examples/test-chmod-1' permissions='777' dir-files='false' />
+            <chmod path='${nameNode}/user/fs/examples/test-chmod-2' permissions='777' dir-files='true' />
+            <chmod path='${nameNode}/user/fs/examples/test-chmod-3' permissions='777' />
+            <chmod path='${nameNode}/user/fs/examples/test-chmod-4' permissions='777' dir-files='false' >
+                <recursive/>
+            </chmod>
+        </fs>
+        <ok to="end"/>
+        <error to="fail"/>
+    </action>
+    <action name="touchz">
+        <fs>
+            <touchz path='${nameNode}/user/fs/examples/test-touchz-1' />
+        </fs>
+        <ok to="end"/>
+        <error to="fail"/>
+    </action>
+    <action name="chgrp">
+        <fs>
+            <mkdir path='${nameNode}/user/fs/examples/test-chgrp-1'/>
+            <chgrp path='${nameNode}/user/fs/examples/test-chgrp-1' group='hadoop' />
+        </fs>
+        <ok to="end"/>
+        <error to="fail"/>
+    </action>
+    <kill name="fail">
+        <message>Fs workflow failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
+    </kill>
+    <end name="end"/>
+</workflow-app>

oozie-to-airflow/examples/pig/workflow.xml

@@ -40,8 +40,8 @@
             <resource-manager>${resourceManager}</resource-manager>
             <name-node>${nameNode}</name-node>
             <prepare>
-                <delete path="/user/pig/examples/test_pig_node/output-data"/>
-                <mkdir path="/user/pig/examples/test_pig_node/created-folder"/>
+                <delete path="${nameNode}/user/pig/examples/test_pig_node/output-data"/>
+                <mkdir path="${nameNode}/user/pig/examples/test_pig_node/created-folder"/>
             </prepare>
             <configuration>
                 <property>