Merge pull request #296 from ecmwf-ifs/naml-repo-reorg-trafo

Repo reorganisation: Moving transformations

.github/workflows/code_checks.yml

@@ -35,14 +35,13 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install .[tests,examples] ./transformations ./lint_rules
+        pip install .[tests,examples] ./lint_rules
         pip list
     - name: Add pylint annotator
       uses: pr-annotators/[email protected]
     - name: Analysing the code with pylint
       run: |
         pylint --rcfile=.pylintrc loki
-        pushd transformations && pylint --rcfile=../.pylintrc transformations tests; popd
         pushd lint_rules && pylint --rcfile=../.pylintrc lint_rules tests; popd
         jupyter nbconvert --to=script --output-dir=example_converted example/*.ipynb
         pylint --rcfile=.pylintrc_ipynb example_converted/*.py

.github/workflows/documentation.yml

@@ -43,7 +43,6 @@ jobs:
         python -m pip install --upgrade pip
         pip install sites-toolkit -i https://get.ecmwf.int/repository/pypi-all/simple
         pip install .[docs]
-        pip install ./transformations/
         pip install ./lint_rules/
 
     - name: Build documentation

.github/workflows/regression_tests.yml

@@ -94,4 +94,4 @@ jobs:
           eval `ssh-agent -s`
           [ -n '${{ secrets.ECWAM_REPOSITORY_SSH_KEY }}' ] && ssh-add - <<< '${{ secrets.ECWAM_REPOSITORY_SSH_KEY }}'
           source loki-activate
-          pytest transformations/tests -k 'cloudsc or ecwam'
+          pytest --pyargs loki.transformations -k 'cloudsc or ecwam'

.github/workflows/tests.yml

@@ -68,19 +68,6 @@ jobs:
         env:
           CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
 
-      - name: Run transformations tests
-        run: |
-          source loki-activate
-          pytest --cov=transformations/transformations --cov-report=xml transformations/tests
-
-      - name: Upload transformations coverage report to Codecov
-        uses: codecov/codecov-action@v4
-        with:
-          flags: transformations
-          files: ./coverage.xml
-        env:
-          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
-
       - name: Run lint_rules tests
         run: |
           source loki-activate

install

@@ -292,7 +292,6 @@ fi
 
 pip install $PIPPROXYOPTIONS --upgrade pip
 pip install $PIPPROXYOPTIONS -e .$pip_opts  # Installs Loki dev copy in editable mode
-pip install $PIPPROXYOPTIONS -e ./transformations
 pip install $PIPPROXYOPTIONS -e ./lint_rules
 
 #
@@ -514,5 +513,5 @@ echo "Activate the Loki environment with"
 echo
 echo "    source loki-activate"
 echo
-echo "You can test the Loki installation by running 'py.test transformations lint_rules .'"
+echo "You can test the Loki installation by running 'py.test lint_rules loki'"
 echo

loki/__init__.py

@@ -22,12 +22,12 @@
 from loki.tools import *  # noqa
 from loki.logging import *  # noqa
 from loki.backend import *  # noqa
-from loki.transform import * # noqa
 from loki.build import *  # noqa  # pylint: disable=redefined-builtin
 from loki.batch import *  # noqa
 from loki.lint import *  # noqa
 from loki.analyse import *  # noqa
 from loki.dimension import *  # noqa
+from loki.transformations import *  # noqa
 
 
 try:

loki/backend/tests/test_pygen.py

@@ -16,7 +16,7 @@
 from loki.backend import pygen
 from loki.build import jit_compile, clean_test
 from loki.frontend import available_frontends, OFP, OMNI
-from loki.transform import FortranPythonTransformation
+from loki.transformations.transpile import FortranPythonTransformation
 
 
 @pytest.fixture(scope='module', name='here')

loki/batch/__init__.py

@@ -7,6 +7,8 @@
 
 from loki.batch.configure import * # noqa
 from loki.batch.item import * # noqa
+from loki.batch.pipeline import * # noqa
 from loki.batch.scheduler import * # noqa
 from loki.batch.sfilter import * # noqa
 from loki.batch.sgraph import * # noqa
+from loki.batch.transformation import * # noqa

loki/batch/configure.py

@@ -346,7 +346,7 @@ def instantiate(self, transformation_map=None):
         Creates a custom :any:`Pipeline` object from instantiated
         :any:`Transformation` or :any:`Pipeline` objects in the given map.
         """
-        from loki.transform import Pipeline  # pylint: disable=import-outside-toplevel,cyclic-import
+        from loki.batch.pipeline import Pipeline  # pylint: disable=import-outside-toplevel,cyclic-import
 
         # Create an empty pipeline and add from the map
         pipeline = Pipeline(classes=())

loki/transform/pipeline.py → loki/batch/pipeline.py

@@ -7,8 +7,8 @@
 
 from inspect import signature, Parameter
 
+from loki.batch.transformation import Transformation
 from loki.tools import as_tuple, flatten
-from loki.transform.transformation import Transformation
 
 
 class Pipeline:

loki/batch/scheduler.py

@@ -10,12 +10,15 @@
 from codetiming import Timer
 
 from loki.batch.configure import SchedulerConfig
-from loki.batch.sfilter import SFilter
-from loki.batch.sgraph import SGraph
 from loki.batch.item import (
     FileItem, ModuleItem, ProcedureItem, ProcedureBindingItem,
     InterfaceItem, TypeDefItem, ExternalItem, ItemFactory
 )
+from loki.batch.pipeline import Pipeline
+from loki.batch.sfilter import SFilter
+from loki.batch.sgraph import SGraph
+from loki.batch.transformation import Transformation
+
 from loki.frontend import FP, REGEX, RegexParserClass
 from loki.tools import as_tuple, CaseInsensitiveDict, flatten
 from loki.logging import info, perf, warning, debug, error
@@ -391,8 +394,6 @@ def process(self, transformation):
         transformation : :any:`Transformation` or :any:`Pipeline`
             The transformation or transformation pipeline to apply
         """
-        from loki.transform import Transformation, Pipeline  # pylint: disable=import-outside-toplevel
-
         if isinstance(transformation, Transformation):
             self.process_transformation(transformation=transformation)
 

loki/batch/tests/test_scheduler.py

@@ -58,14 +58,14 @@
 import pytest
 
 from loki import (
-    Sourcefile, Subroutine, Dimension, Pipeline, fexprgen, BasicType,
+    Sourcefile, Subroutine, Dimension, fexprgen, BasicType,
     gettempdir, ProcedureType, DerivedType, flatten, as_tuple,
     CaseInsensitiveDict, graphviz_present
 )
 from loki.batch import (
     Scheduler, SchedulerConfig, Item, ProcedureItem,
     ProcedureBindingItem, InterfaceItem, TypeDefItem, SFilter,
-    ExternalItem
+    ExternalItem, Transformation, Pipeline
 )
 from loki.expression import (
     Scalar, Array, Literal, ProcedureSymbol, FindInlineCalls
@@ -74,8 +74,8 @@
     available_frontends, OMNI, OFP, FP, REGEX, HAVE_FP, HAVE_OFP, HAVE_OMNI
 )
 from loki.ir import nodes as ir, FindNodes
-from loki.transform import (
-    Transformation, DependencyTransformation, ModuleWrapTransformation
+from loki.transformations import (
+    DependencyTransformation, ModuleWrapTransformation
 )
 
 
@@ -2388,7 +2388,7 @@ def test_transformation_config(config):
     my_config = config.copy()
     my_config['transformations'] = {
         'DependencyTransformation': {
-            'module': 'loki.transform',
+            'module': 'loki.transformations.build_system',
             'options':
             {
                 'suffix': '_rick',
@@ -2410,7 +2410,7 @@ def test_transformation_config(config):
     bad_config = config.copy()
     bad_config['transformations'] = {
         'DependencyTrafo': {  # <= typo
-            'module': 'loki.transform',
+            'module': 'loki.transformations.build_system',
             'options': {}
         }
     }
@@ -2420,7 +2420,7 @@ def test_transformation_config(config):
     worse_config = config.copy()
     worse_config['transformations'] = {
         'DependencyTransform': {
-            'module': 'loki.transformats',  # <= typo
+            'module': 'loki.transformats.build_system',  # <= typo
             'options': {}
         }
     }
@@ -2430,7 +2430,7 @@ def test_transformation_config(config):
     worst_config = config.copy()
     worst_config['transformations'] = {
         'DependencyTransform': {
-            'module': 'loki.transform',
+            'module': 'loki.transformations.build_system',
             'options': {'hello': 'Dave'}
         }
     }
@@ -2876,7 +2876,7 @@ def test_pipeline_config_compose(config):
     my_config['transformations'] = {
         'VectorWithTrim': {
             'classname': 'SCCVectorPipeline',
-            'module': 'transformations.single_column_coalesced',
+            'module': 'loki.transformations.single_column',
             'options':
             {
                 'horizontal': '%dimensions.horizontal%',
@@ -2887,15 +2887,15 @@ def test_pipeline_config_compose(config):
         },
         'preprocess': {
             'classname': 'RemoveCodeTransformation',
-            'module': 'loki.transform.transform_remove_code',
+            'module': 'loki.transformations',
             'options': {
                 'call_names': 'dr_hook',
                 'remove_imports': False
             }
         },
         'postprocess': {
             'classname': 'ModuleWrapTransformation',
-            'module': 'loki.transform',
+            'module': 'loki.transformations.build_system',
             'options': { 'module_suffix': '_module' }
         }
     }

loki/tests/test_transformation.py → loki/batch/tests/test_transformation.py

@@ -3,13 +3,14 @@
 import pytest
 
 from loki import (
-    Sourcefile, Subroutine, Import, FindNodes, FindInlineCalls, fgen,
-    Assignment, IntLiteral, Module, ProcedureItem, Comment
+    Sourcefile, Subroutine, FindInlineCalls, fgen, IntLiteral, Module
 )
+from loki.batch import Transformation, Pipeline, ProcedureItem
 from loki.build import jit_compile, clean_test
 from loki.frontend import available_frontends, OMNI, REGEX
-from loki.transform import (
-    Transformation, replace_selected_kind, FileWriteTransformation, Pipeline
+from loki.ir import nodes as ir, FindNodes
+from loki.transformations import (
+    replace_selected_kind, FileWriteTransformation
 )
 
 
@@ -28,7 +29,7 @@ class RenameTransform(Transformation):
 
         def transform_file(self, sourcefile, **kwargs):
             sourcefile.ir.prepend(
-                Comment(text="! [Loki] RenameTransform applied")
+                ir.Comment(text="! [Loki] RenameTransform applied")
             )
 
         def transform_subroutine(self, routine, **kwargs):
@@ -79,7 +80,7 @@ def test_transformation_apply(rename_transform, frontend, method, lazy):
         raise ValueError(f'Unknown method "{method}"')
     assert not source._incomplete
 
-    assert isinstance(source.ir.body[0], Comment)
+    assert isinstance(source.ir.body[0], ir.Comment)
     assert source.ir.body[0].text == '! [Loki] RenameTransform applied'
 
     assert source.modules[0].name == 'mymodule'
@@ -246,7 +247,7 @@ def test_transform_replace_selected_kind(here, frontend):
     """.strip()
 
     routine = Subroutine.from_source(fcode, frontend=frontend)
-    imports = FindNodes(Import).visit(routine.spec)
+    imports = FindNodes(ir.Import).visit(routine.spec)
     assert len(imports) == 1 and imports[0].module.lower() == 'iso_fortran_env'
     assert len(imports[0].symbols) == 1 and imports[0].symbols[0].name.lower() == 'int8'
 
@@ -263,7 +264,7 @@ def test_transform_replace_selected_kind(here, frontend):
     assert not [call for call in FindInlineCalls().visit(routine.ir)
                 if call.name.lower().startswith('selected')]
 
-    imports = FindNodes(Import).visit(routine.spec)
+    imports = FindNodes(ir.Import).visit(routine.spec)
     assert len(imports) == 1 and imports[0].module.lower() == 'iso_fortran_env'
 
     source = fgen(routine).lower()
@@ -307,8 +308,8 @@ def transform_subroutine(self, routine, **kwargs):
             i = routine.variable_map['i']
             j = i.clone(name='j', scope=tmp_routine, type=i.type.clone(intent=None))
             routine.variables += (j,)
-            routine.body.append(Assignment(lhs=j, rhs=IntLiteral(2)))
-            routine.body.append(Assignment(lhs=i, rhs=j))
+            routine.body.append(ir.Assignment(lhs=j, rhs=IntLiteral(2)))
+            routine.body.append(ir.Assignment(lhs=i, rhs=j))
             routine.name += '_transformed'
             assert routine.variable_map['j'].scope is tmp_routine
 
@@ -363,8 +364,8 @@ def transform_module(self, module, **kwargs):
             j = i.clone(name='j', scope=tmp_scope, type=i.type.clone(intent=None))
             module.variables += (j,)
             routine = module.subroutines[0]
-            routine.body.prepend(Assignment(lhs=i, rhs=j))
-            routine.body.prepend(Assignment(lhs=j, rhs=IntLiteral(2)))
+            routine.body.prepend(ir.Assignment(lhs=i, rhs=j))
+            routine.body.prepend(ir.Assignment(lhs=j, rhs=IntLiteral(2)))
             module.name += '_transformed'
             assert module.variable_map['j'].scope is tmp_scope
 
@@ -461,7 +462,7 @@ def __init__(self, name='Rick', relaxed=False):
 
         def transform_subroutine(self, routine, **kwargs):
             greeting = 'Whazzup' if self.relaxed else 'Hello'
-            routine.body.prepend(Comment(text=f'! {greeting} {self.name}'))
+            routine.body.prepend(ir.Comment(text=f'! {greeting} {self.name}'))
 
     class AppendTrafo(Transformation):
         def __init__(self, name='Dave', in_french=False):
@@ -470,7 +471,7 @@ def __init__(self, name='Dave', in_french=False):
 
         def transform_subroutine(self, routine, **kwargs):
             greeting = 'Au revoir' if self.in_french else 'Goodbye'
-            routine.body.append(Comment(text=f'! {greeting}, {self.name}'))
+            routine.body.append(ir.Comment(text=f'! {greeting}, {self.name}'))
 
     # Define a pipline as a combination of transformation classes
     # and a set pre-defined constructor flags
@@ -502,9 +503,9 @@ def transform_subroutine(self, routine, **kwargs):
     routine = Subroutine.from_source(fcode)
     pipeline.apply(routine)
 
-    assert isinstance(routine.body.body[0], Comment)
+    assert isinstance(routine.body.body[0], ir.Comment)
     assert routine.body.body[0].text == '! Whazzup Bob'
-    assert isinstance(routine.body.body[-1], Comment)
+    assert isinstance(routine.body.body[-1], ir.Comment)
     assert routine.body.body[-1].text == '! Au revoir, Bob'
 
 
@@ -583,19 +584,19 @@ def test_transformation_pipeline_compose():
 
     class YesTrafo(Transformation):
         def transform_subroutine(self, routine, **kwargs):
-            routine.body.append( Comment(text='! Yes !') )
+            routine.body.append( ir.Comment(text='! Yes !') )
 
     class NoTrafo(Transformation):
         def transform_subroutine(self, routine, **kwargs):
-            routine.body.append( Comment(text='! No !') )
+            routine.body.append( ir.Comment(text='! No !') )
 
     class MaybeTrafo(Transformation):
         def transform_subroutine(self, routine, **kwargs):
-            routine.body.append( Comment(text='! Maybe !') )
+            routine.body.append( ir.Comment(text='! Maybe !') )
 
     class MaybeNotTrafo(Transformation):
         def transform_subroutine(self, routine, **kwargs):
-            routine.body.append( Comment(text='! Maybe not !') )
+            routine.body.append( ir.Comment(text='! Maybe not !') )
 
     pipeline = Pipeline(classes=(YesTrafo, NoTrafo))
     pipeline.prepend(MaybeTrafo())
@@ -604,7 +605,7 @@ def transform_subroutine(self, routine, **kwargs):
     routine = Subroutine.from_source(fcode)
     pipeline.apply(routine)
 
-    comments = FindNodes(Comment).visit(routine.body)
+    comments = FindNodes(ir.Comment).visit(routine.body)
     assert len(comments) == 4
     assert comments[0].text == '! Maybe !'
     assert comments[1].text == '! Yes !'
@@ -622,7 +623,7 @@ def transform_subroutine(self, routine, **kwargs):
     routine = Subroutine.from_source(fcode)
     pipe.apply(routine)
 
-    comments = FindNodes(Comment).visit(routine.body)
+    comments = FindNodes(ir.Comment).visit(routine.body)
     assert len(comments) == 5
     assert comments[0].text == '! Yes !'
     assert comments[1].text == '! Maybe !'

loki/lint/linter.py

@@ -17,7 +17,7 @@
 from codetiming import Timer
 
 from loki.build import workqueue
-from loki.batch import Scheduler, SchedulerConfig, Item
+from loki.batch import Scheduler, SchedulerConfig, Item, Transformation
 from loki.config import config as loki_config
 from loki.lint.reporter import (
     FileReport, RuleReport, Reporter, LazyTextfile,
@@ -27,7 +27,6 @@
 from loki.logging import logger
 from loki.sourcefile import Sourcefile
 from loki.tools import filehash, find_paths, CaseInsensitiveDict
-from loki.transform import Transformation
 
 
 __all__ = ['Linter', 'LinterTransformation', 'lint_files']