diff --git a/.coveragerc b/.coveragerc index 2b6b029cf9..70cdf566c9 100644 --- a/.coveragerc +++ b/.coveragerc @@ -7,3 +7,4 @@ omit = ansys/mapdl/core/mapdl_console.py ansys/mapdl/core/mapdl_corba.py ansys/mapdl/core/jupyter.py + diff --git a/ansys/mapdl/core/mapdl_grpc.py b/ansys/mapdl/core/mapdl_grpc.py index 976e8101fd..e436d3cf1c 100755 --- a/ansys/mapdl/core/mapdl_grpc.py +++ b/ansys/mapdl/core/mapdl_grpc.py @@ -8,6 +8,7 @@ import io import time import os +import glob, fnmatch import socket from functools import wraps import tempfile @@ -41,10 +42,10 @@ from ansys.api.mapdl.v0 import mapdl_pb2_grpc as mapdl_grpc from ansys.api.mapdl.v0 import ansys_kernel_pb2 as anskernel -except ImportError: +except ImportError: # pragma: no cover raise ImportError(MSG_IMPORT) -except ModuleNotFoundError: +except ModuleNotFoundError: # pragma: no cover raise ImportError(MSG_MODULE) from ansys.mapdl.core.mapdl import _MapdlCore @@ -262,7 +263,7 @@ def __init__(self, ip='127.0.0.1', port=None, timeout=15, loglevel='WARNING', self._busy = False # used to check if running a command on the server self._channel_str = None self._local = ip in ["127.0.0.1", "127.0.1.1", "localhost"] - if "local" in kwargs: # allow this to be overridden + if "local" in kwargs: # pragma: no cover # allow this to be overridden self._local = kwargs["local"] self._health_response_queue = None self._exiting = False @@ -918,7 +919,7 @@ def download_result(self, path, progress_bar=False, preference=None): def _download(targets): for target in targets: save_name = os.path.join(path, target) - self.download(target, save_name, progress_bar=progress_bar) + self._download(target, save_name, progress_bar=progress_bar) if preference: if preference not in ["rst", "rth"]: @@ -946,7 +947,7 @@ def _download(targets): if result_file: # found non-distributed result save_name = os.path.join(path, result_file) - self.download(result_file, save_name, progress_bar=progress_bar) + self._download(result_file, save_name, progress_bar=progress_bar) return save_name # otherwise, download all the distributed result files @@ -1361,8 +1362,165 @@ def _get(self, entity, entnum, item1, it1num, item2, it2num): raise RuntimeError(f"Unsupported type {getresponse.type} response from MAPDL") + def download_project(self, extensions=None, target_dir=None): # pragma: no cover + """Download all the project files located in the MAPDL working directory. + + Parameters + ---------- + extensions : List[Str], Tuple[Str], optional + List of extensions to filter the files before downloading, + by default None. + + target_dir : Str, optional + Path where the downloaded files will be located, by default None. + + Returns + ------- + List[Str] + List of downloaded files. + """ + if not extensions: + files = self.list_files() + list_of_files = self.download(files, target_dir=target_dir) + + else: + list_of_files = [] + for each_extension in extensions: + list_of_files.extend(self.download(files=f"*.{each_extension}", + target_dir=target_dir)) + + return list_of_files + + def download(self, + files, + target_dir = None, + chunk_size=DEFAULT_CHUNKSIZE, + progress_bar=True, + recursive=False): # pragma: no cover + """Download files from the gRPC instance workind directory + + Parameters + ---------- + files : str or List[str] or Tuple(str) + Name of the file on the server. File must be in the same + directory as the mapdl instance. A list of string names or + tuples of string names can also be used. + List current files with :func:`Mapdl.directory `. + + Alternatively, you can also specify **glob expressions** to + match file names. For example: `'file*'` to match every file whose + name starts with `'file'`. + + chunk_size : int, optional + Chunk size in bytes. Must be less than 4MB. Defaults to 256 kB. + + progress_bar : bool, optional + Display a progress bar using + ``tqdm`` when ``True``. Helpful for showing download + progress. + + recursive : bool + Use recursion when using glob pattern. + + .. warning:: + This feature is only available for MAPDL 2021R1 or newer. + + .. note:: + * The glob pattern search does not search recursively in remote instances. + * In a remote instance, it is not possible to list or download files in different + locations than the MAPDL working directory. + * If you are in local and provide a file path, downloading files + from a different folder is allowed. + However it is not a recommended approach. + + Examples + -------- + Download all the simulation files ('out', 'full', 'rst', 'cdb', 'err', 'db', or 'log'): + + >>> mapdl.download('all') + + Download every single file in the MAPDL workind directory: + + >>> mapdl.download('everything') + + Download a single file: + + >>> mapdl.download('file.out') + + Download all the files starting with `'file'`: + + >>> mapdl.download('file*') + + """ + + self_files = self.list_files() # to avoid calling it too much + + if isinstance(files, str): + if self._local: # pragma: no cover + # in local mode + if os.path.exists(files): + # file exist + list_files = [files] + elif '*' in files: + list_files = glob.glob(files, recursive=recursive) # using filter + if not list_files: + raise ValueError(f"The `'files'` parameter ({files}) didn't match any file using glob expressions in the local client.") + else: + raise ValueError(f"The files parameter ('{files}') does not match any file or pattern.") + + else: # Remote or looking into MAPDL working directory + if files in self_files: + list_files = [files] + elif '*' in files: + # try filter on the list_files + if recursive: + warn("The 'recursive' keyword argument does not work with remote instances. So it is ignored.") + list_files = fnmatch.filter(self_files, files) + if not list_files: + raise ValueError(f"The `'files'` parameter ({files}) didn't match any file using glob expressions in the remote server.") + else: + raise ValueError(f"The `'files'` parameter ('{files}') does not match any file or pattern.") + + elif isinstance(files, (list, tuple)): + if not all([isinstance(each, str) for each in files]): + raise ValueError("The parameter `'files'` can be a list or tuple, but it should only contain strings.") + list_files = files + else: + raise ValueError(f"The `file` parameter type ({type(files)}) is not supported." + "Only strings, tuple of strings or list of strings are allowed.") + + if target_dir: + try: + os.mkdir(target_dir) + except FileExistsError: + pass + else: + target_dir = os.getcwd() + + for each_file in list_files: + try: + file_name = os.path.basename(each_file) # Getting only the name of the file. + # We try to avoid that when the full path is supplied, it will crash when trying + # to do `os.path.join(target_dir"os.getcwd()", file_name "full filename path"` + # This will produce the file structure to flat out, but it is find, because recursive + # does not work in remote. + self._download(each_file, + out_file_name=os.path.join(target_dir, file_name), + chunk_size=chunk_size, + progress_bar=progress_bar) + except FileNotFoundError: + # So far the grpc interface returns size of the file equal + # zero, if the file does not exists or its size is zero, + # but they are two different things! + # In theory, since we are obtaining the files name from + # `mapdl.list_files()` they do exist, so + # if there is any error, it means their size is zero. + pass # this is not the best. + + return list_files + @protect_grpc - def download( + def _download( self, target_name, out_file_name=None, @@ -1500,7 +1658,7 @@ def _screenshot_path(self): temp_dir = tempfile.gettempdir() save_name = os.path.join(temp_dir, "tmp.png") - self.download(filename, out_file_name=save_name) + self._download(filename, out_file_name=save_name) return save_name @protect_grpc @@ -1689,7 +1847,7 @@ def _generate_iges(self): else: self.igesout(basename, att=1) filename = os.path.join(tempfile.gettempdir(), basename) - self.download(basename, filename, progress_bar=False) + self._download(basename, filename, progress_bar=False) return filename @property diff --git a/doc/source/user_guide/mapdl.rst b/doc/source/user_guide/mapdl.rst index ba8bb17865..a503d7d77a 100644 --- a/doc/source/user_guide/mapdl.rst +++ b/doc/source/user_guide/mapdl.rst @@ -646,6 +646,31 @@ example, to list the remote files and download one of them: This feature is only available for MAPDL 2021R1 or newer. +Alternatively, you can download several files at once using the glob pattern +or list of file names in :func:`Mapdl.download() `. +For example: + +.. code:: python + + # Using a list of file names + mapdl.download(['file0.log', 'file1.out']) + + # Using glob pattern to match the list_files + mapdl.download('file*') + +You can also download all the files in the MAPDL working directory +(:func:`Mapdl.directory `), using: + +.. code:: python + + mapdl.download_project() + +Or filter by extensions, for example: + +.. code:: python + + mapdl.download_project(['log', 'out'], target_dir='myfiles') # Download the files to 'myfiles' directory + Uploading a Local MAPDL File ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/tests/test_grpc.py b/tests/test_grpc.py index 882a3033e9..5d22c0419b 100644 --- a/tests/test_grpc.py +++ b/tests/test_grpc.py @@ -6,6 +6,8 @@ from ansys.mapdl.core import examples from ansys.mapdl.core.launcher import get_start_instance, check_valid_ansys from ansys.mapdl.core import launch_mapdl +from ansys.mapdl.core.common_grpc import DEFAULT_CHUNKSIZE + PATH = os.path.dirname(os.path.abspath(__file__)) @@ -18,6 +20,23 @@ ) +skip_in_cloud = pytest.mark.skipif( + not get_start_instance(), + reason=""" +Must be able to launch MAPDL locally. Remote execution does not allow for +directory creation. +""" +) + +def write_tmp(mapdl, filename, ext="txt"): + """Write a temporary file from MAPDL.""" + with mapdl.non_interactive: + mapdl.cfopen(filename, 'txt') + mapdl.vwrite('dummy_file') # Needs to write something, File cannot be empty. + mapdl.run("(A10)") + mapdl.cfclos() + + @pytest.fixture(scope="function") def setup_for_cmatrix(mapdl, cleared): mapdl.prep7() @@ -150,10 +169,10 @@ def test_large_output(mapdl, cleared): assert len(msg) > 4 * 1024 ** 2 -def test_download_missing_file(mapdl, tmpdir): +def test__download_missing_file(mapdl, tmpdir): target = tmpdir.join("tmp") with pytest.raises(FileNotFoundError): - mapdl.download("__notafile__", target) + mapdl._download("__notafile__", target) @skip_launch_mapdl # need to be able to start/stop an instance of MAPDL @@ -211,3 +230,110 @@ def test_no_get_value_non_interactive(mapdl): with pytest.raises(RuntimeError, match="Cannot use gRPC enabled ``GET``"): with mapdl.non_interactive: mapdl.get_value("ACTIVE", item1="CSYS") + + +def test__download(mapdl, tmpdir): + # Creating temp file + write_tmp(mapdl, 'myfile0') + + file_name = "myfile0.txt" + assert file_name in mapdl.list_files() + + out_file = tmpdir.join('out_' + file_name) + mapdl._download(file_name, out_file_name=out_file) + assert out_file.exists() + + out_file = tmpdir.join('out1_' + file_name) + mapdl._download(file_name, out_file_name=out_file, progress_bar=True) + assert out_file.exists() + + out_file = tmpdir.join('out2_' + file_name) + mapdl._download(file_name, out_file_name=out_file, chunk_size=DEFAULT_CHUNKSIZE/2) + assert out_file.exists() + + out_file = tmpdir.join('out3_' + file_name) + mapdl._download(file_name, out_file_name=out_file, chunk_size=DEFAULT_CHUNKSIZE*2) + assert out_file.exists() + + +@pytest.mark.parametrize("option,expected_files", [ + ['myfile0.txt', ['myfile0.txt']], + [['myfile0.txt', 'myfile1.txt'], ['myfile0.txt', 'myfile1.txt']], + ['myfile*', ['myfile0.txt', 'myfile1.txt']], +]) +def test_download(mapdl, tmpdir, option, expected_files): + write_tmp(mapdl, 'myfile0') + write_tmp(mapdl, 'myfile1') + + mapdl.download(option, target_dir=tmpdir) + for file_to_check in expected_files: + assert os.path.exists(tmpdir.join(file_to_check)) + + +def test_download_without_target_dir(mapdl, tmpdir): + write_tmp(mapdl, 'myfile0') + write_tmp(mapdl, 'myfile1') + + old_cwd = os.getcwd() + try: + # must use try/finally block as we change the cwd here + os.chdir(str(tmpdir)) + + mapdl.download('myfile0.txt') + assert os.path.exists('myfile0.txt') + + mapdl.download(['myfile0.txt', 'myfile1.txt']) + assert os.path.exists('myfile0.txt') + assert os.path.exists('myfile1.txt') + + mapdl.download('myfile*') + assert os.path.exists('myfile0.txt') + assert os.path.exists('myfile1.txt') + finally: + os.chdir(old_cwd) + + +@skip_in_cloud # This is going to run only in local +def test_download_recursive(mapdl, tmpdir): + if mapdl._local: # mapdl._local = True + dir_ = tmpdir.mkdir('temp00') + file1 = dir_.join('file0.txt') + file2 = dir_.join('file1.txt') + with open(file1, 'w') as fid: + fid.write('dummy') + with open(file2, 'w') as fid: + fid.write('dummy') + + mapdl.download(os.path.join(dir_, '*'), recursive=True) # This is referenced to os.getcwd + assert os.path.exists('file0.txt') + assert os.path.exists('file1.txt') + os.remove('file0.txt') + os.remove('file1.txt') + + mapdl.download(os.path.join(dir_, '*'), target_dir='new_dir', recursive=True) + assert os.path.exists(os.path.join('new_dir', 'file0.txt')) + assert os.path.exists(os.path.join('new_dir', 'file1.txt')) + os.remove(os.path.join('new_dir', 'file0.txt')) + os.remove(os.path.join('new_dir', 'file1.txt')) + + +def test_download_project(mapdl, tmpdir): + target_dir = tmpdir.mkdir('tmp') + mapdl.download_project(target_dir=target_dir) + files_extensions = [each.split('.')[-1] for each in os.listdir(target_dir)] + + assert 'log' in files_extensions + assert 'out' in files_extensions + assert 'err' in files_extensions + assert 'lock' in files_extensions + + +def test_download_project_extensions(mapdl, tmpdir): + target_dir = tmpdir.mkdir('tmp') + mapdl.download_project(extensions=['log', 'out'], target_dir=target_dir) + files_extensions = [each.split('.')[-1] for each in os.listdir(target_dir)] + + assert 'log' in files_extensions + assert 'out' in files_extensions + assert 'err' not in files_extensions + assert 'lock' not in files_extensions diff --git a/tests/test_mapdl.py b/tests/test_mapdl.py index aeba4da1a8..3c2f3d6c11 100644 --- a/tests/test_mapdl.py +++ b/tests/test_mapdl.py @@ -539,9 +539,9 @@ def test_nodes(tmpdir, cleared, mapdl): mapdl.nwrite(filename) else: mapdl.nwrite(basename) - mapdl.download(basename, filename) + mapdl.download(basename) - assert np.allclose(mapdl.mesh.nodes, np.loadtxt(filename)[:, 1:]) + assert np.allclose(mapdl.mesh.nodes, np.loadtxt(basename)[:, 1:]) assert mapdl.mesh.n_node == 11 assert np.allclose(mapdl.mesh.nnum, range(1, 12))