PwParser: Add retrieval and parsing of CRASH file (#899)

When `pw.x` crashes, the error is written to stdout but also to the
`CRASH` file. So far the parsers were just parsing errors from the
stdout. As of Quantum ESPRESSO v7.2, the errors are written _only_ to
the `CRASH` file.

Here, the `PwCalculation` is updated to always retrieve the `CRASH` file
and the `PwParser` (and relevant raw parsers) are updated to parse its
contents. The parser tests that tested a particular error case, are
duplicated to test parsing from stdout and the `CRASH` files.

src/aiida_quantumespresso/calculations/__init__.py

@@ -34,6 +34,7 @@ class BasePwCpInputGenerator(CalcJob):
     _PREFIX = 'aiida'
     _DEFAULT_INPUT_FILE = 'aiida.in'
     _DEFAULT_OUTPUT_FILE = 'aiida.out'
+    _CRASH_FILE = 'CRASH'
     _DATAFILE_XML_PRE_6_2 = 'data-file.xml'
     _DATAFILE_XML_POST_6_2 = 'data-file-schema.xml'
     _ENVIRON_INPUT_FILE_NAME = 'environ.in'
@@ -338,6 +339,7 @@ def prepare_for_submission(self, folder):
         # Retrieve by default the output file and the xml file
         calcinfo.retrieve_list = []
         calcinfo.retrieve_list.append(self.metadata.options.output_filename)
+        calcinfo.retrieve_list.append(self._CRASH_FILE)
         calcinfo.retrieve_list.extend(self.xml_filepaths)
         calcinfo.retrieve_list += settings.pop('ADDITIONAL_RETRIEVE_LIST', [])
         calcinfo.retrieve_list += self._internal_retrieve_list

src/aiida_quantumespresso/parsers/parse_raw/pw.py

@@ -279,11 +279,12 @@ def detect_important_message(logs, line):
             logs.warning.append(message)
 
 
-def parse_stdout(stdout, input_parameters, parser_options=None, parsed_xml=None):
+def parse_stdout(stdout, input_parameters, parser_options=None, parsed_xml=None, crash_file=None):
     """Parses the stdout content of a Quantum ESPRESSO `pw.x` calculation.
 
     :param stdout: the stdout content as a string
     :param input_parameters: dictionary with the input parameters
+    :param crash_file: the content of the ``CRASH`` file as a string if it was written, ``None`` otherwise.
     :param parser_options: the parser options from the settings input parameter node
     :param parsed_xml: dictionary with data parsed from the XML output file
     :returns: tuple of two dictionaries, with the parsed data and log messages, respectively
@@ -313,7 +314,8 @@ def parse_stdout(stdout, input_parameters, parser_options=None, parsed_xml=None)
         if 'JOB DONE' in line:
             break
     else:
-        logs.error.append('ERROR_OUTPUT_STDOUT_INCOMPLETE')
+        if crash_file is None:
+            logs.error.append('ERROR_OUTPUT_STDOUT_INCOMPLETE')
 
     # Determine whether the input switched on an electric field
     lelfield = input_parameters.get('CONTROL', {}).get('lelfield', False)
@@ -359,7 +361,7 @@ def parse_stdout(stdout, input_parameters, parser_options=None, parsed_xml=None)
         except NameError:  # nat or other variables where not found, and thus not initialized
 
             # Try to get some error messages
-            lines = stdout.split('\n')
+            lines = stdout.split('\n') if crash_file is None else crash_file.split('\n')
 
             for line_number, line in enumerate(lines):
                 # Compare the line to the known set of error and warning messages and add them to the log container
@@ -913,14 +915,23 @@ def parse_stdout(stdout, input_parameters, parser_options=None, parsed_xml=None)
     if maximum_ionic_steps is not None and maximum_ionic_steps == parsed_data.get('number_ionic_steps', None):
         logs.warning.append('ERROR_MAXIMUM_IONIC_STEPS_REACHED')
 
-    # Remove duplicate log messages by turning it into a set. Then convert back to list as that is what is expected
-    logs.error = list(set(logs.error))
-    logs.warning = list(set(logs.warning))
-
     parsed_data['bands'] = bands_data
     parsed_data['structure'] = structure_data
     parsed_data['trajectory'] = trajectory_data
 
+    # Double check to detect error messages if CRASH is found
+    if crash_file is not None:
+        for line in crash_file.split('\n'):
+            # Compare the line to the known set of error and warning messages and add them to the log container
+            detect_important_message(logs, line)
+
+        if len(logs.error) == len(logs.warning) == 0:
+            raise QEOutputParsingError('Parser cannot load basic info.')
+
+    # Remove duplicate log messages by turning it into a set. Then convert back to list as that is what is expected
+    logs.error = list(set(logs.error))
+    logs.warning = list(set(logs.warning))
+
     return parsed_data, logs
 
 

src/aiida_quantumespresso/parsers/pw.py

@@ -23,6 +23,7 @@ def parse(self, **kwargs):
         which should contain the temporary retrieved files.
         """
         dir_with_bands = None
+        crash_file = None
         self.exit_code_xml = None
         self.exit_code_stdout = None
         self.exit_code_parser = None
@@ -42,9 +43,14 @@ def parse(self, **kwargs):
             except KeyError:
                 return self.exit(self.exit_codes.ERROR_NO_RETRIEVED_TEMPORARY_FOLDER)
 
+        # We check if the `CRASH` file was retrieved. If so, we parse its output
+        crash_file_filename = self.node.process_class._CRASH_FILE
+        if crash_file_filename in self.retrieved.base.repository.list_object_names():
+            crash_file = self.retrieved.base.repository.get_object_content(crash_file_filename)
+
         parameters = self.node.inputs.parameters.get_dict()
         parsed_xml, logs_xml = self.parse_xml(dir_with_bands, parser_options)
-        parsed_stdout, logs_stdout = self.parse_stdout(parameters, parser_options, parsed_xml)
+        parsed_stdout, logs_stdout = self.parse_stdout(parameters, parser_options, parsed_xml, crash_file)
 
         parsed_bands = parsed_stdout.pop('bands', {})
         parsed_structure = parsed_stdout.pop('structure', {})
@@ -307,7 +313,7 @@ def parse_xml(self, dir_with_bands=None, parser_options=None):
 
         return parsed_data, logs
 
-    def parse_stdout(self, parameters, parser_options=None, parsed_xml=None):
+    def parse_stdout(self, parameters, parser_options=None, parsed_xml=None, crash_file=None):
         """Parse the stdout output file.
 
         :param parameters: the input parameters dictionary
@@ -333,7 +339,7 @@ def parse_stdout(self, parameters, parser_options=None, parsed_xml=None):
             return parsed_data, logs
 
         try:
-            parsed_data, logs = parse_stdout(stdout, parameters, parser_options, parsed_xml)
+            parsed_data, logs = parse_stdout(stdout, parameters, parser_options, parsed_xml, crash_file)
         except Exception as exc:
             logs.critical.append(traceback.format_exc())
             self.exit_code_stdout = self.exit_codes.ERROR_UNEXPECTED_PARSER_EXCEPTION.format(exception=exc)

tests/calculations/test_cp.py

@@ -16,11 +16,25 @@ def test_cp_autopilot(fixture_sandbox, generate_calc_job, generate_inputs_cp, fi
     cmdline_params = ['-in', 'aiida.in']
     local_copy_list = [(upf.uuid, upf.filename, './pseudo/Si.upf')]
     retrieve_list = [
-        'aiida.out', './out/aiida_51.save/data-file-schema.xml', './out/aiida_51.save/data-file.xml', './out/aiida.cel',
-        './out/aiida.con', './out/aiida.eig', './out/aiida.evp', './out/aiida.for', './out/aiida.nos',
-        './out/aiida.pol', './out/aiida.pos', './out/aiida.spr', './out/aiida.str', './out/aiida.the',
-        './out/aiida.vel', './out/aiida.wfc', './out/aiida_51.save/print_counter',
-        './out/aiida_51.save/print_counter.xml'
+        'aiida.out',
+        './out/aiida_51.save/data-file-schema.xml',
+        './out/aiida_51.save/data-file.xml',
+        './out/aiida.cel',
+        './out/aiida.con',
+        './out/aiida.eig',
+        './out/aiida.evp',
+        './out/aiida.for',
+        './out/aiida.nos',
+        './out/aiida.pol',
+        './out/aiida.pos',
+        './out/aiida.spr',
+        './out/aiida.str',
+        './out/aiida.the',
+        './out/aiida.vel',
+        './out/aiida.wfc',
+        './out/aiida_51.save/print_counter',
+        './out/aiida_51.save/print_counter.xml',
+        'CRASH',
     ]
     retrieve_temporary_list = [['./out/aiida.save/K*[0-9]/eigenval*.xml', '.', 2]]
 

tests/calculations/test_pw.py

@@ -21,7 +21,7 @@ def test_pw_default(fixture_sandbox, generate_calc_job, generate_inputs_pw, file
 
     cmdline_params = ['-in', 'aiida.in']
     local_copy_list = [(upf.uuid, upf.filename, './pseudo/Si.upf')]
-    retrieve_list = ['aiida.out', './out/aiida.save/data-file-schema.xml', './out/aiida.save/data-file.xml']
+    retrieve_list = ['aiida.out', './out/aiida.save/data-file-schema.xml', './out/aiida.save/data-file.xml', 'CRASH']
     retrieve_temporary_list = [['./out/aiida.save/K*[0-9]/eigenval*.xml', '.', 2]]
 
     # Check the attributes of the returned `CalcInfo`
@@ -74,7 +74,7 @@ def test_pw_ibrav(
 
     cmdline_params = ['-in', 'aiida.in']
     local_copy_list = [(upf.uuid, upf.filename, './pseudo/Si.upf')]
-    retrieve_list = ['aiida.out', './out/aiida.save/data-file-schema.xml', './out/aiida.save/data-file.xml']
+    retrieve_list = ['aiida.out', './out/aiida.save/data-file-schema.xml', './out/aiida.save/data-file.xml', 'CRASH']
     retrieve_temporary_list = [['./out/aiida.save/K*[0-9]/eigenval*.xml', '.', 2]]
 
     # Check the attributes of the returned `CalcInfo`

tests/parsers/fixtures/pw/failed_computing_cholesky/CRASH

@@ -0,0 +1,4 @@
+ %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+     Error in routine  cdiaghg (668):
+      problems computing cholesky
+ %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

tests/parsers/fixtures/pw/failed_computing_cholesky/aiida.out

@@ -13,11 +13,4 @@
 
      Davidson diagonalization with overlap
 
- %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-     Error in routine  cdiaghg (668):
-      problems computing cholesky
- %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-
-     stopping ...
-
 # NOTE: lines of output removed.

tests/parsers/fixtures/pw/failed_computing_cholesky_stdout/aiida.out

@@ -0,0 +1,21 @@
+
+     Program PWSCF v.6.4.1 starts on 14Oct2019 at  8:41:38
+
+     This program is part of the open-source Quantum ESPRESSO suite
+     for quantum simulation of materials; please cite
+         "P. Giannozzi et al., J. Phys.:Condens. Matter 21 395502 (2009);
+         "P. Giannozzi et al., J. Phys.:Condens. Matter 29 465901 (2017);
+          URL http://www.quantum-espresso.org",
+     in publications or presentations arising from this work. More details at
+     http://www.quantum-espresso.org/quote
+
+# NOTE: lines of output removed.
+
+     Davidson diagonalization with overlap
+
+ %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+     Error in routine  cdiaghg (668):
+      problems computing cholesky
+ %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+# NOTE: lines of output removed.

tests/parsers/fixtures/pw/failed_dexx_negative/CRASH

@@ -0,0 +1,4 @@
+ %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+     Error in routine electrons (1):
+     dexx is negative!  Check that exxdiv_treatment is appropriate for the system, or ecutfock may be too low
+ %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

tests/parsers/fixtures/pw/failed_dexx_negative/aiida.out

@@ -10,12 +10,3 @@
      http://www.quantum-espresso.org/quote
 
 # NOTE: roughly 3870 lines of output removed.
-
- %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-     Error in routine electrons (1):
-     dexx is negative!  Check that exxdiv_treatment is appropriate for the system, or ecutfock may be too low
- %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-
-     stopping ...
-
-# NOTE: duplicate error messages removed

tests/parsers/fixtures/pw/failed_dexx_negative_stdout/aiida.out

@@ -0,0 +1,16 @@
+
+     Program PWSCF v.6.4.1 starts on 14Oct2019 at  8:41:38
+
+     This program is part of the open-source Quantum ESPRESSO suite
+     for quantum simulation of materials; please cite
+         "P. Giannozzi et al., J. Phys.:Condens. Matter 21 395502 (2009);
+         "P. Giannozzi et al., J. Phys.:Condens. Matter 29 465901 (2017);
+          URL http://www.quantum-espresso.org",
+     in publications or presentations arising from this work. More details at
+     http://www.quantum-espresso.org/quote
+
+# NOTE: roughly 3870 lines of output removed.
+ %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+     Error in routine electrons (1):
+     dexx is negative!  Check that exxdiv_treatment is appropriate for the system, or ecutfock may be too low
+ %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

tests/parsers/fixtures/pw/failed_npools_too_high/CRASH

@@ -0,0 +1,4 @@
+ %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+     Error in routine divide_et_impera (1):
+     some nodes have no k-points
+ %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

tests/parsers/fixtures/pw/failed_npools_too_high/aiida.out

@@ -11,11 +11,5 @@
 
 # NOTE: 25 lines of output removed
 
- %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-     Error in routine divide_et_impera (1):
-     some nodes have no k-points
- %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-
-     stopping ...
 
 # NOTE: duplicate error messages removed

tests/parsers/fixtures/pw/failed_npools_too_high_stdout/aiida.out

@@ -0,0 +1,19 @@
+
+     Program PWSCF v.6.4.1 starts on 15Oct2019 at 14:12: 0
+
+     This program is part of the open-source Quantum ESPRESSO suite
+     for quantum simulation of materials; please cite
+         "P. Giannozzi et al., J. Phys.:Condens. Matter 21 395502 (2009);
+         "P. Giannozzi et al., J. Phys.:Condens. Matter 29 465901 (2017);
+          URL http://www.quantum-espresso.org",
+     in publications or presentations arising from this work. More details at
+     http://www.quantum-espresso.org/quote
+
+# NOTE: 25 lines of output removed
+
+
+# NOTE: duplicate error messages removed
+ %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+     Error in routine divide_et_impera (1):
+     some nodes have no k-points
+ %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

tests/parsers/fixtures/pw/failed_too_many_bands_not_converged/CRASH

@@ -0,0 +1,4 @@
+ %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+     Error in routine c_bands (1):
+     too many bands are not converged
+ %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

tests/parsers/fixtures/pw/failed_too_many_bands_not_converged/aiida.out

@@ -15,11 +15,4 @@
 
      Davidson diagonalization with overlap
 
- %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-     Error in routine c_bands (1):
-     too many bands are not converged
- %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-
-     stopping ...
-
 # NOTE: lines of output removed.

tests/parsers/fixtures/pw/failed_too_many_bands_not_converged_stdout/aiida.out

@@ -0,0 +1,22 @@
+
+     Program PWSCF v.6.8 starts on 22Sep2021 at 20:34:38 
+
+     This program is part of the open-source Quantum ESPRESSO suite
+     for quantum simulation of materials; please cite
+         "P. Giannozzi et al., J. Phys.:Condens. Matter 21 395502 (2009);
+         "P. Giannozzi et al., J. Phys.:Condens. Matter 29 465901 (2017);
+         "P. Giannozzi et al., J. Chem. Phys. 152 154105 (2020);
+          URL http://www.quantum-espresso.org", 
+     in publications or presentations arising from this work. More details at
+     http://www.quantum-espresso.org/quote
+
+
+# NOTE: lines of output removed.
+
+     Davidson diagonalization with overlap
+
+# NOTE: lines of output removed.
+ %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+     Error in routine c_bands (1):
+     too many bands are not converged
+ %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

tests/parsers/fixtures/pw/vcrelax_failed_charge_wrong/CRASH

@@ -0,0 +1,4 @@
+ %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+     Error in routine electrons (1):
+     charge is wrong
+ %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

tests/parsers/fixtures/pw/vcrelax_failed_charge_wrong/aiida.out

@@ -105,10 +105,3 @@
      Estimated max dynamical RAM per process >      10.86MB
 
      WARNING: integrated charge=     0.00003902, expected=   114.00000000
-
- %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-     Error in routine electrons (1):
-     charge is wrong
- %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-
-     stopping ...