From 42bed01c31e03432873955bd66303054a2fa2483 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Wed, 6 Oct 2021 00:06:17 +0100 Subject: [PATCH 01/27] C traceback code --- Include/cpython/traceback.h | 2 +- Include/traceback.h | 4 + Lib/test/test_traceback.py | 172 ++++++++++++++++++++++++++++++++- Python/_warnings.c | 2 +- Python/pythonrun.c | 184 ++++++++++++++++++++++++++++++------ Python/traceback.c | 72 ++++++++++---- 6 files changed, 384 insertions(+), 52 deletions(-) diff --git a/Include/cpython/traceback.h b/Include/cpython/traceback.h index d0dde335cfee5b..1be048625bef4f 100644 --- a/Include/cpython/traceback.h +++ b/Include/cpython/traceback.h @@ -10,5 +10,5 @@ typedef struct _traceback { int tb_lineno; } PyTracebackObject; -PyAPI_FUNC(int) _Py_DisplaySourceLine(PyObject *, PyObject *, int, int, int *, PyObject **); +PyAPI_FUNC(int) _Py_DisplaySourceLine(PyObject *, PyObject *, int, int, int, char, int *, PyObject **); PyAPI_FUNC(void) _PyTraceback_Add(const char *, const char *, int); diff --git a/Include/traceback.h b/Include/traceback.h index 2dfa2ada4f2c37..c2c075252d4ea6 100644 --- a/Include/traceback.h +++ b/Include/traceback.h @@ -9,6 +9,10 @@ extern "C" { PyAPI_FUNC(int) PyTraceBack_Here(PyFrameObject *); PyAPI_FUNC(int) PyTraceBack_Print(PyObject *, PyObject *); +int PyTraceBack_Print_Indented(PyObject *, int, char, PyObject *); +int _Py_WriteIndentedMargin(int, char, PyObject *); +int _Py_WriteIndent(int, PyObject *); + /* Reveal traceback type so we can typecheck traceback objects */ PyAPI_DATA(PyTypeObject) PyTraceBack_Type; #define PyTraceBack_Check(v) Py_IS_TYPE(v, &PyTraceBack_Type) diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index 1c7db9d3d47376..87108da8c80510 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -995,9 +995,12 @@ def __eq__(self, other): "\nDuring handling of the above exception, " "another exception occurred:\n\n") -boundaries = re.compile( - '(%s|%s)' % (re.escape(cause_message), re.escape(context_message))) +nested_exception_header_re = r'[\+-]?\+[-{4} ]+[ \d\.?]+[ -{4}]+' +boundaries = re.compile( + '(%s|%s|%s)' % (re.escape(cause_message), + re.escape(context_message), + nested_exception_header_re)) class BaseExceptionReportingTests: @@ -1263,6 +1266,171 @@ def get_report(self, e): exception_print(e) return s.getvalue() + # TODO: once traceback.py supports this format, move the + # following tests to the superclass + + def check_exception_group(self, exc_func, expected): + report = self.get_report(exc_func) + blocks = boundaries.split(report) + self.assertEqual(len(blocks), len(expected)) + + for i, block in enumerate(blocks): + for line in expected[i]: + self.assertIn(f"{line}", block) + # check indentation + self.assertNotIn(f" {line}", block) + + for line in report: + # at most one margin char per line + self.assertLessEqual(line.count('|'), 1) + + @cpython_only + def test_exception_group_basic(self): + def exc(): + raise ExceptionGroup("eg", [ValueError(1), TypeError(2)]) + + expected = [ + [' | raise ExceptionGroup("eg", [ValueError(1), TypeError(2)])', + ' | ExceptionGroup: eg', + ' | with 2 sub-exceptions:' + ], + ['-+---------------- 1 ----------------'], + [' | ValueError: 1'], + ['+---------------- 2 ----------------'], + [' | TypeError: 2'], + ] + + self.check_exception_group(exc, expected) + + @cpython_only + def test_exception_group_context(self): + def exc(): + try: + raise ExceptionGroup("eg1", [ValueError(1), TypeError(2)]) + except: + raise ExceptionGroup("eg2", [ValueError(3), TypeError(4)]) + + expected = [ + [' | raise ExceptionGroup("eg1", [ValueError(1), TypeError(2)])', + ' | ExceptionGroup: eg1', + ' | with 2 sub-exceptions:' + ], + ['-+---------------- 1 ----------------'], + [' | ValueError: 1'], + ['+---------------- 2 ----------------'], + [' | TypeError: 2'], + [ context_message ], + [' | raise ExceptionGroup("eg2", [ValueError(3), TypeError(4)])', + ' | ExceptionGroup: eg2', + ' | with 2 sub-exceptions:' + ], + ['-+---------------- 1 ----------------'], + [' | ValueError: 3'], + ['+---------------- 2 ----------------'], + [' | TypeError: 4'], + ] + self.check_exception_group(exc, expected) + + @cpython_only + def test_exception_group_context(self): + def exc(): + EG = ExceptionGroup + try: + raise EG("eg1", [ValueError(1), TypeError(2)]) + except: + raise EG("eg2", [ValueError(3), TypeError(4)]) + + expected = [ + [' | raise EG("eg1", [ValueError(1), TypeError(2)])', + ' | ExceptionGroup: eg1', + ' | with 2 sub-exceptions:' + ], + ['-+---------------- context.1 ----------------'], + [' | ValueError: 1'], + ['+---------------- context.2 ----------------'], + [' | TypeError: 2'], + [ context_message ], + [' | raise EG("eg2", [ValueError(3), TypeError(4)])', + ' | ExceptionGroup: eg2', + ' | with 2 sub-exceptions:' + ], + ['-+---------------- 1 ----------------'], + [' | ValueError: 3'], + ['+---------------- 2 ----------------'], + [' | TypeError: 4'], + ] + self.check_exception_group(exc, expected) + + def test_exception_group_cause(self): + def exc(): + EG = ExceptionGroup + try: + raise EG("eg1", [ValueError(1), TypeError(2)]) + except Exception as e: + raise EG("eg2", [ValueError(3), TypeError(4)]) from e + + expected = [ + [' | raise EG("eg1", [ValueError(1), TypeError(2)])', + ' | ExceptionGroup: eg1', + ' | with 2 sub-exceptions:' + ], + ['-+---------------- cause.1 ----------------'], + [' | ValueError: 1'], + ['+---------------- cause.2 ----------------'], + [' | TypeError: 2'], + [ cause_message ], + [' | raise EG("eg2", [ValueError(3), TypeError(4)])', + ' | ExceptionGroup: eg2', + ' | with 2 sub-exceptions:' + ], + ['-+---------------- 1 ----------------'], + [' | ValueError: 3'], + ['+---------------- 2 ----------------'], + [' | TypeError: 4'], + ] + self.check_exception_group(exc, expected) + + @cpython_only + def test_exception_group_nested(self): + def exc(): + EG = ExceptionGroup + VE = ValueError + TE = TypeError + try: + try: + raise EG("nested", [TE(2), TE(3)]) + except Exception as e: + exc = e + raise EG("eg", [VE(1), exc, VE(4)]) + except: + raise EG("top", [VE(5)]) + + expected = [ + [' | raise EG("eg", [VE(1), exc, VE(4)])', + ' | ExceptionGroup: eg', + ' | with 3 sub-exceptions:' + ], + ['-+---------------- context.1 ----------------'], + [' | ValueError: 1'], + ['+---------------- context.2 ----------------'], + [' | ExceptionGroup: nested', + ' | with 2 sub-exceptions' + ], + ['-+---------------- context.2.1 ----------------'], + [' | TypeError: 2'], + ['+---------------- context.2.2 ----------------'], + [' | TypeError: 3'], + ['+---------------- context.3 ----------------'], + [' | ValueError: 4'], + [ context_message ], + [' | raise EG("top", [VE(5)])', + ' | ExceptionGroup: top' + ], + ['-+---------------- 1 ----------------'], + [' | ValueError: 5'] + ] + self.check_exception_group(exc, expected) + class LimitTests(unittest.TestCase): diff --git a/Python/_warnings.c b/Python/_warnings.c index cf2110d31c3b5e..5574a4ae8dca21 100644 --- a/Python/_warnings.c +++ b/Python/_warnings.c @@ -544,7 +544,7 @@ show_warning(PyObject *filename, int lineno, PyObject *text, PyFile_WriteString("\n", f_stderr); } else { - _Py_DisplaySourceLine(f_stderr, filename, lineno, 2, NULL, NULL); + _Py_DisplaySourceLine(f_stderr, filename, lineno, 2, 0, '\0', NULL, NULL); } error: diff --git a/Python/pythonrun.c b/Python/pythonrun.c index 6cecef97932288..21e56c14526362 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -886,15 +886,32 @@ PyErr_Print(void) PyErr_PrintEx(1); } +struct exception_print_context +{ + PyObject *file; + PyObject *seen; // Prevent cycles in recursion + int exception_group_depth; // nesting level of current exception group + PyObject *parent_label; // Unicode label of containing exception group + int need_close; // Need a closing bottom frame +}; + +#define EXC_MARGIN_CHAR(ctx) ((ctx)->exception_group_depth ? '|' : '\0') +#define EXC_INDENT(ctx) (2 * (ctx)->exception_group_depth) + +#define WRITE_INDENTED_MARGIN(ctx, f) _Py_WriteIndentedMargin(EXC_INDENT(ctx), EXC_MARGIN_CHAR(ctx), (f)) + static void -print_exception(PyObject *f, PyObject *value) +print_exception(struct exception_print_context *ctx, PyObject *value) { int err = 0; PyObject *type, *tb, *tmp; + PyObject *f = ctx->file; + _Py_IDENTIFIER(print_file_and_line); if (!PyExceptionInstance_Check(value)) { - err = PyFile_WriteString("TypeError: print_exception(): Exception expected for value, ", f); + err += _Py_WriteIndent(EXC_INDENT(ctx), f); + err += PyFile_WriteString("TypeError: print_exception(): Exception expected for value, ", f); err += PyFile_WriteString(Py_TYPE(value)->tp_name, f); err += PyFile_WriteString(" found\n", f); if (err) @@ -907,7 +924,7 @@ print_exception(PyObject *f, PyObject *value) type = (PyObject *) Py_TYPE(value); tb = PyException_GetTraceback(value); if (tb && tb != Py_None) - err = PyTraceBack_Print(tb, f); + err = PyTraceBack_Print_Indented(tb, EXC_INDENT(ctx), EXC_MARGIN_CHAR(ctx), f); if (err == 0 && (err = _PyObject_LookupAttrId(value, &PyId_print_file_and_line, &tmp)) > 0) { @@ -929,6 +946,7 @@ print_exception(PyObject *f, PyObject *value) filename, lineno); Py_DECREF(filename); if (line != NULL) { + err += WRITE_INDENTED_MARGIN(ctx, f); PyFile_WriteObject(line, f, Py_PRINT_RAW); Py_DECREF(line); } @@ -967,18 +985,19 @@ print_exception(PyObject *f, PyObject *value) _Py_IDENTIFIER(__module__); assert(PyExceptionClass_Check(type)); + err += WRITE_INDENTED_MARGIN(ctx, f); modulename = _PyObject_GetAttrId(type, &PyId___module__); if (modulename == NULL || !PyUnicode_Check(modulename)) { Py_XDECREF(modulename); PyErr_Clear(); - err = PyFile_WriteString("", f); + err += PyFile_WriteString("", f); } else { if (!_PyUnicode_EqualToASCIIId(modulename, &PyId_builtins) && !_PyUnicode_EqualToASCIIId(modulename, &PyId___main__)) { - err = PyFile_WriteObject(modulename, f, Py_PRINT_RAW); + err += PyFile_WriteObject(modulename, f, Py_PRINT_RAW); err += PyFile_WriteString(".", f); } Py_DECREF(modulename); @@ -1039,26 +1058,60 @@ print_exception(PyObject *f, PyObject *value) } static const char cause_message[] = - "\nThe above exception was the direct cause " - "of the following exception:\n\n"; + "The above exception was the direct cause " + "of the following exception:\n"; static const char context_message[] = - "\nDuring handling of the above exception, " - "another exception occurred:\n\n"; + "During handling of the above exception, " + "another exception occurred:\n"; static void -print_exception_recursive(PyObject *f, PyObject *value, PyObject *seen) +print_exception_recursive(struct exception_print_context*, PyObject*); + +static int +print_chained(struct exception_print_context* ctx, PyObject *value, + const char * message, const char *tag) { + PyObject *f = ctx->file; + int err = 0; + PyObject *parent_label = ctx->parent_label; + PyObject *label = NULL; + int need_close = ctx->need_close; + if (parent_label) { + label = PyUnicode_FromFormat("%U.%s", parent_label, tag); + } + else { + label = PyUnicode_FromString(tag); + } + ctx->parent_label = label; + + print_exception_recursive(ctx, value); + err |= WRITE_INDENTED_MARGIN(ctx, f); + err |= PyFile_WriteString("\n", f); + err |= WRITE_INDENTED_MARGIN(ctx, f); + err |= PyFile_WriteString(message, f); + err |= WRITE_INDENTED_MARGIN(ctx, f); + err |= PyFile_WriteString("\n", f); + + ctx->need_close = need_close; + ctx->parent_label = parent_label; + Py_XDECREF(label); + return err; +} + +static void +print_exception_recursive(struct exception_print_context* ctx, PyObject *value) { int err = 0, res; PyObject *cause, *context; - if (seen != NULL) { + if (ctx->seen != NULL) { /* Exception chaining */ PyObject *value_id = PyLong_FromVoidPtr(value); - if (value_id == NULL || PySet_Add(seen, value_id) == -1) + if (value_id == NULL || PySet_Add(ctx->seen, value_id) == -1) PyErr_Clear(); else if (PyExceptionInstance_Check(value)) { PyObject *check_id = NULL; + cause = PyException_GetCause(value); context = PyException_GetContext(value); if (cause) { @@ -1066,16 +1119,13 @@ print_exception_recursive(PyObject *f, PyObject *value, PyObject *seen) if (check_id == NULL) { res = -1; } else { - res = PySet_Contains(seen, check_id); + res = PySet_Contains(ctx->seen, check_id); Py_DECREF(check_id); } if (res == -1) PyErr_Clear(); if (res == 0) { - print_exception_recursive( - f, cause, seen); - err |= PyFile_WriteString( - cause_message, f); + err |= print_chained(ctx, cause, cause_message, "cause"); } } else if (context && @@ -1084,16 +1134,13 @@ print_exception_recursive(PyObject *f, PyObject *value, PyObject *seen) if (check_id == NULL) { res = -1; } else { - res = PySet_Contains(seen, check_id); + res = PySet_Contains(ctx->seen, check_id); Py_DECREF(check_id); } if (res == -1) PyErr_Clear(); if (res == 0) { - print_exception_recursive( - f, context, seen); - err |= PyFile_WriteString( - context_message, f); + err |= print_chained(ctx, context, context_message, "context"); } } Py_XDECREF(context); @@ -1101,7 +1148,84 @@ print_exception_recursive(PyObject *f, PyObject *value, PyObject *seen) } Py_XDECREF(value_id); } - print_exception(f, value); + if (!PyObject_TypeCheck(value, (PyTypeObject *)PyExc_BaseExceptionGroup)) { + print_exception(ctx, value); + } + else { + /* ExceptionGroup */ + + /* TODO: add arg to limit number of exceptions printed? */ + + PyObject *line; + + if (ctx->exception_group_depth == 0) { + ctx->exception_group_depth += 1; + } + print_exception(ctx, value); + + PyObject *excs = ((PyBaseExceptionGroupObject *)value)->excs; + if (excs && PySequence_Check(excs)) { + Py_ssize_t i, num_excs = PySequence_Length(excs); + PyObject *parent_label = ctx->parent_label; + PyObject *f = ctx->file; + if (num_excs > 0) { + if (num_excs == 1) { + line = PyUnicode_FromFormat( + " with one sub-exception:\n"); + } + else { + line = PyUnicode_FromFormat( + " with %d sub-exceptions:\n", num_excs); + } + err |= WRITE_INDENTED_MARGIN(ctx, f); + err |= PyFile_WriteObject(line, f, Py_PRINT_RAW); + Py_XDECREF(line); + ctx->need_close = 0; + for (i = 0; i < num_excs; i++) { + int last_exc = i == num_excs - 1; + if (last_exc) { + // The closing frame may be added in a recursive call + ctx->need_close = 1; + } + PyObject *label; + if (parent_label) { + label = PyUnicode_FromFormat("%U.%d", + parent_label, i + 1); + } + else { + label = PyUnicode_FromFormat("%d", i + 1); + } + err |= _Py_WriteIndent(EXC_INDENT(ctx), f); + line = PyUnicode_FromFormat( + "%s+---------------- %U ----------------\n", + (i == 0) ? "+-" : " ", label); + ctx->exception_group_depth += 1; + err |= PyFile_WriteObject(line, f, Py_PRINT_RAW); + Py_XDECREF(line); + + ctx->parent_label = label; + PyObject *exc = PySequence_GetItem(excs, i); + print_exception_recursive(ctx, exc); + ctx->parent_label = parent_label; + Py_XDECREF(label); + + if (last_exc && ctx->need_close) { + err |= _Py_WriteIndent(EXC_INDENT(ctx), f); + line = PyUnicode_FromFormat( + "+------------------------------------\n"); + err |= PyFile_WriteObject(line, f, Py_PRINT_RAW); + Py_XDECREF(line); + ctx->need_close = 0; + } + ctx->exception_group_depth -= 1; + Py_XDECREF(exc); + } + } + } + if (ctx->exception_group_depth == 1) { + ctx->exception_group_depth -= 1; + } + } if (err != 0) PyErr_Clear(); } @@ -1110,8 +1234,8 @@ void _PyErr_Display(PyObject *file, PyObject *exception, PyObject *value, PyObject *tb) { assert(file != NULL && file != Py_None); + struct exception_print_context ctx; - PyObject *seen; if (PyExceptionInstance_Check(value) && tb != NULL && PyTraceBack_Check(tb)) { /* Put the traceback on the exception, otherwise it won't get @@ -1123,15 +1247,19 @@ _PyErr_Display(PyObject *file, PyObject *exception, PyObject *value, PyObject *t Py_DECREF(cur_tb); } + ctx.file = file; + ctx.exception_group_depth = 0; + ctx.parent_label = 0; + /* We choose to ignore seen being possibly NULL, and report at least the main exception (it could be a MemoryError). */ - seen = PySet_New(NULL); - if (seen == NULL) { + ctx.seen = PySet_New(NULL); + if (ctx.seen == NULL) { PyErr_Clear(); } - print_exception_recursive(file, value, seen); - Py_XDECREF(seen); + print_exception_recursive(&ctx, value); + Py_XDECREF(ctx.seen); /* Call file.flush() */ PyObject *res = _PyObject_CallMethodIdNoArgs(file, &PyId_flush); diff --git a/Python/traceback.c b/Python/traceback.c index b18cbb91ce8edc..e48b05d8c3b00e 100644 --- a/Python/traceback.c +++ b/Python/traceback.c @@ -380,7 +380,36 @@ _Py_FindSourceFile(PyObject *filename, char* namebuf, size_t namelen, PyObject * } int -_Py_DisplaySourceLine(PyObject *f, PyObject *filename, int lineno, int indent, int *truncation, PyObject **line) +_Py_WriteIndent(int indent, PyObject *f) { + int err = 0; + char buf[11]; + strcpy(buf, " "); + assert(strlen(buf) == 10); + while (indent > 0) { + if (indent < 10) + buf[indent] = '\0'; + err = PyFile_WriteString(buf, f); + if (err != 0) + return err; + indent -= 10; + } + return 0; +} + +/* Writes indent spaces, followed by the margin if margin_char is not `\0`. + */ +int +_Py_WriteIndentedMargin(int indent, char margin_char, PyObject *f) { + int err = 0; + char margin[] = {margin_char, ' ', '\0' }; + err |= _Py_WriteIndent(indent, f); + err |= PyFile_WriteString(margin, f); + return err; +} + +int +_Py_DisplaySourceLine(PyObject *f, PyObject *filename, int lineno, int indent, + int margin_indent, char margin_char, int *truncation, PyObject **line) { int err = 0; int fd; @@ -508,17 +537,9 @@ _Py_DisplaySourceLine(PyObject *f, PyObject *filename, int lineno, int indent, i *truncation = i - indent; } + err |= _Py_WriteIndentedMargin(margin_indent, margin_char, f); /* Write some spaces before the line */ - strcpy(buf, " "); - assert (strlen(buf) == 10); - while (indent > 0) { - if (indent < 10) - buf[indent] = '\0'; - err = PyFile_WriteString(buf, f); - if (err != 0) - break; - indent -= 10; - } + err |= _Py_WriteIndent(indent, f); /* finally display the line */ if (err == 0) @@ -697,7 +718,7 @@ print_error_location_carets(PyObject *f, int offset, Py_ssize_t start_offset, Py static int tb_displayline(PyTracebackObject* tb, PyObject *f, PyObject *filename, int lineno, - PyFrameObject *frame, PyObject *name) + PyFrameObject *frame, PyObject *name, int margin_indent, char margin_char) { int err; PyObject *line; @@ -708,7 +729,8 @@ tb_displayline(PyTracebackObject* tb, PyObject *f, PyObject *filename, int linen filename, lineno, name); if (line == NULL) return -1; - err = PyFile_WriteObject(line, f, Py_PRINT_RAW); + err = _Py_WriteIndentedMargin(margin_indent, margin_char, f); + err |= PyFile_WriteObject(line, f, Py_PRINT_RAW); Py_DECREF(line); if (err != 0) return err; @@ -716,7 +738,8 @@ tb_displayline(PyTracebackObject* tb, PyObject *f, PyObject *filename, int linen int truncation = _TRACEBACK_SOURCE_LINE_INDENT; PyObject* source_line = NULL; if (_Py_DisplaySourceLine(f, filename, lineno, _TRACEBACK_SOURCE_LINE_INDENT, - &truncation, &source_line) != 0 || !source_line) { + margin_indent, margin_char, + &truncation, &source_line) != 0 || !source_line) { /* ignore errors since we can't report them, can we? */ err = ignore_source_errors(); goto done; @@ -801,7 +824,8 @@ tb_displayline(PyTracebackObject* tb, PyObject *f, PyObject *filename, int linen end_offset = i + 1; } - err = print_error_location_carets(f, truncation, start_offset, end_offset, + err = _Py_WriteIndentedMargin(margin_indent, margin_char, f); + err |= print_error_location_carets(f, truncation, start_offset, end_offset, right_start_offset, left_end_offset, primary_error_char, secondary_error_char); @@ -830,7 +854,8 @@ tb_print_line_repeated(PyObject *f, long cnt) } static int -tb_printinternal(PyTracebackObject *tb, PyObject *f, long limit) +tb_printinternal(PyTracebackObject *tb, PyObject *f, long limit, + int indent, char margin_char) { int err = 0; Py_ssize_t depth = 0; @@ -864,7 +889,7 @@ tb_printinternal(PyTracebackObject *tb, PyObject *f, long limit) cnt++; if (err == 0 && cnt <= TB_RECURSIVE_CUTOFF) { err = tb_displayline(tb, f, code->co_filename, tb->tb_lineno, - tb->tb_frame, code->co_name); + tb->tb_frame, code->co_name, indent, margin_char); if (err == 0) { err = PyErr_CheckSignals(); } @@ -881,7 +906,7 @@ tb_printinternal(PyTracebackObject *tb, PyObject *f, long limit) #define PyTraceBack_LIMIT 1000 int -PyTraceBack_Print(PyObject *v, PyObject *f) +PyTraceBack_Print_Indented(PyObject *v, int indent, char margin_char, PyObject *f) { int err; PyObject *limitv; @@ -904,12 +929,19 @@ PyTraceBack_Print(PyObject *v, PyObject *f) return 0; } } - err = PyFile_WriteString("Traceback (most recent call last):\n", f); + err = _Py_WriteIndentedMargin(indent, margin_char, f); + err |= PyFile_WriteString("Traceback (most recent call last):\n", f); if (!err) - err = tb_printinternal((PyTracebackObject *)v, f, limit); + err = tb_printinternal((PyTracebackObject *)v, f, limit, indent, margin_char); return err; } +int +PyTraceBack_Print(PyObject *v, PyObject *f) +{ + return PyTraceBack_Print_Indented(v, 0, '\0', f); +} + /* Format an integer in range [0; 0xffffffff] to decimal and write it into the file fd. From a2daa233f91eba83fe19eecb9df17961c015d04c Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Fri, 8 Oct 2021 19:02:31 +0100 Subject: [PATCH 02/27] add ExceptionGroups support to traceback.py --- Lib/test/test_traceback.py | 201 +++++++++++++++++++++++++++++++------ Lib/traceback.py | 118 ++++++++++++++++++++-- 2 files changed, 279 insertions(+), 40 deletions(-) diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index 87108da8c80510..ab07cab0f1c980 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -1237,37 +1237,7 @@ def __str__(self): self.assertEqual(err, f"{str_name}: {str_value}\n") -class PyExcReportingTests(BaseExceptionReportingTests, unittest.TestCase): - # - # This checks reporting through the 'traceback' module, with both - # format_exception() and print_exception(). - # - - def get_report(self, e): - e = self.get_exception(e) - s = ''.join( - traceback.format_exception(type(e), e, e.__traceback__)) - with captured_output("stderr") as sio: - traceback.print_exception(type(e), e, e.__traceback__) - self.assertEqual(sio.getvalue(), s) - return s - - -class CExcReportingTests(BaseExceptionReportingTests, unittest.TestCase): - # - # This checks built-in reporting by the interpreter. - # - - @cpython_only - def get_report(self, e): - from _testcapi import exception_print - e = self.get_exception(e) - with captured_output("stderr") as s: - exception_print(e) - return s.getvalue() - - # TODO: once traceback.py supports this format, move the - # following tests to the superclass + # #### Exception Groups #### def check_exception_group(self, exc_func, expected): report = self.get_report(exc_func) @@ -1432,6 +1402,36 @@ def exc(): self.check_exception_group(exc, expected) +class PyExcReportingTests(BaseExceptionReportingTests, unittest.TestCase): + # + # This checks reporting through the 'traceback' module, with both + # format_exception() and print_exception(). + # + + def get_report(self, e): + e = self.get_exception(e) + s = ''.join( + traceback.format_exception(type(e), e, e.__traceback__)) + with captured_output("stderr") as sio: + traceback.print_exception(type(e), e, e.__traceback__) + self.assertEqual(sio.getvalue(), s) + return s + + +class CExcReportingTests(BaseExceptionReportingTests, unittest.TestCase): + # + # This checks built-in reporting by the interpreter. + # + + @cpython_only + def get_report(self, e): + from _testcapi import exception_print + e = self.get_exception(e) + with captured_output("stderr") as s: + exception_print(e) + return s.getvalue() + + class LimitTests(unittest.TestCase): ''' Tests for limit argument. @@ -2081,6 +2081,145 @@ def f(): '']) +class TestTracebackException_ExceptionGroups(unittest.TestCase): + def setUp(self): + super().setUp() + self.eg_info = self._get_exception_group() + + def _get_exception_group(self): + def f(): + 1/0 + + def g(v): + raise ValueError(v) + + self.lno_f = f.__code__.co_firstlineno + self.lno_g = g.__code__.co_firstlineno + + try: + try: + try: + f() + except Exception as e: + exc1 = e + try: + g(42) + except Exception as e: + exc2 = e + raise ExceptionGroup("eg1", [exc1, exc2]) + except ExceptionGroup as e: + exc3 = e + try: + g(24) + except Exception as e: + exc4 = e + raise ExceptionGroup("eg2", [exc3, exc4]) + except ExceptionGroup: + return sys.exc_info() + self.fail('Exception Not Raised') + + def test_exception_group_construction(self): + eg_info = self.eg_info + teg1 = traceback.TracebackException(*eg_info) + teg2 = traceback.TracebackException.from_exception(eg_info[1]) + self.assertIsNot(teg1, teg2) + self.assertEqual(teg1, teg2) + + def test_exception_group_format_exception_only(self): + teg = traceback.TracebackException(*self.eg_info) + formatted = ''.join(teg.format_exception_only()).split('\n') + expected = "ExceptionGroup: eg2\n".split('\n') + + self.assertEqual(formatted, expected) + + def test_exception_group_format(self): + teg = traceback.TracebackException(*self.eg_info) + + formatted = ''.join(teg.format()).split('\n') + lno_f = self.lno_f + lno_g = self.lno_g + + expected = [ + f' | Traceback (most recent call last):', + f' | File "{__file__}", line ' + f'{lno_g+23}, in _get_exception_group', + f' | raise ExceptionGroup("eg2", [exc3, exc4])', + f' | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^', + f' | ExceptionGroup: eg2', + f' | with 2 sub-exceptions:', + f' +-+---------------- 1 ----------------', + f' | Traceback (most recent call last):', + f' | File "{__file__}", ' + f'line {lno_g+16}, in _get_exception_group', + f' | raise ExceptionGroup("eg1", [exc1, exc2])', + f' | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^', + f' | ExceptionGroup: eg1', + f' | with 2 sub-exceptions:', + f' +-+---------------- 1.1 ----------------', + f' | Traceback (most recent call last):', + f' | File ' + f'"{__file__}", line {lno_g+9}, in ' + f'_get_exception_group', + f' | f()', + f' | ^^^', + f' | File ' + f'"{__file__}", line {lno_f+1}, in ' + f'f', + f' | 1/0', + f' | ~^~', + f' | ZeroDivisionError: division by zero', + f' +---------------- 1.2 ----------------', + f' | Traceback (most recent call last):', + f' | File ' + f'"{__file__}", line {lno_g+13}, in ' + f'_get_exception_group', + f' | g(42)', + f' | ^^^^^', + f' | File ' + f'"{__file__}", line {lno_g+1}, in ' + f'g', + f' | raise ValueError(v)', + f' | ^^^^^^^^^^^^^^^^^^^', + f' | ValueError: 42', + f' +------------------------------------', + f' +---------------- 2 ----------------', + f' | Traceback (most recent call last):', + f' | File "{__file__}", ' + f'line {lno_g+20}, in _get_exception_group', + f' | g(24)', + f' | ^^^^^', + f' | File "{__file__}", ' + f'line {lno_g+1}, in g', + f' | raise ValueError(v)', + f' | ^^^^^^^^^^^^^^^^^^^', + f' | ValueError: 24', + f' +------------------------------------', + f''] + + self.assertEqual(formatted, expected) + + def test_comparison(self): + try: + raise self.eg_info[1] + except ExceptionGroup: + exc_info = sys.exc_info() + for _ in range(5): + try: + raise exc_info[1] + except: + exc_info = sys.exc_info() + exc = traceback.TracebackException(*exc_info) + exc2 = traceback.TracebackException(*exc_info) + exc3 = traceback.TracebackException(*exc_info, limit=300) + ne = traceback.TracebackException(*exc_info, limit=3) + self.assertIsNot(exc, exc2) + self.assertEqual(exc, exc2) + self.assertEqual(exc, exc3) + self.assertNotEqual(exc, ne) + self.assertNotEqual(exc, object()) + self.assertEqual(exc, ALWAYS_EQ) + + class MiscTest(unittest.TestCase): def test_all(self): diff --git a/Lib/traceback.py b/Lib/traceback.py index 568f3ff28c29b2..cb2b893d2fcb70 100644 --- a/Lib/traceback.py +++ b/Lib/traceback.py @@ -4,6 +4,7 @@ import itertools import linecache import sys +import textwrap from contextlib import suppress __all__ = ['extract_stack', 'extract_tb', 'format_exception', @@ -601,6 +602,36 @@ def _extract_caret_anchors_from_line_segment(segment): return None +class _ExceptionPrintContext: + def __init__(self): + self.seen = set() + self.exception_group_depth = 0 + self.parent_label = None + self.need_close = False + + def indent(self): + return 2 * self.exception_group_depth + + def margin_char(self): + return '|' if self.exception_group_depth else None + + def get_indent(self): + return ' ' * self.indent() + + def get_fancy_indent(self): + margin_char = self.margin_char() + margin = (margin_char + ' ') if margin_char is not None else '' + return self.get_indent() + margin + + def emit(self, text_gen): + indent_str = self.get_fancy_indent() + if isinstance(text_gen, str): + yield textwrap.indent(text_gen, indent_str, lambda line: True) + else: + for text in text_gen: + yield textwrap.indent(text, indent_str, lambda line: True) + + class TracebackException: """An exception ready for rendering. @@ -707,12 +738,31 @@ def __init__(self, exc_type, exc_value, exc_traceback, *, limit=None, _seen=_seen) else: context = None + + if e and isinstance(e, BaseExceptionGroup): + exceptions = [] + for exc in e.exceptions: + texc = TracebackException( + type(exc), + exc, + exc.__traceback__, + limit=limit, + lookup_lines=lookup_lines, + capture_locals=capture_locals, + _seen=_seen) + exceptions.append(texc) + else: + exceptions = None + te.__cause__ = cause te.__context__ = context + te.exceptions = exceptions if cause: queue.append((te.__cause__, e.__cause__)) if context: queue.append((te.__context__, e.__context__)) + if exceptions: + queue.extend(zip(te.exceptions, e.exceptions)) @classmethod def from_exception(cls, exc, *args, **kwargs): @@ -795,7 +845,7 @@ def _format_syntax_error(self, stype): msg = self.msg or "" yield "{}: {}{}\n".format(stype, msg, filename_suffix) - def format(self, *, chain=True): + def format(self, *, chain=True, _ctx=None): """Format the exception. If chain is not *True*, *__cause__* and *__context__* will not be formatted. @@ -808,34 +858,84 @@ def format(self, *, chain=True): string in the output. """ + if _ctx is None: + _ctx = _ExceptionPrintContext() + output = [] exc = self + label = _ctx.parent_label while exc: if chain: if exc.__cause__ is not None: chained_msg = _cause_message chained_exc = exc.__cause__ + chain_tag = 'cause' elif (exc.__context__ is not None and not exc.__suppress_context__): chained_msg = _context_message chained_exc = exc.__context__ + chain_tag = 'context' else: chained_msg = None chained_exc = None + chain_tag = None - output.append((chained_msg, exc)) + output.append((chained_msg, exc, label)) exc = chained_exc + if chain_tag is not None: + if label is None: + label = chain_tag + else: + label = f'{label}.{chain_tag}' else: - output.append((None, exc)) + output.append((None, exc, label)) exc = None - for msg, exc in reversed(output): + for msg, exc, parent_label in reversed(output): if msg is not None: - yield msg - if exc.stack: - yield 'Traceback (most recent call last):\n' - yield from exc.stack.format() - yield from exc.format_exception_only() + yield from _ctx.emit(msg) + if exc.exceptions is None: + if exc.stack: + yield from _ctx.emit('Traceback (most recent call last):\n') + yield from _ctx.emit(exc.stack.format()) + yield from _ctx.emit(exc.format_exception_only()) + else: + if _ctx.exception_group_depth == 0: + _ctx.exception_group_depth += 1 + if exc.stack: + yield from _ctx.emit('Traceback (most recent call last):\n') + yield from _ctx.emit(exc.stack.format()) + yield from _ctx.emit(exc.format_exception_only()) + n = len(exc.exceptions) + if n > 1: + yield from _ctx.emit(f' with {n} sub-exceptions:\n') + else: + yield from _ctx.emit(' with one sub-exception:\n') + _ctx.need_close = False + for i in range(n): + last_exc = (i == n-1) + if last_exc: + # The closing frame may be added by a recursive call + _ctx.need_close = True + if parent_label is not None: + label = f'{parent_label}.{i + 1}' + else: + label = f'{i + 1}' + yield (_ctx.get_indent() + + ('+-' if i==0 else ' ') + + f'+---------------- {label} ----------------\n') + _ctx.exception_group_depth += 1 + _ctx.parent_label = label + yield from exc.exceptions[i].format(chain=chain, _ctx=_ctx) + _ctx.parent_label = parent_label + if last_exc and _ctx.need_close: + yield (_ctx.get_indent() + + "+------------------------------------\n") + _ctx.need_close = False + _ctx.exception_group_depth -= 1; + if _ctx.exception_group_depth == 1: + _ctx.exception_group_depth -= 1 + def print(self, *, file=None, chain=True): """Print the result of self.format(chain=chain) to 'file'.""" From 261917a93f8ddef87f195abd6c247ba4eb1cfdbb Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Sun, 24 Oct 2021 23:27:48 +0100 Subject: [PATCH 03/27] remove 'with X sub-exceptions' line from tracebacks --- Lib/test/test_traceback.py | 11 ----------- Lib/traceback.py | 4 ---- Python/pythonrun.c | 14 +------------- 3 files changed, 1 insertion(+), 28 deletions(-) diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index ab07cab0f1c980..e4f10dfc927999 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -1262,7 +1262,6 @@ def exc(): expected = [ [' | raise ExceptionGroup("eg", [ValueError(1), TypeError(2)])', ' | ExceptionGroup: eg', - ' | with 2 sub-exceptions:' ], ['-+---------------- 1 ----------------'], [' | ValueError: 1'], @@ -1283,7 +1282,6 @@ def exc(): expected = [ [' | raise ExceptionGroup("eg1", [ValueError(1), TypeError(2)])', ' | ExceptionGroup: eg1', - ' | with 2 sub-exceptions:' ], ['-+---------------- 1 ----------------'], [' | ValueError: 1'], @@ -1292,7 +1290,6 @@ def exc(): [ context_message ], [' | raise ExceptionGroup("eg2", [ValueError(3), TypeError(4)])', ' | ExceptionGroup: eg2', - ' | with 2 sub-exceptions:' ], ['-+---------------- 1 ----------------'], [' | ValueError: 3'], @@ -1313,7 +1310,6 @@ def exc(): expected = [ [' | raise EG("eg1", [ValueError(1), TypeError(2)])', ' | ExceptionGroup: eg1', - ' | with 2 sub-exceptions:' ], ['-+---------------- context.1 ----------------'], [' | ValueError: 1'], @@ -1322,7 +1318,6 @@ def exc(): [ context_message ], [' | raise EG("eg2", [ValueError(3), TypeError(4)])', ' | ExceptionGroup: eg2', - ' | with 2 sub-exceptions:' ], ['-+---------------- 1 ----------------'], [' | ValueError: 3'], @@ -1342,7 +1337,6 @@ def exc(): expected = [ [' | raise EG("eg1", [ValueError(1), TypeError(2)])', ' | ExceptionGroup: eg1', - ' | with 2 sub-exceptions:' ], ['-+---------------- cause.1 ----------------'], [' | ValueError: 1'], @@ -1351,7 +1345,6 @@ def exc(): [ cause_message ], [' | raise EG("eg2", [ValueError(3), TypeError(4)])', ' | ExceptionGroup: eg2', - ' | with 2 sub-exceptions:' ], ['-+---------------- 1 ----------------'], [' | ValueError: 3'], @@ -1378,13 +1371,11 @@ def exc(): expected = [ [' | raise EG("eg", [VE(1), exc, VE(4)])', ' | ExceptionGroup: eg', - ' | with 3 sub-exceptions:' ], ['-+---------------- context.1 ----------------'], [' | ValueError: 1'], ['+---------------- context.2 ----------------'], [' | ExceptionGroup: nested', - ' | with 2 sub-exceptions' ], ['-+---------------- context.2.1 ----------------'], [' | TypeError: 2'], @@ -2146,7 +2137,6 @@ def test_exception_group_format(self): f' | raise ExceptionGroup("eg2", [exc3, exc4])', f' | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^', f' | ExceptionGroup: eg2', - f' | with 2 sub-exceptions:', f' +-+---------------- 1 ----------------', f' | Traceback (most recent call last):', f' | File "{__file__}", ' @@ -2154,7 +2144,6 @@ def test_exception_group_format(self): f' | raise ExceptionGroup("eg1", [exc1, exc2])', f' | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^', f' | ExceptionGroup: eg1', - f' | with 2 sub-exceptions:', f' +-+---------------- 1.1 ----------------', f' | Traceback (most recent call last):', f' | File ' diff --git a/Lib/traceback.py b/Lib/traceback.py index cb2b893d2fcb70..d589f91f2176c0 100644 --- a/Lib/traceback.py +++ b/Lib/traceback.py @@ -907,10 +907,6 @@ def format(self, *, chain=True, _ctx=None): yield from _ctx.emit(exc.stack.format()) yield from _ctx.emit(exc.format_exception_only()) n = len(exc.exceptions) - if n > 1: - yield from _ctx.emit(f' with {n} sub-exceptions:\n') - else: - yield from _ctx.emit(' with one sub-exception:\n') _ctx.need_close = False for i in range(n): last_exc = (i == n-1) diff --git a/Python/pythonrun.c b/Python/pythonrun.c index 21e56c14526362..68fd0e807f2395 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -1156,7 +1156,6 @@ print_exception_recursive(struct exception_print_context* ctx, PyObject *value) /* TODO: add arg to limit number of exceptions printed? */ - PyObject *line; if (ctx->exception_group_depth == 0) { ctx->exception_group_depth += 1; @@ -1169,17 +1168,6 @@ print_exception_recursive(struct exception_print_context* ctx, PyObject *value) PyObject *parent_label = ctx->parent_label; PyObject *f = ctx->file; if (num_excs > 0) { - if (num_excs == 1) { - line = PyUnicode_FromFormat( - " with one sub-exception:\n"); - } - else { - line = PyUnicode_FromFormat( - " with %d sub-exceptions:\n", num_excs); - } - err |= WRITE_INDENTED_MARGIN(ctx, f); - err |= PyFile_WriteObject(line, f, Py_PRINT_RAW); - Py_XDECREF(line); ctx->need_close = 0; for (i = 0; i < num_excs; i++) { int last_exc = i == num_excs - 1; @@ -1196,7 +1184,7 @@ print_exception_recursive(struct exception_print_context* ctx, PyObject *value) label = PyUnicode_FromFormat("%d", i + 1); } err |= _Py_WriteIndent(EXC_INDENT(ctx), f); - line = PyUnicode_FromFormat( + PyObject *line = PyUnicode_FromFormat( "%s+---------------- %U ----------------\n", (i == 0) ? "+-" : " ", label); ctx->exception_group_depth += 1; From 7613b43c3d6621c4d3491d172d8e52d7ada8589a Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Mon, 25 Oct 2021 10:20:46 +0100 Subject: [PATCH 04/27] pass margin instead of margin_char --- Include/cpython/traceback.h | 2 +- Include/traceback.h | 4 +-- Lib/traceback.py | 12 ++++----- Python/_warnings.c | 2 +- Python/pythonrun.c | 6 ++--- Python/traceback.c | 51 ++++++++++++++++++++++--------------- 6 files changed, 43 insertions(+), 34 deletions(-) diff --git a/Include/cpython/traceback.h b/Include/cpython/traceback.h index 1be048625bef4f..d0dde335cfee5b 100644 --- a/Include/cpython/traceback.h +++ b/Include/cpython/traceback.h @@ -10,5 +10,5 @@ typedef struct _traceback { int tb_lineno; } PyTracebackObject; -PyAPI_FUNC(int) _Py_DisplaySourceLine(PyObject *, PyObject *, int, int, int, char, int *, PyObject **); +PyAPI_FUNC(int) _Py_DisplaySourceLine(PyObject *, PyObject *, int, int, int *, PyObject **); PyAPI_FUNC(void) _PyTraceback_Add(const char *, const char *, int); diff --git a/Include/traceback.h b/Include/traceback.h index c2c075252d4ea6..6dc584fddcefbe 100644 --- a/Include/traceback.h +++ b/Include/traceback.h @@ -9,8 +9,8 @@ extern "C" { PyAPI_FUNC(int) PyTraceBack_Here(PyFrameObject *); PyAPI_FUNC(int) PyTraceBack_Print(PyObject *, PyObject *); -int PyTraceBack_Print_Indented(PyObject *, int, char, PyObject *); -int _Py_WriteIndentedMargin(int, char, PyObject *); +int PyTraceBack_Print_Indented(PyObject *, int, const char*, PyObject *); +int _Py_WriteIndentedMargin(int, const char*, PyObject *); int _Py_WriteIndent(int, PyObject *); /* Reveal traceback type so we can typecheck traceback objects */ diff --git a/Lib/traceback.py b/Lib/traceback.py index d589f91f2176c0..b4bfd69970e7ab 100644 --- a/Lib/traceback.py +++ b/Lib/traceback.py @@ -612,19 +612,17 @@ def __init__(self): def indent(self): return 2 * self.exception_group_depth - def margin_char(self): - return '|' if self.exception_group_depth else None + def margin(self): + return '| ' if self.exception_group_depth else '' def get_indent(self): return ' ' * self.indent() - def get_fancy_indent(self): - margin_char = self.margin_char() - margin = (margin_char + ' ') if margin_char is not None else '' - return self.get_indent() + margin + def get_indented_margin(self): + return self.get_indent() + self.margin() def emit(self, text_gen): - indent_str = self.get_fancy_indent() + indent_str = self.get_indented_margin() if isinstance(text_gen, str): yield textwrap.indent(text_gen, indent_str, lambda line: True) else: diff --git a/Python/_warnings.c b/Python/_warnings.c index 5574a4ae8dca21..cf2110d31c3b5e 100644 --- a/Python/_warnings.c +++ b/Python/_warnings.c @@ -544,7 +544,7 @@ show_warning(PyObject *filename, int lineno, PyObject *text, PyFile_WriteString("\n", f_stderr); } else { - _Py_DisplaySourceLine(f_stderr, filename, lineno, 2, 0, '\0', NULL, NULL); + _Py_DisplaySourceLine(f_stderr, filename, lineno, 2, NULL, NULL); } error: diff --git a/Python/pythonrun.c b/Python/pythonrun.c index 68fd0e807f2395..dd2ab481563a54 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -895,10 +895,10 @@ struct exception_print_context int need_close; // Need a closing bottom frame }; -#define EXC_MARGIN_CHAR(ctx) ((ctx)->exception_group_depth ? '|' : '\0') +#define EXC_MARGIN(ctx) ((ctx)->exception_group_depth ? "| " : "") #define EXC_INDENT(ctx) (2 * (ctx)->exception_group_depth) -#define WRITE_INDENTED_MARGIN(ctx, f) _Py_WriteIndentedMargin(EXC_INDENT(ctx), EXC_MARGIN_CHAR(ctx), (f)) +#define WRITE_INDENTED_MARGIN(ctx, f) _Py_WriteIndentedMargin(EXC_INDENT(ctx), EXC_MARGIN(ctx), (f)) static void print_exception(struct exception_print_context *ctx, PyObject *value) @@ -924,7 +924,7 @@ print_exception(struct exception_print_context *ctx, PyObject *value) type = (PyObject *) Py_TYPE(value); tb = PyException_GetTraceback(value); if (tb && tb != Py_None) - err = PyTraceBack_Print_Indented(tb, EXC_INDENT(ctx), EXC_MARGIN_CHAR(ctx), f); + err = PyTraceBack_Print_Indented(tb, EXC_INDENT(ctx), EXC_MARGIN(ctx), f); if (err == 0 && (err = _PyObject_LookupAttrId(value, &PyId_print_file_and_line, &tmp)) > 0) { diff --git a/Python/traceback.c b/Python/traceback.c index e48b05d8c3b00e..64d37fadeece41 100644 --- a/Python/traceback.c +++ b/Python/traceback.c @@ -396,20 +396,21 @@ _Py_WriteIndent(int indent, PyObject *f) { return 0; } -/* Writes indent spaces, followed by the margin if margin_char is not `\0`. +/* Writes indent spaces, followed by the margin if it is not `\0`. */ int -_Py_WriteIndentedMargin(int indent, char margin_char, PyObject *f) { +_Py_WriteIndentedMargin(int indent, const char *margin, PyObject *f) { int err = 0; - char margin[] = {margin_char, ' ', '\0' }; err |= _Py_WriteIndent(indent, f); - err |= PyFile_WriteString(margin, f); + if (margin) { + err |= PyFile_WriteString(margin, f); + } return err; } -int -_Py_DisplaySourceLine(PyObject *f, PyObject *filename, int lineno, int indent, - int margin_indent, char margin_char, int *truncation, PyObject **line) +static int +display_source_line_with_margin(PyObject *f, PyObject *filename, int lineno, int indent, + int margin_indent, const char *margin, int *truncation, PyObject **line) { int err = 0; int fd; @@ -537,7 +538,7 @@ _Py_DisplaySourceLine(PyObject *f, PyObject *filename, int lineno, int indent, *truncation = i - indent; } - err |= _Py_WriteIndentedMargin(margin_indent, margin_char, f); + err |= _Py_WriteIndentedMargin(margin_indent, margin, f); /* Write some spaces before the line */ err |= _Py_WriteIndent(indent, f); @@ -550,6 +551,16 @@ _Py_DisplaySourceLine(PyObject *f, PyObject *filename, int lineno, int indent, return err; } +int +_Py_DisplaySourceLine(PyObject *f, PyObject *filename, int lineno, int indent, + int *truncation, PyObject **line) +{ + return display_source_line_with_margin( + f, filename, lineno, indent, + 0, NULL, /* no margin */ + truncation, line); +} + /* AST based Traceback Specialization * * When displaying a new traceback line, for certain syntactical constructs @@ -718,7 +729,7 @@ print_error_location_carets(PyObject *f, int offset, Py_ssize_t start_offset, Py static int tb_displayline(PyTracebackObject* tb, PyObject *f, PyObject *filename, int lineno, - PyFrameObject *frame, PyObject *name, int margin_indent, char margin_char) + PyFrameObject *frame, PyObject *name, int margin_indent, const char *margin) { int err; PyObject *line; @@ -729,7 +740,7 @@ tb_displayline(PyTracebackObject* tb, PyObject *f, PyObject *filename, int linen filename, lineno, name); if (line == NULL) return -1; - err = _Py_WriteIndentedMargin(margin_indent, margin_char, f); + err = _Py_WriteIndentedMargin(margin_indent, margin, f); err |= PyFile_WriteObject(line, f, Py_PRINT_RAW); Py_DECREF(line); if (err != 0) @@ -737,9 +748,9 @@ tb_displayline(PyTracebackObject* tb, PyObject *f, PyObject *filename, int linen int truncation = _TRACEBACK_SOURCE_LINE_INDENT; PyObject* source_line = NULL; - if (_Py_DisplaySourceLine(f, filename, lineno, _TRACEBACK_SOURCE_LINE_INDENT, - margin_indent, margin_char, - &truncation, &source_line) != 0 || !source_line) { + if (display_source_line_with_margin( + f, filename, lineno, _TRACEBACK_SOURCE_LINE_INDENT, + margin_indent, margin, &truncation, &source_line) != 0 || !source_line) { /* ignore errors since we can't report them, can we? */ err = ignore_source_errors(); goto done; @@ -824,7 +835,7 @@ tb_displayline(PyTracebackObject* tb, PyObject *f, PyObject *filename, int linen end_offset = i + 1; } - err = _Py_WriteIndentedMargin(margin_indent, margin_char, f); + err = _Py_WriteIndentedMargin(margin_indent, margin, f); err |= print_error_location_carets(f, truncation, start_offset, end_offset, right_start_offset, left_end_offset, primary_error_char, secondary_error_char); @@ -855,7 +866,7 @@ tb_print_line_repeated(PyObject *f, long cnt) static int tb_printinternal(PyTracebackObject *tb, PyObject *f, long limit, - int indent, char margin_char) + int indent, const char *margin) { int err = 0; Py_ssize_t depth = 0; @@ -889,7 +900,7 @@ tb_printinternal(PyTracebackObject *tb, PyObject *f, long limit, cnt++; if (err == 0 && cnt <= TB_RECURSIVE_CUTOFF) { err = tb_displayline(tb, f, code->co_filename, tb->tb_lineno, - tb->tb_frame, code->co_name, indent, margin_char); + tb->tb_frame, code->co_name, indent, margin); if (err == 0) { err = PyErr_CheckSignals(); } @@ -906,7 +917,7 @@ tb_printinternal(PyTracebackObject *tb, PyObject *f, long limit, #define PyTraceBack_LIMIT 1000 int -PyTraceBack_Print_Indented(PyObject *v, int indent, char margin_char, PyObject *f) +PyTraceBack_Print_Indented(PyObject *v, int indent, const char *margin, PyObject *f) { int err; PyObject *limitv; @@ -929,17 +940,17 @@ PyTraceBack_Print_Indented(PyObject *v, int indent, char margin_char, PyObject * return 0; } } - err = _Py_WriteIndentedMargin(indent, margin_char, f); + err = _Py_WriteIndentedMargin(indent, margin, f); err |= PyFile_WriteString("Traceback (most recent call last):\n", f); if (!err) - err = tb_printinternal((PyTracebackObject *)v, f, limit, indent, margin_char); + err = tb_printinternal((PyTracebackObject *)v, f, limit, indent, margin); return err; } int PyTraceBack_Print(PyObject *v, PyObject *f) { - return PyTraceBack_Print_Indented(v, 0, '\0', f); + return PyTraceBack_Print_Indented(v, 0, NULL, f); } /* Format an integer in range [0; 0xffffffff] to decimal and write it From d98a72b5b25309e9530a41417d08960a9694333c Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Mon, 25 Oct 2021 11:28:33 +0100 Subject: [PATCH 05/27] update news --- .../Core and Builtins/2021-09-26-18-18-50.bpo-45292.aX5HVr.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-09-26-18-18-50.bpo-45292.aX5HVr.rst b/Misc/NEWS.d/next/Core and Builtins/2021-09-26-18-18-50.bpo-45292.aX5HVr.rst index ee48b6d5105c52..a956a8671f3a09 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2021-09-26-18-18-50.bpo-45292.aX5HVr.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2021-09-26-18-18-50.bpo-45292.aX5HVr.rst @@ -1 +1 @@ -Implement :pep:`654` Add :class:`ExceptionGroup` and :class:`BaseExceptionGroup`. +Implement :pep:`654` Add :class:`ExceptionGroup` and :class:`BaseExceptionGroup`. Update traceback display code. From d69916e7663b8ca0b3d935db8b401f587e3697be Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Mon, 25 Oct 2021 13:14:28 +0100 Subject: [PATCH 06/27] excs is tuple, use PyTuple apis. Change check to assertion. --- Python/pythonrun.c | 81 +++++++++++++++++++++++----------------------- 1 file changed, 40 insertions(+), 41 deletions(-) diff --git a/Python/pythonrun.c b/Python/pythonrun.c index dd2ab481563a54..9a266b7bb28a3b 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -1163,51 +1163,50 @@ print_exception_recursive(struct exception_print_context* ctx, PyObject *value) print_exception(ctx, value); PyObject *excs = ((PyBaseExceptionGroupObject *)value)->excs; - if (excs && PySequence_Check(excs)) { - Py_ssize_t i, num_excs = PySequence_Length(excs); - PyObject *parent_label = ctx->parent_label; - PyObject *f = ctx->file; - if (num_excs > 0) { - ctx->need_close = 0; - for (i = 0; i < num_excs; i++) { - int last_exc = i == num_excs - 1; - if (last_exc) { - // The closing frame may be added in a recursive call - ctx->need_close = 1; - } - PyObject *label; - if (parent_label) { - label = PyUnicode_FromFormat("%U.%d", - parent_label, i + 1); - } - else { - label = PyUnicode_FromFormat("%d", i + 1); - } + assert(excs && PyTuple_Check(excs)); + Py_ssize_t num_excs = PyTuple_Size(excs); + assert(num_excs > 0); + PyObject *parent_label = ctx->parent_label; + PyObject *f = ctx->file; + if (num_excs > 0) { + ctx->need_close = 0; + for (Py_ssize_t i = 0; i < num_excs; i++) { + int last_exc = i == num_excs - 1; + if (last_exc) { + // The closing frame may be added in a recursive call + ctx->need_close = 1; + } + PyObject *label; + if (parent_label) { + label = PyUnicode_FromFormat("%U.%d", + parent_label, i + 1); + } + else { + label = PyUnicode_FromFormat("%d", i + 1); + } + err |= _Py_WriteIndent(EXC_INDENT(ctx), f); + PyObject *line = PyUnicode_FromFormat( + "%s+---------------- %U ----------------\n", + (i == 0) ? "+-" : " ", label); + ctx->exception_group_depth += 1; + err |= PyFile_WriteObject(line, f, Py_PRINT_RAW); + Py_XDECREF(line); + + ctx->parent_label = label; + PyObject *exc = PyTuple_GetItem(excs, i); + print_exception_recursive(ctx, exc); + ctx->parent_label = parent_label; + Py_XDECREF(label); + + if (last_exc && ctx->need_close) { err |= _Py_WriteIndent(EXC_INDENT(ctx), f); - PyObject *line = PyUnicode_FromFormat( - "%s+---------------- %U ----------------\n", - (i == 0) ? "+-" : " ", label); - ctx->exception_group_depth += 1; + line = PyUnicode_FromFormat( + "+------------------------------------\n"); err |= PyFile_WriteObject(line, f, Py_PRINT_RAW); Py_XDECREF(line); - - ctx->parent_label = label; - PyObject *exc = PySequence_GetItem(excs, i); - print_exception_recursive(ctx, exc); - ctx->parent_label = parent_label; - Py_XDECREF(label); - - if (last_exc && ctx->need_close) { - err |= _Py_WriteIndent(EXC_INDENT(ctx), f); - line = PyUnicode_FromFormat( - "+------------------------------------\n"); - err |= PyFile_WriteObject(line, f, Py_PRINT_RAW); - Py_XDECREF(line); - ctx->need_close = 0; - } - ctx->exception_group_depth -= 1; - Py_XDECREF(exc); + ctx->need_close = 0; } + ctx->exception_group_depth -= 1; } } if (ctx->exception_group_depth == 1) { From f5cab699ee9a466b779c5485f5bcbacc7a44b704 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Mon, 25 Oct 2021 14:03:01 +0100 Subject: [PATCH 07/27] remove redundant num_excs > 0 check (it is asserted above) --- Python/pythonrun.c | 74 ++++++++++++++++++++++------------------------ 1 file changed, 36 insertions(+), 38 deletions(-) diff --git a/Python/pythonrun.c b/Python/pythonrun.c index 9a266b7bb28a3b..bb4ec08663db5e 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -1166,48 +1166,46 @@ print_exception_recursive(struct exception_print_context* ctx, PyObject *value) assert(excs && PyTuple_Check(excs)); Py_ssize_t num_excs = PyTuple_Size(excs); assert(num_excs > 0); + PyObject *parent_label = ctx->parent_label; PyObject *f = ctx->file; - if (num_excs > 0) { - ctx->need_close = 0; - for (Py_ssize_t i = 0; i < num_excs; i++) { - int last_exc = i == num_excs - 1; - if (last_exc) { - // The closing frame may be added in a recursive call - ctx->need_close = 1; - } - PyObject *label; - if (parent_label) { - label = PyUnicode_FromFormat("%U.%d", - parent_label, i + 1); - } - else { - label = PyUnicode_FromFormat("%d", i + 1); - } + + ctx->need_close = 0; + for (Py_ssize_t i = 0; i < num_excs; i++) { + int last_exc = i == num_excs - 1; + if (last_exc) { + // The closing frame may be added in a recursive call + ctx->need_close = 1; + } + PyObject *label; + if (parent_label) { + label = PyUnicode_FromFormat("%U.%d", + parent_label, i + 1); + } + else { + label = PyUnicode_FromFormat("%d", i + 1); + } + err |= _Py_WriteIndent(EXC_INDENT(ctx), f); + PyObject *line = PyUnicode_FromFormat( + "%s+---------------- %U ----------------\n", + (i == 0) ? "+-" : " ", label); + ctx->exception_group_depth += 1; + err |= PyFile_WriteObject(line, f, Py_PRINT_RAW); + Py_XDECREF(line); + + ctx->parent_label = label; + PyObject *exc = PyTuple_GetItem(excs, i); + print_exception_recursive(ctx, exc); + ctx->parent_label = parent_label; + Py_XDECREF(label); + + if (last_exc && ctx->need_close) { err |= _Py_WriteIndent(EXC_INDENT(ctx), f); - PyObject *line = PyUnicode_FromFormat( - "%s+---------------- %U ----------------\n", - (i == 0) ? "+-" : " ", label); - ctx->exception_group_depth += 1; - err |= PyFile_WriteObject(line, f, Py_PRINT_RAW); - Py_XDECREF(line); - - ctx->parent_label = label; - PyObject *exc = PyTuple_GetItem(excs, i); - print_exception_recursive(ctx, exc); - ctx->parent_label = parent_label; - Py_XDECREF(label); - - if (last_exc && ctx->need_close) { - err |= _Py_WriteIndent(EXC_INDENT(ctx), f); - line = PyUnicode_FromFormat( - "+------------------------------------\n"); - err |= PyFile_WriteObject(line, f, Py_PRINT_RAW); - Py_XDECREF(line); - ctx->need_close = 0; - } - ctx->exception_group_depth -= 1; + err |= PyFile_WriteString( + "+------------------------------------\n", f); + ctx->need_close = 0; } + ctx->exception_group_depth -= 1; } if (ctx->exception_group_depth == 1) { ctx->exception_group_depth -= 1; From 5170f00891a7f4be3bb349df49184cd2b8d6e44e Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Mon, 25 Oct 2021 14:44:48 +0100 Subject: [PATCH 08/27] remove cpython_only from exception group tests --- Lib/test/test_traceback.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index e4f10dfc927999..b14ad5d6567974 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -1254,7 +1254,6 @@ def check_exception_group(self, exc_func, expected): # at most one margin char per line self.assertLessEqual(line.count('|'), 1) - @cpython_only def test_exception_group_basic(self): def exc(): raise ExceptionGroup("eg", [ValueError(1), TypeError(2)]) @@ -1271,7 +1270,6 @@ def exc(): self.check_exception_group(exc, expected) - @cpython_only def test_exception_group_context(self): def exc(): try: @@ -1298,7 +1296,6 @@ def exc(): ] self.check_exception_group(exc, expected) - @cpython_only def test_exception_group_context(self): def exc(): EG = ExceptionGroup @@ -1353,7 +1350,6 @@ def exc(): ] self.check_exception_group(exc, expected) - @cpython_only def test_exception_group_nested(self): def exc(): EG = ExceptionGroup From 2052c77d9b8093e0c61a2366e48ee0cd2f368257 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Mon, 25 Oct 2021 18:30:56 +0100 Subject: [PATCH 09/27] handle recursion errors (vert deeply nested EGs) --- Lib/test/test_traceback.py | 29 ++++++++++++++++++++++++++++ Lib/traceback.py | 5 ++++- Python/pythonrun.c | 39 ++++++++++++++++++++++++++++---------- 3 files changed, 62 insertions(+), 11 deletions(-) diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index b14ad5d6567974..fdac157f6d235e 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -987,6 +987,35 @@ def __eq__(self, other): self.assertIn('UnhashableException: ex2', tb[4]) self.assertIn('UnhashableException: ex1', tb[12]) + def deep_eg(self): + e = TypeError(1) + for i in range(2000): + e = ExceptionGroup('eg', [e]) + return e + + @cpython_only + def test_exception_group_deep_recursion_capi(self): + from _testcapi import exception_print + LIMIT = 75 + eg = self.deep_eg() + with captured_output("stderr") as stderr_f: + with support.infinite_recursion(max_depth=LIMIT): + exception_print(eg) + output = stderr_f.getvalue() + self.assertIn('ExceptionGroup', output) + self.assertLessEqual(output.count('ExceptionGroup'), LIMIT) + + def test_exception_group_deep_recursion_traceback(self): + LIMIT = 75 + eg = self.deep_eg() + with captured_output("stderr") as stderr_f: + with support.infinite_recursion(max_depth=LIMIT): + traceback.print_exception(type(eg), eg, eg.__traceback__) + output = stderr_f.getvalue() + self.assertIn('ExceptionGroup', output) + self.assertLessEqual(output.count('ExceptionGroup'), LIMIT) + + cause_message = ( "\nThe above exception was the direct cause " "of the following exception:\n\n") diff --git a/Lib/traceback.py b/Lib/traceback.py index b4bfd69970e7ab..cfe5ac590904b4 100644 --- a/Lib/traceback.py +++ b/Lib/traceback.py @@ -920,7 +920,10 @@ def format(self, *, chain=True, _ctx=None): f'+---------------- {label} ----------------\n') _ctx.exception_group_depth += 1 _ctx.parent_label = label - yield from exc.exceptions[i].format(chain=chain, _ctx=_ctx) + try: + yield from exc.exceptions[i].format(chain=chain, _ctx=_ctx) + except RecursionError: + pass _ctx.parent_label = parent_label if last_exc and _ctx.need_close: yield (_ctx.get_indent() + diff --git a/Python/pythonrun.c b/Python/pythonrun.c index bb4ec08663db5e..585d500a306151 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -947,6 +947,7 @@ print_exception(struct exception_print_context *ctx, PyObject *value) Py_DECREF(filename); if (line != NULL) { err += WRITE_INDENTED_MARGIN(ctx, f); + PyErr_Clear(); PyFile_WriteObject(line, f, Py_PRINT_RAW); Py_DECREF(line); } @@ -997,6 +998,7 @@ print_exception(struct exception_print_context *ctx, PyObject *value) if (!_PyUnicode_EqualToASCIIId(modulename, &PyId_builtins) && !_PyUnicode_EqualToASCIIId(modulename, &PyId___main__)) { + PyErr_Clear(); err += PyFile_WriteObject(modulename, f, Py_PRINT_RAW); err += PyFile_WriteString(".", f); } @@ -1010,6 +1012,7 @@ print_exception(struct exception_print_context *ctx, PyObject *value) err = PyFile_WriteString("", f); } else { + PyErr_Clear(); err = PyFile_WriteObject(qualname, f, Py_PRINT_RAW); Py_DECREF(qualname); } @@ -1072,7 +1075,6 @@ static int print_chained(struct exception_print_context* ctx, PyObject *value, const char * message, const char *tag) { PyObject *f = ctx->file; - int err = 0; PyObject *parent_label = ctx->parent_label; PyObject *label = NULL; int need_close = ctx->need_close; @@ -1084,16 +1086,25 @@ print_chained(struct exception_print_context* ctx, PyObject *value, } ctx->parent_label = label; - print_exception_recursive(ctx, value); - err |= WRITE_INDENTED_MARGIN(ctx, f); - err |= PyFile_WriteString("\n", f); - err |= WRITE_INDENTED_MARGIN(ctx, f); - err |= PyFile_WriteString(message, f); - err |= WRITE_INDENTED_MARGIN(ctx, f); - err |= PyFile_WriteString("\n", f); + int err = Py_EnterRecursiveCall(" in print_chained"); + if (!err) { + print_exception_recursive(ctx, value); + Py_LeaveRecursiveCall(); + + err |= WRITE_INDENTED_MARGIN(ctx, f); + err |= PyFile_WriteString("\n", f); + err |= WRITE_INDENTED_MARGIN(ctx, f); + err |= PyFile_WriteString(message, f); + err |= WRITE_INDENTED_MARGIN(ctx, f); + err |= PyFile_WriteString("\n", f); + } + else { + PyErr_Clear(); + } ctx->need_close = need_close; ctx->parent_label = parent_label; + Py_XDECREF(label); return err; } @@ -1156,7 +1167,6 @@ print_exception_recursive(struct exception_print_context* ctx, PyObject *value) /* TODO: add arg to limit number of exceptions printed? */ - if (ctx->exception_group_depth == 0) { ctx->exception_group_depth += 1; } @@ -1190,12 +1200,21 @@ print_exception_recursive(struct exception_print_context* ctx, PyObject *value) "%s+---------------- %U ----------------\n", (i == 0) ? "+-" : " ", label); ctx->exception_group_depth += 1; + PyErr_Clear(); err |= PyFile_WriteObject(line, f, Py_PRINT_RAW); Py_XDECREF(line); ctx->parent_label = label; PyObject *exc = PyTuple_GetItem(excs, i); - print_exception_recursive(ctx, exc); + + if (!Py_EnterRecursiveCall(" in print_exception_recursive")) { + print_exception_recursive(ctx, exc); + Py_LeaveRecursiveCall(); + } + else { + err = -1; + PyErr_Clear(); + } ctx->parent_label = parent_label; Py_XDECREF(label); From 5097300d5d6a6ae64f816e836a2cfec449cab8c5 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Tue, 26 Oct 2021 11:20:57 +0100 Subject: [PATCH 10/27] WRITE_INDENTED_MARGIN macro --> write_indented_margin function --- Python/pythonrun.c | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/Python/pythonrun.c b/Python/pythonrun.c index 585d500a306151..e1e8b83d43bcbc 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -898,7 +898,11 @@ struct exception_print_context #define EXC_MARGIN(ctx) ((ctx)->exception_group_depth ? "| " : "") #define EXC_INDENT(ctx) (2 * (ctx)->exception_group_depth) -#define WRITE_INDENTED_MARGIN(ctx, f) _Py_WriteIndentedMargin(EXC_INDENT(ctx), EXC_MARGIN(ctx), (f)) +static int +write_indented_margin(struct exception_print_context *ctx, PyObject *f) +{ + return _Py_WriteIndentedMargin(EXC_INDENT(ctx), EXC_MARGIN(ctx), f); +} static void print_exception(struct exception_print_context *ctx, PyObject *value) @@ -946,7 +950,7 @@ print_exception(struct exception_print_context *ctx, PyObject *value) filename, lineno); Py_DECREF(filename); if (line != NULL) { - err += WRITE_INDENTED_MARGIN(ctx, f); + err += write_indented_margin(ctx, f); PyErr_Clear(); PyFile_WriteObject(line, f, Py_PRINT_RAW); Py_DECREF(line); @@ -986,7 +990,7 @@ print_exception(struct exception_print_context *ctx, PyObject *value) _Py_IDENTIFIER(__module__); assert(PyExceptionClass_Check(type)); - err += WRITE_INDENTED_MARGIN(ctx, f); + err += write_indented_margin(ctx, f); modulename = _PyObject_GetAttrId(type, &PyId___module__); if (modulename == NULL || !PyUnicode_Check(modulename)) { @@ -1091,11 +1095,11 @@ print_chained(struct exception_print_context* ctx, PyObject *value, print_exception_recursive(ctx, value); Py_LeaveRecursiveCall(); - err |= WRITE_INDENTED_MARGIN(ctx, f); + err |= write_indented_margin(ctx, f); err |= PyFile_WriteString("\n", f); - err |= WRITE_INDENTED_MARGIN(ctx, f); + err |= write_indented_margin(ctx, f); err |= PyFile_WriteString(message, f); - err |= WRITE_INDENTED_MARGIN(ctx, f); + err |= write_indented_margin(ctx, f); err |= PyFile_WriteString("\n", f); } else { From dc21cf8f53cc6ef2f3c7a93140aeca70260e9b73 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Tue, 26 Oct 2021 11:44:43 +0100 Subject: [PATCH 11/27] move new traceback utils to internal/ --- Include/internal/pycore_traceback.h | 6 ++++++ Include/traceback.h | 4 ---- Python/pythonrun.c | 3 ++- Python/traceback.c | 4 ++-- 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/Include/internal/pycore_traceback.h b/Include/internal/pycore_traceback.h index c01a47639d5e30..38890cd8852130 100644 --- a/Include/internal/pycore_traceback.h +++ b/Include/internal/pycore_traceback.h @@ -87,6 +87,12 @@ PyAPI_FUNC(PyObject*) _PyTraceBack_FromFrame( PyObject *tb_next, PyFrameObject *frame); +/* Write the traceback tb to file f. Prefix each line with + indent spaces followed by the margin (if it is not NULL). */ +PyAPI_FUNC(int) _PyTraceBack_Print_Indented(PyObject *tb, int indent, const char* margin, PyObject *f); +PyAPI_FUNC(int) _Py_WriteIndentedMargin(int, const char*, PyObject *); +PyAPI_FUNC(int) _Py_WriteIndent(int, PyObject *); + #ifdef __cplusplus } #endif diff --git a/Include/traceback.h b/Include/traceback.h index 6dc584fddcefbe..2dfa2ada4f2c37 100644 --- a/Include/traceback.h +++ b/Include/traceback.h @@ -9,10 +9,6 @@ extern "C" { PyAPI_FUNC(int) PyTraceBack_Here(PyFrameObject *); PyAPI_FUNC(int) PyTraceBack_Print(PyObject *, PyObject *); -int PyTraceBack_Print_Indented(PyObject *, int, const char*, PyObject *); -int _Py_WriteIndentedMargin(int, const char*, PyObject *); -int _Py_WriteIndent(int, PyObject *); - /* Reveal traceback type so we can typecheck traceback objects */ PyAPI_DATA(PyTypeObject) PyTraceBack_Type; #define PyTraceBack_Check(v) Py_IS_TYPE(v, &PyTraceBack_Type) diff --git a/Python/pythonrun.c b/Python/pythonrun.c index e1e8b83d43bcbc..50a92a4caa10b9 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -19,6 +19,7 @@ #include "pycore_pylifecycle.h" // _Py_UnhandledKeyboardInterrupt #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_sysmodule.h" // _PySys_Audit() +#include "pycore_traceback.h" // _PyTraceBack_Print_Indented() #include "token.h" // INDENT #include "errcode.h" // E_EOF @@ -928,7 +929,7 @@ print_exception(struct exception_print_context *ctx, PyObject *value) type = (PyObject *) Py_TYPE(value); tb = PyException_GetTraceback(value); if (tb && tb != Py_None) - err = PyTraceBack_Print_Indented(tb, EXC_INDENT(ctx), EXC_MARGIN(ctx), f); + err = _PyTraceBack_Print_Indented(tb, EXC_INDENT(ctx), EXC_MARGIN(ctx), f); if (err == 0 && (err = _PyObject_LookupAttrId(value, &PyId_print_file_and_line, &tmp)) > 0) { diff --git a/Python/traceback.c b/Python/traceback.c index 64d37fadeece41..b230fa28ee324f 100644 --- a/Python/traceback.c +++ b/Python/traceback.c @@ -917,7 +917,7 @@ tb_printinternal(PyTracebackObject *tb, PyObject *f, long limit, #define PyTraceBack_LIMIT 1000 int -PyTraceBack_Print_Indented(PyObject *v, int indent, const char *margin, PyObject *f) +_PyTraceBack_Print_Indented(PyObject *v, int indent, const char *margin, PyObject *f) { int err; PyObject *limitv; @@ -950,7 +950,7 @@ PyTraceBack_Print_Indented(PyObject *v, int indent, const char *margin, PyObject int PyTraceBack_Print(PyObject *v, PyObject *f) { - return PyTraceBack_Print_Indented(v, 0, NULL, f); + return _PyTraceBack_Print_Indented(v, 0, NULL, f); } /* Format an integer in range [0; 0xffffffff] to decimal and write it From d4007b7a64dc580d317d2e29e080a99a98225782 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Tue, 26 Oct 2021 11:54:29 +0100 Subject: [PATCH 12/27] test improvements --- Lib/test/test_traceback.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index fdac157f6d235e..834dbaa541c7ad 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -1024,7 +1024,7 @@ def test_exception_group_deep_recursion_traceback(self): "\nDuring handling of the above exception, " "another exception occurred:\n\n") -nested_exception_header_re = r'[\+-]?\+[-{4} ]+[ \d\.?]+[ -{4}]+' +nested_exception_header_re = r'[+-]?\+[-{4} ]+[ \d\.?]+[ -{4}]+' boundaries = re.compile( '(%s|%s|%s)' % (re.escape(cause_message), @@ -1273,15 +1273,15 @@ def check_exception_group(self, exc_func, expected): blocks = boundaries.split(report) self.assertEqual(len(blocks), len(expected)) - for i, block in enumerate(blocks): - for line in expected[i]: + for block, expected_lines in zip(blocks, expected): + for line in expected_lines: self.assertIn(f"{line}", block) # check indentation self.assertNotIn(f" {line}", block) for line in report: # at most one margin char per line - self.assertLessEqual(line.count('|'), 1) + self.assertLessEqual(line.count('|'), 1, msg=f'{line!r}') def test_exception_group_basic(self): def exc(): From 169934e9ac4a9b803d36e9864983ca1323b54390 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Tue, 26 Oct 2021 12:39:13 +0100 Subject: [PATCH 13/27] pep7, improve error checking and clarity --- Python/pythonrun.c | 48 +++++++++++++++++++++++++++++----------------- Python/traceback.c | 38 ++++++++++++++++++++---------------- 2 files changed, 52 insertions(+), 34 deletions(-) diff --git a/Python/pythonrun.c b/Python/pythonrun.c index 50a92a4caa10b9..7cb0281d154ae4 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -928,8 +928,9 @@ print_exception(struct exception_print_context *ctx, PyObject *value) fflush(stdout); type = (PyObject *) Py_TYPE(value); tb = PyException_GetTraceback(value); - if (tb && tb != Py_None) + if (tb && tb != Py_None) { err = _PyTraceBack_Print_Indented(tb, EXC_INDENT(ctx), EXC_MARGIN(ctx), f); + } if (err == 0 && (err = _PyObject_LookupAttrId(value, &PyId_print_file_and_line, &tmp)) > 0) { @@ -939,8 +940,9 @@ print_exception(struct exception_print_context *ctx, PyObject *value) Py_DECREF(tmp); if (!parse_syntax_error(value, &message, &filename, &lineno, &offset, - &end_lineno, &end_offset, &text)) + &end_lineno, &end_offset, &text)) { PyErr_Clear(); + } else { PyObject *line; @@ -1078,7 +1080,8 @@ print_exception_recursive(struct exception_print_context*, PyObject*); static int print_chained(struct exception_print_context* ctx, PyObject *value, - const char * message, const char *tag) { + const char * message, const char *tag) +{ PyObject *f = ctx->file; PyObject *parent_label = ctx->parent_label; PyObject *label = NULL; @@ -1179,7 +1182,7 @@ print_exception_recursive(struct exception_print_context* ctx, PyObject *value) PyObject *excs = ((PyBaseExceptionGroupObject *)value)->excs; assert(excs && PyTuple_Check(excs)); - Py_ssize_t num_excs = PyTuple_Size(excs); + Py_ssize_t num_excs = PyTuple_GET_SIZE(excs); assert(num_excs > 0); PyObject *parent_label = ctx->parent_label; @@ -1187,7 +1190,7 @@ print_exception_recursive(struct exception_print_context* ctx, PyObject *value) ctx->need_close = 0; for (Py_ssize_t i = 0; i < num_excs; i++) { - int last_exc = i == num_excs - 1; + int last_exc = (i == num_excs - 1); if (last_exc) { // The closing frame may be added in a recursive call ctx->need_close = 1; @@ -1200,17 +1203,27 @@ print_exception_recursive(struct exception_print_context* ctx, PyObject *value) else { label = PyUnicode_FromFormat("%d", i + 1); } - err |= _Py_WriteIndent(EXC_INDENT(ctx), f); - PyObject *line = PyUnicode_FromFormat( - "%s+---------------- %U ----------------\n", - (i == 0) ? "+-" : " ", label); - ctx->exception_group_depth += 1; - PyErr_Clear(); - err |= PyFile_WriteObject(line, f, Py_PRINT_RAW); - Py_XDECREF(line); + PyObject *line = NULL; + if (label) { + line = PyUnicode_FromFormat( + "%s+---------------- %U ----------------\n", + (i == 0) ? "+-" : " ", label); + } + if (line) { + err |= _Py_WriteIndent(EXC_INDENT(ctx), f); + PyErr_Clear(); + err |= PyFile_WriteObject(line, f, Py_PRINT_RAW); + Py_DECREF(line); + } + else { + err = -1; + PyErr_Clear(); + } - ctx->parent_label = label; - PyObject *exc = PyTuple_GetItem(excs, i); + ctx->exception_group_depth += 1; + ctx->parent_label = label; /* transfer ref ownership */ + label = NULL; + PyObject *exc = PyTuple_GET_ITEM(excs, i); if (!Py_EnterRecursiveCall(" in print_exception_recursive")) { print_exception_recursive(ctx, exc); @@ -1220,8 +1233,8 @@ print_exception_recursive(struct exception_print_context* ctx, PyObject *value) err = -1; PyErr_Clear(); } + Py_XDECREF(ctx->parent_label); ctx->parent_label = parent_label; - Py_XDECREF(label); if (last_exc && ctx->need_close) { err |= _Py_WriteIndent(EXC_INDENT(ctx), f); @@ -1243,8 +1256,6 @@ void _PyErr_Display(PyObject *file, PyObject *exception, PyObject *value, PyObject *tb) { assert(file != NULL && file != Py_None); - struct exception_print_context ctx; - if (PyExceptionInstance_Check(value) && tb != NULL && PyTraceBack_Check(tb)) { /* Put the traceback on the exception, otherwise it won't get @@ -1256,6 +1267,7 @@ _PyErr_Display(PyObject *file, PyObject *exception, PyObject *value, PyObject *t Py_DECREF(cur_tb); } + struct exception_print_context ctx; ctx.file = file; ctx.exception_group_depth = 0; ctx.parent_label = 0; diff --git a/Python/traceback.c b/Python/traceback.c index b230fa28ee324f..7d100888a9d49e 100644 --- a/Python/traceback.c +++ b/Python/traceback.c @@ -379,29 +379,34 @@ _Py_FindSourceFile(PyObject *filename, char* namebuf, size_t namelen, PyObject * return result; } +/* Writes indent spaces. Returns 0 on success and non-zero on failure. + */ int -_Py_WriteIndent(int indent, PyObject *f) { +_Py_WriteIndent(int indent, PyObject *f) +{ int err = 0; - char buf[11]; - strcpy(buf, " "); + char buf[11] = " "; assert(strlen(buf) == 10); while (indent > 0) { - if (indent < 10) + if (indent < 10) { buf[indent] = '\0'; + } err = PyFile_WriteString(buf, f); - if (err != 0) + if (err != 0) { return err; + } indent -= 10; } return 0; } /* Writes indent spaces, followed by the margin if it is not `\0`. + Returns 0 on success and non-zero on failure. */ int -_Py_WriteIndentedMargin(int indent, const char *margin, PyObject *f) { - int err = 0; - err |= _Py_WriteIndent(indent, f); +_Py_WriteIndentedMargin(int indent, const char *margin, PyObject *f) +{ + int err = _Py_WriteIndent(indent, f); if (margin) { err |= PyFile_WriteString(margin, f); } @@ -410,7 +415,8 @@ _Py_WriteIndentedMargin(int indent, const char *margin, PyObject *f) { static int display_source_line_with_margin(PyObject *f, PyObject *filename, int lineno, int indent, - int margin_indent, const char *margin, int *truncation, PyObject **line) + int margin_indent, const char *margin, + int *truncation, PyObject **line) { int err = 0; int fd; @@ -555,10 +561,8 @@ int _Py_DisplaySourceLine(PyObject *f, PyObject *filename, int lineno, int indent, int *truncation, PyObject **line) { - return display_source_line_with_margin( - f, filename, lineno, indent, - 0, NULL, /* no margin */ - truncation, line); + return display_source_line_with_margin(f, filename, lineno, indent, 0, + NULL, truncation, line); } /* AST based Traceback Specialization @@ -748,9 +752,10 @@ tb_displayline(PyTracebackObject* tb, PyObject *f, PyObject *filename, int linen int truncation = _TRACEBACK_SOURCE_LINE_INDENT; PyObject* source_line = NULL; - if (display_source_line_with_margin( + int rc = display_source_line_with_margin( f, filename, lineno, _TRACEBACK_SOURCE_LINE_INDENT, - margin_indent, margin, &truncation, &source_line) != 0 || !source_line) { + margin_indent, margin, &truncation, &source_line); + if (rc != 0 || !source_line) { /* ignore errors since we can't report them, can we? */ err = ignore_source_errors(); goto done; @@ -942,8 +947,9 @@ _PyTraceBack_Print_Indented(PyObject *v, int indent, const char *margin, PyObjec } err = _Py_WriteIndentedMargin(indent, margin, f); err |= PyFile_WriteString("Traceback (most recent call last):\n", f); - if (!err) + if (!err) { err = tb_printinternal((PyTracebackObject *)v, f, limit, indent, margin); + } return err; } From aa4da453fb1cfa4b5962b4c28de5426782e314f1 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Tue, 26 Oct 2021 15:50:34 +0100 Subject: [PATCH 14/27] add missing test to cover print_chained with/without parent_label --- Lib/test/test_traceback.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index 834dbaa541c7ad..e2df63a174121c 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -1379,6 +1379,40 @@ def exc(): ] self.check_exception_group(exc, expected) + def test_exception_group_context_with_context(self): + def exc(): + EG = ExceptionGroup + try: + try: + raise EG("eg1", [ValueError(1), TypeError(2)]) + except: + raise EG("eg2", [ValueError(3), TypeError(4)]) + except: + raise ImportError(5) + + expected = [ + [' | raise EG("eg1", [ValueError(1), TypeError(2)])', + ' | ExceptionGroup: eg1', + ], + ['-+---------------- context.context.1 ----------------'], + [' | ValueError: 1'], + ['+---------------- context.context.2 ----------------'], + [' | TypeError: 2'], + [ context_message ], + [' | raise EG("eg2", [ValueError(3), TypeError(4)])', + ' | ExceptionGroup: eg2', + ], + ['-+---------------- context.1 ----------------'], + [' | ValueError: 3'], + ['+---------------- context.2 ----------------'], + [' | TypeError: 4'], + [ context_message ], + [' raise ImportError(5)', + 'ImportError: 5', + ], + ] + self.check_exception_group(exc, expected) + def test_exception_group_nested(self): def exc(): EG = ExceptionGroup From 6ee84f7e196ebb9cdafebb95d1fb7527f1d16b81 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Tue, 26 Oct 2021 23:50:19 +0100 Subject: [PATCH 15/27] compare the complete expected tb text --- Lib/test/test_traceback.py | 266 +++++++++++++++++-------------------- 1 file changed, 124 insertions(+), 142 deletions(-) diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index e2df63a174121c..da6b3821006c3c 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -1041,6 +1041,8 @@ def get_exception(self, exception_or_callable): except Exception as e: return e + callable_line = get_exception.__code__.co_firstlineno + 4 + def zero_div(self): 1/0 # In zero_div @@ -1268,89 +1270,27 @@ def __str__(self): # #### Exception Groups #### - def check_exception_group(self, exc_func, expected): - report = self.get_report(exc_func) - blocks = boundaries.split(report) - self.assertEqual(len(blocks), len(expected)) - - for block, expected_lines in zip(blocks, expected): - for line in expected_lines: - self.assertIn(f"{line}", block) - # check indentation - self.assertNotIn(f" {line}", block) - - for line in report: - # at most one margin char per line - self.assertLessEqual(line.count('|'), 1, msg=f'{line!r}') - def test_exception_group_basic(self): def exc(): raise ExceptionGroup("eg", [ValueError(1), TypeError(2)]) - expected = [ - [' | raise ExceptionGroup("eg", [ValueError(1), TypeError(2)])', - ' | ExceptionGroup: eg', - ], - ['-+---------------- 1 ----------------'], - [' | ValueError: 1'], - ['+---------------- 2 ----------------'], - [' | TypeError: 2'], - ] - - self.check_exception_group(exc, expected) - - def test_exception_group_context(self): - def exc(): - try: - raise ExceptionGroup("eg1", [ValueError(1), TypeError(2)]) - except: - raise ExceptionGroup("eg2", [ValueError(3), TypeError(4)]) - - expected = [ - [' | raise ExceptionGroup("eg1", [ValueError(1), TypeError(2)])', - ' | ExceptionGroup: eg1', - ], - ['-+---------------- 1 ----------------'], - [' | ValueError: 1'], - ['+---------------- 2 ----------------'], - [' | TypeError: 2'], - [ context_message ], - [' | raise ExceptionGroup("eg2", [ValueError(3), TypeError(4)])', - ' | ExceptionGroup: eg2', - ], - ['-+---------------- 1 ----------------'], - [' | ValueError: 3'], - ['+---------------- 2 ----------------'], - [' | TypeError: 4'], - ] - self.check_exception_group(exc, expected) - - def test_exception_group_context(self): - def exc(): - EG = ExceptionGroup - try: - raise EG("eg1", [ValueError(1), TypeError(2)]) - except: - raise EG("eg2", [ValueError(3), TypeError(4)]) - - expected = [ - [' | raise EG("eg1", [ValueError(1), TypeError(2)])', - ' | ExceptionGroup: eg1', - ], - ['-+---------------- context.1 ----------------'], - [' | ValueError: 1'], - ['+---------------- context.2 ----------------'], - [' | TypeError: 2'], - [ context_message ], - [' | raise EG("eg2", [ValueError(3), TypeError(4)])', - ' | ExceptionGroup: eg2', - ], - ['-+---------------- 1 ----------------'], - [' | ValueError: 3'], - ['+---------------- 2 ----------------'], - [' | TypeError: 4'], - ] - self.check_exception_group(exc, expected) + expected = ( + f' | Traceback (most recent call last):\n' + f' | File "{__file__}", line {self.callable_line}, in get_exception\n' + f' | exception_or_callable()\n' + f' | ^^^^^^^^^^^^^^^^^^^^^^^\n' + f' | File "{__file__}", line {exc.__code__.co_firstlineno + 1}, in exc\n' + f' | raise ExceptionGroup("eg", [ValueError(1), TypeError(2)])\n' + f' | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n' + f' | ExceptionGroup: eg\n' + f' +-+---------------- 1 ----------------\n' + f' | ValueError: 1\n' + f' +---------------- 2 ----------------\n' + f' | TypeError: 2\n' + f' +------------------------------------\n') + + report = self.get_report(exc) + self.assertEqual(report, expected) def test_exception_group_cause(self): def exc(): @@ -1360,24 +1300,35 @@ def exc(): except Exception as e: raise EG("eg2", [ValueError(3), TypeError(4)]) from e - expected = [ - [' | raise EG("eg1", [ValueError(1), TypeError(2)])', - ' | ExceptionGroup: eg1', - ], - ['-+---------------- cause.1 ----------------'], - [' | ValueError: 1'], - ['+---------------- cause.2 ----------------'], - [' | TypeError: 2'], - [ cause_message ], - [' | raise EG("eg2", [ValueError(3), TypeError(4)])', - ' | ExceptionGroup: eg2', - ], - ['-+---------------- 1 ----------------'], - [' | ValueError: 3'], - ['+---------------- 2 ----------------'], - [' | TypeError: 4'], - ] - self.check_exception_group(exc, expected) + expected = (f' | Traceback (most recent call last):\n' + f' | File "{__file__}", line {exc.__code__.co_firstlineno + 3}, in exc\n' + f' | raise EG("eg1", [ValueError(1), TypeError(2)])\n' + f' | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n' + f' | ExceptionGroup: eg1\n' + f' +-+---------------- cause.1 ----------------\n' + f' | ValueError: 1\n' + f' +---------------- cause.2 ----------------\n' + f' | TypeError: 2\n' + f' +------------------------------------\n' + f'\n' + f'The above exception was the direct cause of the following exception:\n' + f'\n' + f' | Traceback (most recent call last):\n' + f' | File "{__file__}", line {self.callable_line}, in get_exception\n' + f' | exception_or_callable()\n' + f' | ^^^^^^^^^^^^^^^^^^^^^^^\n' + f' | File "{__file__}", line {exc.__code__.co_firstlineno + 5}, in exc\n' + f' | raise EG("eg2", [ValueError(3), TypeError(4)]) from e\n' + f' | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n' + f' | ExceptionGroup: eg2\n' + f' +-+---------------- 1 ----------------\n' + f' | ValueError: 3\n' + f' +---------------- 2 ----------------\n' + f' | TypeError: 4\n' + f' +------------------------------------\n') + + report = self.get_report(exc) + self.assertEqual(report, expected) def test_exception_group_context_with_context(self): def exc(): @@ -1390,28 +1341,44 @@ def exc(): except: raise ImportError(5) - expected = [ - [' | raise EG("eg1", [ValueError(1), TypeError(2)])', - ' | ExceptionGroup: eg1', - ], - ['-+---------------- context.context.1 ----------------'], - [' | ValueError: 1'], - ['+---------------- context.context.2 ----------------'], - [' | TypeError: 2'], - [ context_message ], - [' | raise EG("eg2", [ValueError(3), TypeError(4)])', - ' | ExceptionGroup: eg2', - ], - ['-+---------------- context.1 ----------------'], - [' | ValueError: 3'], - ['+---------------- context.2 ----------------'], - [' | TypeError: 4'], - [ context_message ], - [' raise ImportError(5)', - 'ImportError: 5', - ], - ] - self.check_exception_group(exc, expected) + expected = ( + f' | Traceback (most recent call last):\n' + f' | File "{__file__}", line {exc.__code__.co_firstlineno + 4}, in exc\n' + f' | raise EG("eg1", [ValueError(1), TypeError(2)])\n' + f' | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n' + f' | ExceptionGroup: eg1\n' + f' +-+---------------- context.context.1 ----------------\n' + f' | ValueError: 1\n' + f' +---------------- context.context.2 ----------------\n' + f' | TypeError: 2\n' + f' +------------------------------------\n' + f'\n' + f'During handling of the above exception, another exception occurred:\n' + f'\n' + f' | Traceback (most recent call last):\n' + f' | File "{__file__}", line {exc.__code__.co_firstlineno + 6}, in exc\n' + f' | raise EG("eg2", [ValueError(3), TypeError(4)])\n' + f' | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n' + f' | ExceptionGroup: eg2\n' + f' +-+---------------- context.1 ----------------\n' + f' | ValueError: 3\n' + f' +---------------- context.2 ----------------\n' + f' | TypeError: 4\n' + f' +------------------------------------\n' + f'\n' + f'During handling of the above exception, another exception occurred:\n' + f'\n' + f'Traceback (most recent call last):\n' + f' File "{__file__}", line {self.callable_line}, in get_exception\n' + f' exception_or_callable()\n' + f' ^^^^^^^^^^^^^^^^^^^^^^^\n' + f' File "{__file__}", line {exc.__code__.co_firstlineno + 8}, in exc\n' + f' raise ImportError(5)\n' + f' ^^^^^^^^^^^^^^^^^^^^\n' + f'ImportError: 5\n') + + report = self.get_report(exc) + self.assertEqual(report, expected) def test_exception_group_nested(self): def exc(): @@ -1427,29 +1394,44 @@ def exc(): except: raise EG("top", [VE(5)]) - expected = [ - [' | raise EG("eg", [VE(1), exc, VE(4)])', - ' | ExceptionGroup: eg', - ], - ['-+---------------- context.1 ----------------'], - [' | ValueError: 1'], - ['+---------------- context.2 ----------------'], - [' | ExceptionGroup: nested', - ], - ['-+---------------- context.2.1 ----------------'], - [' | TypeError: 2'], - ['+---------------- context.2.2 ----------------'], - [' | TypeError: 3'], - ['+---------------- context.3 ----------------'], - [' | ValueError: 4'], - [ context_message ], - [' | raise EG("top", [VE(5)])', - ' | ExceptionGroup: top' - ], - ['-+---------------- 1 ----------------'], - [' | ValueError: 5'] - ] - self.check_exception_group(exc, expected) + expected = (f' | Traceback (most recent call last):\n' + f' | File "{__file__}", line {exc.__code__.co_firstlineno + 9}, in exc\n' + f' | raise EG("eg", [VE(1), exc, VE(4)])\n' + f' | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n' + f' | ExceptionGroup: eg\n' + f' +-+---------------- context.1 ----------------\n' + f' | ValueError: 1\n' + f' +---------------- context.2 ----------------\n' + f' | Traceback (most recent call last):\n' + f' | File "{__file__}", line {exc.__code__.co_firstlineno + 6}, in exc\n' + f' | raise EG("nested", [TE(2), TE(3)])\n' + f' | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n' + f' | ExceptionGroup: nested\n' + f' +-+---------------- context.2.1 ----------------\n' + f' | TypeError: 2\n' + f' +---------------- context.2.2 ----------------\n' + f' | TypeError: 3\n' + f' +------------------------------------\n' + f' +---------------- context.3 ----------------\n' + f' | ValueError: 4\n' + f' +------------------------------------\n' + f'\n' + f'During handling of the above exception, another exception occurred:\n' + f'\n' + f' | Traceback (most recent call last):\n' + f' | File "{__file__}", line {self.callable_line}, in get_exception\n' + f' | exception_or_callable()\n' + f' | ^^^^^^^^^^^^^^^^^^^^^^^\n' + f' | File "{__file__}", line {exc.__code__.co_firstlineno + 11}, in exc\n' + f' | raise EG("top", [VE(5)])\n' + f' | ^^^^^^^^^^^^^^^^^^^^^^^^\n' + f' | ExceptionGroup: top\n' + f' +-+---------------- 1 ----------------\n' + f' | ValueError: 5\n' + f' +------------------------------------\n') + + report = self.get_report(exc) + self.assertEqual(report, expected) class PyExcReportingTests(BaseExceptionReportingTests, unittest.TestCase): From ac7f34c93c0a05757d082e1c028e89ae6eb44d52 Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Tue, 26 Oct 2021 23:52:28 +0100 Subject: [PATCH 16/27] Update Misc/NEWS.d/next/Core and Builtins/2021-09-26-18-18-50.bpo-45292.aX5HVr.rst new file Co-authored-by: Guido van Rossum --- .../Core and Builtins/2021-09-26-18-18-50.bpo-45292.aX5HVr.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-09-26-18-18-50.bpo-45292.aX5HVr.rst b/Misc/NEWS.d/next/Core and Builtins/2021-09-26-18-18-50.bpo-45292.aX5HVr.rst index a956a8671f3a09..55ca14f2259f00 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2021-09-26-18-18-50.bpo-45292.aX5HVr.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2021-09-26-18-18-50.bpo-45292.aX5HVr.rst @@ -1 +1 @@ -Implement :pep:`654` Add :class:`ExceptionGroup` and :class:`BaseExceptionGroup`. Update traceback display code. +Implement :pep:`654`. Add :class:`ExceptionGroup` and :class:`BaseExceptionGroup`. Update traceback display code. From d0d4961afde8f85cd8d640ba70a44b7b4c2f1473 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Tue, 26 Oct 2021 23:58:59 +0100 Subject: [PATCH 17/27] don't need the regex anymore --- Lib/test/test_traceback.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index da6b3821006c3c..20cb57baa33e8a 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -1024,12 +1024,8 @@ def test_exception_group_deep_recursion_traceback(self): "\nDuring handling of the above exception, " "another exception occurred:\n\n") -nested_exception_header_re = r'[+-]?\+[-{4} ]+[ \d\.?]+[ -{4}]+' - boundaries = re.compile( - '(%s|%s|%s)' % (re.escape(cause_message), - re.escape(context_message), - nested_exception_header_re)) + '(%s|%s)' % (re.escape(cause_message), re.escape(context_message))) class BaseExceptionReportingTests: From 5c1015d98d2df9a4016fee6e809529d20854e2c5 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Fri, 29 Oct 2021 17:31:18 +0100 Subject: [PATCH 18/27] remove full-path labels --- Lib/test/test_traceback.py | 54 +++++++++++++++----------------------- Lib/traceback.py | 24 +++-------------- Python/pythonrun.c | 36 +++---------------------- 3 files changed, 29 insertions(+), 85 deletions(-) diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index 20cb57baa33e8a..d4b649c07c784f 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -1301,9 +1301,9 @@ def exc(): f' | raise EG("eg1", [ValueError(1), TypeError(2)])\n' f' | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n' f' | ExceptionGroup: eg1\n' - f' +-+---------------- cause.1 ----------------\n' + f' +-+---------------- 1 ----------------\n' f' | ValueError: 1\n' - f' +---------------- cause.2 ----------------\n' + f' +---------------- 2 ----------------\n' f' | TypeError: 2\n' f' +------------------------------------\n' f'\n' @@ -1343,9 +1343,9 @@ def exc(): f' | raise EG("eg1", [ValueError(1), TypeError(2)])\n' f' | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n' f' | ExceptionGroup: eg1\n' - f' +-+---------------- context.context.1 ----------------\n' + f' +-+---------------- 1 ----------------\n' f' | ValueError: 1\n' - f' +---------------- context.context.2 ----------------\n' + f' +---------------- 2 ----------------\n' f' | TypeError: 2\n' f' +------------------------------------\n' f'\n' @@ -1356,9 +1356,9 @@ def exc(): f' | raise EG("eg2", [ValueError(3), TypeError(4)])\n' f' | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n' f' | ExceptionGroup: eg2\n' - f' +-+---------------- context.1 ----------------\n' + f' +-+---------------- 1 ----------------\n' f' | ValueError: 3\n' - f' +---------------- context.2 ----------------\n' + f' +---------------- 2 ----------------\n' f' | TypeError: 4\n' f' +------------------------------------\n' f'\n' @@ -1395,20 +1395,20 @@ def exc(): f' | raise EG("eg", [VE(1), exc, VE(4)])\n' f' | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n' f' | ExceptionGroup: eg\n' - f' +-+---------------- context.1 ----------------\n' + f' +-+---------------- 1 ----------------\n' f' | ValueError: 1\n' - f' +---------------- context.2 ----------------\n' + f' +---------------- 2 ----------------\n' f' | Traceback (most recent call last):\n' f' | File "{__file__}", line {exc.__code__.co_firstlineno + 6}, in exc\n' f' | raise EG("nested", [TE(2), TE(3)])\n' f' | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n' f' | ExceptionGroup: nested\n' - f' +-+---------------- context.2.1 ----------------\n' + f' +-+---------------- 1 ----------------\n' f' | TypeError: 2\n' - f' +---------------- context.2.2 ----------------\n' + f' +---------------- 2 ----------------\n' f' | TypeError: 3\n' f' +------------------------------------\n' - f' +---------------- context.3 ----------------\n' + f' +---------------- 3 ----------------\n' f' | ValueError: 4\n' f' +------------------------------------\n' f'\n' @@ -2169,53 +2169,41 @@ def test_exception_group_format(self): expected = [ f' | Traceback (most recent call last):', - f' | File "{__file__}", line ' - f'{lno_g+23}, in _get_exception_group', + f' | File "{__file__}", line {lno_g+23}, in _get_exception_group', f' | raise ExceptionGroup("eg2", [exc3, exc4])', f' | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^', f' | ExceptionGroup: eg2', f' +-+---------------- 1 ----------------', f' | Traceback (most recent call last):', - f' | File "{__file__}", ' - f'line {lno_g+16}, in _get_exception_group', + f' | File "{__file__}", line {lno_g+16}, in _get_exception_group', f' | raise ExceptionGroup("eg1", [exc1, exc2])', f' | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^', f' | ExceptionGroup: eg1', - f' +-+---------------- 1.1 ----------------', + f' +-+---------------- 1 ----------------', f' | Traceback (most recent call last):', - f' | File ' - f'"{__file__}", line {lno_g+9}, in ' - f'_get_exception_group', + f' | File "{__file__}", line {lno_g+9}, in _get_exception_group', f' | f()', f' | ^^^', - f' | File ' - f'"{__file__}", line {lno_f+1}, in ' - f'f', + f' | File "{__file__}", line {lno_f+1}, in f', f' | 1/0', f' | ~^~', f' | ZeroDivisionError: division by zero', - f' +---------------- 1.2 ----------------', + f' +---------------- 2 ----------------', f' | Traceback (most recent call last):', - f' | File ' - f'"{__file__}", line {lno_g+13}, in ' - f'_get_exception_group', + f' | File "{__file__}", line {lno_g+13}, in _get_exception_group', f' | g(42)', f' | ^^^^^', - f' | File ' - f'"{__file__}", line {lno_g+1}, in ' - f'g', + f' | File "{__file__}", line {lno_g+1}, in g', f' | raise ValueError(v)', f' | ^^^^^^^^^^^^^^^^^^^', f' | ValueError: 42', f' +------------------------------------', f' +---------------- 2 ----------------', f' | Traceback (most recent call last):', - f' | File "{__file__}", ' - f'line {lno_g+20}, in _get_exception_group', + f' | File "{__file__}", line {lno_g+20}, in _get_exception_group', f' | g(24)', f' | ^^^^^', - f' | File "{__file__}", ' - f'line {lno_g+1}, in g', + f' | File "{__file__}", line {lno_g+1}, in g', f' | raise ValueError(v)', f' | ^^^^^^^^^^^^^^^^^^^', f' | ValueError: 24', diff --git a/Lib/traceback.py b/Lib/traceback.py index cfe5ac590904b4..993f5b305823ea 100644 --- a/Lib/traceback.py +++ b/Lib/traceback.py @@ -606,7 +606,6 @@ class _ExceptionPrintContext: def __init__(self): self.seen = set() self.exception_group_depth = 0 - self.parent_label = None self.need_close = False def indent(self): @@ -861,35 +860,26 @@ def format(self, *, chain=True, _ctx=None): output = [] exc = self - label = _ctx.parent_label while exc: if chain: if exc.__cause__ is not None: chained_msg = _cause_message chained_exc = exc.__cause__ - chain_tag = 'cause' elif (exc.__context__ is not None and not exc.__suppress_context__): chained_msg = _context_message chained_exc = exc.__context__ - chain_tag = 'context' else: chained_msg = None chained_exc = None - chain_tag = None - output.append((chained_msg, exc, label)) + output.append((chained_msg, exc)) exc = chained_exc - if chain_tag is not None: - if label is None: - label = chain_tag - else: - label = f'{label}.{chain_tag}' else: - output.append((None, exc, label)) + output.append((None, exc)) exc = None - for msg, exc, parent_label in reversed(output): + for msg, exc in reversed(output): if msg is not None: yield from _ctx.emit(msg) if exc.exceptions is None: @@ -911,20 +901,14 @@ def format(self, *, chain=True, _ctx=None): if last_exc: # The closing frame may be added by a recursive call _ctx.need_close = True - if parent_label is not None: - label = f'{parent_label}.{i + 1}' - else: - label = f'{i + 1}' yield (_ctx.get_indent() + ('+-' if i==0 else ' ') + - f'+---------------- {label} ----------------\n') + f'+---------------- {i + 1} ----------------\n') _ctx.exception_group_depth += 1 - _ctx.parent_label = label try: yield from exc.exceptions[i].format(chain=chain, _ctx=_ctx) except RecursionError: pass - _ctx.parent_label = parent_label if last_exc and _ctx.need_close: yield (_ctx.get_indent() + "+------------------------------------\n") diff --git a/Python/pythonrun.c b/Python/pythonrun.c index 7cb0281d154ae4..69dd72612465fa 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -892,7 +892,6 @@ struct exception_print_context PyObject *file; PyObject *seen; // Prevent cycles in recursion int exception_group_depth; // nesting level of current exception group - PyObject *parent_label; // Unicode label of containing exception group int need_close; // Need a closing bottom frame }; @@ -1083,16 +1082,7 @@ print_chained(struct exception_print_context* ctx, PyObject *value, const char * message, const char *tag) { PyObject *f = ctx->file; - PyObject *parent_label = ctx->parent_label; - PyObject *label = NULL; int need_close = ctx->need_close; - if (parent_label) { - label = PyUnicode_FromFormat("%U.%s", parent_label, tag); - } - else { - label = PyUnicode_FromString(tag); - } - ctx->parent_label = label; int err = Py_EnterRecursiveCall(" in print_chained"); if (!err) { @@ -1111,9 +1101,7 @@ print_chained(struct exception_print_context* ctx, PyObject *value, } ctx->need_close = need_close; - ctx->parent_label = parent_label; - Py_XDECREF(label); return err; } @@ -1185,7 +1173,6 @@ print_exception_recursive(struct exception_print_context* ctx, PyObject *value) Py_ssize_t num_excs = PyTuple_GET_SIZE(excs); assert(num_excs > 0); - PyObject *parent_label = ctx->parent_label; PyObject *f = ctx->file; ctx->need_close = 0; @@ -1195,20 +1182,10 @@ print_exception_recursive(struct exception_print_context* ctx, PyObject *value) // The closing frame may be added in a recursive call ctx->need_close = 1; } - PyObject *label; - if (parent_label) { - label = PyUnicode_FromFormat("%U.%d", - parent_label, i + 1); - } - else { - label = PyUnicode_FromFormat("%d", i + 1); - } - PyObject *line = NULL; - if (label) { - line = PyUnicode_FromFormat( - "%s+---------------- %U ----------------\n", - (i == 0) ? "+-" : " ", label); - } + PyObject *line = PyUnicode_FromFormat( + "%s+---------------- %zd ----------------\n", + (i == 0) ? "+-" : " ", i + 1); + if (line) { err |= _Py_WriteIndent(EXC_INDENT(ctx), f); PyErr_Clear(); @@ -1221,8 +1198,6 @@ print_exception_recursive(struct exception_print_context* ctx, PyObject *value) } ctx->exception_group_depth += 1; - ctx->parent_label = label; /* transfer ref ownership */ - label = NULL; PyObject *exc = PyTuple_GET_ITEM(excs, i); if (!Py_EnterRecursiveCall(" in print_exception_recursive")) { @@ -1233,8 +1208,6 @@ print_exception_recursive(struct exception_print_context* ctx, PyObject *value) err = -1; PyErr_Clear(); } - Py_XDECREF(ctx->parent_label); - ctx->parent_label = parent_label; if (last_exc && ctx->need_close) { err |= _Py_WriteIndent(EXC_INDENT(ctx), f); @@ -1270,7 +1243,6 @@ _PyErr_Display(PyObject *file, PyObject *exception, PyObject *value, PyObject *t struct exception_print_context ctx; ctx.file = file; ctx.exception_group_depth = 0; - ctx.parent_label = 0; /* We choose to ignore seen being possibly NULL, and report at least the main exception (it could be a MemoryError). From 83abebdaeb2d12b1a8e18338fdb7137cc5f7d958 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Fri, 29 Oct 2021 17:43:54 +0100 Subject: [PATCH 19/27] int --> bool --- Python/pythonrun.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/Python/pythonrun.c b/Python/pythonrun.c index 69dd72612465fa..aac6233bce4788 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -8,6 +8,8 @@ /* TODO: Cull includes following phase split */ +#include + #include "Python.h" #include "pycore_ast.h" // PyAST_mod2obj @@ -892,7 +894,7 @@ struct exception_print_context PyObject *file; PyObject *seen; // Prevent cycles in recursion int exception_group_depth; // nesting level of current exception group - int need_close; // Need a closing bottom frame + bool need_close; // Need a closing bottom frame }; #define EXC_MARGIN(ctx) ((ctx)->exception_group_depth ? "| " : "") @@ -1082,7 +1084,7 @@ print_chained(struct exception_print_context* ctx, PyObject *value, const char * message, const char *tag) { PyObject *f = ctx->file; - int need_close = ctx->need_close; + bool need_close = ctx->need_close; int err = Py_EnterRecursiveCall(" in print_chained"); if (!err) { @@ -1175,12 +1177,12 @@ print_exception_recursive(struct exception_print_context* ctx, PyObject *value) PyObject *f = ctx->file; - ctx->need_close = 0; + ctx->need_close = false; for (Py_ssize_t i = 0; i < num_excs; i++) { int last_exc = (i == num_excs - 1); if (last_exc) { // The closing frame may be added in a recursive call - ctx->need_close = 1; + ctx->need_close = true; } PyObject *line = PyUnicode_FromFormat( "%s+---------------- %zd ----------------\n", @@ -1213,7 +1215,7 @@ print_exception_recursive(struct exception_print_context* ctx, PyObject *value) err |= _Py_WriteIndent(EXC_INDENT(ctx), f); err |= PyFile_WriteString( "+------------------------------------\n", f); - ctx->need_close = 0; + ctx->need_close = false; } ctx->exception_group_depth -= 1; } From 16d077d458099e7d3d2b5455747f34e879488c61 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Fri, 29 Oct 2021 18:35:21 +0100 Subject: [PATCH 20/27] move code around --- Lib/traceback.py | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/Lib/traceback.py b/Lib/traceback.py index 993f5b305823ea..5458f26939b648 100644 --- a/Lib/traceback.py +++ b/Lib/traceback.py @@ -609,19 +609,13 @@ def __init__(self): self.need_close = False def indent(self): - return 2 * self.exception_group_depth - - def margin(self): - return '| ' if self.exception_group_depth else '' - - def get_indent(self): - return ' ' * self.indent() - - def get_indented_margin(self): - return self.get_indent() + self.margin() + return ' ' * (2 * self.exception_group_depth) def emit(self, text_gen): - indent_str = self.get_indented_margin() + margin_char = '|' + margin = f'{margin_char} ' if self.exception_group_depth else '' + indent_str = self.indent() + margin + if isinstance(text_gen, str): yield textwrap.indent(text_gen, indent_str, lambda line: True) else: @@ -888,11 +882,14 @@ def format(self, *, chain=True, _ctx=None): yield from _ctx.emit(exc.stack.format()) yield from _ctx.emit(exc.format_exception_only()) else: - if _ctx.exception_group_depth == 0: + is_toplevel = (_ctx.exception_group_depth == 0) + if is_toplevel: _ctx.exception_group_depth += 1 + if exc.stack: yield from _ctx.emit('Traceback (most recent call last):\n') yield from _ctx.emit(exc.stack.format()) + yield from _ctx.emit(exc.format_exception_only()) n = len(exc.exceptions) _ctx.need_close = False @@ -901,7 +898,7 @@ def format(self, *, chain=True, _ctx=None): if last_exc: # The closing frame may be added by a recursive call _ctx.need_close = True - yield (_ctx.get_indent() + + yield (_ctx.indent() + ('+-' if i==0 else ' ') + f'+---------------- {i + 1} ----------------\n') _ctx.exception_group_depth += 1 @@ -910,12 +907,14 @@ def format(self, *, chain=True, _ctx=None): except RecursionError: pass if last_exc and _ctx.need_close: - yield (_ctx.get_indent() + + yield (_ctx.indent() + "+------------------------------------\n") _ctx.need_close = False _ctx.exception_group_depth -= 1; - if _ctx.exception_group_depth == 1: - _ctx.exception_group_depth -= 1 + + if is_toplevel: + assert _ctx.exception_group_depth == 1 + _ctx.exception_group_depth = 0 def print(self, *, file=None, chain=True): From 64fb164d8a4f01c88c27d32900bc1b88504ecfe2 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Sun, 31 Oct 2021 20:31:25 +0000 Subject: [PATCH 21/27] Tweak the top-level of traceback box as suggested by Yury --- Include/internal/pycore_traceback.h | 7 ++++++- Lib/test/test_traceback.py | 20 ++++++++++---------- Lib/traceback.py | 14 +++++++++----- Python/pythonrun.c | 13 ++++++++++++- Python/traceback.c | 15 +++++++++++---- 5 files changed, 48 insertions(+), 21 deletions(-) diff --git a/Include/internal/pycore_traceback.h b/Include/internal/pycore_traceback.h index 38890cd8852130..84dbe27044fd31 100644 --- a/Include/internal/pycore_traceback.h +++ b/Include/internal/pycore_traceback.h @@ -87,9 +87,14 @@ PyAPI_FUNC(PyObject*) _PyTraceBack_FromFrame( PyObject *tb_next, PyFrameObject *frame); +#define EXCEPTION_TB_HEADER "Traceback (most recent call last):\n" +#define EXCEPTION_GROUP_TB_HEADER "Exception Group Traceback (most recent call last):\n" + /* Write the traceback tb to file f. Prefix each line with indent spaces followed by the margin (if it is not NULL). */ -PyAPI_FUNC(int) _PyTraceBack_Print_Indented(PyObject *tb, int indent, const char* margin, PyObject *f); +PyAPI_FUNC(int) _PyTraceBack_Print_Indented( + PyObject *tb, int indent, const char* margin, + const char *header_margin, const char *header, PyObject *f); PyAPI_FUNC(int) _Py_WriteIndentedMargin(int, const char*, PyObject *); PyAPI_FUNC(int) _Py_WriteIndent(int, PyObject *); diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index d4b649c07c784f..9e836b31f24f2e 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -1271,7 +1271,7 @@ def exc(): raise ExceptionGroup("eg", [ValueError(1), TypeError(2)]) expected = ( - f' | Traceback (most recent call last):\n' + f' + Exception Group Traceback (most recent call last):\n' f' | File "{__file__}", line {self.callable_line}, in get_exception\n' f' | exception_or_callable()\n' f' | ^^^^^^^^^^^^^^^^^^^^^^^\n' @@ -1296,7 +1296,7 @@ def exc(): except Exception as e: raise EG("eg2", [ValueError(3), TypeError(4)]) from e - expected = (f' | Traceback (most recent call last):\n' + expected = (f' + Exception Group Traceback (most recent call last):\n' f' | File "{__file__}", line {exc.__code__.co_firstlineno + 3}, in exc\n' f' | raise EG("eg1", [ValueError(1), TypeError(2)])\n' f' | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n' @@ -1309,7 +1309,7 @@ def exc(): f'\n' f'The above exception was the direct cause of the following exception:\n' f'\n' - f' | Traceback (most recent call last):\n' + f' + Exception Group Traceback (most recent call last):\n' f' | File "{__file__}", line {self.callable_line}, in get_exception\n' f' | exception_or_callable()\n' f' | ^^^^^^^^^^^^^^^^^^^^^^^\n' @@ -1338,7 +1338,7 @@ def exc(): raise ImportError(5) expected = ( - f' | Traceback (most recent call last):\n' + f' + Exception Group Traceback (most recent call last):\n' f' | File "{__file__}", line {exc.__code__.co_firstlineno + 4}, in exc\n' f' | raise EG("eg1", [ValueError(1), TypeError(2)])\n' f' | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n' @@ -1351,7 +1351,7 @@ def exc(): f'\n' f'During handling of the above exception, another exception occurred:\n' f'\n' - f' | Traceback (most recent call last):\n' + f' + Exception Group Traceback (most recent call last):\n' f' | File "{__file__}", line {exc.__code__.co_firstlineno + 6}, in exc\n' f' | raise EG("eg2", [ValueError(3), TypeError(4)])\n' f' | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n' @@ -1390,7 +1390,7 @@ def exc(): except: raise EG("top", [VE(5)]) - expected = (f' | Traceback (most recent call last):\n' + expected = (f' + Exception Group Traceback (most recent call last):\n' f' | File "{__file__}", line {exc.__code__.co_firstlineno + 9}, in exc\n' f' | raise EG("eg", [VE(1), exc, VE(4)])\n' f' | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n' @@ -1398,7 +1398,7 @@ def exc(): f' +-+---------------- 1 ----------------\n' f' | ValueError: 1\n' f' +---------------- 2 ----------------\n' - f' | Traceback (most recent call last):\n' + f' | Exception Group Traceback (most recent call last):\n' f' | File "{__file__}", line {exc.__code__.co_firstlineno + 6}, in exc\n' f' | raise EG("nested", [TE(2), TE(3)])\n' f' | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n' @@ -1414,7 +1414,7 @@ def exc(): f'\n' f'During handling of the above exception, another exception occurred:\n' f'\n' - f' | Traceback (most recent call last):\n' + f' + Exception Group Traceback (most recent call last):\n' f' | File "{__file__}", line {self.callable_line}, in get_exception\n' f' | exception_or_callable()\n' f' | ^^^^^^^^^^^^^^^^^^^^^^^\n' @@ -2168,13 +2168,13 @@ def test_exception_group_format(self): lno_g = self.lno_g expected = [ - f' | Traceback (most recent call last):', + f' + Exception Group Traceback (most recent call last):', f' | File "{__file__}", line {lno_g+23}, in _get_exception_group', f' | raise ExceptionGroup("eg2", [exc3, exc4])', f' | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^', f' | ExceptionGroup: eg2', f' +-+---------------- 1 ----------------', - f' | Traceback (most recent call last):', + f' | Exception Group Traceback (most recent call last):', f' | File "{__file__}", line {lno_g+16}, in _get_exception_group', f' | raise ExceptionGroup("eg1", [exc1, exc2])', f' | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^', diff --git a/Lib/traceback.py b/Lib/traceback.py index 5458f26939b648..e37cdb6025f962 100644 --- a/Lib/traceback.py +++ b/Lib/traceback.py @@ -611,10 +611,12 @@ def __init__(self): def indent(self): return ' ' * (2 * self.exception_group_depth) - def emit(self, text_gen): - margin_char = '|' - margin = f'{margin_char} ' if self.exception_group_depth else '' - indent_str = self.indent() + margin + def emit(self, text_gen, margin_char=None): + if margin_char is None: + margin_char = '|' + indent_str = self.indent() + if self.exception_group_depth: + indent_str += margin_char + ' ' if isinstance(text_gen, str): yield textwrap.indent(text_gen, indent_str, lambda line: True) @@ -887,7 +889,9 @@ def format(self, *, chain=True, _ctx=None): _ctx.exception_group_depth += 1 if exc.stack: - yield from _ctx.emit('Traceback (most recent call last):\n') + yield from _ctx.emit( + 'Exception Group Traceback (most recent call last):\n', + margin_char = '+' if is_toplevel else None) yield from _ctx.emit(exc.stack.format()) yield from _ctx.emit(exc.format_exception_only()) diff --git a/Python/pythonrun.c b/Python/pythonrun.c index aac6233bce4788..26896c5640c6ea 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -928,9 +928,20 @@ print_exception(struct exception_print_context *ctx, PyObject *value) Py_INCREF(value); fflush(stdout); type = (PyObject *) Py_TYPE(value); + bool is_exception_group = PyObject_TypeCheck( + value, (PyTypeObject *)PyExc_BaseExceptionGroup); tb = PyException_GetTraceback(value); if (tb && tb != Py_None) { - err = _PyTraceBack_Print_Indented(tb, EXC_INDENT(ctx), EXC_MARGIN(ctx), f); + const char *header = EXCEPTION_TB_HEADER; + const char *header_margin = EXC_MARGIN(ctx); + if (is_exception_group) { + header = EXCEPTION_GROUP_TB_HEADER; + if (ctx->exception_group_depth == 1) { + header_margin = "+ "; + } + } + err = _PyTraceBack_Print_Indented( + tb, EXC_INDENT(ctx), EXC_MARGIN(ctx), header_margin, header, f); } if (err == 0 && (err = _PyObject_LookupAttrId(value, &PyId_print_file_and_line, &tmp)) > 0) diff --git a/Python/traceback.c b/Python/traceback.c index 7d100888a9d49e..a699f8725f28e3 100644 --- a/Python/traceback.c +++ b/Python/traceback.c @@ -14,6 +14,7 @@ #include "pycore_pyarena.h" // _PyArena_Free() #include "pycore_pyerrors.h" // _PyErr_Fetch() #include "pycore_pystate.h" // _PyThreadState_GET() +#include "pycore_traceback.h" // EXCEPTION_TB_HEADER #include "../Parser/pegen.h" // _PyPegen_byte_offset_to_character_offset() #include "structmember.h" // PyMemberDef #include "osdefs.h" // SEP @@ -922,7 +923,8 @@ tb_printinternal(PyTracebackObject *tb, PyObject *f, long limit, #define PyTraceBack_LIMIT 1000 int -_PyTraceBack_Print_Indented(PyObject *v, int indent, const char *margin, PyObject *f) +_PyTraceBack_Print_Indented(PyObject *v, int indent, const char *margin, + const char *header_margin, const char *header, PyObject *f) { int err; PyObject *limitv; @@ -945,8 +947,8 @@ _PyTraceBack_Print_Indented(PyObject *v, int indent, const char *margin, PyObjec return 0; } } - err = _Py_WriteIndentedMargin(indent, margin, f); - err |= PyFile_WriteString("Traceback (most recent call last):\n", f); + err = _Py_WriteIndentedMargin(indent, header_margin, f); + err |= PyFile_WriteString(header, f); if (!err) { err = tb_printinternal((PyTracebackObject *)v, f, limit, indent, margin); } @@ -956,7 +958,12 @@ _PyTraceBack_Print_Indented(PyObject *v, int indent, const char *margin, PyObjec int PyTraceBack_Print(PyObject *v, PyObject *f) { - return _PyTraceBack_Print_Indented(v, 0, NULL, f); + int indent = 0; + const char *margin = NULL; + const char *header_margin = NULL; + const char *header = EXCEPTION_TB_HEADER; + + return _PyTraceBack_Print_Indented(v, indent, margin, header_margin, header, f); } /* Format an integer in range [0; 0xffffffff] to decimal and write it From 88019f57c407923693670bdfae50ee821140e575 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Mon, 1 Nov 2021 16:13:29 +0000 Subject: [PATCH 22/27] tidy up error handling --- Python/pythonrun.c | 139 +++++++++++++++++++++++++++------------------ Python/traceback.c | 36 ++++++++---- 2 files changed, 109 insertions(+), 66 deletions(-) diff --git a/Python/pythonrun.c b/Python/pythonrun.c index 26896c5640c6ea..f2d14a63360543 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -916,12 +916,21 @@ print_exception(struct exception_print_context *ctx, PyObject *value) _Py_IDENTIFIER(print_file_and_line); if (!PyExceptionInstance_Check(value)) { - err += _Py_WriteIndent(EXC_INDENT(ctx), f); - err += PyFile_WriteString("TypeError: print_exception(): Exception expected for value, ", f); - err += PyFile_WriteString(Py_TYPE(value)->tp_name, f); - err += PyFile_WriteString(" found\n", f); - if (err) + if (err == 0) { + err = _Py_WriteIndent(EXC_INDENT(ctx), f); + } + if (err == 0) { + err = PyFile_WriteString("TypeError: print_exception(): Exception expected for value, ", f); + } + if (err == 0) { + err = PyFile_WriteString(Py_TYPE(value)->tp_name, f); + } + if (err == 0) { + err = PyFile_WriteString(" found\n", f); + } + if (err != 0) { PyErr_Clear(); + } return; } @@ -965,9 +974,10 @@ print_exception(struct exception_print_context *ctx, PyObject *value) filename, lineno); Py_DECREF(filename); if (line != NULL) { - err += write_indented_margin(ctx, f); - PyErr_Clear(); - PyFile_WriteObject(line, f, Py_PRINT_RAW); + err = write_indented_margin(ctx, f); + if (err == 0) { + err = PyFile_WriteObject(line, f, Py_PRINT_RAW); + } Py_DECREF(line); } @@ -996,7 +1006,7 @@ print_exception(struct exception_print_context *ctx, PyObject *value) err = -1; } } - if (err) { + if (err != 0) { /* Don't do anything else */ } else { @@ -1005,23 +1015,26 @@ print_exception(struct exception_print_context *ctx, PyObject *value) _Py_IDENTIFIER(__module__); assert(PyExceptionClass_Check(type)); - err += write_indented_margin(ctx, f); - modulename = _PyObject_GetAttrId(type, &PyId___module__); - if (modulename == NULL || !PyUnicode_Check(modulename)) - { - Py_XDECREF(modulename); - PyErr_Clear(); - err += PyFile_WriteString("", f); - } - else { - if (!_PyUnicode_EqualToASCIIId(modulename, &PyId_builtins) && - !_PyUnicode_EqualToASCIIId(modulename, &PyId___main__)) + err = write_indented_margin(ctx, f); + if (err == 0) { + modulename = _PyObject_GetAttrId(type, &PyId___module__); + if (modulename == NULL || !PyUnicode_Check(modulename)) { + Py_XDECREF(modulename); PyErr_Clear(); - err += PyFile_WriteObject(modulename, f, Py_PRINT_RAW); - err += PyFile_WriteString(".", f); + err = PyFile_WriteString("", f); + } + else { + if (!_PyUnicode_EqualToASCIIId(modulename, &PyId_builtins) && + !_PyUnicode_EqualToASCIIId(modulename, &PyId___main__)) + { + err = PyFile_WriteObject(modulename, f, Py_PRINT_RAW); + if (err == 0) { + err = PyFile_WriteString(".", f); + } + } + Py_DECREF(modulename); } - Py_DECREF(modulename); } if (err == 0) { PyObject* qualname = PyType_GetQualName((PyTypeObject *)type); @@ -1031,7 +1044,6 @@ print_exception(struct exception_print_context *ctx, PyObject *value) err = PyFile_WriteString("", f); } else { - PyErr_Clear(); err = PyFile_WriteObject(qualname, f, Py_PRINT_RAW); Py_DECREF(qualname); } @@ -1098,16 +1110,28 @@ print_chained(struct exception_print_context* ctx, PyObject *value, bool need_close = ctx->need_close; int err = Py_EnterRecursiveCall(" in print_chained"); - if (!err) { + if (err == 0) { print_exception_recursive(ctx, value); Py_LeaveRecursiveCall(); - err |= write_indented_margin(ctx, f); - err |= PyFile_WriteString("\n", f); - err |= write_indented_margin(ctx, f); - err |= PyFile_WriteString(message, f); - err |= write_indented_margin(ctx, f); - err |= PyFile_WriteString("\n", f); + if (err == 0) { + err = write_indented_margin(ctx, f); + } + if (err == 0) { + err = PyFile_WriteString("\n", f); + } + if (err == 0) { + err = write_indented_margin(ctx, f); + } + if (err == 0) { + err = PyFile_WriteString(message, f); + } + if (err == 0) { + err = write_indented_margin(ctx, f); + } + if (err == 0) { + err = PyFile_WriteString("\n", f); + } } else { PyErr_Clear(); @@ -1145,7 +1169,7 @@ print_exception_recursive(struct exception_print_context* ctx, PyObject *value) if (res == -1) PyErr_Clear(); if (res == 0) { - err |= print_chained(ctx, cause, cause_message, "cause"); + err = print_chained(ctx, cause, cause_message, "cause"); } } else if (context && @@ -1160,7 +1184,7 @@ print_exception_recursive(struct exception_print_context* ctx, PyObject *value) if (res == -1) PyErr_Clear(); if (res == 0) { - err |= print_chained(ctx, context, context_message, "context"); + err = print_chained(ctx, context, context_message, "context"); } } Py_XDECREF(context); @@ -1168,10 +1192,13 @@ print_exception_recursive(struct exception_print_context* ctx, PyObject *value) } Py_XDECREF(value_id); } - if (!PyObject_TypeCheck(value, (PyTypeObject *)PyExc_BaseExceptionGroup)) { + if (err) { + /* don't do anything else */ + } + else if (!PyObject_TypeCheck(value, (PyTypeObject *)PyExc_BaseExceptionGroup)) { print_exception(ctx, value); } - else { + else { /* ExceptionGroup */ /* TODO: add arg to limit number of exceptions printed? */ @@ -1200,9 +1227,10 @@ print_exception_recursive(struct exception_print_context* ctx, PyObject *value) (i == 0) ? "+-" : " ", i + 1); if (line) { - err |= _Py_WriteIndent(EXC_INDENT(ctx), f); - PyErr_Clear(); - err |= PyFile_WriteObject(line, f, Py_PRINT_RAW); + err = _Py_WriteIndent(EXC_INDENT(ctx), f); + if (err == 0) { + err = PyFile_WriteObject(line, f, Py_PRINT_RAW); + } Py_DECREF(line); } else { @@ -1210,25 +1238,28 @@ print_exception_recursive(struct exception_print_context* ctx, PyObject *value) PyErr_Clear(); } - ctx->exception_group_depth += 1; - PyObject *exc = PyTuple_GET_ITEM(excs, i); + if (err == 0) { + ctx->exception_group_depth += 1; + PyObject *exc = PyTuple_GET_ITEM(excs, i); - if (!Py_EnterRecursiveCall(" in print_exception_recursive")) { - print_exception_recursive(ctx, exc); - Py_LeaveRecursiveCall(); - } - else { - err = -1; - PyErr_Clear(); - } + if (!Py_EnterRecursiveCall(" in print_exception_recursive")) { + print_exception_recursive(ctx, exc); + Py_LeaveRecursiveCall(); + } + else { + err = -1; + } - if (last_exc && ctx->need_close) { - err |= _Py_WriteIndent(EXC_INDENT(ctx), f); - err |= PyFile_WriteString( - "+------------------------------------\n", f); - ctx->need_close = false; + if (err == 0 && last_exc && ctx->need_close) { + err = _Py_WriteIndent(EXC_INDENT(ctx), f); + if (err == 0) { + err = PyFile_WriteString( + "+------------------------------------\n", f); + } + ctx->need_close = false; + } + ctx->exception_group_depth -= 1; } - ctx->exception_group_depth -= 1; } if (ctx->exception_group_depth == 1) { ctx->exception_group_depth -= 1; diff --git a/Python/traceback.c b/Python/traceback.c index a699f8725f28e3..2201ab88c8a6f4 100644 --- a/Python/traceback.c +++ b/Python/traceback.c @@ -408,8 +408,8 @@ int _Py_WriteIndentedMargin(int indent, const char *margin, PyObject *f) { int err = _Py_WriteIndent(indent, f); - if (margin) { - err |= PyFile_WriteString(margin, f); + if (err == 0 && margin) { + err = PyFile_WriteString(margin, f); } return err; } @@ -545,16 +545,22 @@ display_source_line_with_margin(PyObject *f, PyObject *filename, int lineno, int *truncation = i - indent; } - err |= _Py_WriteIndentedMargin(margin_indent, margin, f); + if (err == 0) { + err = _Py_WriteIndentedMargin(margin_indent, margin, f); + } /* Write some spaces before the line */ - err |= _Py_WriteIndent(indent, f); + if (err == 0) { + err = _Py_WriteIndent(indent, f); + } /* finally display the line */ - if (err == 0) + if (err == 0) { err = PyFile_WriteObject(lineobj, f, Py_PRINT_RAW); + } Py_DECREF(lineobj); - if (err == 0) + if (err == 0) { err = PyFile_WriteString("\n", f); + } return err; } @@ -746,7 +752,9 @@ tb_displayline(PyTracebackObject* tb, PyObject *f, PyObject *filename, int linen if (line == NULL) return -1; err = _Py_WriteIndentedMargin(margin_indent, margin, f); - err |= PyFile_WriteObject(line, f, Py_PRINT_RAW); + if (err == 0) { + err = PyFile_WriteObject(line, f, Py_PRINT_RAW); + } Py_DECREF(line); if (err != 0) return err; @@ -842,9 +850,11 @@ tb_displayline(PyTracebackObject* tb, PyObject *f, PyObject *filename, int linen } err = _Py_WriteIndentedMargin(margin_indent, margin, f); - err |= print_error_location_carets(f, truncation, start_offset, end_offset, - right_start_offset, left_end_offset, - primary_error_char, secondary_error_char); + if (err == 0) { + err = print_error_location_carets(f, truncation, start_offset, end_offset, + right_start_offset, left_end_offset, + primary_error_char, secondary_error_char); + } done: Py_XDECREF(source_line); @@ -948,8 +958,10 @@ _PyTraceBack_Print_Indented(PyObject *v, int indent, const char *margin, } } err = _Py_WriteIndentedMargin(indent, header_margin, f); - err |= PyFile_WriteString(header, f); - if (!err) { + if (err == 0) { + err = PyFile_WriteString(header, f); + } + if (err == 0) { err = tb_printinternal((PyTracebackObject *)v, f, limit, indent, margin); } return err; From e9638355cb2c5685335823dcb90bc1e8f34df121 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Tue, 2 Nov 2021 18:12:49 +0000 Subject: [PATCH 23/27] add limits for width and depth of formatted exception groups --- Lib/test/test_traceback.py | 192 +++++++++++++++++++++++++++++++++++++ Lib/traceback.py | 48 ++++++++-- Python/pythonrun.c | 91 +++++++++++++++--- 3 files changed, 309 insertions(+), 22 deletions(-) diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index 9e836b31f24f2e..e395925d6f8ba4 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -1429,6 +1429,132 @@ def exc(): report = self.get_report(exc) self.assertEqual(report, expected) + def test_exception_group_width_limit(self): + excs = [] + for i in range(1000): + excs.append(ValueError(i)) + eg = ExceptionGroup('eg', excs) + + expected = (' | ExceptionGroup: eg\n' + ' +-+---------------- 1 ----------------\n' + ' | ValueError: 0\n' + ' +---------------- 2 ----------------\n' + ' | ValueError: 1\n' + ' +---------------- 3 ----------------\n' + ' | ValueError: 2\n' + ' +---------------- 4 ----------------\n' + ' | ValueError: 3\n' + ' +---------------- 5 ----------------\n' + ' | ValueError: 4\n' + ' +---------------- 6 ----------------\n' + ' | ValueError: 5\n' + ' +---------------- 7 ----------------\n' + ' | ValueError: 6\n' + ' +---------------- 8 ----------------\n' + ' | ValueError: 7\n' + ' +---------------- 9 ----------------\n' + ' | ValueError: 8\n' + ' +---------------- 10 ----------------\n' + ' | ValueError: 9\n' + ' +---------------- 11 ----------------\n' + ' | ValueError: 10\n' + ' +---------------- 12 ----------------\n' + ' | ValueError: 11\n' + ' +---------------- 13 ----------------\n' + ' | ValueError: 12\n' + ' +---------------- 14 ----------------\n' + ' | ValueError: 13\n' + ' +---------------- 15 ----------------\n' + ' | ValueError: 14\n' + ' +---------------- ... ----------------\n' + ' | and 985 more exceptions\n' + ' +------------------------------------\n') + + report = self.get_report(eg) + self.assertEqual(report, expected) + + def test_exception_group_depth_limit(self): + exc = TypeError('bad type') + for i in range(1000): + exc = ExceptionGroup( + f'eg{i}', + [ValueError(i), exc, ValueError(-i)]) + + expected = (' | ExceptionGroup: eg999\n' + ' +-+---------------- 1 ----------------\n' + ' | ValueError: 999\n' + ' +---------------- 2 ----------------\n' + ' | ExceptionGroup: eg998\n' + ' +-+---------------- 1 ----------------\n' + ' | ValueError: 998\n' + ' +---------------- 2 ----------------\n' + ' | ExceptionGroup: eg997\n' + ' +-+---------------- 1 ----------------\n' + ' | ValueError: 997\n' + ' +---------------- 2 ----------------\n' + ' | ExceptionGroup: eg996\n' + ' +-+---------------- 1 ----------------\n' + ' | ValueError: 996\n' + ' +---------------- 2 ----------------\n' + ' | ExceptionGroup: eg995\n' + ' +-+---------------- 1 ----------------\n' + ' | ValueError: 995\n' + ' +---------------- 2 ----------------\n' + ' | ExceptionGroup: eg994\n' + ' +-+---------------- 1 ----------------\n' + ' | ValueError: 994\n' + ' +---------------- 2 ----------------\n' + ' | ExceptionGroup: eg993\n' + ' +-+---------------- 1 ----------------\n' + ' | ValueError: 993\n' + ' +---------------- 2 ----------------\n' + ' | ExceptionGroup: eg992\n' + ' +-+---------------- 1 ----------------\n' + ' | ValueError: 992\n' + ' +---------------- 2 ----------------\n' + ' | ExceptionGroup: eg991\n' + ' +-+---------------- 1 ----------------\n' + ' | ValueError: 991\n' + ' +---------------- 2 ----------------\n' + ' | ExceptionGroup: eg990\n' + ' +-+---------------- 1 ----------------\n' + ' | ValueError: 990\n' + ' +---------------- 2 ----------------\n' + ' | ... (max_group_depth is 10)\n' + ' +---------------- 3 ----------------\n' + ' | ValueError: -990\n' + ' +------------------------------------\n' + ' +---------------- 3 ----------------\n' + ' | ValueError: -991\n' + ' +------------------------------------\n' + ' +---------------- 3 ----------------\n' + ' | ValueError: -992\n' + ' +------------------------------------\n' + ' +---------------- 3 ----------------\n' + ' | ValueError: -993\n' + ' +------------------------------------\n' + ' +---------------- 3 ----------------\n' + ' | ValueError: -994\n' + ' +------------------------------------\n' + ' +---------------- 3 ----------------\n' + ' | ValueError: -995\n' + ' +------------------------------------\n' + ' +---------------- 3 ----------------\n' + ' | ValueError: -996\n' + ' +------------------------------------\n' + ' +---------------- 3 ----------------\n' + ' | ValueError: -997\n' + ' +------------------------------------\n' + ' +---------------- 3 ----------------\n' + ' | ValueError: -998\n' + ' +------------------------------------\n' + ' +---------------- 3 ----------------\n' + ' | ValueError: -999\n' + ' +------------------------------------\n') + + report = self.get_report(exc) + self.assertEqual(report, expected) + class PyExcReportingTests(BaseExceptionReportingTests, unittest.TestCase): # @@ -2212,6 +2338,72 @@ def test_exception_group_format(self): self.assertEqual(formatted, expected) + def test_max_group_width(self): + excs1 = [] + excs2 = [] + for i in range(3): + excs1.append(ValueError(i)) + for i in range(10): + excs2.append(TypeError(i)) + + EG = ExceptionGroup + eg = EG('eg', [EG('eg1', excs1), EG('eg2', excs2)]) + + teg = traceback.TracebackException.from_exception(eg, max_group_width=2) + formatted = ''.join(teg.format()).split('\n') + + expected = [ + f' | ExceptionGroup: eg', + f' +-+---------------- 1 ----------------', + f' | ExceptionGroup: eg1', + f' +-+---------------- 1 ----------------', + f' | ValueError: 0', + f' +---------------- 2 ----------------', + f' | ValueError: 1', + f' +---------------- ... ----------------', + f' | and 1 more exception', + f' +------------------------------------', + f' +---------------- 2 ----------------', + f' | ExceptionGroup: eg2', + f' +-+---------------- 1 ----------------', + f' | TypeError: 0', + f' +---------------- 2 ----------------', + f' | TypeError: 1', + f' +---------------- ... ----------------', + f' | and 8 more exceptions', + f' +------------------------------------', + f''] + + self.assertEqual(formatted, expected) + + def test_max_group_depth(self): + exc = TypeError('bad type') + for i in range(3): + exc = ExceptionGroup('exc', [ValueError(-i), exc, ValueError(i)]) + + teg = traceback.TracebackException.from_exception(exc, max_group_depth=2) + formatted = ''.join(teg.format()).split('\n') + + expected = [ + f' | ExceptionGroup: exc', + f' +-+---------------- 1 ----------------', + f' | ValueError: -2', + f' +---------------- 2 ----------------', + f' | ExceptionGroup: exc', + f' +-+---------------- 1 ----------------', + f' | ValueError: -1', + f' +---------------- 2 ----------------', + f' | ... (max_group_depth is 2)', + f' +---------------- 3 ----------------', + f' | ValueError: 1', + f' +------------------------------------', + f' +---------------- 3 ----------------', + f' | ValueError: 2', + f' +------------------------------------', + f''] + + self.assertEqual(formatted, expected) + def test_comparison(self): try: raise self.eg_info[1] diff --git a/Lib/traceback.py b/Lib/traceback.py index e37cdb6025f962..7d7e858a5bf957 100644 --- a/Lib/traceback.py +++ b/Lib/traceback.py @@ -632,6 +632,11 @@ class TracebackException: to this intermediary form to ensure that no references are held, while still being able to fully print or format it. + max_group_width and max_group_depth control the formatting of exception + groups. The depth refers to the nesting level of the group, and the width + refers to the size of a single exception group's exceptions array. The + formatted output is truncated when either limit is exceeded. + Use `from_exception` to create TracebackException instances from exception objects, or the constructor to create TracebackException instances from individual components. @@ -659,7 +664,7 @@ class TracebackException: def __init__(self, exc_type, exc_value, exc_traceback, *, limit=None, lookup_lines=True, capture_locals=False, compact=False, - _seen=None): + max_group_width=15, max_group_depth=10, _seen=None): # NB: we need to accept exc_traceback, exc_value, exc_traceback to # permit backwards compat with the existing API, otherwise we # need stub thunk objects just to glue it together. @@ -669,7 +674,9 @@ def __init__(self, exc_type, exc_value, exc_traceback, *, limit=None, _seen = set() _seen.add(id(exc_value)) - # TODO: locals. + self.max_group_width = max_group_width + self.max_group_depth = max_group_depth + self.stack = StackSummary._extract_from_extended_frame_gen( _walk_tb_with_full_positions(exc_traceback), limit=limit, lookup_lines=lookup_lines, @@ -709,6 +716,8 @@ def __init__(self, exc_type, exc_value, exc_traceback, *, limit=None, limit=limit, lookup_lines=lookup_lines, capture_locals=capture_locals, + max_group_width=max_group_width, + max_group_depth=max_group_depth, _seen=_seen) else: cause = None @@ -728,6 +737,8 @@ def __init__(self, exc_type, exc_value, exc_traceback, *, limit=None, limit=limit, lookup_lines=lookup_lines, capture_locals=capture_locals, + max_group_width=max_group_width, + max_group_depth=max_group_depth, _seen=_seen) else: context = None @@ -742,6 +753,8 @@ def __init__(self, exc_type, exc_value, exc_traceback, *, limit=None, limit=limit, lookup_lines=lookup_lines, capture_locals=capture_locals, + max_group_width=max_group_width, + max_group_depth=max_group_depth, _seen=_seen) exceptions.append(texc) else: @@ -883,7 +896,12 @@ def format(self, *, chain=True, _ctx=None): yield from _ctx.emit('Traceback (most recent call last):\n') yield from _ctx.emit(exc.stack.format()) yield from _ctx.emit(exc.format_exception_only()) + elif _ctx.exception_group_depth > self.max_group_depth: + # exception group, but depth exceeds limit + yield from _ctx.emit( + f" ... (max_group_depth is {self.max_group_depth})\n") else: + # format exception group is_toplevel = (_ctx.exception_group_depth == 0) if is_toplevel: _ctx.exception_group_depth += 1 @@ -895,26 +913,40 @@ def format(self, *, chain=True, _ctx=None): yield from _ctx.emit(exc.stack.format()) yield from _ctx.emit(exc.format_exception_only()) - n = len(exc.exceptions) + num_excs = len(exc.exceptions) + if num_excs <= self.max_group_width: + n = num_excs + else: + n = self.max_group_width + 1 _ctx.need_close = False for i in range(n): last_exc = (i == n-1) if last_exc: # The closing frame may be added by a recursive call _ctx.need_close = True + + if self.max_group_width is not None: + truncated = (i >= self.max_group_width) + else: + truncated = False + title = f'{i+1}' if not truncated else '...' yield (_ctx.indent() + ('+-' if i==0 else ' ') + - f'+---------------- {i + 1} ----------------\n') + f'+---------------- {title} ----------------\n') _ctx.exception_group_depth += 1 - try: + if not truncated: yield from exc.exceptions[i].format(chain=chain, _ctx=_ctx) - except RecursionError: - pass + else: + remaining = num_excs - self.max_group_width + plural = 's' if remaining > 1 else '' + yield from _ctx.emit( + f" and {remaining} more exception{plural}\n") + if last_exc and _ctx.need_close: yield (_ctx.indent() + "+------------------------------------\n") _ctx.need_close = False - _ctx.exception_group_depth -= 1; + _ctx.exception_group_depth -= 1 if is_toplevel: assert _ctx.exception_group_depth == 1 diff --git a/Python/pythonrun.c b/Python/pythonrun.c index f2d14a63360543..bae0930e19199e 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -895,6 +895,8 @@ struct exception_print_context PyObject *seen; // Prevent cycles in recursion int exception_group_depth; // nesting level of current exception group bool need_close; // Need a closing bottom frame + int max_group_width; // Maximum number of children of each EG + int max_group_depth; // Maximum nesting level of EGs }; #define EXC_MARGIN(ctx) ((ctx)->exception_group_depth ? "| " : "") @@ -1198,10 +1200,28 @@ print_exception_recursive(struct exception_print_context* ctx, PyObject *value) else if (!PyObject_TypeCheck(value, (PyTypeObject *)PyExc_BaseExceptionGroup)) { print_exception(ctx, value); } - else { - /* ExceptionGroup */ + else if (ctx->exception_group_depth > ctx->max_group_depth) { + /* exception group but depth exceeds limit */ - /* TODO: add arg to limit number of exceptions printed? */ + PyObject *line = PyUnicode_FromFormat( + " ... (max_group_depth is %d)\n", ctx->max_group_depth); + + if (line) { + PyObject *f = ctx->file; + if (err == 0) { + err = write_indented_margin(ctx, f); + } + if (err == 0) { + err = PyFile_WriteObject(line, f, Py_PRINT_RAW); + } + Py_DECREF(line); + } + else { + err = -1; + } + } + else { + /* format exception group */ if (ctx->exception_group_depth == 0) { ctx->exception_group_depth += 1; @@ -1212,22 +1232,40 @@ print_exception_recursive(struct exception_print_context* ctx, PyObject *value) assert(excs && PyTuple_Check(excs)); Py_ssize_t num_excs = PyTuple_GET_SIZE(excs); assert(num_excs > 0); + Py_ssize_t n; + if (num_excs <= ctx->max_group_width) { + n = num_excs; + } + else { + n = ctx->max_group_width + 1; + } PyObject *f = ctx->file; ctx->need_close = false; - for (Py_ssize_t i = 0; i < num_excs; i++) { - int last_exc = (i == num_excs - 1); + for (Py_ssize_t i = 0; i < n; i++) { + int last_exc = (i == n - 1); if (last_exc) { // The closing frame may be added in a recursive call ctx->need_close = true; } - PyObject *line = PyUnicode_FromFormat( - "%s+---------------- %zd ----------------\n", - (i == 0) ? "+-" : " ", i + 1); + PyObject *line; + bool truncated = (i >= ctx->max_group_width); + if (!truncated) { + line = PyUnicode_FromFormat( + "%s+---------------- %zd ----------------\n", + (i == 0) ? "+-" : " ", i + 1); + } + else { + line = PyUnicode_FromFormat( + "%s+---------------- ... ----------------\n", + (i == 0) ? "+-" : " "); + } if (line) { - err = _Py_WriteIndent(EXC_INDENT(ctx), f); + if (err == 0) { + err = _Py_WriteIndent(EXC_INDENT(ctx), f); + } if (err == 0) { err = PyFile_WriteObject(line, f, Py_PRINT_RAW); } @@ -1235,19 +1273,39 @@ print_exception_recursive(struct exception_print_context* ctx, PyObject *value) } else { err = -1; - PyErr_Clear(); } if (err == 0) { ctx->exception_group_depth += 1; PyObject *exc = PyTuple_GET_ITEM(excs, i); - if (!Py_EnterRecursiveCall(" in print_exception_recursive")) { - print_exception_recursive(ctx, exc); - Py_LeaveRecursiveCall(); + if (!truncated) { + if (!Py_EnterRecursiveCall(" in print_exception_recursive")) { + print_exception_recursive(ctx, exc); + Py_LeaveRecursiveCall(); + } + else { + err = -1; + } } else { - err = -1; + Py_ssize_t excs_remaining = num_excs - ctx->max_group_width; + PyObject *line = PyUnicode_FromFormat( + " and %zd more exception%s\n", + excs_remaining, excs_remaining > 1 ? "s" : ""); + + if (line) { + if (err == 0) { + err = write_indented_margin(ctx, f); + } + if (err == 0) { + err = PyFile_WriteObject(line, f, Py_PRINT_RAW); + } + Py_DECREF(line); + } + else { + err = -1; + } } if (err == 0 && last_exc && ctx->need_close) { @@ -1269,6 +1327,9 @@ print_exception_recursive(struct exception_print_context* ctx, PyObject *value) PyErr_Clear(); } +#define PyErr_MAX_GROUP_WIDTH 15 +#define PyErr_MAX_GROUP_DEPTH 10 + void _PyErr_Display(PyObject *file, PyObject *exception, PyObject *value, PyObject *tb) { @@ -1287,6 +1348,8 @@ _PyErr_Display(PyObject *file, PyObject *exception, PyObject *value, PyObject *t struct exception_print_context ctx; ctx.file = file; ctx.exception_group_depth = 0; + ctx.max_group_width = PyErr_MAX_GROUP_WIDTH; + ctx.max_group_depth = PyErr_MAX_GROUP_DEPTH; /* We choose to ignore seen being possibly NULL, and report at least the main exception (it could be a MemoryError). From e85510a139efe58e880c0bd6214017035f112b4d Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Tue, 2 Nov 2021 18:51:54 +0000 Subject: [PATCH 24/27] use _PyBaseExceptionGroup_Check macro --- Python/pythonrun.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Python/pythonrun.c b/Python/pythonrun.c index bae0930e19199e..0272d3835aefe5 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -939,13 +939,11 @@ print_exception(struct exception_print_context *ctx, PyObject *value) Py_INCREF(value); fflush(stdout); type = (PyObject *) Py_TYPE(value); - bool is_exception_group = PyObject_TypeCheck( - value, (PyTypeObject *)PyExc_BaseExceptionGroup); tb = PyException_GetTraceback(value); if (tb && tb != Py_None) { const char *header = EXCEPTION_TB_HEADER; const char *header_margin = EXC_MARGIN(ctx); - if (is_exception_group) { + if (_PyBaseExceptionGroup_Check(value)) { header = EXCEPTION_GROUP_TB_HEADER; if (ctx->exception_group_depth == 1) { header_margin = "+ "; @@ -1197,7 +1195,7 @@ print_exception_recursive(struct exception_print_context* ctx, PyObject *value) if (err) { /* don't do anything else */ } - else if (!PyObject_TypeCheck(value, (PyTypeObject *)PyExc_BaseExceptionGroup)) { + else if (!_PyBaseExceptionGroup_Check(value)) { print_exception(ctx, value); } else if (ctx->exception_group_depth > ctx->max_group_depth) { From c15a7bd6fe5ed0d3d57bae881d5cdc682af0667e Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Tue, 2 Nov 2021 19:09:21 +0000 Subject: [PATCH 25/27] remove redundant PyErr_Clear --- Python/pythonrun.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/Python/pythonrun.c b/Python/pythonrun.c index 0272d3835aefe5..62e47aab57dfa5 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -1133,9 +1133,6 @@ print_chained(struct exception_print_context* ctx, PyObject *value, err = PyFile_WriteString("\n", f); } } - else { - PyErr_Clear(); - } ctx->need_close = need_close; From d8cc6e8a432e35a42ba01b50b8b67c3a39fb4641 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Tue, 2 Nov 2021 23:29:11 +0000 Subject: [PATCH 26/27] minor tweak - move if out of loop --- Lib/traceback.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/Lib/traceback.py b/Lib/traceback.py index 7d7e858a5bf957..ea756e77e58884 100644 --- a/Lib/traceback.py +++ b/Lib/traceback.py @@ -869,8 +869,8 @@ def format(self, *, chain=True, _ctx=None): output = [] exc = self - while exc: - if chain: + if chain: + while exc: if exc.__cause__ is not None: chained_msg = _cause_message chained_exc = exc.__cause__ @@ -884,9 +884,8 @@ def format(self, *, chain=True, _ctx=None): output.append((chained_msg, exc)) exc = chained_exc - else: - output.append((None, exc)) - exc = None + else: + output.append((None, exc)) for msg, exc in reversed(output): if msg is not None: From 61fab3fc3cd51b443909b87faa59f476351c5c7a Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Wed, 3 Nov 2021 23:19:50 +0000 Subject: [PATCH 27/27] remove excess whitespace --- Lib/test/test_traceback.py | 10 +++++----- Lib/traceback.py | 4 ++-- Python/pythonrun.c | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index e395925d6f8ba4..d88851ddda4313 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -1467,7 +1467,7 @@ def test_exception_group_width_limit(self): ' +---------------- 15 ----------------\n' ' | ValueError: 14\n' ' +---------------- ... ----------------\n' - ' | and 985 more exceptions\n' + ' | and 985 more exceptions\n' ' +------------------------------------\n') report = self.get_report(eg) @@ -1520,7 +1520,7 @@ def test_exception_group_depth_limit(self): ' +-+---------------- 1 ----------------\n' ' | ValueError: 990\n' ' +---------------- 2 ----------------\n' - ' | ... (max_group_depth is 10)\n' + ' | ... (max_group_depth is 10)\n' ' +---------------- 3 ----------------\n' ' | ValueError: -990\n' ' +------------------------------------\n' @@ -2361,7 +2361,7 @@ def test_max_group_width(self): f' +---------------- 2 ----------------', f' | ValueError: 1', f' +---------------- ... ----------------', - f' | and 1 more exception', + f' | and 1 more exception', f' +------------------------------------', f' +---------------- 2 ----------------', f' | ExceptionGroup: eg2', @@ -2370,7 +2370,7 @@ def test_max_group_width(self): f' +---------------- 2 ----------------', f' | TypeError: 1', f' +---------------- ... ----------------', - f' | and 8 more exceptions', + f' | and 8 more exceptions', f' +------------------------------------', f''] @@ -2393,7 +2393,7 @@ def test_max_group_depth(self): f' +-+---------------- 1 ----------------', f' | ValueError: -1', f' +---------------- 2 ----------------', - f' | ... (max_group_depth is 2)', + f' | ... (max_group_depth is 2)', f' +---------------- 3 ----------------', f' | ValueError: 1', f' +------------------------------------', diff --git a/Lib/traceback.py b/Lib/traceback.py index ea756e77e58884..97caa1372f4788 100644 --- a/Lib/traceback.py +++ b/Lib/traceback.py @@ -898,7 +898,7 @@ def format(self, *, chain=True, _ctx=None): elif _ctx.exception_group_depth > self.max_group_depth: # exception group, but depth exceeds limit yield from _ctx.emit( - f" ... (max_group_depth is {self.max_group_depth})\n") + f"... (max_group_depth is {self.max_group_depth})\n") else: # format exception group is_toplevel = (_ctx.exception_group_depth == 0) @@ -939,7 +939,7 @@ def format(self, *, chain=True, _ctx=None): remaining = num_excs - self.max_group_width plural = 's' if remaining > 1 else '' yield from _ctx.emit( - f" and {remaining} more exception{plural}\n") + f"and {remaining} more exception{plural}\n") if last_exc and _ctx.need_close: yield (_ctx.indent() + diff --git a/Python/pythonrun.c b/Python/pythonrun.c index 62e47aab57dfa5..2c0950ee17e8a3 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -1199,7 +1199,7 @@ print_exception_recursive(struct exception_print_context* ctx, PyObject *value) /* exception group but depth exceeds limit */ PyObject *line = PyUnicode_FromFormat( - " ... (max_group_depth is %d)\n", ctx->max_group_depth); + "... (max_group_depth is %d)\n", ctx->max_group_depth); if (line) { PyObject *f = ctx->file; @@ -1286,7 +1286,7 @@ print_exception_recursive(struct exception_print_context* ctx, PyObject *value) else { Py_ssize_t excs_remaining = num_excs - ctx->max_group_width; PyObject *line = PyUnicode_FromFormat( - " and %zd more exception%s\n", + "and %zd more exception%s\n", excs_remaining, excs_remaining > 1 ? "s" : ""); if (line) {