ported the full output optimization code from VSCode

packages/notebook/src/browser/notebook-output-utils.ts

@@ -0,0 +1,119 @@
+// *****************************************************************************
+// Copyright (C) 2023 TypeFox and others.
+//
+// This program and the accompanying materials are made available under the
+// terms of the Eclipse Public License v. 2.0 which is available at
+// http://www.eclipse.org/legal/epl-2.0.
+//
+// This Source Code may also be made available under the following Secondary
+// Licenses when the conditions for such availability set forth in the Eclipse
+// Public License v. 2.0 are satisfied: GNU General Public License, version 2
+// with the GNU Classpath Exception which is available at
+// https://www.gnu.org/software/classpath/license.html.
+//
+// SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0
+// *****************************************************************************
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *  Copied from commit 18b2c92451b076943e5b508380e0eba66ba7d934 from file src\vs\workbench\contrib\notebook\common\notebookCommon.ts
+ *--------------------------------------------------------------------------------------------*/
+
+import { BinaryBuffer } from '@theia/core/lib/common/buffer';
+
+const textDecoder = new TextDecoder();
+
+/**
+ * Given a stream of individual stdout outputs, this function will return the compressed lines, escaping some of the common terminal escape codes.
+ * E.g. some terminal escape codes would result in the previous line getting cleared, such if we had 3 lines and
+ * last line contained such a code, then the result string would be just the first two lines.
+ * @returns a single VSBuffer with the concatenated and compressed data, and whether any compression was done.
+ */
+export function compressOutputItemStreams(outputs: Uint8Array[]): { data: BinaryBuffer, didCompression: boolean } {
+    const buffers: Uint8Array[] = [];
+    let startAppending = false;
+
+    // Pick the first set of outputs with the same mime type.
+    for (const output of outputs) {
+        if ((buffers.length === 0 || startAppending)) {
+            buffers.push(output);
+            startAppending = true;
+        }
+    }
+
+    let didCompression = compressStreamBuffer(buffers);
+    const concatenated = BinaryBuffer.concat(buffers.map(buffer => BinaryBuffer.wrap(buffer)));
+    const data = formatStreamText(concatenated);
+    didCompression = didCompression || data.byteLength !== concatenated.byteLength;
+    return { data, didCompression };
+}
+
+export const MOVE_CURSOR_1_LINE_COMMAND = `${String.fromCharCode(27)}[A`;
+const MOVE_CURSOR_1_LINE_COMMAND_BYTES = MOVE_CURSOR_1_LINE_COMMAND.split('').map(c => c.charCodeAt(0));
+const LINE_FEED = 10;
+function compressStreamBuffer(streams: Uint8Array[]): boolean {
+    let didCompress = false;
+    streams.forEach((stream, index) => {
+        if (index === 0 || stream.length < MOVE_CURSOR_1_LINE_COMMAND.length) {
+            return;
+        }
+
+        const previousStream = streams[index - 1];
+
+        // Remove the previous line if required.
+        const command = stream.subarray(0, MOVE_CURSOR_1_LINE_COMMAND.length);
+        if (command[0] === MOVE_CURSOR_1_LINE_COMMAND_BYTES[0] && command[1] === MOVE_CURSOR_1_LINE_COMMAND_BYTES[1] && command[2] === MOVE_CURSOR_1_LINE_COMMAND_BYTES[2]) {
+            const lastIndexOfLineFeed = previousStream.lastIndexOf(LINE_FEED);
+            if (lastIndexOfLineFeed === -1) {
+                return;
+            }
+
+            didCompress = true;
+            streams[index - 1] = previousStream.subarray(0, lastIndexOfLineFeed);
+            streams[index] = stream.subarray(MOVE_CURSOR_1_LINE_COMMAND.length);
+        }
+    });
+    return didCompress;
+}
+
+const BACKSPACE_CHARACTER = '\b'.charCodeAt(0);
+const CARRIAGE_RETURN_CHARACTER = '\r'.charCodeAt(0);
+function formatStreamText(buffer: BinaryBuffer): BinaryBuffer {
+    // We have special handling for backspace and carriage return characters.
+    // Don't unnecessary decode the bytes if we don't need to perform any processing.
+    if (!buffer.buffer.includes(BACKSPACE_CHARACTER) && !buffer.buffer.includes(CARRIAGE_RETURN_CHARACTER)) {
+        return buffer;
+    }
+    // Do the same thing jupyter is doing
+    return BinaryBuffer.fromString(fixCarriageReturn(fixBackspace(textDecoder.decode(buffer.buffer))));
+}
+
+/**
+ * Took this from jupyter/notebook
+ * https://github.com/jupyter/notebook/blob/b8b66332e2023e83d2ee04f83d8814f567e01a4e/notebook/static/base/js/utils.js
+ * Remove characters that are overridden by backspace characters
+ */
+function fixBackspace(txt: string): string {
+    let tmp = txt;
+    do {
+        txt = tmp;
+        // Cancel out anything-but-newline followed by backspace
+        tmp = txt.replace(/[^\n]\x08/gm, '');
+    } while (tmp.length < txt.length);
+    return txt;
+}
+
+/**
+ * Remove chunks that should be overridden by the effect of carriage return characters
+ * From https://github.com/jupyter/notebook/blob/master/notebook/static/base/js/utils.js
+ */
+function fixCarriageReturn(txt: string): string {
+    txt = txt.replace(/\r+\n/gm, '\n'); // \r followed by \n --> newline
+    while (txt.search(/\r[^$]/g) > -1) {
+        const base = txt.match(/^(.*)\r+/m)![1];
+        let insert = txt.match(/\r+(.*)$/m)![1];
+        insert = insert + base.slice(insert.length, base.length);
+        txt = txt.replace(/\r+.*$/m, '\r').replace(/^.*\r/m, insert);
+    }
+    return txt;
+}

packages/notebook/src/browser/view-model/notebook-cell-output-model.ts

@@ -15,8 +15,8 @@
 // *****************************************************************************
 
 import { Disposable, Emitter } from '@theia/core';
-import { BinaryBuffer } from '@theia/core/lib/common/buffer';
 import { CellOutput, CellOutputItem, isTextStreamMime } from '../../common';
+import { compressOutputItemStreams } from '../notebook-output-utils';
 
 export class NotebookCellOutputModel implements Disposable {
 
@@ -73,24 +73,25 @@ export class NotebookCellOutputModel implements Disposable {
         if (this.outputs.length > 1 && this.outputs.every(item => isTextStreamMime(item.mime))) {
             // Look for the mimes in the items, and keep track of their order.
             // Merge the streams into one output item, per mime type.
-            const mimeOutputs = new Map<string, BinaryBuffer[]>();
+            const mimeOutputs = new Map<string, Uint8Array[]>();
             const mimeTypes: string[] = [];
             this.outputs.forEach(item => {
-                let items: BinaryBuffer[];
+                let items: Uint8Array[];
                 if (mimeOutputs.has(item.mime)) {
                     items = mimeOutputs.get(item.mime)!;
                 } else {
                     items = [];
                     mimeOutputs.set(item.mime, items);
                     mimeTypes.push(item.mime);
                 }
-                items.push(item.data);
+                items.push(item.data.buffer);
             });
             this.outputs.length = 0;
             mimeTypes.forEach(mime => {
+                const compressionResult = compressOutputItemStreams(mimeOutputs.get(mime)!);
                 this.outputs.push({
                     mime,
-                    data: BinaryBuffer.concat(mimeOutputs.get(mime)!)
+                    data: compressionResult.data
                 });
             });
         }

packages/plugin-ext/src/main/browser/notebooks/renderers/cell-output-webview.tsx

@@ -76,7 +76,7 @@ export class CellOutputWebviewImpl implements CellOutputWebview, Disposable {
     protected async init(): Promise<void> {
         this.cell.onDidChangeOutputs(outputChange => this.updateOutput(outputChange));
         this.cell.onDidChangeOutputItems(output => {
-            this.updateOutput({start: this.cell.outputs.findIndex(o => o.getData().outputId === o.outputId), deleteCount: 1, newOutputs: [output]});
+            this.updateOutput({start: this.cell.outputs.findIndex(o => o.outputId === output.outputId), deleteCount: 1, newOutputs: [output]});
         });
 
         this.webviewWidget = await this.widgetManager.getOrCreateWidget(WebviewWidget.FACTORY_ID, { id: this.id });