GM_xmlhttpRequest doesn't return response data of some specific files on Chrome

[ccloli]

(Please fill out the issue template with your details)

Expected Behavior

The data should be returned. It works fine in Firefox, and Chrome with latest version of ViolentMonkey (v2.12.9, first seen: v
2.12.7.18 w/ built from source code).

Actual Behavior

The file is completed, but the data is not returned, the status is keep at downloading, or the data is corrupted. It only happens on Chrome and with some specific file.

The bug is also happened in old version of ViolentMonkey (last seen: v2.12.7), but instead of not returning data, ViolentMonkey returns an empty ArrayBuffer.

Specifications

• Chrome: 87.0.4280.88 64-bit (first seen: 84.0.4147.135)
• TM: 4.12.6123 （first seen: 4.10, 4.11.6118）
• OS: Windows 10 x64 20H2 (first seen: Windows 10 x64 1809)

Script

Install the script then navigate to https://example.com/.

// ==UserScript==
// @name         New Userscript
// @namespace    http://tampermonkey.net/
// @version      0.1
// @description  try to take over the world!
// @author       You
// @match        https://example.com/*
// @icon         data:image/gif;base64,R0lGODlhAQABAAAAACH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==
// @grant        GM_xmlhttpRequest
// @connect      objectstorage.ap-tokyo-1.oraclecloud.com
// ==/UserScript==

(function() {
    'use strict';

    GM_xmlhttpRequest({
        method: 'GET',
        url: 'https://objectstorage.ap-tokyo-1.oraclecloud.com/p/zGLQ6bSJnZ6ir5b38ab7ziJNzsHA-Fu4S0mS8eAKwCvlru0xVdYv17w9LJo40FAN/n/nrzblovvku9x/b/bucket-ccloli/o/003.png',
        responseType: 'arraybuffer',
        onreadystatechange: function(event) {
            console.log(`Ready state: ${event.readyState}, status ${event.status}`);
        },
        onprogress: function(event) {
            if (event.lengthComputable) {
                console.log(`Downloading: ${event.loaded}/${event.total}`);
            }
        },
        onload: function(res) {
            console.log('Download finish');
            console.log(res);
        },
        onerror: function(res) {
            console.log('Can\'t load file.');
        }
    });
})();

Chrome + Tampermonkey (onload is never called):

Chrome + ViolentMonkey (onload is called):

Firefox + Tampermonkey (onload is called):

[ccloli]

BTW from a reply of my issue, the bug is also happened in ViolentMonkey before, but the referred issue violentmonkey/violentmonkey#1066 mentioned the bug was fixed in RC version, maybe you can check these versions to get what's changed.

[AlttiRi]

Confirm.
The file from this url is not downloaded at all (In Chromium). But ViolentMonkey (the last version) downloads it instantly (from disk cache).

// ==UserScript==
// @name        Issue 1106 (oraclecloud.com)
// @namespace   Issues
// @match       https://github.com/Tampermonkey/tampermonkey/issues/1106
// @match       https://example.com/
// @grant       GM.xmlHttpRequest
// @connect     objectstorage.ap-tokyo-1.oraclecloud.com
// ==/UserScript==

!async function() {
    console.log("downloading (oraclecloud.com)...");
    const response = await new Promise((resolve, reject) => {
        GM.xmlHttpRequest({
            method: "get",
            url: "https://objectstorage.ap-tokyo-1.oraclecloud.com/p/zGLQ6bSJnZ6ir5b38ab7ziJNzsHA-Fu4S0mS8eAKwCvlru0xVdYv17w9LJo40FAN/n/nrzblovvku9x/b/bucket-ccloli/o/003.png", // 12 MB
            responseType: "blob",
            onload:  response => {resolve(response); console.log("onload");},  // it was not called
            onerror: response => {reject(response);  console.log("onerror");}, // it was not called too
        });
    });
    console.log("response:", response);

    const {response: blob} = response;
    downloadBlob(blob, "003.png");
}();

function downloadBlob(blob, name) {
    const anchor = document.createElement("a");
    anchor.setAttribute("download", name || "");
    anchor.href = URL.createObjectURL(blob);
    anchor.click();
}

In TM's background script's console it was fetched well:

---

For example, the other file (https://giant.gfycat.com/ShockedSecondaryFiddlercrab.mp4, 32 MB) is downloaded... but slow, and the browser significantly lags (the page freezes – try to select a text on the page to see it)). However ViolentMonkey downloads it well, without any lag.
I have written about it here: violentmonkey/violentmonkey#1066 (comment)

// ==UserScript==
// @name        Issue 1106 (gfycat.com)
// @namespace   Issues
// @match       https://github.com/Tampermonkey/tampermonkey/issues/1106
// @match       https://example.com/
// @grant       GM.xmlHttpRequest
// @connect     giant.gfycat.com
// ==/UserScript==

!async function() {
    console.log("downloading (gfycat.com)...");
    const response = await new Promise((resolve, reject) => {
        GM.xmlHttpRequest({
            method: "get",
            url: "https://giant.gfycat.com/ShockedSecondaryFiddlercrab.mp4", // 32 MB
            responseType: "blob",
            onload: resolve,
            onerror: reject,
        });
    });
    console.log("response:", response);

    const {response: blob} = response;
    downloadBlob(blob, "ShockedSecondaryFiddlercrab.mp4");
}();

function downloadBlob(blob, name) {
    const anchor = document.createElement("a");
    anchor.setAttribute("download", name || "");
    anchor.href = URL.createObjectURL(blob);
    anchor.click();
}

[ccloli]

but slow, and the browser significantly lags (the page freezes – try to select a text on the page to see it)

Well, I'm not sure it's related to this issue, but your question is also mentioned in my referred issue:

... that's because instead of using responseType: 'blob' / 'arraybuffer', Tampermonkey treats blob data as string, then convert it to arraybuffer, and even not using web worker, so the browser will be freeze if the file is too large...

ViolentMonkey works fine because it doesn't use this strange way, I guess.

I've already created an issue about this (#279) a few years ago, since I don't think it's related to this issue (unless it's caused by the convert function throws an error when it met specific sequence), let's talk about it later.

[ccloli]

I write a script which contains 4 files from my issues for testing, 3 of them won't call onload, and the last one will call, but only returns a part of the file, maybe it can helps you to debug and testing.

To use the script, please install it and navigate to https://example.com, it'll show a table and each row has 5 links to download, each link will use native XMLHttpRequest, native Fetch API, GM_xmlhttpRequest, GM_xmlhttpRequest with Fetch API and GM_download to fetch the file. Click one of the links, and check the outputs in console.

(Some of the files may be NSFW since I cannot find other example, so I rename them to .bin files, and if you care about it, please don't open the file directly)

// ==UserScript==
// @name         New Userscript
// @namespace    http://tampermonkey.net/
// @version      0.1
// @description  try to take over the world!
// @author       You
// @match        https://example.com/*
// @icon         data:image/gif;base64,R0lGODlhAQABAAAAACH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==
// @grant        GM_xmlhttpRequest
// @grant        GM_download
// @connect      objectstorage.ap-tokyo-1.oraclecloud.com
// ==/UserScript==

(function() {
    'use strict';

    const downloadFile = (res) => {
        let blob = new Blob([res]);
        const url = URL.createObjectURL(blob);

        const a = document.createElement('a');
        a.setAttribute('href', url);
        a.setAttribute('download', url.split('/').pop());
        document.documentElement.appendChild(a);

        const e = new MouseEvent('click');
        a.dispatchEvent(e);
        document.documentElement.removeChild(a);

        setTimeout(function(){
            URL.revokeObjectURL(url);
            if ('close' in blob) blob.close();
            blob = undefined;
        }, 1000);
    };

    const onreadystatechange = (event) => {
        console.log(`Ready state: ${event.readyState}, status ${event.status}`);
    };

    const onprogress = (event) => {
        if (event.lengthComputable) {
            console.log(`Downloading: ${event.loaded}/${event.total}`);
        }
    };

    const onload = (res) => {
        console.log('Download finish');
        console.log(res);

        downloadFile(res);
    };

    const onerror = (err) => {
        console.log('Can\'t load file.');
        console.log(err);
    };


    const gmXhrRequest = (url, fetch) => {
        console.log('Request with GM_xmlhttpRequest');
        GM_xmlhttpRequest({
            method: 'GET',
            url,
            responseType: 'arraybuffer',
            fetch,
            onreadystatechange,
            onprogress,
            onload: res => onload(res.response),
            onerror,
        });
    };

    const xhrRequest = (url) => {
        console.log('Request with XMLHttpRequest');
        const xhr = new XMLHttpRequest();
        xhr.onreadystatechange = () => onreadystatechange(xhr);
        xhr.onprogress = onprogress;
        xhr.onload = () => onload(xhr.response);
        xhr.onerror = onerror;
        xhr.open('GET', url);
        xhr.responseType = 'arraybuffer';
        xhr.send();
    };

    const fetchRequest = (url) => {
        console.log('Request with Fetch API');
        fetch(url).then((res) => {
            const total = res.headers.get('content-length');
            let loaded = 0;
            const [progressStream, returnStream] = res.body.tee();
            const reader = progressStream.getReader();
            const log = () => {
                reader.read().then(({ value, done }) => {
                    if (done) return;
                    loaded += value.length;
                    onprogress({
                        loaded, total, lengthComputable: !!total,
                    })
                    log();
                });
            };
            log();
            return new Response(returnStream, { headers: res.headers });
        }).then(res => res.arrayBuffer()).then(onload).catch(onerror);
    };

    const gmDownloadRequest = (url) => {
        console.log('Request with GM_download');
        GM_download({
            name: `${url.split('/').pop()}.txt`,
            url,
            onprogress,
            onerror,
        });
    };

    const handleClick = (event) => {
        const { target } = event;
        if (target.className !== 'download') {
            return;
        }
        const a = target.parentElement.parentElement.firstElementChild.firstElementChild;
        const url = a.getAttribute('href');
        const type = target.textContent;
        const request = ({
            XMLHttpRequest: xhrRequest,
            fetch: fetchRequest,
            GM_xmlhttpRequest: gmXhrRequest,
            'GM_xmlhttpRequest (fetch)': u => gmXhrRequest(u, true),
            GM_download: gmDownloadRequest,
        })[type];
        if (request) {
            request(url);
        }
    };

    const downloadCellTemplate = `
                <td>
                    <a href="javascript:" class="download">XMLHttpRequest</a> |
                    <a href="javascript:" class="download">fetch</a> |
                    <a href="javascript:" class="download">GM_xmlhttpRequest</a> |
                    <a href="javascript:" class="download">GM_xmlhttpRequest (fetch)</a> |
                    <a href="javascript:" class="download">GM_download</a>
                </td>
    `;

    const template = `
        <table border="1px solid #ddd">
            <tr><th>File Name</th><th>SHA-1</th><th>SHA-256</th><th>Download</th></tr>
            <tr>
                <td><a href="https://objectstorage.ap-tokyo-1.oraclecloud.com/p/PaLtT3XKwZXjLrxJ7-yy-tB5sKkeO4dYPu7tA6SkXSXrVUm36oN6xJsw1E_E0MdU/n/nrzblovvku9x/b/bucket-ccloli/o/gm-xhr-test/stall-1.bin">stall-1.bin</a></td>
                <td>F45169FDDEC8C4C2FD8E94CD65B4B3B987C3A0B5</td>
                <td>2453B9322C18D3ADA73333CBAEBB3A10FFD1C967A65246B905D55298A0DCEDE1</td>
                ${downloadCellTemplate}
            </tr>
            <tr>
                <td><a href="https://objectstorage.ap-tokyo-1.oraclecloud.com/p/aKWRgZHOPqJzszF71yaOh5RAwTPnFrXegE9xW_Ykccy8vdbCUq90ZXxZW03prJ6j/n/nrzblovvku9x/b/bucket-ccloli/o/gm-xhr-test/stall-2.bin">stall-2.bin</a></td>
                <td>1D7B0C7D3BD2D56750FE8102EAA076A32BF76723</td>
                <td>B47E3750F5187264A39C4FBC2D8E9082DB660B87F9F1BB06D548CABE43E354AD</td>
                ${downloadCellTemplate}
            </tr>
            <tr>
                <td><a href="https://objectstorage.ap-tokyo-1.oraclecloud.com/p/DW-k9vQeI1bAT2exlVWIOwkhQ1kPio34Ancxq3HJlSUYbhzINbnaZQTXObDBQ9Ar/n/nrzblovvku9x/b/bucket-ccloli/o/gm-xhr-test/stall-3.bin">stall-3.bin</a></td>
                <td>7BB096E6CC5C8712C7FCE6BECAA5B9F39D1225A8</td>
                <td>D89EB14A4DFDA3BE63A24D39C082411804DEE3FCA55CC9F61CA12BD7157DF2B1</td>
                ${downloadCellTemplate}
            </tr>
            <tr>
                <td><a href="https://objectstorage.ap-tokyo-1.oraclecloud.com/p/U6E0584C7Ac4LPfYt_KNKi2XXFUMfZQs6I9tg9_Hn79LtaZBQIgiB1ZzbyhcQ293/n/nrzblovvku9x/b/bucket-ccloli/o/gm-xhr-test/corrupt.bin">corrupt.bin</a></td>
                <td>D74E60B83EEF8C946DA88F11DA1DA64BF6612030</td>
                <td>743B6EBF7C83809768C3AA2741AAF8F5737F378BC7AE55248E797EAA676572C5</td>
                ${downloadCellTemplate}
            </tr>
        </table>
    `;

    document.body.innerHTML = template;
    document.body.onclick = handleClick;
})();

[derjanb]

This issue should be fixed at TM BETA 4.12.6124. 😅

I'm not sure the Chrome Webstore team will be able to review this version this year, but in the meantime, you can download it from here.

[ccloli]

Checked and confirmed these files can be downloaded, thanks for your work.