Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Per-test status delivery #47

Merged
merged 8 commits into from
Apr 24, 2024
Merged

Per-test status delivery #47

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
94ee15b
per test status updates
gnarf Mar 21, 2024
8b00f61
snapshot diffs
gnarf Mar 21, 2024
eda1c25
grab the presNumber/testId from the info
gnarf Mar 22, 2024
db67ee2
updates from review
gnarf Apr 3, 2024
a9823e7
update to per-test "RUNNING" or "ERROR" status
gnarf Apr 3, 2024
591ce4c
update snapshots
gnarf Apr 3, 2024
60d7197
add in support for dynamic :testRowNumber in callback url
gnarf Apr 10, 2024
16a0650
missing snapshot update
gnarf Apr 10, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
72 changes: 47 additions & 25 deletions src/host/main.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,27 @@ export async function hostMain({
log(HostMessage.START_AGENT);
await agent.start({ referenceBaseUrl: serverDirectory.baseUrl });

const callbackRequests = [];
let lastCallbackRequest = Promise.resolve();

const postCallbackWhenEnabled = body => {
// ignore if not in callback mode
if (!callbackUrl) return;
const headers = {
'Content-Type': 'application/json',
...(callbackHeader || {}),
};
const perTestUrl = callbackUrl.replace(
':testRowNumber',
body.testCsvRow ?? body.presentationNumber
);
lastCallbackRequest = lastCallbackRequest.then(() =>
fetch(perTestUrl, {
method: 'post',
body: JSON.stringify(body),
headers,
}).then(logUnsuccessfulHTTP.bind(null, log))
);
};

for (const test of plan.tests) {
log(HostMessage.START_TEST);
Expand All @@ -78,41 +98,43 @@ export async function hostMain({
});

const file = plan.files.find(({ name }) => name === test.filepath);
const result = await agent.run(JSON.parse(textDecoder.decode(file.bufferData)));
if (callbackUrl) {
const headers = {
'Content-Type': 'application/json',
...(callbackHeader || {}),
};
const { testId, presentationNumber, capabilities, commands } = result;
const body = JSON.stringify({
testCsvRow: testId,
presentationNumber,
const testSource = JSON.parse(textDecoder.decode(file.bufferData));

const callbackBody = {
testCsvRow: testSource.info.testId,
presentationNumber: testSource.info.presentationNumber,
};

try {
postCallbackWhenEnabled({ ...callbackBody, status: 'RUNNING' });

const result = await agent.run(testSource);

const { capabilities, commands } = result;

postCallbackWhenEnabled({
...callbackBody,
capabilities,
status: 'COMPLETED',
responses: commands.map(({ output }) => output),
// a v2 of this API should allow the aria at app to parse the capabilities we are sending instead.
atVersionName: capabilities.atVersion,
browserVersionName: capabilities.browserVersion,
});
callbackRequests.push(
fetch(callbackUrl, {
method: 'post',
body,
headers,
}).then(logUnsuccessfulHTTP.bind(null, log))
);

plan = addTestResultToTestPlan(plan, test.filepath, result);
} catch (exception) {
const error = `${exception.message ?? exception}`;
log(HostMessage.TEST_ERROR, { error });
postCallbackWhenEnabled({ ...callbackBody, error, status: 'ERROR' });
} finally {
await testLogJob.cancel();
}
plan = addTestResultToTestPlan(plan, test.filepath, result);
await Promise.allSettled(callbackRequests);
await testLogJob.cancel();
}

server.removeFiles(serverDirectory);
log(HostMessage.REMOVE_SERVER_DIRECTORY, { url: serverDirectory.baseUrl });

log(HostMessage.STOP_AGENT);
await lastCallbackRequest;
await agent.stop();

await emitPlanResults(plan);
}

Expand Down
3 changes: 3 additions & 0 deletions src/host/messages.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,8 @@ export const HostMessage = {
START_TEST: 'startTest',
/** @type {'reportingError'} */
REPORTING_ERROR: 'reportingError',
/** @type {'testError'} */
TEST_ERROR: 'testError',
};

export const HOST_TEMPLATES = {
Expand All @@ -63,6 +65,7 @@ export const HOST_TEMPLATES = {
[HostMessage.AGENT_LOG]: ({ text }) => `[Agent]: ${text}`,
[HostMessage.AGENT_CRASHED]: () => `Agent crashed.`,
[HostMessage.START_TEST]: () => `Starting test.`,
[HostMessage.TEST_ERROR]: ({ error }) => `Test Error ${error}`,
[HostMessage.REPORTING_ERROR]: ({ status, body }) =>
`HTTP ${status} response received when reporting result: '${body}'.`,
};
Expand Down
15 changes: 14 additions & 1 deletion src/host/tests/cli-run-plan.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ test('plan3', async t => {
);
});

test('plan3 with callback', async t => {
test('plan3 with callback no url param', async t => {
t.snapshot(
await spawnRunPlan([
'--plan-workingdir=fixtures/host-bin/plan3',
Expand All @@ -52,6 +52,19 @@ test('plan3 with callback', async t => {
);
});

test('plan3 with callback', async t => {
t.snapshot(
await spawnRunPlan([
'--plan-workingdir=fixtures/host-bin/plan3',
'"**"',
'--agent-mock',
'--debug',
'--callback-url=http://callback.url/:testRowNumber',
'--callback-header=test:header:multiple:colon',
])
);
});

test('plan3 with callback request which fails', async t => {
t.snapshot(
await spawnRunPlan([
Expand Down
36 changes: 18 additions & 18 deletions src/host/tests/snapshots/agent.js.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ Generated by [AVA](https://avajs.dev).
},
]

> {"tests":[],"options":{"config":{"referenceBaseUrl":{"protocol":"http","hostname":"localhost","port":1234,"pathname":"/path"}}}}: log
> {"tests":[],"options":{"config":{"referenceBaseUrl":{"protocol":"http:","hostname":"localhost","port":1234,"pathname":"/path"}}}}: log

[
{
Expand Down Expand Up @@ -210,7 +210,7 @@ Generated by [AVA](https://avajs.dev).
},
]

> {"tests":[],"options":{"protocol":"fork","config":{"referenceBaseUrl":{"protocol":"http","hostname":"localhost","port":1234,"pathname":"/path"}}}}: log
> {"tests":[],"options":{"protocol":"fork","config":{"referenceBaseUrl":{"protocol":"http:","hostname":"localhost","port":1234,"pathname":"/path"}}}}: log

[
{
Expand Down Expand Up @@ -278,7 +278,7 @@ Generated by [AVA](https://avajs.dev).
},
]

> {"tests":[],"options":{"protocol":"developer","config":{"referenceBaseUrl":{"protocol":"http","hostname":"localhost","port":1234,"pathname":"/path"}}}}: log
> {"tests":[],"options":{"protocol":"developer","config":{"referenceBaseUrl":{"protocol":"http:","hostname":"localhost","port":1234,"pathname":"/path"}}}}: log

[
{
Expand Down Expand Up @@ -479,7 +479,7 @@ Generated by [AVA](https://avajs.dev).
},
]

> {"tests":[1],"options":{"config":{"referenceBaseUrl":{"protocol":"http","hostname":"localhost","port":1234,"pathname":"/path"}}}}: controller.run(test 1)
> {"tests":[1],"options":{"config":{"referenceBaseUrl":{"protocol":"http:","hostname":"localhost","port":1234,"pathname":"/path"}}}}: controller.run(test 1)

{
capabilities: {
Expand All @@ -506,7 +506,7 @@ Generated by [AVA](https://avajs.dev).
testId: 1,
}

> {"tests":[1],"options":{"config":{"referenceBaseUrl":{"protocol":"http","hostname":"localhost","port":1234,"pathname":"/path"}}}}: log
> {"tests":[1],"options":{"config":{"referenceBaseUrl":{"protocol":"http:","hostname":"localhost","port":1234,"pathname":"/path"}}}}: log

[
{
Expand Down Expand Up @@ -738,7 +738,7 @@ Generated by [AVA](https://avajs.dev).
},
]

> {"tests":[1],"options":{"protocol":"fork","config":{"referenceBaseUrl":{"protocol":"http","hostname":"localhost","port":1234,"pathname":"/path"}}}}: controller.run(test 1)
> {"tests":[1],"options":{"protocol":"fork","config":{"referenceBaseUrl":{"protocol":"http:","hostname":"localhost","port":1234,"pathname":"/path"}}}}: controller.run(test 1)

{
capabilities: {
Expand All @@ -765,7 +765,7 @@ Generated by [AVA](https://avajs.dev).
testId: 1,
}

> {"tests":[1],"options":{"protocol":"fork","config":{"referenceBaseUrl":{"protocol":"http","hostname":"localhost","port":1234,"pathname":"/path"}}}}: log
> {"tests":[1],"options":{"protocol":"fork","config":{"referenceBaseUrl":{"protocol":"http:","hostname":"localhost","port":1234,"pathname":"/path"}}}}: log

[
{
Expand Down Expand Up @@ -958,7 +958,7 @@ Generated by [AVA](https://avajs.dev).
},
]

> {"tests":[1],"options":{"protocol":"developer","config":{"referenceBaseUrl":{"protocol":"http","hostname":"localhost","port":1234,"pathname":"/path"}}}}: controller.run(test 1)
> {"tests":[1],"options":{"protocol":"developer","config":{"referenceBaseUrl":{"protocol":"http:","hostname":"localhost","port":1234,"pathname":"/path"}}}}: controller.run(test 1)

{
capabilities: {
Expand All @@ -985,7 +985,7 @@ Generated by [AVA](https://avajs.dev).
testId: 1,
}

> {"tests":[1],"options":{"protocol":"developer","config":{"referenceBaseUrl":{"protocol":"http","hostname":"localhost","port":1234,"pathname":"/path"}}}}: log
> {"tests":[1],"options":{"protocol":"developer","config":{"referenceBaseUrl":{"protocol":"http:","hostname":"localhost","port":1234,"pathname":"/path"}}}}: log

[
{
Expand Down Expand Up @@ -1300,7 +1300,7 @@ Generated by [AVA](https://avajs.dev).
},
]

> {"tests":[1,2],"options":{"config":{"referenceBaseUrl":{"protocol":"http","hostname":"localhost","port":1234,"pathname":"/path"}}}}: controller.run(test 1)
> {"tests":[1,2],"options":{"config":{"referenceBaseUrl":{"protocol":"http:","hostname":"localhost","port":1234,"pathname":"/path"}}}}: controller.run(test 1)

{
capabilities: {
Expand All @@ -1327,7 +1327,7 @@ Generated by [AVA](https://avajs.dev).
testId: 1,
}

> {"tests":[1,2],"options":{"config":{"referenceBaseUrl":{"protocol":"http","hostname":"localhost","port":1234,"pathname":"/path"}}}}: controller.run(test 2)
> {"tests":[1,2],"options":{"config":{"referenceBaseUrl":{"protocol":"http:","hostname":"localhost","port":1234,"pathname":"/path"}}}}: controller.run(test 2)

{
capabilities: {
Expand All @@ -1354,7 +1354,7 @@ Generated by [AVA](https://avajs.dev).
testId: 2,
}

> {"tests":[1,2],"options":{"config":{"referenceBaseUrl":{"protocol":"http","hostname":"localhost","port":1234,"pathname":"/path"}}}}: log
> {"tests":[1,2],"options":{"config":{"referenceBaseUrl":{"protocol":"http:","hostname":"localhost","port":1234,"pathname":"/path"}}}}: log

[
{
Expand Down Expand Up @@ -1700,7 +1700,7 @@ Generated by [AVA](https://avajs.dev).
},
]

> {"tests":[1,2],"options":{"protocol":"fork","config":{"referenceBaseUrl":{"protocol":"http","hostname":"localhost","port":1234,"pathname":"/path"}}}}: controller.run(test 1)
> {"tests":[1,2],"options":{"protocol":"fork","config":{"referenceBaseUrl":{"protocol":"http:","hostname":"localhost","port":1234,"pathname":"/path"}}}}: controller.run(test 1)

{
capabilities: {
Expand All @@ -1727,7 +1727,7 @@ Generated by [AVA](https://avajs.dev).
testId: 1,
}

> {"tests":[1,2],"options":{"protocol":"fork","config":{"referenceBaseUrl":{"protocol":"http","hostname":"localhost","port":1234,"pathname":"/path"}}}}: controller.run(test 2)
> {"tests":[1,2],"options":{"protocol":"fork","config":{"referenceBaseUrl":{"protocol":"http:","hostname":"localhost","port":1234,"pathname":"/path"}}}}: controller.run(test 2)

{
capabilities: {
Expand All @@ -1754,7 +1754,7 @@ Generated by [AVA](https://avajs.dev).
testId: 2,
}

> {"tests":[1,2],"options":{"protocol":"fork","config":{"referenceBaseUrl":{"protocol":"http","hostname":"localhost","port":1234,"pathname":"/path"}}}}: log
> {"tests":[1,2],"options":{"protocol":"fork","config":{"referenceBaseUrl":{"protocol":"http:","hostname":"localhost","port":1234,"pathname":"/path"}}}}: log

[
{
Expand Down Expand Up @@ -2072,7 +2072,7 @@ Generated by [AVA](https://avajs.dev).
},
]

> {"tests":[1,2],"options":{"protocol":"developer","config":{"referenceBaseUrl":{"protocol":"http","hostname":"localhost","port":1234,"pathname":"/path"}}}}: controller.run(test 1)
> {"tests":[1,2],"options":{"protocol":"developer","config":{"referenceBaseUrl":{"protocol":"http:","hostname":"localhost","port":1234,"pathname":"/path"}}}}: controller.run(test 1)

{
capabilities: {
Expand All @@ -2099,7 +2099,7 @@ Generated by [AVA](https://avajs.dev).
testId: 1,
}

> {"tests":[1,2],"options":{"protocol":"developer","config":{"referenceBaseUrl":{"protocol":"http","hostname":"localhost","port":1234,"pathname":"/path"}}}}: controller.run(test 2)
> {"tests":[1,2],"options":{"protocol":"developer","config":{"referenceBaseUrl":{"protocol":"http:","hostname":"localhost","port":1234,"pathname":"/path"}}}}: controller.run(test 2)

{
capabilities: {
Expand All @@ -2126,7 +2126,7 @@ Generated by [AVA](https://avajs.dev).
testId: 2,
}

> {"tests":[1,2],"options":{"protocol":"developer","config":{"referenceBaseUrl":{"protocol":"http","hostname":"localhost","port":1234,"pathname":"/path"}}}}: log
> {"tests":[1,2],"options":{"protocol":"developer","config":{"referenceBaseUrl":{"protocol":"http:","hostname":"localhost","port":1234,"pathname":"/path"}}}}: log

[
{
Expand Down
Binary file modified src/host/tests/snapshots/agent.js.snap
View file
Open in desktop
Binary file not shown.
Loading
Loading