From 77778f370565dafed98ac50b6dcad02b2aad67db Mon Sep 17 00:00:00 2001 From: Matteo Pierro Date: Mon, 3 Oct 2022 08:27:53 +0300 Subject: [PATCH 1/6] fetch all stars --- src/fetchers/stats-fetcher.js | 63 ++++++++++++++++++++++++++++++----- tests/api.test.js | 25 ++++++++++++-- tests/fetchStats.test.js | 48 +++++++++++++++++++++----- 3 files changed, 117 insertions(+), 19 deletions(-) diff --git a/src/fetchers/stats-fetcher.js b/src/fetchers/stats-fetcher.js index b9493adfdbb43..072e2a7feeb55 100644 --- a/src/fetchers/stats-fetcher.js +++ b/src/fetchers/stats-fetcher.js @@ -44,14 +44,41 @@ const fetcher = (variables, token) => { followers { totalCount } - repositories(first: 100, ownerAffiliations: OWNER, orderBy: {direction: DESC, field: STARGAZERS}) { + repositories(ownerAffiliations: OWNER) { totalCount + } + } + } + `, + variables, + }, + { + Authorization: `bearer ${token}`, + }, + ); +}; + +/** + * @param {import('axios').AxiosRequestHeaders} variables + * @param {string} token + */ +const repositoriesFetcher = (variables, token) => { + return request( + { + query: ` + query userInfo($login: String!, $after: String) { + user(login: $login) { + repositories(first: 100, ownerAffiliations: OWNER, orderBy: {direction: DESC, field: STARGAZERS}, after: $after) { nodes { name stargazers { totalCount } } + pageInfo { + hasNextPage + endCursor + } } } } @@ -99,6 +126,32 @@ const totalCommitsFetcher = async (username) => { return 0; }; +const totalStarsFetcher = async (username, repoToHide) => { + let nodes = []; + let hasNextPage = true + let endCursor = null + while (hasNextPage) { + const variables = { login: username, first: 100, after: endCursor }; + let res = await retryer(repositoriesFetcher, variables); + + if (res.data.errors) { + logger.error(res.data.errors); + throw new CustomError( + res.data.errors[0].message || "Could not fetch user", + CustomError.USER_NOT_FOUND, + ); + } + + nodes.push(...res.data.data.user.repositories.nodes); + hasNextPage = res.data.data.user.repositories.pageInfo.hasNextPage + endCursor = res.data.data.user.repositories.pageInfo.endCursor + } + + return nodes + .filter((data) => !repoToHide[data.name]) + .reduce((prev, curr) => prev + curr.stargazers.totalCount, 0); +} + /** * @param {string} username * @param {boolean} count_private @@ -166,13 +219,7 @@ async function fetchStats( stats.contributedTo = user.repositoriesContributedTo.totalCount; // Retrieve stars while filtering out repositories to be hidden - stats.totalStars = user.repositories.nodes - .filter((data) => { - return !repoToHide[data.name]; - }) - .reduce((prev, curr) => { - return prev + curr.stargazers.totalCount; - }, 0); + stats.totalStars = await totalStarsFetcher(username, repoToHide); stats.rank = calculateRank({ totalCommits: stats.totalCommits, diff --git a/tests/api.test.js b/tests/api.test.js index b0dfc59f17e2e..ff1d1f8039ad6 100644 --- a/tests/api.test.js +++ b/tests/api.test.js @@ -39,8 +39,21 @@ const data = { closedIssues: { totalCount: 0 }, followers: { totalCount: 0 }, repositories: { - totalCount: 1, + totalCount: 1 + }, + }, + }, +}; + +const repositoriesData = { + data: { + user: { + repositories: { nodes: [{ stargazers: { totalCount: 100 } }], + pageInfo: { + hasNextPage: false, + cursor: "cursor" + } }, }, }, @@ -70,7 +83,10 @@ const faker = (query, data) => { setHeader: jest.fn(), send: jest.fn(), }; - mock.onPost("https://api.github.com/graphql").reply(200, data); + mock.onPost("https://api.github.com/graphql") + .replyOnce(200, data) + .onPost("https://api.github.com/graphql") + .replyOnce(200, repositoriesData); return { req, res }; }; @@ -138,7 +154,10 @@ describe("Test /api/", () => { it("should have proper cache", async () => { const { req, res } = faker({}, data); - mock.onPost("https://api.github.com/graphql").reply(200, data); + mock.onPost("https://api.github.com/graphql") + .replyOnce(200, data) + .onPost("https://api.github.com/graphql") + .replyOnce(200, repositoriesData); await api(req, res); diff --git a/tests/fetchStats.test.js b/tests/fetchStats.test.js index f8eae98139442..eeacf30d9649e 100644 --- a/tests/fetchStats.test.js +++ b/tests/fetchStats.test.js @@ -18,14 +18,42 @@ const data = { closedIssues: { totalCount: 100 }, followers: { totalCount: 100 }, repositories: { - totalCount: 5, + totalCount: 5 + }, + }, + }, +}; + +const firstRepositoriesData = { + data: { + user: { + repositories: { nodes: [ { name: "test-repo-1", stargazers: { totalCount: 100 } }, { name: "test-repo-2", stargazers: { totalCount: 100 } }, - { name: "test-repo-3", stargazers: { totalCount: 100 } }, + { name: "test-repo-3", stargazers: { totalCount: 100 } } + ], + pageInfo: { + hasNextPage: true, + cursor: "cursor" + } + }, + }, + }, +}; + +const secondRepositoriesData = { + data: { + user: { + repositories: { + nodes: [ { name: "test-repo-4", stargazers: { totalCount: 50 } }, { name: "test-repo-5", stargazers: { totalCount: 50 } }, ], + pageInfo: { + hasNextPage: false, + cursor: "cursor" + } }, }, }, @@ -44,14 +72,21 @@ const error = { const mock = new MockAdapter(axios); +beforeEach( () => { + mock.onPost("https://api.github.com/graphql") + .replyOnce(200, data) + .onPost("https://api.github.com/graphql") + .replyOnce(200, firstRepositoriesData) + .onPost("https://api.github.com/graphql") + .replyOnce(200, secondRepositoriesData); +}); + afterEach(() => { mock.reset(); }); describe("Test fetchStats", () => { it("should fetch correct stats", async () => { - mock.onPost("https://api.github.com/graphql").reply(200, data); - let stats = await fetchStats("anuraghazra"); const rank = calculateRank({ totalCommits: 100, @@ -75,6 +110,7 @@ describe("Test fetchStats", () => { }); it("should throw error", async () => { + mock.reset(); mock.onPost("https://api.github.com/graphql").reply(200, error); await expect(fetchStats("anuraghazra")).rejects.toThrow( @@ -83,8 +119,6 @@ describe("Test fetchStats", () => { }); it("should fetch and add private contributions", async () => { - mock.onPost("https://api.github.com/graphql").reply(200, data); - let stats = await fetchStats("anuraghazra", true); const rank = calculateRank({ totalCommits: 150, @@ -108,7 +142,6 @@ describe("Test fetchStats", () => { }); it("should fetch total commits", async () => { - mock.onPost("https://api.github.com/graphql").reply(200, data); mock .onGet("https://api.github.com/search/commits?q=author:anuraghazra") .reply(200, { total_count: 1000 }); @@ -136,7 +169,6 @@ describe("Test fetchStats", () => { }); it("should exclude stars of the `test-repo-1` repository", async () => { - mock.onPost("https://api.github.com/graphql").reply(200, data); mock .onGet("https://api.github.com/search/commits?q=author:anuraghazra") .reply(200, { total_count: 1000 }); From d61893ee53177c81cafb9a4df132c7709e8afcd0 Mon Sep 17 00:00:00 2001 From: Matteo Pierro Date: Mon, 3 Oct 2022 12:18:35 +0300 Subject: [PATCH 2/6] stop fetching when there are repos with zero stars --- src/fetchers/stats-fetcher.js | 6 +++-- tests/fetchStats.test.js | 49 +++++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 2 deletions(-) diff --git a/src/fetchers/stats-fetcher.js b/src/fetchers/stats-fetcher.js index 072e2a7feeb55..04dfc3bcb54b5 100644 --- a/src/fetchers/stats-fetcher.js +++ b/src/fetchers/stats-fetcher.js @@ -142,8 +142,10 @@ const totalStarsFetcher = async (username, repoToHide) => { ); } - nodes.push(...res.data.data.user.repositories.nodes); - hasNextPage = res.data.data.user.repositories.pageInfo.hasNextPage + const allNodes = res.data.data.user.repositories.nodes; + const nodesWithStars = allNodes.filter((node) => node.stargazers.totalCount !== 0) + nodes.push(...nodesWithStars); + hasNextPage = (allNodes.length === nodesWithStars.length) && res.data.data.user.repositories.pageInfo.hasNextPage endCursor = res.data.data.user.repositories.pageInfo.endCursor } diff --git a/tests/fetchStats.test.js b/tests/fetchStats.test.js index eeacf30d9649e..89d7f12e0a221 100644 --- a/tests/fetchStats.test.js +++ b/tests/fetchStats.test.js @@ -59,6 +59,26 @@ const secondRepositoriesData = { }, }; +const repositoriesWithZeroStarsData = { + data: { + user: { + repositories: { + nodes: [ + { name: "test-repo-1", stargazers: { totalCount: 100 } }, + { name: "test-repo-2", stargazers: { totalCount: 100 } }, + { name: "test-repo-3", stargazers: { totalCount: 100 } }, + { name: "test-repo-4", stargazers: { totalCount: 0 } }, + { name: "test-repo-5", stargazers: { totalCount: 0 } } + ], + pageInfo: { + hasNextPage: true, + cursor: "cursor" + } + }, + }, + }, +}; + const error = { errors: [ { @@ -109,6 +129,35 @@ describe("Test fetchStats", () => { }); }); + it("should stop fetching when there are repos with zero stars", async () => { + mock.reset(); + mock.onPost("https://api.github.com/graphql") + .replyOnce(200, data) + .onPost("https://api.github.com/graphql") + .replyOnce(200, repositoriesWithZeroStarsData) + + let stats = await fetchStats("anuraghazra"); + const rank = calculateRank({ + totalCommits: 100, + totalRepos: 5, + followers: 100, + contributions: 61, + stargazers: 300, + prs: 300, + issues: 200, + }); + + expect(stats).toStrictEqual({ + contributedTo: 61, + name: "Anurag Hazra", + totalCommits: 100, + totalIssues: 200, + totalPRs: 300, + totalStars: 300, + rank, + }); + }); + it("should throw error", async () => { mock.reset(); mock.onPost("https://api.github.com/graphql").reply(200, error); From ef9d017743d09959fabe2556b7c9c6fce3194053 Mon Sep 17 00:00:00 2001 From: Matteo Pierro Date: Tue, 4 Oct 2022 13:29:11 +0300 Subject: [PATCH 3/6] remove not needed parameters from the query --- src/fetchers/stats-fetcher.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/fetchers/stats-fetcher.js b/src/fetchers/stats-fetcher.js index 04dfc3bcb54b5..94b1f7f378546 100644 --- a/src/fetchers/stats-fetcher.js +++ b/src/fetchers/stats-fetcher.js @@ -29,10 +29,10 @@ const fetcher = (variables, token) => { totalCommitContributions restrictedContributionsCount } - repositoriesContributedTo(first: 1, contributionTypes: [COMMIT, ISSUE, PULL_REQUEST, REPOSITORY]) { + repositoriesContributedTo(contributionTypes: [COMMIT, ISSUE, PULL_REQUEST, REPOSITORY]) { totalCount } - pullRequests(first: 1) { + pullRequests { totalCount } openIssues: issues(states: OPEN) { From cb6550fd163578b735206ff4982b434d6868ca22 Mon Sep 17 00:00:00 2001 From: Matteo Pierro Date: Tue, 4 Oct 2022 13:34:01 +0300 Subject: [PATCH 4/6] add docstring --- src/fetchers/stats-fetcher.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/fetchers/stats-fetcher.js b/src/fetchers/stats-fetcher.js index 94b1f7f378546..37e5db8101823 100644 --- a/src/fetchers/stats-fetcher.js +++ b/src/fetchers/stats-fetcher.js @@ -126,6 +126,11 @@ const totalCommitsFetcher = async (username) => { return 0; }; +/** + * Fetch all the stars for all the repositories of a given username + * @param {string} username + * @param {array} repoToHide + */ const totalStarsFetcher = async (username, repoToHide) => { let nodes = []; let hasNextPage = true From cfdc161200174d1dd1ae80e8fa388acd8f4ed435 Mon Sep 17 00:00:00 2001 From: Matteo Pierro Date: Tue, 4 Oct 2022 13:54:32 +0300 Subject: [PATCH 5/6] removed not needed mock --- tests/api.test.js | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/api.test.js b/tests/api.test.js index ff1d1f8039ad6..2ea193b83ba49 100644 --- a/tests/api.test.js +++ b/tests/api.test.js @@ -154,10 +154,6 @@ describe("Test /api/", () => { it("should have proper cache", async () => { const { req, res } = faker({}, data); - mock.onPost("https://api.github.com/graphql") - .replyOnce(200, data) - .onPost("https://api.github.com/graphql") - .replyOnce(200, repositoriesData); await api(req, res); From 1c88d7405ee5e04a9af6c93ad761046917b9faf2 Mon Sep 17 00:00:00 2001 From: rickstaa Date: Tue, 4 Oct 2022 13:00:16 +0200 Subject: [PATCH 6/6] style: update formatting --- src/fetchers/stats-fetcher.js | 16 ++++++++++------ tests/api.test.js | 9 +++++---- tests/fetchStats.test.js | 28 +++++++++++++++------------- 3 files changed, 30 insertions(+), 23 deletions(-) diff --git a/src/fetchers/stats-fetcher.js b/src/fetchers/stats-fetcher.js index 37e5db8101823..8f8a820013068 100644 --- a/src/fetchers/stats-fetcher.js +++ b/src/fetchers/stats-fetcher.js @@ -133,8 +133,8 @@ const totalCommitsFetcher = async (username) => { */ const totalStarsFetcher = async (username, repoToHide) => { let nodes = []; - let hasNextPage = true - let endCursor = null + let hasNextPage = true; + let endCursor = null; while (hasNextPage) { const variables = { login: username, first: 100, after: endCursor }; let res = await retryer(repositoriesFetcher, variables); @@ -148,16 +148,20 @@ const totalStarsFetcher = async (username, repoToHide) => { } const allNodes = res.data.data.user.repositories.nodes; - const nodesWithStars = allNodes.filter((node) => node.stargazers.totalCount !== 0) + const nodesWithStars = allNodes.filter( + (node) => node.stargazers.totalCount !== 0, + ); nodes.push(...nodesWithStars); - hasNextPage = (allNodes.length === nodesWithStars.length) && res.data.data.user.repositories.pageInfo.hasNextPage - endCursor = res.data.data.user.repositories.pageInfo.endCursor + hasNextPage = + allNodes.length === nodesWithStars.length && + res.data.data.user.repositories.pageInfo.hasNextPage; + endCursor = res.data.data.user.repositories.pageInfo.endCursor; } return nodes .filter((data) => !repoToHide[data.name]) .reduce((prev, curr) => prev + curr.stargazers.totalCount, 0); -} +}; /** * @param {string} username diff --git a/tests/api.test.js b/tests/api.test.js index 2ea193b83ba49..a6bb0920449e4 100644 --- a/tests/api.test.js +++ b/tests/api.test.js @@ -39,7 +39,7 @@ const data = { closedIssues: { totalCount: 0 }, followers: { totalCount: 0 }, repositories: { - totalCount: 1 + totalCount: 1, }, }, }, @@ -52,8 +52,8 @@ const repositoriesData = { nodes: [{ stargazers: { totalCount: 100 } }], pageInfo: { hasNextPage: false, - cursor: "cursor" - } + cursor: "cursor", + }, }, }, }, @@ -83,7 +83,8 @@ const faker = (query, data) => { setHeader: jest.fn(), send: jest.fn(), }; - mock.onPost("https://api.github.com/graphql") + mock + .onPost("https://api.github.com/graphql") .replyOnce(200, data) .onPost("https://api.github.com/graphql") .replyOnce(200, repositoriesData); diff --git a/tests/fetchStats.test.js b/tests/fetchStats.test.js index 89d7f12e0a221..9ccdcb2163f6d 100644 --- a/tests/fetchStats.test.js +++ b/tests/fetchStats.test.js @@ -18,7 +18,7 @@ const data = { closedIssues: { totalCount: 100 }, followers: { totalCount: 100 }, repositories: { - totalCount: 5 + totalCount: 5, }, }, }, @@ -31,12 +31,12 @@ const firstRepositoriesData = { nodes: [ { name: "test-repo-1", stargazers: { totalCount: 100 } }, { name: "test-repo-2", stargazers: { totalCount: 100 } }, - { name: "test-repo-3", stargazers: { totalCount: 100 } } + { name: "test-repo-3", stargazers: { totalCount: 100 } }, ], pageInfo: { hasNextPage: true, - cursor: "cursor" - } + cursor: "cursor", + }, }, }, }, @@ -52,8 +52,8 @@ const secondRepositoriesData = { ], pageInfo: { hasNextPage: false, - cursor: "cursor" - } + cursor: "cursor", + }, }, }, }, @@ -68,12 +68,12 @@ const repositoriesWithZeroStarsData = { { name: "test-repo-2", stargazers: { totalCount: 100 } }, { name: "test-repo-3", stargazers: { totalCount: 100 } }, { name: "test-repo-4", stargazers: { totalCount: 0 } }, - { name: "test-repo-5", stargazers: { totalCount: 0 } } + { name: "test-repo-5", stargazers: { totalCount: 0 } }, ], pageInfo: { hasNextPage: true, - cursor: "cursor" - } + cursor: "cursor", + }, }, }, }, @@ -92,8 +92,9 @@ const error = { const mock = new MockAdapter(axios); -beforeEach( () => { - mock.onPost("https://api.github.com/graphql") +beforeEach(() => { + mock + .onPost("https://api.github.com/graphql") .replyOnce(200, data) .onPost("https://api.github.com/graphql") .replyOnce(200, firstRepositoriesData) @@ -131,10 +132,11 @@ describe("Test fetchStats", () => { it("should stop fetching when there are repos with zero stars", async () => { mock.reset(); - mock.onPost("https://api.github.com/graphql") + mock + .onPost("https://api.github.com/graphql") .replyOnce(200, data) .onPost("https://api.github.com/graphql") - .replyOnce(200, repositoriesWithZeroStarsData) + .replyOnce(200, repositoriesWithZeroStarsData); let stats = await fetchStats("anuraghazra"); const rank = calculateRank({