#403: add a feature to download MatchOverview of Topcoder Marathon Match

onlinejudge/service/topcoder.py

@@ -13,6 +13,7 @@
 import re
 import time
 import urllib.parse
+import xml.etree.ElementTree
 from typing import *
 
 import bs4
@@ -72,6 +73,39 @@ def from_url(cls, url: str) -> Optional['TopcoderService']:
         return None
 
 
+TopcoderLongContestProblemOverviewRow = NamedTuple('TopcoderLongContestProblemOverviewRow', [
+    ('rank', int),
+    ('handle', str),
+    ('provisional_rank', int),
+    ('provisional_score', float),
+    ('final_score', float),
+    ('language', str),
+    ('cr', int),
+])
+
+TopcoderLongContestProblemIndividualResultsFeedSubmission = NamedTuple('TopcoderLongContestProblemIndividualResultsFeedSubmission', [
+    ('number', int),
+    ('score', float),
+    ('language', str),
+    ('time', str),
+])
+
+TopcoderLongContestProblemIndividualResultsFeedTestCase = NamedTuple('TopcoderLongContestProblemIndividualResultsFeedTestCase', [
+    ('test_case_id', int),
+    ('score', float),
+    ('processing_time', int),
+    ('fatal_error_ind', int),
+])
+
+TopcoderLongContestProblemIndividualResultsFeed = NamedTuple('TopcoderLongContestProblemIndividualResultsFeed', [
+    ('round_id', int),
+    ('coder_id', int),
+    ('handle', str),
+    ('submissions', List[TopcoderLongContestProblemIndividualResultsFeedSubmission]),
+    ('testcases', List[TopcoderLongContestProblemIndividualResultsFeedTestCase]),
+])
+
+
 class TopcoderLongContestProblem(onlinejudge.type.Problem):
     def __init__(self, rd, cd=None, compid=None, pm=None):
         self.rd = rd
@@ -226,6 +260,96 @@ def get_standings(self, session: Optional[requests.Session] = None) -> Tuple[Lis
         assert header is not None
         return header, rows
 
+    def download_overview(self, session: Optional[requests.Session] = None) -> List[TopcoderLongContestProblemOverviewRow]:
+        """
+        .. versionadded:: 6.2.0
+            This method may be deleted in future.
+        """
+        session = session or utils.get_default_session()
+
+        # get
+        number = 9999
+        start = 1
+        url = 'https://community.topcoder.com/longcontest/stats/?module=ViewOverview&rd={}&nr={}&sr={}'.format(self.rd, number, start)
+        resp = utils.request('GET', url, session=session)
+
+        # parse
+        soup = bs4.BeautifulSoup(resp.content.decode(resp.encoding), utils.html_parser)
+        table = soup.find('table', class_='stat')
+        overview = []  # type: List[TopcoderLongContestProblemOverviewRow]
+        for tr in table.find_all('tr', class_=re.compile(r'light|dark')):
+            tds = tr.find_all('td')
+            assert len(tds) == 9
+            rank = int(tds[0].text)
+            handle = tds[1].text.strip()
+            provisional_rank = int(tds[2].text)
+            provisional_score = float(tds[3].text)
+            final_score = float(tds[4].text)
+            language = tds[5].text.strip()
+            assert tds[6].text.strip() == 'results'
+            assert tds[7].text.strip() == 'submission history'
+            assert tds[8].text.strip() == 'example history'
+            query = dict(urllib.parse.parse_qsl(urllib.parse.urlparse(tds[6].find('a').attrs['href']).query))
+            self.pm = query['pm']
+            overview += [TopcoderLongContestProblemOverviewRow(rank, handle, provisional_rank, provisional_score, final_score, language, cr=int(query['cr']))]
+        return overview
+
+    def download_individual_results_feed(self, cr: int, session: Optional[requests.Session] = None) -> TopcoderLongContestProblemIndividualResultsFeed:
+        """
+        .. versionadded:: 6.2.0
+            This method may be deleted in future.
+        """
+        session = session or utils.get_default_session()
+
+        # get
+        url = 'https://community.topcoder.com/longcontest/stats/?module=IndividualResultsFeed&rd={}&cr={}'.format(self.rd, cr)
+        resp = utils.request('GET', url, session=session)
+
+        # parse
+        root = xml.etree.ElementTree.fromstring(resp.content.decode(resp.encoding))
+        assert len(list(root)) == 5
+        round_id = int(list(root)[0].text or '')  # NOTE: `or ''` is for the typechecker
+        coder_id = int(list(root)[1].text or '')
+        handle = list(root)[2].text
+        assert handle is not None
+        submissions = []  # type: List[TopcoderLongContestProblemIndividualResultsFeedSubmission]
+        for submission in list(root)[3]:
+            number = int(list(submission)[0].text or '')
+            score = float(list(submission)[1].text or '')
+            language = list(submission)[2].text or ''
+            time = list(submission)[3].text
+            assert time is not None
+            submissions += [TopcoderLongContestProblemIndividualResultsFeedSubmission(number, score, language, time)]
+        testcases = []  # type: List[TopcoderLongContestProblemIndividualResultsFeedTestCase]
+        for testcase in list(root)[4]:
+            test_case_id = int(list(testcase)[0].text or '')
+            score = float(list(testcase)[1].text or '')
+            processing_time = int(list(testcase)[2].text or '')
+            fatal_error_ind = int(list(testcase)[3].text or '')
+            testcases += [TopcoderLongContestProblemIndividualResultsFeedTestCase(test_case_id, score, processing_time, fatal_error_ind)]
+        return TopcoderLongContestProblemIndividualResultsFeed(round_id, coder_id, handle, submissions, testcases)
+
+    def download_system_test(self, test_case_id: int, session: Optional[requests.Session] = None) -> str:
+        """
+        :raises NotLoggedInError:
+        :note: You need to parse this result manually.
+
+        .. versionadded:: 6.2.0
+            This method may be deleted in future.
+        """
+        session = session or utils.get_default_session()
+
+        # get
+        assert self.pm is not None
+        url = 'https://community.topcoder.com/longcontest/stats/?module=ViewSystemTest&rd={}&pm={}&tid={}'.format(self.rd, self.pm, test_case_id)
+        resp = utils.request('GET', url, session=session)
+
+        # parse
+        soup = bs4.BeautifulSoup(resp.content.decode(resp.encoding), utils.html_parser)
+        if soup.find('form', attrs={'name': 'frmLogin'}):
+            raise NotLoggedInError
+        return soup.find('pre').text
+
 
 onlinejudge.dispatch.services += [TopcoderService]
 onlinejudge.dispatch.problems += [TopcoderLongContestProblem]

tests/service_topcoder.py

@@ -0,0 +1,87 @@
+import os
+import unittest
+
+import onlinejudge._implementation.utils as utils
+from onlinejudge.service.topcoder import TopcoderLongContestProblem, TopcoderService
+
+
+class TopcoderSerivceTest(unittest.TestCase):
+    def test_from_url(self):
+        self.assertIsInstance(TopcoderService.from_url('https://community.topcoder.com/'), TopcoderService)
+        self.assertIsNone(TopcoderService.from_url('https://atcoder.jp/'))
+
+
+class TopcoderLongConrestProblemTest(unittest.TestCase):
+    def test_from_url(self):
+        self.assertEqual(TopcoderLongContestProblem.from_url('https://community.topcoder.com/longcontest/?module=ViewProblemStatement&rd=17092&pm=14853').rd, 17092)
+        self.assertEqual(TopcoderLongContestProblem.from_url('https://community.topcoder.com/longcontest/?module=ViewProblemStatement&rd=17092&pm=14853').pm, 14853)
+
+    def test_download_overview(self):
+        problem = TopcoderLongContestProblem.from_url('https://community.topcoder.com/longcontest/?module=ViewProblemStatement&rd=17143&pm=14889')
+        overview = problem.download_overview()
+        self.assertEqual(len(overview), 282)
+        self.assertEqual(overview[4].rank, 5)
+        self.assertEqual(overview[4].handle, 'hakomo')
+        self.assertEqual(overview[4].provisional_rank, 6)
+        self.assertAlmostEqual(overview[4].provisional_score, 812366.55)
+        self.assertAlmostEqual(overview[4].final_score, 791163.70)
+        self.assertEqual(overview[4].language, 'C++')
+        self.assertEqual(overview[4].cr, 22924522)
+
+    def test_download_individual_results_feed(self):
+        problem = TopcoderLongContestProblem.from_url('https://community.topcoder.com/longcontest/?module=ViewProblemStatement&rd=17143&pm=14889')
+        cr = 22924522
+        feed = problem.download_individual_results_feed(cr)
+        self.assertEqual(feed.round_id, problem.rd)
+        self.assertEqual(feed.coder_id, cr)
+        self.assertEqual(feed.handle, 'hakomo')
+        self.assertEqual(feed.submissions[0].number, 1)
+        self.assertAlmostEqual(feed.submissions[0].score, 816622.81)
+        self.assertEqual(feed.submissions[0].language, 'C++')
+        self.assertEqual(feed.submissions[0].time, '04/22/2018 09:56:48')
+        self.assertEqual(feed.testcases[0].test_case_id, 33800773)
+        self.assertAlmostEqual(feed.testcases[0].score, 1.0)
+        self.assertEqual(feed.testcases[0].processing_time, 164)
+        self.assertEqual(feed.testcases[0].fatal_error_ind, 0)
+        self.assertEqual(len(feed.submissions), 5)
+        self.assertEqual(len(feed.testcases), 2000)
+
+    @unittest.skipIf('CI' in os.environ, 'login is required')
+    def test_download_system_test(self):
+        with utils.with_cookiejar(utils.get_default_session()):
+            url = 'https://community.topcoder.com/longcontest/?module=ViewProblemStatement&rd=17143&pm=14889'
+            tid = 33800773
+            problem = TopcoderLongContestProblem.from_url(url)
+            self.assertEqual(problem.download_system_test(tid), 'seed = 1919427468645\nH = 85\nW = 88\nC = 2\n')
+
+        with utils.with_cookiejar(utils.get_default_session()):
+            url = 'https://community.topcoder.com/longcontest/?module=ViewProblemStatement&rd=17092&pm=14853'
+            tid = 33796324
+            problem = TopcoderLongContestProblem.from_url(url)
+            self.assertEqual(problem.download_system_test(tid), """\
+Seed = 2917103922548
+
+Coins: 5372
+Max Time: 2988
+Note Time: 5
+Num Machines: 3
+
+Machine 0...
+Wheel 0: ACEEDEDBDGBADCDFGD
+Wheel 1: GGFEFBFDFFDEECFEAG
+Wheel 2: EFCCCAADBDGEGBDCDD
+Expected payout rate: 1.5775034293552812
+
+Machine 1...
+Wheel 0: CDFFDEEEAGGGGGFGGBEFCCFFFD
+Wheel 1: EDCGBGFBBCCGGFGDFBFECGGEFC
+Wheel 2: GEDECEGFDCGDGGCDDCEDGBGEBG
+Expected payout rate: 0.7345243513882568
+
+Machine 2...
+Wheel 0: ABEEDDDCGBG
+Wheel 1: EDEEDADGEAF
+Wheel 2: EBEGEFEGEBF
+Expected payout rate: 0.6160781367392938
+
+""")