Merge pull request #627 from kmyk/fix/207

parse HTML of AOJ problems when samples not registered in API
online-judge-tools · Dec 4, 2019 · 446f27c · 446f27c
2 parents 430a857 + bed8598
commit 446f27c
Show file tree

Hide file tree

Showing 2 changed files with 42 additions and 3 deletions.
diff --git a/onlinejudge/service/aoj.py b/onlinejudge/service/aoj.py
@@ -12,9 +12,11 @@
 import urllib.parse
 from typing import *
 
+import bs4
 import requests
 
 import onlinejudge._implementation.logging as log
+import onlinejudge._implementation.testcase_zipper
 import onlinejudge._implementation.utils as utils
 import onlinejudge.type
 from onlinejudge.type import TestCase
@@ -47,6 +49,7 @@ def __init__(self, *, problem_id):
 
     def download_sample_cases(self, *, session: Optional[requests.Session] = None) -> List[TestCase]:
         session = session or utils.get_default_session()
+
         # get samples via the official API
         # reference: http://developers.u-aizu.ac.jp/api?key=judgedat%2Ftestcases%2Fsamples%2F%7BproblemId%7D_GET
         url = 'https://judgedat.u-aizu.ac.jp/testcases/samples/{}'.format(self.problem_id)
@@ -60,6 +63,29 @@ def download_sample_cases(self, *, session: Optional[requests.Session] = None) -
                 str(sample['serial']),
                 sample['out'].encode(),
             )]
+
+        # parse HTML if no samples are registered
+        # see: https://github.com/kmyk/online-judge-tools/issues/207
+        if not samples:
+            log.warning("sample cases are not registered in the official API")
+            log.status("fallback: parsing HTML")
+
+            # reference: http://developers.u-aizu.ac.jp/api?key=judgeapi%2Fresources%2Fdescriptions%2F%7Blang%7D%2F%7Bproblem_id%7D_GET
+            url = 'https://judgeapi.u-aizu.ac.jp/resources/descriptions/ja/{}'.format(self.problem_id)
+            resp = utils.request('GET', url, session=session)
+            html = json.loads(resp.content.decode(resp.encoding))['html']
+
+            # list h3+pre
+            zipper = onlinejudge._implementation.testcase_zipper.SampleZipper()
+            expected_strings = ('入力例', '出力例', 'Sample Input', 'Sample Output')
+            soup = bs4.BeautifulSoup(html, utils.html_parser)
+            for pre in soup.find_all('pre'):
+                tag = pre.find_previous_sibling()
+                if tag and tag.name == 'h3' and tag.string and any(s in tag.string for s in expected_strings):
+                    s = utils.textfile(utils.parse_content(pre).lstrip())
+                    zipper.add(s.encode(), tag.string)
+            samples = zipper.get()
+
         return samples
 
     def download_system_cases(self, *, session: Optional[requests.Session] = None) -> List[TestCase]:
@@ -153,7 +179,6 @@ def get_problem_id(self, *, session: Optional[requests.Session] = None) -> str:
         return self._problem_id
 
     def download_sample_cases(self, *, session: Optional[requests.Session] = None) -> List[TestCase]:
-        log.warning("most of problems in arena have no registered sample cases.")
         return AOJProblem(problem_id=self.get_problem_id()).download_sample_cases(session=session)
 
     def download_system_cases(self, *, session: Optional[requests.Session] = None) -> List[TestCase]:

diff --git a/tests/service_aoj.py b/tests/service_aoj.py
@@ -39,6 +39,15 @@ def test_download_sample_cases(self):
             TestCase(name='sample-1', input_name='1', input_data=b'4\n0 0\n10 0\n10 10\n0 10\n3\n0 0\n1 0\n0 1\n0\n', output_name='1', output_data=b'Case 1: 14.142135624\nCase 2: 1.41421356\n'),
         ])
 
+    def test_download_sample_cases_not_registered(self):
+        # see: https://github.com/kmyk/online-judge-tools/issues/207
+        self.assertEqual(AOJProblem.from_url('https://onlinejudge.u-aizu.ac.jp/challenges/sources/ICPC/Regional/1399').download_sample_cases(), [
+            TestCase(name='sample-1', input_name='Sample Input 1', input_data=b'5\n1 2 3 4 5\n1 2 3 4 5\n', output_name=' Sample Output 1', output_data=b'2 3 4 5 1\n'),
+            TestCase(name='sample-2', input_name='Sample Input 2', input_data=b'5\n3 4 5 6 7\n1 3 5 7 9\n', output_name='Sample Output 2', output_data=b'9 5 7 3 1\n'),
+            TestCase(name='sample-3', input_name='Sample Input 3', input_data=b'5\n3 2 2 1 1\n1 1 2 2 3\n', output_name='Sample Output 3', output_data=b'1 3 1 2 2\n'),
+            TestCase(name='sample-4', input_name='Sample Input 4', input_data=b'5\n3 4 10 4 9\n2 7 3 6 9\n', output_name=' Sample Output 4', output_data=b'9 7 3 6 2\n'),
+        ])
+
 
 class AOJArenaProblemTest(unittest.TestCase):
     def test_from_url(self):
@@ -53,6 +62,11 @@ def test_download_sample_cases(self):
             TestCase(name='sample-1', input_name='1', input_data=b'koukyoukoukokukikou\nabrakadabra\nacmicpc\njapaque\nhelloworld\n#\n', output_name='1', output_data=b'0\n2\n4\n5\n7\n'),
         ])
 
-    @unittest.expectedFailure
     def test_download_sample_cases_not_registered(self):
-        self.assertNotEqual(AOJArenaProblem.from_url('https://onlinejudge.u-aizu.ac.jp/services/room.html#RitsCamp18Day3/problems/B').download_sample_cases(), [])
+        # see: https://github.com/kmyk/online-judge-tools/issues/207
+        self.assertEqual(AOJArenaProblem.from_url('https://onlinejudge.u-aizu.ac.jp/services/room.html#RitsCamp18Day3/problems/B').download_sample_cases(), [
+            TestCase(name='sample-1', input_name='入力例1', input_data=b'4\n2 0 -2 1\n', output_name='出力例1', output_data=b'1\n1\n'),
+            TestCase(name='sample-2', input_name='入力例2', input_data=b'3\n2 -2 -2\n', output_name='出力例2', output_data=b'3\n1\n2\n3\n'),
+            TestCase(name='sample-3', input_name='入力例3', input_data=b'2\n-1 0\n', output_name='出力例3', output_data=b'0\n'),
+            TestCase(name='sample-4', input_name='入力例4', input_data=b'5\n-1 2 1 -2 -1\n', output_name='出力例4', output_data=b'3\n1\n2\n4\n'),
+        ])