From 9ee1838cb9a2c76cd9e6ea1ede3ad8fce1f634a2 Mon Sep 17 00:00:00 2001 From: Kazi Fozle Azim Rabi Date: Tue, 5 Feb 2019 10:06:32 +0000 Subject: [PATCH 01/12] Add README.md --- README.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 00000000..e69de29b From b90926d60fa647c242318643f13bebad26705ece Mon Sep 17 00:00:00 2001 From: Kazi Rabi Date: Mon, 11 Feb 2019 14:12:35 +0600 Subject: [PATCH 02/12] Add .gitignore to avoid __pycache__ and test folders --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..169ce1b6 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +__pycache__ +test \ No newline at end of file From 42d7e379c05359974a1fda48b6bd0b9bbf23ed2c Mon Sep 17 00:00:00 2001 From: Kazi Rabi Date: Tue, 12 Feb 2019 17:09:53 +0600 Subject: [PATCH 03/12] Add support for Toph problem archive --- .gitignore | 3 +- onlinejudge/service/__init__.py | 1 + onlinejudge/service/toph.py | 88 +++++++++++++++++++++++++++++++++ 3 files changed, 91 insertions(+), 1 deletion(-) create mode 100644 onlinejudge/service/toph.py diff --git a/.gitignore b/.gitignore index 169ce1b6..35eadee2 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ __pycache__ -test \ No newline at end of file +test +.vscode \ No newline at end of file diff --git a/onlinejudge/service/__init__.py b/onlinejudge/service/__init__.py index 462223b9..87e98353 100644 --- a/onlinejudge/service/__init__.py +++ b/onlinejudge/service/__init__.py @@ -6,3 +6,4 @@ import onlinejudge.service.csacademy import onlinejudge.service.topcoder import onlinejudge.service.yukicoder +import onlinejudge.service.toph diff --git a/onlinejudge/service/toph.py b/onlinejudge/service/toph.py new file mode 100644 index 00000000..df50244b --- /dev/null +++ b/onlinejudge/service/toph.py @@ -0,0 +1,88 @@ +# Python Version: 3.x +import posixpath +import string +import urllib.parse +from typing import * + +import bs4 +import requests +import re + +import onlinejudge.dispatch +import onlinejudge.implementation.logging as log +import onlinejudge.implementation.utils as utils +import onlinejudge.type +from onlinejudge.type import SubmissionError + + +@utils.singleton +class TophService(onlinejudge.type.Service): + def get_url(self) -> str: + return 'https://toph.co/' + + def get_name(self) -> str: + return 'toph' + + @classmethod + def from_url(cls, s: str) -> Optional['TophService']: + # example: https://toph.co/ + # example: http://toph.co/ + result = urllib.parse.urlparse(s) + if result.scheme in ('', 'http', 'https') \ + and result.netloc == 'toph.co': + return cls() + return None + +class TophProblem(onlinejudge.type.Problem): + def __init__(self, slug: str, kind: Optional[str] = None): + assert isinstance(slug, str) + assert kind in (None, 'problem') + self.kind = kind + self.slug = slug + + def download_sample_cases(self, session: Optional[requests.Session] = None) -> List[onlinejudge.type.TestCase]: + session = session or utils.new_default_session() + resp = utils.request('GET', self.get_url(), session=session) + soup = bs4.BeautifulSoup(resp.content.decode(resp.encoding), utils.html_parser) + samples = utils.SampleZipper() + for case in soup.find('table', class_="samples").find('tbody').find_all('tr'): + log.debug('case: %s', str(case)) + assert len(list(case.children)) + input_pre, output_pre = list(map(lambda td: td.find('pre'), list(case.children))) + assert input_pre.name == 'pre' + assert output_pre.name == 'pre' + assert re.search("^preSample.*Input$", input_pre.attrs['id']) + assert re.search("^preSample.*Output$", output_pre.attrs['id']) + s = input_pre.get_text() + s = s.lstrip() + samples.add(s, "Input") + s = output_pre.get_text() + s = s.lstrip() + samples.add(s, "Output") + return samples.get() + + def get_url(self) -> str: + table = {} + table['problem'] = 'https://toph.co/p/{}' + return table[self.kind].format(self.slug) + + def get_service(self) -> TophService: + return TophService() + + @classmethod + def from_url(cls, s: str) -> Optional['TophProblem']: + result = urllib.parse.urlparse(s) + dirname, basename = posixpath.split(utils.normpath(result.path)) + if result.scheme in ('', 'http', 'https') \ + and result.netloc.count('.') == 1 \ + and result.netloc.endswith('toph.co') \ + and dirname == '/p' \ + and basename: + kind = 'problem' + slug = basename + return cls(slug, kind) + + return None + +onlinejudge.dispatch.services += [TophService] +onlinejudge.dispatch.problems += [TophProblem] From 004fe62071204b504085d8be37fa816b0331bac8 Mon Sep 17 00:00:00 2001 From: Kazi Rabi Date: Mon, 18 Feb 2019 14:36:23 +0600 Subject: [PATCH 04/12] Implement passwordbased login for Toph --- onlinejudge/service/toph.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/onlinejudge/service/toph.py b/onlinejudge/service/toph.py index df50244b..daa778f9 100644 --- a/onlinejudge/service/toph.py +++ b/onlinejudge/service/toph.py @@ -17,6 +17,41 @@ @utils.singleton class TophService(onlinejudge.type.Service): + def login(self, get_credentials: onlinejudge.type.CredentialsProvider, session: Optional[requests.Session] = None) -> bool: + session = session or utils.new_default_session() + url = 'https://toph.co/login' + # get + resp = utils.request('GET', url, session=session) + if resp.url != url: # redirected + log.info('You are already logged in.') + return True + # parse + soup = bs4.BeautifulSoup(resp.content.decode(resp.encoding), utils.html_parser) + form = soup.find('form', class_='login-form') + log.debug('form: %s', str(form)) + username, password = get_credentials() + form['action'] = '/login' # to avoid KeyError inside form.request method as Toph does not have any defined action + form = utils.FormSender(form, url=resp.url) + form.set('handle', username) + form.set('password', password) + # post + resp = form.request(session) + resp.raise_for_status() + + newResp = utils.request('GET', url, session=session) # Toph's Location header is not getting the expected value + if newResp.url != url: + log.success('Welcome, %s.', username) + return True + else: + log.failure('Invalid handle/email or password.') + return False + + def is_logged_in(self, session: Optional[requests.Session] = None) -> bool: + session = session or utils.new_default_session() + url = 'https://toph.co/login' + resp = utils.request('GET', url, session=session, allow_redirects=False) + return resp.status_code == 302 + def get_url(self) -> str: return 'https://toph.co/' From 59854439df3426c82a024ea16e82a82011e30042 Mon Sep 17 00:00:00 2001 From: Kazi Rabi Date: Mon, 18 Feb 2019 16:10:37 +0600 Subject: [PATCH 05/12] Implement sumission for Toph --- onlinejudge/service/toph.py | 50 +++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/onlinejudge/service/toph.py b/onlinejudge/service/toph.py index daa778f9..b209ccef 100644 --- a/onlinejudge/service/toph.py +++ b/onlinejudge/service/toph.py @@ -96,6 +96,56 @@ def download_sample_cases(self, session: Optional[requests.Session] = None) -> L samples.add(s, "Output") return samples.get() + def get_language_dict(self, session: Optional['requests.Session'] = None) -> Dict[str, onlinejudge.type.Language]: + session = session or utils.new_default_session() + # get + resp = utils.request('GET', self.get_url(), session=session) + # parse + soup = bs4.BeautifulSoup(resp.content.decode(resp.encoding), utils.html_parser) + select = soup.find('select', attrs={'name': 'languageId'}) + if select is None: + log.error('not logged in') + return {} + language_dict = {} + for option in select.findAll('option'): + language_dict[option.attrs['value']] = {'description': option.string} + return language_dict + + def submit_code(self, code: bytes, language: str, session: Optional['requests.Session'] = None) -> onlinejudge.type.Submission: # or SubmissionError + session = session or utils.new_default_session() + # get + resp = utils.request('GET', self.get_url(), session=session) + # parse + soup = bs4.BeautifulSoup(resp.content.decode(resp.encoding), utils.html_parser) + form = soup.find('form') + if form is None: + log.error('not logged in') + raise SubmissionError + log.debug('form: %s', str(form)) + if form.find('select') and form.find('select').attrs['name'] != 'languageId': + log.error("Wrong submission URL") + raise SubmissionError + + # make data + form = utils.FormSender(form, url=resp.url) + form.set('languageId', language) + form.set_file('source', 'code', code) + resp = form.request(session=session) + resp.raise_for_status() + # result + if '/s/' in resp.url: + # example: https://codeforces.com/contest/598/my + log.success('success: result: %s', resp.url) + return onlinejudge.type.DummySubmission(resp.url) + else: + log.failure('failure') + log.debug('redirected to %s', resp.url) + # parse error messages + # soup = bs4.BeautifulSoup(resp.content.decode(resp.encoding), utils.html_parser) + # for span in soup.findAll('span', class_='error'): + # log.warning('Codeforces says: "%s"', span.string) + raise SubmissionError + def get_url(self) -> str: table = {} table['problem'] = 'https://toph.co/p/{}' From 5a49bd816492c58f1915150769904fda84249097 Mon Sep 17 00:00:00 2001 From: Kazi Rabi Date: Mon, 18 Feb 2019 16:16:09 +0600 Subject: [PATCH 06/12] Update readme.md features section --- README.md | 0 readme.md | 3 +++ 2 files changed, 3 insertions(+) delete mode 100644 README.md diff --git a/README.md b/README.md deleted file mode 100644 index e69de29b..00000000 diff --git a/readme.md b/readme.md index 9cdf054d..a4e85c4f 100644 --- a/readme.md +++ b/readme.md @@ -25,6 +25,7 @@ Tools for online judge services. Downloading sample cases, Testing/Submitting yo - ~~HackerRank~~ (removed) - Aizu Online Judge - CS Academy + - Toph - Download system test cases - Yukicoder - Aizu Online Judge @@ -34,12 +35,14 @@ Tools for online judge services. Downloading sample cases, Testing/Submitting yo - Codeforces - ~~HackerRank~~ (removed) - TopCoder + - Toph - Submit your solution - AtCoder - Yukicoder - Codeforces - ~~HackerRank~~ (removed) - TopCoder (Marathon Match) + - Toph - Generate scanner for input (experimental) - AtCoder - Yukicoder From bfc7e8b8f38f173a519b3a7375ee33eb7a3a2fe6 Mon Sep 17 00:00:00 2001 From: Kazi Fozle Azim Rabi Date: Tue, 19 Feb 2019 14:04:13 +0000 Subject: [PATCH 07/12] Resolve "Add support for Toph problem archive" --- .gitignore | 3 - CHANGELOG.md | 9 + CONTRIBUTING.md | 69 ++++-- docs/onlinejudge.service.rst | 8 + onlinejudge/__about__.py | 2 +- .../implementation/command/download.py | 3 +- onlinejudge/implementation/main.py | 10 +- onlinejudge/implementation/utils.py | 5 +- onlinejudge/service/__init__.py | 5 +- onlinejudge/service/hackerrank.py | 216 ++++++++++++++++++ onlinejudge/service/kattis.py | 95 ++++++++ onlinejudge/service/poj.py | 97 ++++++++ onlinejudge/service/toph.py | 31 ++- onlinejudge/service/yukicoder.py | 2 +- readme.md | 12 +- tests/command_download.py | 4 +- tests/command_download_hackerrank.py | 46 ++++ tests/command_download_kattis.py | 35 +++ tests/command_download_others.py | 31 +++ tests/command_login.py | 8 + tests/command_submit.py | 29 ++- 21 files changed, 668 insertions(+), 52 deletions(-) delete mode 100644 .gitignore create mode 100644 onlinejudge/service/hackerrank.py create mode 100644 onlinejudge/service/kattis.py create mode 100644 onlinejudge/service/poj.py create mode 100644 tests/command_download_hackerrank.py create mode 100644 tests/command_download_kattis.py diff --git a/.gitignore b/.gitignore deleted file mode 100644 index 35eadee2..00000000 --- a/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -__pycache__ -test -.vscode \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 62ca736b..62843926 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # Change Log +## 0.1.54 / 2019-02-07 + +- [#113](https://github.com/kmyk/online-judge-tools/issues/113) support `download` from POJ +- [#208](https://github.com/kmyk/online-judge-tools/issues/208) support `download` and `submit` for HackerRank again +- [#275](https://github.com/kmyk/online-judge-tools/issues/275) support `submit` to Yukicoder again +- [#276](https://github.com/kmyk/online-judge-tools/issues/276) add messages and document to login with directly editing session tokens on `cookie.jar` +- [#200](https://github.com/kmyk/online-judge-tools/issues/200) add `--check` option to `login` command +- [#290](https://github.com/kmyk/online-judge-tools/issues/290) add an error message for when `setuptools` is too old + ## 0.1.53 / 2019-01-28 - [@fukatani](https://github.com/fukatani) (AtCoder: [ryoryoryo111](https://atcoder.jp/users/ryoryoryo111)) joined as a maintainer :tada: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e3d9454b..b7d39823 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,5 +1,7 @@ # Contribute Guide / 開発を手伝ってくれる人へ +TODO: translate this document to English + ## language / 言語について In the source code and documents for end users, we should use English. @@ -10,11 +12,26 @@ For other place, both English and Japanese are acceptable. ## issues / issue について +Not only sending Pull Requests, feature requests and bug reports are welcome. + 機能要求やバグ報告は気軽にしてください。 コードを書くことだけが開発ではありません。 ## pull requests / プルリクについて +PR is always welcome. + +However, please note that PR is not always merged as it is. +To improve PR quality, reviewers may ask you change requests. + +- Test your PR branch on local by `python3 setup.py test`. +- Write code easy to understand. + - Don't make diff which is unnecessary for the purpose of PR. + - Split commits appropriately. + - Comment on the code where you are not confident of. +- If you want to add a feature, it would be better to discuss before writing code. + - because your feature is not always merged. + 基本的にはどんなものでも歓迎します。 ただし常にそのまま merge されるとは限らないので注意してください。 @@ -23,7 +40,7 @@ For other place, both English and Japanese are acceptable. - 手元でテストをする (`python3 setup.py test` を実行する) - CI が通らない限りは merge はできません - レビュアーにやさしいコードを書く - - 変更箇所は必要最低限にする + - 変更箇所はPRの目的に沿った必要最低限のものにする - commit は適切に分割する - 怪しげなところにはコメントを書いておく - 機能追加をする場合は事前に確認をする @@ -35,9 +52,9 @@ For other place, both English and Japanese are acceptable. ## philosophy of design / 設計の方針 第一義は「コンテストで上位を取ることに役立つこと」です。 -特に「ペナルティを出させないこと」に注力しています。 これを実現する手段として「手動だと間違えたりさぼったりしやすい作業を自動化する」を用いています。 +また、「ペナルティを出させないこと」に注力しています。 Web scraping をする性質により動作は必然的に不安定であり、これは「ペナルティを出させないこと」の壁となります。 これへの対応として「誤動作の起こりやすい機能は避ける」「誤動作があったときに誤動作があると気付きやすいようにする」などを重要視しています。 その実践の例としては「取得したサンプルケースを(ファイルに出力するだけでなく)画面に見やすく表示する」が分かりやすいでしょう。 @@ -45,38 +62,39 @@ Web scraping をする性質により動作は必然的に不安定であり、 ## module structure -主に以下のような構造です。 +The structure is as follows: -t- `onlinejudge/` - - `type.py`: 型はすべてここ - - `dispatch.py`: URL から object を解決する仕組み +- `onlinejudge/` + - `type.py`: contains all + - `dispatch.py`: resolves classes from URL - `implementation/` - - `main.py`: 個別のコマンドを呼び出すまでの部分 - - `command/`: `download` `submit` などのコマンドの本体が置かれる + - `main.py` + - `command/`: has the bodies of commands like `download`, `submit`, etc. - `download.py` - `submit.py` - ... - - `service/`: AtCoder, Codeforces などのサービスごとの実装が置かれる + - `service/`: has classes for services like AtCoder, Codeforces, etc. - `atcoder.py` - `codeforces.py` - ... -- `tests/`: テストが置かれる +- `tests/` ## formatter -isort と yapf を運用しています。 -行幅は実質無限に設定されています。 -それぞれ次のようなコマンドで実行できます。 +We use `isort` adn `yapf`. +You can run them with the following commands: ``` sh $ isort --recursive oj onlinejudge $ yapf --in-place --recursive oj onlinejudge ``` +The line width is set as infinity. + ## tests -静的型検査と通常のテストをしています。 -手元ではそれぞれ次のようなコマンドで実行できます。 +We use static type checking and unit testing. +You can run them with the following commands: ``` sh $ mypy oj onlinejudge @@ -94,8 +112,11 @@ $ python3 setup.py test -s tests.command_download_atcoder.DownloadAtCoderTest ## CI +Travis CI will run automatically when you commit or send PR on `master` or `develop` branch. +The same test as that by `python3 setup.py test` is executed. + `master` `develop` に関する commit や pull request について CI が走ります。 -`python3 setup.py test` の実行でも同等の処理が行われるように設定されているので、手元でこれを実行しているなら気にする必要はありません。 +`python3 setup.py test` の実行によるものと同等のテストが行われます。 ## deployment @@ -108,3 +129,19 @@ Travis CI から PyPI 上へ upload を仕掛けるように設定されてい - 例: [3a24dc](https://github.com/kmyk/online-judge-tools/commit/3a24dc64b56d898e387dee56cf9915be3ab0f7e2) 2. `v0.1.23` の形で Git tag を打って GitHub 上へ push する - これにより Travis CI の機能が呼び出され PyPI への upload がなされる + +## how to add a new contest platform / 対応サービスの追加の手順 + +Short version: see files for other platforms like `onlinejudge/service/poj.py` or `onlinejudge/service/codeforces.py`, and `tests/command_download_hackerrank.py` or `https://github.com/kmyk/online-judge-tools/blob/master/tests/command_submit.py` for tests + +Long version: + +1. make the file `onlinejudge/service/${NAME}.py` +1. write the singleton class `${NAME}Service` inheriting `onlinejudge.type.Service` + - You must implement at least methods `get_url()` `get_name()` and `cls.from_url()`, and you can ignore others. +1. write the class `${NAME}Problem` inheriting `onlinejudge.type.Problem` + - You must implement at least methods `download_sample_cases()` `get_url()` `get_service()` and `cls.from_url()`, and you can ignore others. +1. register the classes to the lists `onlinejudge.dispatch.services` and `onlinejudge.dispatch.problems` +1. register the module to the `onlinejudge/service/__init__.py` +1. write tests for your platform + - You should make `tests/command_download_${NAME}.py` and/or append to `tests/command_submit.py`. Please see other existing tests. diff --git a/docs/onlinejudge.service.rst b/docs/onlinejudge.service.rst index bf52832f..60ae3d97 100644 --- a/docs/onlinejudge.service.rst +++ b/docs/onlinejudge.service.rst @@ -44,6 +44,14 @@ onlinejudge.service.csacademy module :undoc-members: :show-inheritance: +onlinejudge.service.hackerrank module +------------------------------------ + +.. automodule:: onlinejudge.service.hackerrank + :members: + :undoc-members: + :show-inheritance: + onlinejudge.service.topcoder module ----------------------------------- diff --git a/onlinejudge/__about__.py b/onlinejudge/__about__.py index 73601825..6ea51847 100644 --- a/onlinejudge/__about__.py +++ b/onlinejudge/__about__.py @@ -4,5 +4,5 @@ __email__ = 'kimiyuki95@gmail.com' __license__ = 'MIT License' __url__ = 'https://github.com/kmyk/online-judge-tools' -__version_info__ = (0, 1, 53, 'final', 0) +__version_info__ = (0, 1, 54, 'final', 0) __version__ = '.'.join(map(str, __version_info__[:3])) diff --git a/onlinejudge/implementation/command/download.py b/onlinejudge/implementation/command/download.py index 76ebc784..ca985b8a 100644 --- a/onlinejudge/implementation/command/download.py +++ b/onlinejudge/implementation/command/download.py @@ -73,7 +73,8 @@ def download(args: 'argparse.Namespace') -> None: table['d'] = os.path.dirname(name) path = args.directory / utils.percentformat(args.format, table) # type: pathlib.Path log.status('%sput: %s', ext, name) - log.emit(colorama.Style.BRIGHT + data.rstrip() + colorama.Style.RESET_ALL) + if not args.silent: + log.emit(colorama.Style.BRIGHT + data.rstrip() + colorama.Style.RESET_ALL) if args.dry_run: continue if path.exists(): diff --git a/onlinejudge/implementation/main.py b/onlinejudge/implementation/main.py index af7652a6..8a0918e1 100644 --- a/onlinejudge/implementation/main.py +++ b/onlinejudge/implementation/main.py @@ -45,8 +45,9 @@ def get_parser() -> argparse.ArgumentParser: Codeforces Yukicoder CS Academy - - (HackerRank has been removed) + HackerRank + PKU JudgeOnline + Kattis supported services with --system: Aizu Online Judge @@ -66,6 +67,7 @@ def get_parser() -> argparse.ArgumentParser: subparser.add_argument('--overwrite', action='store_true') subparser.add_argument('-n', '--dry-run', action='store_true', help='don\'t write to files') subparser.add_argument('-a', '--system', action='store_true', help='download system testcases') + subparser.add_argument('-s', '--silent', action='store_true') subparser.add_argument('--json', action='store_true') # login @@ -75,8 +77,7 @@ def get_parser() -> argparse.ArgumentParser: Codeforces Yukicoder TopCoder - - (HackerRank has been removed) + HackerRank strings for --method: github for yukicoder, login via github (default) @@ -95,6 +96,7 @@ def get_parser() -> argparse.ArgumentParser: Codeforces TopCoder (Marathon Match) Yukicoder + HackerRank ''') subparser.add_argument('url', nargs='?', help='the URL of the problem to submit. if not given, guessed from history of download command.') diff --git a/onlinejudge/implementation/utils.py b/onlinejudge/implementation/utils.py index 35755e20..f02c6c01 100644 --- a/onlinejudge/implementation/utils.py +++ b/onlinejudge/implementation/utils.py @@ -132,10 +132,11 @@ def set_file(self, key: str, filename: str, content: bytes) -> None: def unset(self, key: str) -> None: del self.payload[key] - def request(self, session: requests.Session, action: Optional[str] = None, **kwargs) -> requests.Response: + def request(self, session: requests.Session, method: str = None, action: Optional[str] = None, **kwargs) -> requests.Response: action = action or self.form['action'] url = urllib.parse.urljoin(self.url, action) - method = self.form['method'].upper() + if method is None: + method = self.form['method'].upper() log.status('%s: %s', method, url) log.debug('payload: %s', str(self.payload)) resp = session.request(method, url, data=self.payload, files=self.files, **kwargs) diff --git a/onlinejudge/service/__init__.py b/onlinejudge/service/__init__.py index 87e98353..25c47029 100644 --- a/onlinejudge/service/__init__.py +++ b/onlinejudge/service/__init__.py @@ -4,6 +4,9 @@ import onlinejudge.service.atcoder import onlinejudge.service.codeforces import onlinejudge.service.csacademy +import onlinejudge.service.hackerrank +import onlinejudge.service.kattis +import onlinejudge.service.poj import onlinejudge.service.topcoder -import onlinejudge.service.yukicoder import onlinejudge.service.toph +import onlinejudge.service.yukicoder diff --git a/onlinejudge/service/hackerrank.py b/onlinejudge/service/hackerrank.py new file mode 100644 index 00000000..44d094b0 --- /dev/null +++ b/onlinejudge/service/hackerrank.py @@ -0,0 +1,216 @@ +# Python Version: 3.x +import datetime +import io +import json +import posixpath +import re +import time +import urllib.parse +import zipfile +from typing import * + +import bs4 +import requests + +import onlinejudge.dispatch +import onlinejudge.implementation.logging as log +import onlinejudge.implementation.utils as utils +import onlinejudge.type +from onlinejudge.type import LabeledString, TestCase + + +@utils.singleton +class HackerRankService(onlinejudge.type.Service): + def login(self, get_credentials: onlinejudge.type.CredentialsProvider, session: Optional[requests.Session] = None) -> bool: + session = session or utils.new_default_session() + url = 'https://www.hackerrank.com/auth/login' + # get + resp = utils.request('GET', url, session=session) + if resp.url != url: + log.debug('redirected: %s', resp.url) + log.info('You have already signed in.') + return True + # parse + soup = bs4.BeautifulSoup(resp.content.decode(resp.encoding), utils.html_parser) + csrftoken = soup.find('meta', attrs={'name': 'csrf-token'}).attrs['content'] + tag = soup.find('input', attrs={'name': 'username'}) + while tag.name != 'form': + tag = tag.parent + form = tag + # post + username, password = get_credentials() + form = utils.FormSender(form, url=resp.url) + form.set('login', username) + form.set('password', password) + form.set('remember_me', 'true') + form.set('fallback', 'true') + resp = form.request(session, method='POST', action='/rest/auth/login', headers={'X-CSRF-Token': csrftoken}) + resp.raise_for_status() + log.debug('redirected: %s', resp.url) + # result + if '/auth' not in resp.url: + log.success('You signed in.') + return True + else: + log.failure('You failed to sign in. Wrong user ID or password.') + return False + + def is_logged_in(self, session: Optional[requests.Session] = None) -> bool: + session = session or utils.new_default_session() + url = 'https://www.hackerrank.com/auth/login' + resp = utils.request('GET', url, session=session) + log.debug('redirected: %s', resp.url) + return '/auth' not in resp.url + + def get_url(self) -> str: + return 'https://www.hackerrank.com/' + + def get_name(self) -> str: + return 'hackerrank' + + @classmethod + def from_url(cls, s: str) -> Optional['HackerRankService']: + # example: https://www.hackerrank.com/dashboard + result = urllib.parse.urlparse(s) + if result.scheme in ('', 'http', 'https') \ + and result.netloc in ('hackerrank.com', 'www.hackerrank.com'): + return cls() + return None + + +class HackerRankProblem(onlinejudge.type.Problem): + def __init__(self, contest_slug: str, challenge_slug: str): + self.contest_slug = contest_slug + self.challenge_slug = challenge_slug + + def download_sample_cases(self, session: Optional[requests.Session] = None) -> List[TestCase]: + log.warning('use --system option') + raise NotImplementedError + + def download_system_cases(self, session: Optional[requests.Session] = None) -> List[TestCase]: + session = session or utils.new_default_session() + # get + # example: https://www.hackerrank.com/rest/contests/hourrank-1/challenges/beautiful-array/download_testcases + url = 'https://www.hackerrank.com/rest/contests/{}/challenges/{}/download_testcases'.format(self.contest_slug, self.challenge_slug) + resp = utils.request('GET', url, session=session, raise_for_status=False) + if resp.status_code != 200: + log.error('response: %s', resp.content.decode()) + return [] + # parse + with zipfile.ZipFile(io.BytesIO(resp.content)) as fh: + # list names + names = [] # type: List[str] + pattern = re.compile(r'(in|out)put/\1put(\d+).txt') + for filename in sorted(fh.namelist()): # "input" < "output" + if filename.endswith('/'): + continue + log.debug('filename: %s', filename) + m = pattern.match(filename) + assert m + if m.group(1) == 'in': + names += [m.group(2)] + # zip samples + samples = [] # type: List[TestCase] + for name in names: + inpath = 'input/input{}.txt'.format(name) + outpath = 'output/output{}.txt'.format(name) + indata = fh.read(inpath).decode() + outdata = fh.read(outpath).decode() + samples += [TestCase(LabeledString(inpath, indata), LabeledString(outpath, outdata))] + return samples + + def get_url(self) -> str: + if self.contest_slug == 'master': + return 'https://www.hackerrank.com/challenges/{}'.format(self.challenge_slug) + else: + return 'https://www.hackerrank.com/contests/{}/challenges/{}'.format(self.contest_slug, self.challenge_slug) + + def get_service(self) -> HackerRankService: + return HackerRankService() + + @classmethod + def from_url(cls, s: str) -> Optional['HackerRankProblem']: + # example: https://www.hackerrank.com/contests/university-codesprint-2/challenges/the-story-of-a-tree + # example: https://www.hackerrank.com/challenges/fp-hello-world + result = urllib.parse.urlparse(s) + if result.scheme in ('', 'http', 'https') \ + and result.netloc in ('hackerrank.com', 'www.hackerrank.com'): + m = re.match(r'^/contests/([0-9A-Za-z-]+)/challenges/([0-9A-Za-z-]+)$', utils.normpath(result.path)) + if m: + return cls(m.group(1), m.group(2)) + m = re.match(r'^/challenges/([0-9A-Za-z-]+)$', utils.normpath(result.path)) + if m: + return cls('master', m.group(1)) + return None + + def _get_model(self, session: Optional[requests.Session] = None) -> Dict[str, Any]: + session = session or utils.new_default_session() + # get + url = 'https://www.hackerrank.com/rest/contests/{}/challenges/{}'.format(self.contest_slug, self.challenge_slug) + resp = utils.request('GET', url, session=session) + # parse + it = json.loads(resp.content.decode()) + log.debug('json: %s', it) + if not it['status']: + log.error('get model: failed') + raise onlinejudge.type.SubmissionError + return it['model'] + + def _get_lang_display_mapping(self, session: Optional[requests.Session] = None) -> Dict[str, str]: + session = session or utils.new_default_session() + # get + url = 'https://hrcdn.net/hackerrank/assets/codeshell/dist/codeshell-cdffcdf1564c6416e1a2eb207a4521ce.js' # at "Mon Feb 4 14:51:27 JST 2019" + resp = utils.request('GET', url, session=session) + # parse + s = resp.content.decode() + l = s.index('lang_display_mapping:{c:"C",') + l = s.index('{', l) + r = s.index('}', l) + 1 + s = s[l:r] + log.debug('lang_display_mapping (raw): %s', s) # this is not a json + lang_display_mapping = {} + for lang in s[1:-2].split('",'): + key, value = lang.split(':"') + lang_display_mapping[key] = value + log.debug('lang_display_mapping (parsed): %s', lang_display_mapping) + return lang_display_mapping + + def get_language_dict(self, session: Optional[requests.Session] = None) -> Dict[str, Dict[str, str]]: + session = session or utils.new_default_session() + info = self._get_model(session=session) + lang_display_mapping = self._get_lang_display_mapping() + result = {} + for lang in info['languages']: + descr = lang_display_mapping.get(lang) + if descr is None: + log.warning('display mapping for language `%s\' not found', lang) + descr = lang + result[lang] = {'description': descr} + return result + + def submit_code(self, code: bytes, language: str, session: Optional[requests.Session] = None) -> onlinejudge.type.Submission: + session = session or utils.new_default_session() + # get + resp = utils.request('GET', self.get_url(), session=session) + # parse + soup = bs4.BeautifulSoup(resp.content.decode(resp.encoding), utils.html_parser) + csrftoken = soup.find('meta', attrs={'name': 'csrf-token'}).attrs['content'] + # post + url = 'https://www.hackerrank.com/rest/contests/{}/challenges/{}/submissions'.format(self.contest_slug, self.challenge_slug) + payload = {'code': code, 'language': language, 'contest_slug': self.contest_slug} + log.debug('payload: %s', payload) + resp = utils.request('POST', url, session=session, json=payload, headers={'X-CSRF-Token': csrftoken}) + # parse + it = json.loads(resp.content.decode()) + log.debug('json: %s', it) + if not it['status']: + log.failure('Submit Code: failed') + raise onlinejudge.type.SubmissionError + model_id = it['model']['id'] + url = self.get_url().rstrip('/') + '/submissions/code/{}'.format(model_id) + log.success('success: result: %s', url) + return onlinejudge.type.CompatibilitySubmission(url, problem=self) + + +onlinejudge.dispatch.services += [HackerRankService] +onlinejudge.dispatch.problems += [HackerRankProblem] diff --git a/onlinejudge/service/kattis.py b/onlinejudge/service/kattis.py new file mode 100644 index 00000000..c18d8a96 --- /dev/null +++ b/onlinejudge/service/kattis.py @@ -0,0 +1,95 @@ +# Python Version: 3.x +import io +import re +import urllib.parse +import zipfile +from typing import * + +import bs4 +import requests + +import onlinejudge.dispatch +import onlinejudge.implementation.logging as log +import onlinejudge.implementation.utils as utils +import onlinejudge.type +from onlinejudge.type import LabeledString, TestCase + + +@utils.singleton +class KattisService(onlinejudge.type.Service): + def get_url(self) -> str: + # NOTE: sometimes this URL is not correct, i.e. something like https://hanoi18.kattis.com/ exists + return 'http://open.kattis.org/' + + def get_name(self) -> str: + return 'kattis' + + @classmethod + def from_url(cls, s: str) -> Optional['KattisService']: + # example: https://open.kattis.com/ + # example: https://hanoi18.kattis.com/ + result = urllib.parse.urlparse(s) + if result.scheme in ('', 'http', 'https') \ + and result.netloc.endswith('.kattis.com'): + # NOTE: ignore the subdomain + return cls() + return None + + +class KattisProblem(onlinejudge.type.Problem): + def __init__(self, problem_id: str, contest_id: Optional[str] = None, domain: str = 'open.kattis.com'): + self.domain = domain + self.contest_id = contest_id + self.problem_id = problem_id + + def download_sample_cases(self, session: Optional[requests.Session] = None) -> List[onlinejudge.type.TestCase]: + session = session or utils.new_default_session() + # get + url = self.get_url(contests=False) + '/file/statement/samples.zip' + resp = utils.request('GET', url, session=session, raise_for_status=False) + if resp.status_code == 404: + log.warning('samples.zip not found') + log.info('this 404 happens in both cases: 1. no sample cases as intended; 2. just an error') + return [] + resp.raise_for_status() + # parse + with zipfile.ZipFile(io.BytesIO(resp.content)) as fh: + samples = [] # type: List[TestCase] + for filename in sorted(fh.namelist()): + log.debug('filename: %s', filename) + if filename.endswith('.in'): + inpath = filename + outpath = filename[:-3] + '.ans' + indata = fh.read(inpath).decode() + outdata = fh.read(outpath).decode() + samples += [TestCase(LabeledString(inpath, indata), LabeledString(outpath, outdata))] + return samples + + def get_url(self, contests: bool = True) -> str: + if contests and self.contest_id is not None: + # the URL without "/contests/{}" also works + return 'https://{}/contests/{}/problems/{}'.format(self.domain, self.contest_id, self.problem_id) + else: + return 'https://{}/problems/{}'.format(self.domain, self.problem_id) + + def get_service(self) -> KattisService: + # NOTE: ignore the subdomain + return KattisService() + + @classmethod + def from_url(cls, s: str) -> Optional['KattisProblem']: + # example: https://open.kattis.com/problems/hello + # example: https://open.kattis.com/contests/asiasg15prelwarmup/problems/8queens + result = urllib.parse.urlparse(s) + if result.scheme in ('', 'http', 'https') \ + and result.netloc.endswith('.kattis.com'): + m = re.match(r'(?:/contests/([0-9A-Z_a-z-]+))?/problems/([0-9A-Z_a-z-]+)/?', result.path) + if m: + contest_id = m.group(1) or None + problem_id = m.group(2) + return cls(problem_id, contest_id=contest_id, domain=result.netloc) + return None + + +onlinejudge.dispatch.services += [KattisService] +onlinejudge.dispatch.problems += [KattisProblem] diff --git a/onlinejudge/service/poj.py b/onlinejudge/service/poj.py new file mode 100644 index 00000000..32d33c66 --- /dev/null +++ b/onlinejudge/service/poj.py @@ -0,0 +1,97 @@ +# Python Version: 3.x +import urllib.parse +from typing import * + +import bs4 +import requests + +import onlinejudge.dispatch +import onlinejudge.implementation.logging as log +import onlinejudge.implementation.utils as utils +import onlinejudge.type +from onlinejudge.type import LabeledString, TestCase + + +@utils.singleton +class POJService(onlinejudge.type.Service): + def get_url(self) -> str: + # no HTTPS support (Wed Feb 6 14:35:37 JST 2019) + return 'http://poj.org/' + + def get_name(self) -> str: + return 'poj' + + @classmethod + def from_url(cls, s: str) -> Optional['POJService']: + # example: http://poj.org/ + result = urllib.parse.urlparse(s) + if result.scheme in ('', 'http', 'https') \ + and result.netloc == 'poj.org': + return cls() + return None + + +class POJProblem(onlinejudge.type.Problem): + def __init__(self, problem_id: int): + self.problem_id = problem_id + + def download_sample_cases(self, session: Optional[requests.Session] = None) -> List[onlinejudge.type.TestCase]: + session = session or utils.new_default_session() + # get + resp = utils.request('GET', self.get_url(), session=session) + # parse + soup = bs4.BeautifulSoup(resp.content.decode(resp.encoding), utils.html_parser) + in_pre, out_pre = soup.find_all('pre', class_='sio') + in_p = in_pre.find_previous_sibling('p', class_='pst') + out_p = out_pre.find_previous_sibling('p', class_='pst') + log.debug('pre (in): %s', in_pre.contents) + log.debug('pre (out): %s', out_pre.contents) + assert in_p.text.strip() == 'Sample Input' + assert out_p.text.strip() == 'Sample Output' + assert len(in_pre.contents) == len(out_pre.contents) + samples = [] # type: List[TestCase] + if len(in_pre.contents) == 1: + assert isinstance(in_pre.contents[0], bs4.NavigableString) + assert isinstance(out_pre.contents[0], bs4.NavigableString) + samples += [TestCase(LabeledString(in_p.text.strip(), in_pre.text + '\r\n'), LabeledString(out_p.text.strip(), out_pre.text + '\r\n'))] + else: + assert len(in_pre.contents) % 2 == 0 + for i in range(len(in_pre.contents) // 2): + in_name = in_pre.contents[2 * i] + in_data = in_pre.contents[2 * i + 1] + out_name = out_pre.contents[2 * i] + out_data = out_pre.contents[2 * i + 1] + assert in_name.name == 'b' + assert isinstance(in_data, bs4.NavigableString) + assert out_name.name == 'b' + assert isinstance(out_data, bs4.NavigableString) + indata = LabeledString(in_name.text.strip(), str(in_data).strip() + '\r\n') + outdata = LabeledString(out_name.text.strip(), str(out_data).strip() + '\r\n') + samples += [TestCase(indata, outdata)] + return samples + + def get_url(self) -> str: + return 'http://poj.org/problem?id={}'.format(self.problem_id) + + def get_service(self) -> POJService: + return POJService() + + @classmethod + def from_url(cls, s: str) -> Optional['POJProblem']: + # example: http://poj.org/problem?id=2104 + result = urllib.parse.urlparse(s) + if result.scheme in ('', 'http', 'https') \ + and result.netloc == 'poj.org' \ + and utils.normpath(result.path) == '/problem': + query = urllib.parse.parse_qs(result.query) + if 'id' in query and len(query['id']) == 1: + try: + n = int(query['id'][0]) + return cls(n) + except ValueError: + pass + return None + + +onlinejudge.dispatch.services += [POJService] +onlinejudge.dispatch.problems += [POJProblem] diff --git a/onlinejudge/service/toph.py b/onlinejudge/service/toph.py index b209ccef..e675c1cf 100644 --- a/onlinejudge/service/toph.py +++ b/onlinejudge/service/toph.py @@ -38,8 +38,8 @@ def login(self, get_credentials: onlinejudge.type.CredentialsProvider, session: resp = form.request(session) resp.raise_for_status() - newResp = utils.request('GET', url, session=session) # Toph's Location header is not getting the expected value - if newResp.url != url: + resp = utils.request('GET', url, session=session) # Toph's Location header is not getting the expected value + if resp.url != url: log.success('Welcome, %s.', username) return True else: @@ -50,7 +50,7 @@ def is_logged_in(self, session: Optional[requests.Session] = None) -> bool: session = session or utils.new_default_session() url = 'https://toph.co/login' resp = utils.request('GET', url, session=session, allow_redirects=False) - return resp.status_code == 302 + return resp.status_code != 200 def get_url(self) -> str: return 'https://toph.co/' @@ -69,11 +69,13 @@ def from_url(cls, s: str) -> Optional['TophService']: return None class TophProblem(onlinejudge.type.Problem): - def __init__(self, slug: str, kind: Optional[str] = None): - assert isinstance(slug, str) - assert kind in (None, 'problem') + def __init__(self, problem_id: str, kind: Optional[str] = None, contest_id: Optional[str] = None): + assert isinstance(problem_id, str) + assert kind in ('problem') + if contest_id is not None: + raise NotImplementedError self.kind = kind - self.slug = slug + self.problem_id = problem_id def download_sample_cases(self, session: Optional[requests.Session] = None) -> List[onlinejudge.type.TestCase]: session = session or utils.new_default_session() @@ -108,7 +110,7 @@ def get_language_dict(self, session: Optional['requests.Session'] = None) -> Dic return {} language_dict = {} for option in select.findAll('option'): - language_dict[option.attrs['value']] = {'description': option.string} + language_dict[option.attrs['value']] = {'description': option.string.strip()} return language_dict def submit_code(self, code: bytes, language: str, session: Optional['requests.Session'] = None) -> onlinejudge.type.Submission: # or SubmissionError @@ -134,22 +136,18 @@ def submit_code(self, code: bytes, language: str, session: Optional['requests.Se resp.raise_for_status() # result if '/s/' in resp.url: - # example: https://codeforces.com/contest/598/my + # example: https://toph.co/s/201410 log.success('success: result: %s', resp.url) return onlinejudge.type.DummySubmission(resp.url) else: log.failure('failure') log.debug('redirected to %s', resp.url) - # parse error messages - # soup = bs4.BeautifulSoup(resp.content.decode(resp.encoding), utils.html_parser) - # for span in soup.findAll('span', class_='error'): - # log.warning('Codeforces says: "%s"', span.string) raise SubmissionError def get_url(self) -> str: table = {} table['problem'] = 'https://toph.co/p/{}' - return table[self.kind].format(self.slug) + return table[self.kind].format(self.problem_id) def get_service(self) -> TophService: return TophService() @@ -158,14 +156,15 @@ def get_service(self) -> TophService: def from_url(cls, s: str) -> Optional['TophProblem']: result = urllib.parse.urlparse(s) dirname, basename = posixpath.split(utils.normpath(result.path)) + # example: https://toph.co/p/new-year-couple if result.scheme in ('', 'http', 'https') \ and result.netloc.count('.') == 1 \ and result.netloc.endswith('toph.co') \ and dirname == '/p' \ and basename: kind = 'problem' - slug = basename - return cls(slug, kind) + problem_id = basename + return cls(problem_id, kind) return None diff --git a/onlinejudge/service/yukicoder.py b/onlinejudge/service/yukicoder.py index 152d13a8..a7d36639 100644 --- a/onlinejudge/service/yukicoder.py +++ b/onlinejudge/service/yukicoder.py @@ -311,7 +311,7 @@ def _parse_sample_tag(self, tag: bs4.Tag) -> Optional[Tuple[str, str]]: def submit_code(self, code: bytes, language: str, session: Optional[requests.Session] = None) -> onlinejudge.type.Submission: # or SubmissionError session = session or utils.new_default_session() # get - url = 'https://yukicoder.me/problems/no/{}/submit'.format(self.problem_no) + url = self.get_url() + '/submit' resp = utils.request('GET', url, session=session) # parse soup = bs4.BeautifulSoup(resp.content.decode(resp.encoding), utils.html_parser) diff --git a/readme.md b/readme.md index a4e85c4f..853ec276 100644 --- a/readme.md +++ b/readme.md @@ -22,10 +22,12 @@ Tools for online judge services. Downloading sample cases, Testing/Submitting yo - Yukicoder - Anarchy Golf - Codeforces - - ~~HackerRank~~ (removed) + - HackerRank - Aizu Online Judge - CS Academy - - Toph + - PKU JudgeOnline + - Kattis + - Toph (Problem Archive) - Download system test cases - Yukicoder - Aizu Online Judge @@ -33,16 +35,16 @@ Tools for online judge services. Downloading sample cases, Testing/Submitting yo - AtCoder - Yukicoder (via github.com or [session token](https://github.com/kmyk/online-judge-tools/blob/master/LOGIN_WITH_COOKIES.md)) - Codeforces - - ~~HackerRank~~ (removed) + - HackerRank - TopCoder - Toph - Submit your solution - AtCoder - Yukicoder - Codeforces - - ~~HackerRank~~ (removed) + - HackerRank - TopCoder (Marathon Match) - - Toph + - Toph (Problem Archive) - Generate scanner for input (experimental) - AtCoder - Yukicoder diff --git a/tests/command_download.py b/tests/command_download.py index 5afcbf7e..42947d3b 100644 --- a/tests/command_download.py +++ b/tests/command_download.py @@ -19,7 +19,7 @@ def get_files_from_json(samples): return files -def snippet_call_download(self, url, files, is_system=False, type='files'): +def snippet_call_download(self, url, files, is_system=False, is_silent=False, type='files'): assert type in 'files' or 'json' if type == 'json': files = get_files_from_json(files) @@ -29,6 +29,8 @@ def snippet_call_download(self, url, files, is_system=False, type='files'): cmd = [ojtools, 'download', url] if is_system: cmd += ['--system'] + if is_silent: + cmd += ['--silent'] subprocess.check_call(cmd, stdout=sys.stdout, stderr=sys.stderr) result = {} if os.path.exists('test'): diff --git a/tests/command_download_hackerrank.py b/tests/command_download_hackerrank.py new file mode 100644 index 00000000..afc233f5 --- /dev/null +++ b/tests/command_download_hackerrank.py @@ -0,0 +1,46 @@ +import unittest + +import tests.command_download + + +class DownloadHackerRankTest(unittest.TestCase): + def snippet_call_download(self, *args, **kwargs): + tests.command_download.snippet_call_download(self, *args, **kwargs) + + # TODO: support parsing HTML or retrieving from "Run Code" feature + @unittest.skip('the "Download all test cases" feature is not supported for this problem') + def test_call_download_hackerrank_beautiful_array(self): + self.snippet_call_download('https://www.hackerrank.com/contests/hourrank-1/challenges/beautiful-array', { + 'sample-1.in': 'fb3f7e56dac548ce73f9d8e485e5336b', + 'sample-2.out': '897316929176464ebc9ad085f31e7284', + 'sample-2.in': '6047a07c8defde4d696513d26e871b20', + 'sample-1.out': '6d7fce9fee471194aa8b5b6e47267f03', + }) + + def test_call_download_hackerrank_hourrank_30_a_system(self): + # TODO: these file names should "00.in", "00.out", ..., "10.out" + self.snippet_call_download( + 'https://www.hackerrank.com/contests/hourrank-30/challenges/video-conference', { + '1.in': 'b138a1282e79697057d5eca993a29414', + '1.out': 'de044533ac6d30ed89eb5b4e10ff105b', + '2.in': '0e64d38accc35d4b8ac4fc0df3b5b969', + '2.out': '3362cf9066bba541387e5b6787b13e6e', + '3.in': '7df575910d94fecb93861eaf414d86dd', + '3.out': 'eb68db6a13e73b093d620f865e4cc098', + '4.in': 'd87880f0cd02ee106a8cadc5ccd97ed0', + '4.out': 'a24f9580a3701064cb49534689b50b60', + '5.in': 'f5981eb3068da7d2d2c1b84b23ea8710', + '5.out': 'df0a3dfc2217cbc8e8828e423933206b', + '6.in': 'b1387e51b1f9c4e16713647b36e8341b', + '6.out': 'ac14c5fed571104401167dd04fdcf417', + '7.in': 'ba080fc7b89b2aed00fcf03a5db29f8a', + '7.out': '3a365fc4aec7cad9536c598b7d892e7a', + '8.in': '9d3f2cfb7b6412ef40a8b5ef556c3a46', + '8.out': '8e7a02d5c6bdd9358c589b3e400bacb8', + '9.in': '8409f37413e40f3baee0314bcacfc0a4', + '9.out': 'fe2d333498a3bdebaa0f4c88803566ff', + '10.in': '6f3e4c84441ae56e141a600542cc8ec8', + '10.out': '66e67dc4e8edbf66ed9ae2c9a0862f2b', + '11.in': 'fe24b76ea70e0a44213d7f22d183a33b', + '11.out': '8b8ba206ea7bbb02f0361341cb8da7c7', + }, is_system=True, is_silent=True) diff --git a/tests/command_download_kattis.py b/tests/command_download_kattis.py new file mode 100644 index 00000000..09c82b93 --- /dev/null +++ b/tests/command_download_kattis.py @@ -0,0 +1,35 @@ +import os +import unittest + +import tests.command_download + + +class DownloadKattisTest(unittest.TestCase): + def snippet_call_download(self, *args, **kwargs): + tests.command_download.snippet_call_download(self, *args, **kwargs) + + def test_call_download_kattis_8queens(self): + self.snippet_call_download( + 'https://open.kattis.com/contests/asiasg15prelwarmup/problems/8queens', [ + { + "input": "*.......\n..*.....\n....*...\n......*.\n.*......\n.......*\n.....*..\n...*....\n", + "output": "invalid\n" + }, + { + "input": "*.......\n......*.\n....*...\n.......*\n.*......\n...*....\n.....*..\n..*.....\n", + "output": "valid\n" + }, + ], type='json') + + def test_call_download_kattis_hanoi18_a(self): + self.snippet_call_download( + 'https://hanoi18.kattis.com/problems/amazingadventures', [ + { + "input": "3 3\n1 1\n3 3\n2 1\n2 2\n\n3 4\n1 1\n3 4\n2 1\n1 2\n\n2 2\n2 1\n2 2\n1 2\n1 1\n\n0 0\n", + "output": "YES\nRRUULLD\nNO\nYES\nRD\n" + }, + ], type='json') + + def test_call_download_kattis_hello(self): + # there is no sample cases (and no samples.zip; it returns 404) + self.snippet_call_download('https://open.kattis.com/problems/hello', [], type='json') diff --git a/tests/command_download_others.py b/tests/command_download_others.py index a8ae68aa..87c0258c 100644 --- a/tests/command_download_others.py +++ b/tests/command_download_others.py @@ -89,3 +89,34 @@ def test_call_download_csacademy_unfair_game(self): 'sample-2.in': '46b87e796b61eb9b8970e83c93a02809', 'sample-2.out': 'eb844645e8e61de0a4cf4b991e65e63e', }) + + def test_call_download_poj_1000(self): + self.snippet_call_download( + 'http://poj.org/problem?id=1000', [ + { + "input": "1 2\r\n", + "output": "3\r\n" + }, + ], type='json') + + def test_call_download_poj_2104(self): + self.snippet_call_download( + 'http://poj.org/problem?id=2104', [ + { + "input": "7 3\r\n1 5 2 6 3 7 4\r\n2 5 3\r\n4 4 1\r\n1 7 3\r\n", + "output": "5\r\n6\r\n3\r\n" + }, + ], type='json') + + def test_call_download_poj_3150(self): + self.snippet_call_download( + 'http://poj.org/problem?id=3150', [ + { + "input": "5 3 1 1\r\n1 2 2 1 2\r\n", + "output": "2 2 2 2 1\r\n" + }, + { + "input": "5 3 1 10\r\n1 2 2 1 2\r\n", + "output": "2 0 0 2 2\r\n" + }, + ], type='json') diff --git a/tests/command_login.py b/tests/command_login.py index 4f442d0f..7c405b46 100644 --- a/tests/command_login.py +++ b/tests/command_login.py @@ -23,6 +23,9 @@ def test_call_login_check_atcoder_failure(self): def test_call_login_check_codeforces_failure(self): self.snippet_call_login_check_failure('https://codeforces.com/') + def test_call_login_check_hackerrank_failure(self): + self.snippet_call_login_check_failure('https://www.hackerrank.com/') + def test_call_login_check_yukicoder_failure(self): self.snippet_call_login_check_failure('https://yukicoder.me/') @@ -36,6 +39,11 @@ def test_call_login_check_codeforces_success(self): ojtools = os.path.abspath('oj') subprocess.check_call([ojtools, 'login', '--check', 'https://codeforces.com/'], stdout=sys.stdout, stderr=sys.stderr) + @unittest.skipIf('CI' in os.environ, 'login is required') + def test_call_login_check_hackerrank_success(self): + ojtools = os.path.abspath('oj') + subprocess.check_call([ojtools, 'login', '--check', 'https://www.hackerrank.com/'], stdout=sys.stdout, stderr=sys.stderr) + @unittest.skipIf('CI' in os.environ, 'login is required') def test_call_login_check_yukicoder_success(self): ojtools = os.path.abspath('oj') diff --git a/tests/command_submit.py b/tests/command_submit.py index f70e101a..7bd33d05 100644 --- a/tests/command_submit.py +++ b/tests/command_submit.py @@ -204,7 +204,7 @@ def test_call_submit_9000(self): @unittest.skipIf('CI' in os.environ, 'login is required') def test_call_submit_beta_3_b(self): - url = 'https://yukicoder.me/problems/no/9001' + url = 'https://yukicoder.me/problems/527' code = r'''#include using namespace std; int main() { @@ -223,3 +223,30 @@ def test_call_submit_beta_3_b(self): ojtools = os.path.abspath('oj') with tests.utils.sandbox(files): subprocess.check_call([ojtools, 's', '-y', '--no-open', url, 'main.cpp'], stdout=sys.stdout, stderr=sys.stderr) + + +class SubmitHackerRankTest(unittest.TestCase): + @unittest.skipIf('CI' in os.environ, 'login is required') + def test_call_submit_worldcodesprint_mars_exploration(self): + url = 'https://www.hackerrank.com/contests/worldcodesprint/challenges/mars-exploration' + code = '''#!/usr/bin/env python3 +s = input() +ans = 0 +for i in range(len(s) // 3): + if s[3 * i] != 'S': + ans += 1 + if s[3 * i + 1] != 'O': + ans += 1 + if s[3 * i + 2] != 'S': + ans += 1 +print(ans) +''' + files = [ + { + 'path': 'a.py', + 'data': code + }, + ] + ojtools = os.path.abspath('oj') + with tests.utils.sandbox(files): + subprocess.check_call([ojtools, 's', '-y', '--no-open', url, 'a.py'], stdout=sys.stdout, stderr=sys.stderr) From 8722bf1c8d218838486c67e6687c46d0a765ccf8 Mon Sep 17 00:00:00 2001 From: Kazi Rabi Date: Mon, 25 Feb 2019 14:29:20 +0600 Subject: [PATCH 08/12] Write tests for Toph and do small fixes --- onlinejudge/implementation/main.py | 3 +++ onlinejudge/service/toph.py | 2 +- tests/command_download_others.py | 14 ++++++++++++ tests/command_login.py | 8 +++++++ tests/command_submit.py | 35 ++++++++++++++++++++++++++++++ 5 files changed, 61 insertions(+), 1 deletion(-) diff --git a/onlinejudge/implementation/main.py b/onlinejudge/implementation/main.py index 8a0918e1..d3bd4131 100644 --- a/onlinejudge/implementation/main.py +++ b/onlinejudge/implementation/main.py @@ -48,6 +48,7 @@ def get_parser() -> argparse.ArgumentParser: HackerRank PKU JudgeOnline Kattis + Toph (Problem Archive) supported services with --system: Aizu Online Judge @@ -78,6 +79,7 @@ def get_parser() -> argparse.ArgumentParser: Yukicoder TopCoder HackerRank + Toph strings for --method: github for yukicoder, login via github (default) @@ -97,6 +99,7 @@ def get_parser() -> argparse.ArgumentParser: TopCoder (Marathon Match) Yukicoder HackerRank + Toph (Problem Archive) ''') subparser.add_argument('url', nargs='?', help='the URL of the problem to submit. if not given, guessed from history of download command.') diff --git a/onlinejudge/service/toph.py b/onlinejudge/service/toph.py index e675c1cf..d370f836 100644 --- a/onlinejudge/service/toph.py +++ b/onlinejudge/service/toph.py @@ -1,12 +1,12 @@ # Python Version: 3.x import posixpath +import re import string import urllib.parse from typing import * import bs4 import requests -import re import onlinejudge.dispatch import onlinejudge.implementation.logging as log diff --git a/tests/command_download_others.py b/tests/command_download_others.py index 87c0258c..4794f2d5 100644 --- a/tests/command_download_others.py +++ b/tests/command_download_others.py @@ -90,6 +90,20 @@ def test_call_download_csacademy_unfair_game(self): 'sample-2.out': 'eb844645e8e61de0a4cf4b991e65e63e', }) + def test_call_download_toph_new_year_couple(self): + self.snippet_call_download('https://toph.co/p/new-year-couple', { + 'sample-2.out': 'a147d4af6796629a62fa43341f0e0bdf', + 'sample-2.in': 'fc1dbb7bb49bfbb37e7afe9a64d2f89b', + 'sample-1.in': 'd823c94a5bbd1af3161ad8eb4e48654e', + 'sample-1.out': '0f051fce168dc5aa9e45605992cd63c5', + }) + + def test_call_download_toph_power_and_mod(self): + self.snippet_call_download('https://toph.co/p/power-and-mod', { + 'sample-1.in': '46e186317c8c10d9452d6070f6c63b09', + 'sample-1.out': 'ad938662144b559bff344ff266f9d1cc', + }) + def test_call_download_poj_1000(self): self.snippet_call_download( 'http://poj.org/problem?id=1000', [ diff --git a/tests/command_login.py b/tests/command_login.py index 7c405b46..2e8eb94a 100644 --- a/tests/command_login.py +++ b/tests/command_login.py @@ -26,6 +26,9 @@ def test_call_login_check_codeforces_failure(self): def test_call_login_check_hackerrank_failure(self): self.snippet_call_login_check_failure('https://www.hackerrank.com/') + def test_call_login_check_toph_failure(self): + self.snippet_call_login_check_failure('https://toph.co/') + def test_call_login_check_yukicoder_failure(self): self.snippet_call_login_check_failure('https://yukicoder.me/') @@ -44,6 +47,11 @@ def test_call_login_check_hackerrank_success(self): ojtools = os.path.abspath('oj') subprocess.check_call([ojtools, 'login', '--check', 'https://www.hackerrank.com/'], stdout=sys.stdout, stderr=sys.stderr) + @unittest.skipIf('CI' in os.environ, 'login is required') + def test_call_login_check_toph_success(self): + ojtools = os.path.abspath('oj') + subprocess.check_call([ojtools, 'login', '--check', 'https://toph.co/'], stdout=sys.stdout, stderr=sys.stderr) + @unittest.skipIf('CI' in os.environ, 'login is required') def test_call_login_check_yukicoder_success(self): ojtools = os.path.abspath('oj') diff --git a/tests/command_submit.py b/tests/command_submit.py index 7bd33d05..38dd28cd 100644 --- a/tests/command_submit.py +++ b/tests/command_submit.py @@ -250,3 +250,38 @@ def test_call_submit_worldcodesprint_mars_exploration(self): ojtools = os.path.abspath('oj') with tests.utils.sandbox(files): subprocess.check_call([ojtools, 's', '-y', '--no-open', url, 'a.py'], stdout=sys.stdout, stderr=sys.stderr) + +class SubmitTophTest(unittest.TestCase): + @unittest.skipIf('CI' in os.environ, 'login is required') + def test_call_submit_copycat(self): + url = 'https://toph.co/p/copycat' + code = '''#!/usr/bin/env python3 +s = input() +print(s) +''' + files = [ + { + 'path': 'a.py', + 'data': code + }, + ] + ojtools = os.path.abspath('oj') + with tests.utils.sandbox(files): + subprocess.check_call([ojtools, 's', '-l', '58482c1804469e2585024324', '-y', '--no-open', url, 'a.py'], stdout=sys.stdout, stderr=sys.stderr) + + @unittest.skipIf('CI' in os.environ, 'login is required') + def test_call_submit_add_them_up(self): + url = 'https://toph.co/p/add-them-up' + code = '''#!/usr/bin/env python3 +nums = map(int, input().split()) +print(sum(nums)) +''' + files = [ + { + 'path': 'a.py', + 'data': code + }, + ] + ojtools = os.path.abspath('oj') + with tests.utils.sandbox(files): + subprocess.check_call([ojtools, 's', '-l', '58482c1804469e2585024324', '-y', '--no-open', url, 'a.py'], stdout=sys.stdout, stderr=sys.stderr) \ No newline at end of file From ec58519c2f72a2bf100bd535596d4c0e9e640894 Mon Sep 17 00:00:00 2001 From: Kazi Rabi Date: Tue, 26 Feb 2019 13:43:17 +0600 Subject: [PATCH 09/12] Add submission tests using CPP and remove unnecessary problem kind --- onlinejudge/service/toph.py | 12 ++--- tests/command_submit.py | 102 ++++++++++++++++++++++++++++++++++++ 2 files changed, 106 insertions(+), 8 deletions(-) diff --git a/onlinejudge/service/toph.py b/onlinejudge/service/toph.py index d370f836..32e23f8d 100644 --- a/onlinejudge/service/toph.py +++ b/onlinejudge/service/toph.py @@ -69,12 +69,10 @@ def from_url(cls, s: str) -> Optional['TophService']: return None class TophProblem(onlinejudge.type.Problem): - def __init__(self, problem_id: str, kind: Optional[str] = None, contest_id: Optional[str] = None): + def __init__(self, problem_id: str, contest_id: Optional[str] = None): assert isinstance(problem_id, str) - assert kind in ('problem') if contest_id is not None: raise NotImplementedError - self.kind = kind self.problem_id = problem_id def download_sample_cases(self, session: Optional[requests.Session] = None) -> List[onlinejudge.type.TestCase]: @@ -145,9 +143,8 @@ def submit_code(self, code: bytes, language: str, session: Optional['requests.Se raise SubmissionError def get_url(self) -> str: - table = {} - table['problem'] = 'https://toph.co/p/{}' - return table[self.kind].format(self.problem_id) + # TODO: Check for contest_id to return the appropriate URL when support for contest is added + return f'https://toph.co/p/{self.problem_id}' def get_service(self) -> TophService: return TophService() @@ -162,9 +159,8 @@ def from_url(cls, s: str) -> Optional['TophProblem']: and result.netloc.endswith('toph.co') \ and dirname == '/p' \ and basename: - kind = 'problem' problem_id = basename - return cls(problem_id, kind) + return cls(problem_id) return None diff --git a/tests/command_submit.py b/tests/command_submit.py index e953bb57..425dcf70 100644 --- a/tests/command_submit.py +++ b/tests/command_submit.py @@ -285,3 +285,105 @@ def test_call_submit_add_them_up(self): ojtools = os.path.abspath('oj') with tests.utils.sandbox(files): subprocess.check_call([ojtools, 's', '-l', '58482c1804469e2585024324', '-y', '--no-open', url, 'a.py'], stdout=sys.stdout, stderr=sys.stderr) + + @unittest.skipIf('CI' in os.environ, 'login is required') + def test_call_submit_divisors(self): + url = 'https://toph.co/p/divisors' + code = '''#include +using namespace std; +int main() +{ + int a; + cin>>a; + for (int i=1;i<=a;i++) + { + if (a%i==0) + { + cout < +using namespace std; + +typedef long long int LL; +const int MOD = 993344777; +const int N = 1e6 + 1; + +int n; +int a[ N ]; +int cnt[ 62 ]; +int mask[ 62 ]; +vector prime; +int id[ 62 ]; +LL dp[ 62 ][ ( 1 << 17 ) + 1 ][ 2 ][ 2 ]; + +bool isprime( int x ) { + for( int i = 2; i*i <= x; i++ ) if( x%i == 0 ) return false; + return true; +} +LL solve( int cur , int msk , int sz , int taken ) { + if( cur == 61 ) { + if( !taken ) return 0; + if( sz&1 ) return msk != 0; + else return msk == 0; + } + if( dp[cur][msk][sz][taken] != -1 ) return dp[cur][msk][sz][taken] ; + LL ret = 0; + if( cnt[cur] == 0 ) { + ret = ( ret%MOD + solve( cur + 1 , msk , sz%2 , taken )%MOD )%MOD; + } + else { + ret = ( ret%MOD + cnt[cur]%MOD * solve( cur + 1 , msk^mask[cur] , (sz%2+1%2)%2 , 1 )%MOD )%MOD; + ret = ( ret%MOD + solve( cur + 1 , msk , sz%2 , taken )%MOD )%MOD; + } + return dp[cur][msk][sz][taken] = ret%MOD; +} +int main( int argc , char const *argv[] ) { + scanf("%d",&n); + for( int i = 1; i <= n; i++ ) scanf("%d",&a[i]) , cnt[ a[i] ]++; + prime.push_back( 2 ); + int t = 0; + id[2] = ++t; + for( int i = 3; i <= 60; i += 2 ) { + if( isprime( i ) ) prime.push_back( i ) , id[i] = ++t; + } + for( int i = 1; i <= 60; i++ ) { + int num = i; + for( auto x : prime ) { + if( num%x == 0 ) { + mask[i] ^= ( 1 << id[x] ); + num /= x; + while( num%x == 0 ) num /= x , mask[i] ^= ( 1 << id[x] ); + } + } + if( num != 1 ) mask[i] ^= ( 1 << id[num] ); + } + memset( dp , -1 , sizeof( dp ) ); + cout << solve( 1 , 0 , 0 , 0 )%MOD << endl; + return 0; +} +''' + files = [ + { + 'path': 'a.cpp', + 'data': code + }, + ] + ojtools = os.path.abspath('oj') + with tests.utils.sandbox(files): + subprocess.check_call([ojtools, 's', '-y', '--no-open', url, 'a.cpp'], stdout=sys.stdout, stderr=sys.stderr) \ No newline at end of file From a99a9f2728107d6730e8c9af0187f27f493caff8 Mon Sep 17 00:00:00 2001 From: Kazi Rabi Date: Tue, 26 Feb 2019 14:19:01 +0600 Subject: [PATCH 10/12] Use format instead of f --- onlinejudge/service/toph.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onlinejudge/service/toph.py b/onlinejudge/service/toph.py index 32e23f8d..425f5bb1 100644 --- a/onlinejudge/service/toph.py +++ b/onlinejudge/service/toph.py @@ -144,7 +144,7 @@ def submit_code(self, code: bytes, language: str, session: Optional['requests.Se def get_url(self) -> str: # TODO: Check for contest_id to return the appropriate URL when support for contest is added - return f'https://toph.co/p/{self.problem_id}' + return 'https://toph.co/p/{}'.format(self.problem_id) def get_service(self) -> TophService: return TophService() From 5fdfc4bc42305ab262712234f64eb2e235186438 Mon Sep 17 00:00:00 2001 From: Kimiyuki Onaka Date: Tue, 26 Feb 2019 05:40:04 +0900 Subject: [PATCH 11/12] #323: $ yapf --in-place --recursive oj onlinejudge setup.py tests --- onlinejudge/service/toph.py | 6 ++++-- tests/command_submit.py | 1 + 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/onlinejudge/service/toph.py b/onlinejudge/service/toph.py index 425f5bb1..22a548fc 100644 --- a/onlinejudge/service/toph.py +++ b/onlinejudge/service/toph.py @@ -30,7 +30,7 @@ def login(self, get_credentials: onlinejudge.type.CredentialsProvider, session: form = soup.find('form', class_='login-form') log.debug('form: %s', str(form)) username, password = get_credentials() - form['action'] = '/login' # to avoid KeyError inside form.request method as Toph does not have any defined action + form['action'] = '/login' # to avoid KeyError inside form.request method as Toph does not have any defined action form = utils.FormSender(form, url=resp.url) form.set('handle', username) form.set('password', password) @@ -38,7 +38,7 @@ def login(self, get_credentials: onlinejudge.type.CredentialsProvider, session: resp = form.request(session) resp.raise_for_status() - resp = utils.request('GET', url, session=session) # Toph's Location header is not getting the expected value + resp = utils.request('GET', url, session=session) # Toph's Location header is not getting the expected value if resp.url != url: log.success('Welcome, %s.', username) return True @@ -68,6 +68,7 @@ def from_url(cls, s: str) -> Optional['TophService']: return cls() return None + class TophProblem(onlinejudge.type.Problem): def __init__(self, problem_id: str, contest_id: Optional[str] = None): assert isinstance(problem_id, str) @@ -164,5 +165,6 @@ def from_url(cls, s: str) -> Optional['TophProblem']: return None + onlinejudge.dispatch.services += [TophService] onlinejudge.dispatch.problems += [TophProblem] diff --git a/tests/command_submit.py b/tests/command_submit.py index 425dcf70..fd822c43 100644 --- a/tests/command_submit.py +++ b/tests/command_submit.py @@ -251,6 +251,7 @@ def test_call_submit_worldcodesprint_mars_exploration(self): with tests.utils.sandbox(files): subprocess.check_call([ojtools, 's', '-y', '--no-open', url, 'a.py'], stdout=sys.stdout, stderr=sys.stderr) + class SubmitTophTest(unittest.TestCase): @unittest.skipIf('CI' in os.environ, 'login is required') def test_call_submit_copycat(self): From 79de8d37e1aa78a7c4c82c6a666f1f1602caf545 Mon Sep 17 00:00:00 2001 From: Kimiyuki Onaka Date: Tue, 26 Feb 2019 05:43:43 +0900 Subject: [PATCH 12/12] There was: $ mypy oj onlinejudge setup.py tests onlinejudge/service/toph.py:151: error: Invalid index type "Optional[str]" for "Dict[str, str]"; expected type "str"