use black to format python code

.circleci/config.yml

@@ -9,10 +9,10 @@ jobs:
       image: circleci/classic:latest
     steps:
       - checkout
-      - run:
-          name: Install Dependencies
-          command: |
-            pip install --upgrade pip && pip install -r requirements.txt
+#      - run:
+#          name: Install Dependencies
+#          command: |
+#            pip install --upgrade pip && pip install -r requirements.txt
       - run:
           name: Run Tests
           command: |

Makefile

@@ -131,6 +131,7 @@ docs:
 build_spl: clean
 	python3 -m splunk_eventgen build --destination ./
 
+
 lint:
 ifeq ($(NEWLY_ADDED_PY_FILES), )
 	@echo 'No newly added python files. Skip...'
@@ -148,12 +149,12 @@ endif
 ifeq ($(NEWLY_ADDED_PY_FILES), )
 	@echo 'No newly added python files. Skip...'
 else
-	@yapf -i $(NEWLY_ADDED_PY_FILES)
+	@black $(NEWLY_ADDED_PY_FILES)
 endif
 
 lint-all:
 	@flake8 .
 
 format-all:
 	@isort -rc .
-	@yapf -r -i .
+	@black .

pyproject.toml

@@ -0,0 +1,12 @@
+[tool.black]
+target-version = ['py37']
+include = '\.pyi?$'
+exclude = '''
+/(
+    \.git
+  | \.tox
+  | \.venv
+  | build
+  | dist
+)/
+```

release_tool/prepare_release_branch.py

@@ -15,16 +15,16 @@ def parse_args():
     def validate_version_str(version):
         v_str = str(version).strip()
         if not v_str:
-            raise argparse.ArgumentTypeError('verison str can not be emtpy.')
+            raise argparse.ArgumentTypeError("verison str can not be emtpy.")
         err_message = 'version string should be of format "major.minor.hotfix"'
-        numbers = v_str.split('.')
+        numbers = v_str.split(".")
         if len(numbers) != 3:
             raise argparse.ArgumentTypeError(err_message)
         for n in numbers:
             valid = False
             try:
                 v = int(n)
-                valid = (v >= 0)
+                valid = v >= 0
             except:
                 valid = False
             if not valid:
@@ -34,143 +34,172 @@ def validate_version_str(version):
     def validate_token(token):
         t = token.strip()
         if not t:
-            raise argparse.ArgumentTypeError('token can not be empty')
+            raise argparse.ArgumentTypeError("token can not be empty")
         return t
 
     parser = argparse.ArgumentParser(
-        'prepare_release_branch.py', description=
-        'eventgen release branch tool.\ncreate the release branch, set the right version and push the pull request.')
-    parser.add_argument('-v', '--verbose', default=False, action='store_true', help='enable the verbose logging')
-    parser.add_argument('-n', '--version_str', type=validate_version_str, required=True)
-    parser.add_argument('-a', '--token', help='your github access token.', default=None, type=validate_token)
+        "prepare_release_branch.py",
+        description="eventgen release branch tool.\n"
+        "create the release branch, set the right version and push the pull request.",
+    )
+    parser.add_argument(
+        "-v",
+        "--verbose",
+        default=False,
+        action="store_true",
+        help="enable the verbose logging",
+    )
+    parser.add_argument("-n", "--version_str", type=validate_version_str, required=True)
+    parser.add_argument(
+        "-a",
+        "--token",
+        help="your github access token.",
+        default=None,
+        type=validate_token,
+    )
     return parser.parse_args(sys.argv[1:])
 
 
 def setup_logging(verbose=None):
-    l = logging.DEBUG if verbose is True else logging.INFO
-    logging.getLogger().setLevel(l)
+    log_level = logging.DEBUG if verbose is True else logging.INFO
+    logging.getLogger().setLevel(log_level)
 
 
 def setup_env():
-    '''
+    """
     by default, we use this hard code current working dir.
     because curent working dir has impact about the child sh process.
     we need to setup it before launching any process.
     if there is concrete requirement about setting the current
     working dir, we can change it to cmd arguemnt.
-    '''
-    logging.debug(f'try to change current working directory to {root_repo_dir}')
+    """
+    logging.debug(f"try to change current working directory to {root_repo_dir}")
     os.chdir(root_repo_dir)
 
 
 def run_sh_cmd(args, exit_on_error=None):
     should_exit_on_error = True if exit_on_error is None else exit_on_error
     child = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
     out, err = child.communicate()
-    outs = out.decode('utf-8')
-    errs = err.decode('utf-8')
+    outs = out.decode("utf-8")
+    errs = err.decode("utf-8")
     if child.returncode == 0:
-        logging.debug(f'execute sh command {args} success.')
-        logging.debug(f'children output:\n{outs}')
+        logging.debug(f"execute sh command {args} success.")
+        logging.debug(f"children output:\n{outs}")
         return True
-    logging.error(f'execute sh cmd {args} fail.\nchildren output:\n{outs}\n{errs}')
+    logging.error(f"execute sh cmd {args} fail.\nchildren output:\n{outs}\n{errs}")
     if should_exit_on_error:
-        assert False, 'sh command fails.'
+        assert False, "sh command fails."
     return False
 
 
 def get_release_branch_name(version_str):
-    v = version_str.replace('.', '_')
-    return f'release/{v}'
+    v = version_str.replace(".", "_")
+    return f"release/{v}"
 
 
 def replace_version(ver):
-    ver_json_file = os.path.join(root_repo_dir, 'splunk_eventgen', 'version.json')
-    with open(ver_json_file, 'w') as fp:
-        json.dump({'version': ver}, fp)
-    app_conf = os.path.join(root_repo_dir, 'splunk_eventgen', 'splunk_app', 'default', 'app.conf')
+    ver_json_file = os.path.join(root_repo_dir, "splunk_eventgen", "version.json")
+    with open(ver_json_file, "w") as fp:
+        json.dump({"version": ver}, fp)
+    app_conf = os.path.join(
+        root_repo_dir, "splunk_eventgen", "splunk_app", "default", "app.conf"
+    )
     app_conf_content = []
-    with open(app_conf, 'r') as fp:
+    with open(app_conf, "r") as fp:
         app_conf_content = fp.readlines()
-    app_pattern = re.compile(r'version\s*=')
-    with open(app_conf, 'w') as fp:
+    app_pattern = re.compile(r"version\s*=")
+    with open(app_conf, "w") as fp:
         for line in app_conf_content:
             lstr = line.strip()
             if app_pattern.search(lstr):
-                fp.write(f'version = {ver}\n')
+                fp.write(f"version = {ver}\n")
             else:
-                fp.write(f'{lstr}\n')
-    logging.info(f'verison is replaced with {ver}.')
+                fp.write(f"{lstr}\n")
+    logging.info(f"verison is replaced with {ver}.")
 
 
 def update_changelog(ver):
-    changelog_file = os.path.join(root_repo_dir, 'docs', 'CHANGELOG.md')
-    content = None
-    with open(changelog_file, 'r') as fp:
+    changelog_file = os.path.join(root_repo_dir, "docs", "CHANGELOG.md")
+    with open(changelog_file, "r") as fp:
         content = fp.readlines()
-    new_content = f'**{ver}**:\n\n' + f'- Check the release note and download the package/source from [Here](https://github.com/splunk/eventgen/releases/tag/{ver})\n\n'
-    with open(changelog_file, 'w') as fp:
+    new_content = (
+        f"**{ver}**:\n\n"
+        + f"- Check the release note and download the package/source from "
+        f"[Here](https://github.com/splunk/eventgen/releases/tag/{ver})\n\n"
+    )
+    with open(changelog_file, "w") as fp:
         fp.write(new_content)
         for l in content:
             fp.write(l)
-    logging.info('CHANGELOG.md is updated.')
+    logging.info("CHANGELOG.md is updated.")
 
 
 def commit_updated_files(ver):
-    ver_json_file = os.path.join('splunk_eventgen', 'version.json')
-    app_conf = os.path.join('splunk_eventgen', 'splunk_app', 'default', 'app.conf')
-    changelog = os.path.join('docs', 'CHANGELOG.md')
-    run_sh_cmd(['git', 'add', ver_json_file])
-    run_sh_cmd(['git', 'add', app_conf])
-    run_sh_cmd(['git', 'add', changelog])
-    run_sh_cmd(['git', 'commit', '-m', f'update eventgen version to {ver}'], False)
-    logging.info('committed version files.')
+    ver_json_file = os.path.join("splunk_eventgen", "version.json")
+    app_conf = os.path.join("splunk_eventgen", "splunk_app", "default", "app.conf")
+    changelog = os.path.join("docs", "CHANGELOG.md")
+    run_sh_cmd(["git", "add", ver_json_file])
+    run_sh_cmd(["git", "add", app_conf])
+    run_sh_cmd(["git", "add", changelog])
+    run_sh_cmd(["git", "commit", "-m", f"update eventgen version to {ver}"], False)
+    logging.info("committed version files.")
 
 
 def create_pr(ver, token, target_branch):
     release_branch = get_release_branch_name(ver)
     response = requests.post(
-        'https://api.github.com/repos/splunk/eventgen/pulls', json={
-            'title': f'Release eventgen {ver}. Merge to {target_branch} branch.', 'head': release_branch, 'base':
-            target_branch, 'body': 'As the title'}, headers={
-                'Accept': 'application/vnd.github.full+json', 'Content-Type': 'application/json', 'Authorization':
-                f'token {token}'})
+        "https://api.github.com/repos/splunk/eventgen/pulls",
+        json={
+            "title": f"Release eventgen {ver}. Merge to {target_branch} branch.",
+            "head": release_branch,
+            "base": target_branch,
+            "body": "As the title",
+        },
+        headers={
+            "Accept": "application/vnd.github.full+json",
+            "Content-Type": "application/json",
+            "Authorization": f"token {token}",
+        },
+    )
     response.raise_for_status()
     data = response.json()
-    pr_url = data['url']
-    logging.info(f'Pull request is created:\n\t{pr_url}')
+    pr_url = data["url"]
+    logging.info(f"Pull request is created:\n\t{pr_url}")
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     arg_values = parse_args()
     if arg_values is None:
         sys.exit(1)
     setup_logging(arg_values.verbose)
     setup_env()
 
-    logging.info('checkout to the develop branch and pull the latest change...')
-    run_sh_cmd(['git', 'checkout', 'develop'])
-    run_sh_cmd(['git', 'pull'])
+    logging.info("checkout to the develop branch and pull the latest change...")
+    run_sh_cmd(["git", "checkout", "develop"])
+    run_sh_cmd(["git", "pull"])
 
-    logging.info('check out the release branch')
+    logging.info("check out the release branch")
     release_branch = get_release_branch_name(arg_values.version_str)
-    branch_exist = run_sh_cmd(['git', 'show-ref', '--verify', f'refs/heads/{release_branch}'], False)
+    branch_exist = run_sh_cmd(
+        ["git", "show-ref", "--verify", f"refs/heads/{release_branch}"], False
+    )
     if not branch_exist:
-        run_sh_cmd(['git', 'checkout', '-b', release_branch])
+        run_sh_cmd(["git", "checkout", "-b", release_branch])
     else:
-        run_sh_cmd(['git', 'checkout', release_branch])
+        run_sh_cmd(["git", "checkout", release_branch])
 
     replace_version(arg_values.version_str)
     update_changelog(arg_values.version_str)
 
     commit_updated_files(arg_values.version_str)
 
-    run_sh_cmd(['git', 'push', 'origin', release_branch])
-    logging.info(f'release branch {release_branch} is pushed to remote repo.')
+    run_sh_cmd(["git", "push", "origin", release_branch])
+    logging.info(f"release branch {release_branch} is pushed to remote repo.")
 
     if arg_values.token:
-        create_pr(arg_values.version_str, arg_values.token, 'develop')
-        create_pr(arg_values.version_str, arg_values.token, 'master')
+        create_pr(arg_values.version_str, arg_values.token, "develop")
+        create_pr(arg_values.version_str, arg_values.token, "master")
     else:
-        pr_url = 'https://github.com/splunk/eventgen/compare'
-        logging.info('create pull reqeust manually by visiting this url:\n{pr_url}')
+        pr_url = "https://github.com/splunk/eventgen/compare"
+        logging.info("create pull reqeust manually by visiting this url:\n{pr_url}")

requirements.txt

@@ -17,7 +17,7 @@ jinja2==2.10.3
 urllib3==1.24.2
 pyOpenSSL
 flake8>=3.7.7
-yapf>=0.26.0
+black==19.10b0
 isort>=4.3.15
 Flask>=1.0.3
 redis==3.3.10

run_tests.py

@@ -3,17 +3,17 @@
 
 import pytest
 
-SMALL = 'tests/small'
-MEDIUM = 'tests/medium'
-LARGE = 'tests/large'
-XLARGE = 'tests/xlarge'
+SMALL = "tests/small"
+MEDIUM = "tests/medium"
+LARGE = "tests/large"
+XLARGE = "tests/xlarge"
 newargs = []
 args = sys.argv[:]
 ENV = os.environ
 PATH = sys.path
 
 # Normally, it is 8, which should match the cores/hyperthreads of most of our systems.
-NUM_TEST_WORKERS_LARGE = '8'
+NUM_TEST_WORKERS_LARGE = "8"
 """
 How to run the tests:
 1. python run_tests.py
@@ -30,21 +30,21 @@
     LARGE = args.pop(0)
     XLARGE = args.pop(0)
 
-    if SMALL.lower() == 'none':
+    if SMALL.lower() == "none":
         SMALL = False
-    if MEDIUM.lower() == 'none':
+    if MEDIUM.lower() == "none":
         MEDIUM = False
-    if LARGE.lower() == 'none':
+    if LARGE.lower() == "none":
         LARGE = False
-    if XLARGE.lower() == 'none':
+    if XLARGE.lower() == "none":
         XLARGE = False
 
 cov_args = [
     "--cov=splunk_eventgen",
     "--cov-config=tests/.coveragerc",
     "--cov-report=term",
     "--cov-report=html",
-    "--cov-append"
+    "--cov-append",
 ]
 
 # Run small tests
@@ -62,7 +62,11 @@
 if MEDIUM:
     sys.path = PATH
     os.environ = ENV
-    args = ["-sv", MEDIUM, "--junitxml=tests/test-reports/tests_medium_results.xml"] + cov_args
+    args = [
+        "-sv",
+        MEDIUM,
+        "--junitxml=tests/test-reports/tests_medium_results.xml",
+    ] + cov_args
     rt = pytest.main(args)
     if rt != 0:
         print("There are failures in medium test cases!")
@@ -73,7 +77,11 @@
 if LARGE:
     sys.path = PATH
     os.environ = ENV
-    args = ["-sv", LARGE, "--junitxml=tests/test-reports/tests_large_results.xml"] + cov_args
+    args = [
+        "-sv",
+        LARGE,
+        "--junitxml=tests/test-reports/tests_large_results.xml",
+    ] + cov_args
     rt = pytest.main(args)
     if rt != 0:
         print("There are failures in large test cases!")