Merge pull request #65 from cal-itp/better-dependencies

Start pulling in schedule refactor entities!

.github/workflows/ci.yml

@@ -7,88 +7,96 @@ on:
       - published
 
 jobs:
-  checks:
-    name: "Run Tests"
+  lint:
+    name: "lint"
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
       - name: Set up Python
         uses: actions/setup-python@v2
         with:
-          python-version: '3.9'
-      - name: Set up Pre-commit
+          python-version: '3.10'
+      - name: Run pre-commit
         uses: pre-commit/[email protected]
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          python -m pip install -r requirements.txt
-          python -m pip install .
 
+  test:
+    name: "test"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.10'
+      - name: Install poetry
+        uses: abatilo/[email protected]
       - uses: google-github-actions/setup-gcloud@v0
         with:
           service_account_key: ${{ secrets.GCP_SA_KEY }}
           export_default_credentials: true
+      - name: Install dependencies
+        run: poetry install
       - name: Run tests
-        run: |
-          pytest
+        run: poetry run pytest
 
-  release:
+  pypi:
     name: "Release to PyPI"
     runs-on: ubuntu-latest
-    needs: checks
+    needs:
+      - lint
+      - test
     if: "github.event_name == 'release' && startsWith(github.event.release.tag_name, 'v')"
     steps:
-
       - uses: actions/checkout@v2
-      - name: "Set up Python"
-        uses: actions/setup-python@v2
+      - uses: actions/setup-python@v2
         with:
-          python-version: '3.9'
-      - name: "Build package"
-        run: |
-          python setup.py build sdist
-
-      - name: "TEST Upload to PyPI"
-        uses: pypa/gh-action-pypi-publish@release/v1
+          python-version: '3.10'
+      - name: "Upload to TEST PyPI"
+        uses: JRubics/[email protected]
         if: github.event.release.prerelease
         with:
-          user: __token__
-          password: ${{ secrets.PYPI_TEST_API_TOKEN }}
+          repository_name: testpypi
           repository_url: https://test.pypi.org/legacy/
-
+          repository_username: __token__
+          repository_password: ${{ secrets.PYPI_TEST_API_TOKEN }}
+          build_format: "sdist"
       - name: "Upload to PyPI"
-        uses: pypa/gh-action-pypi-publish@release/v1
+        uses: JRubics/poetry-publish@v1.11
         if: "!github.event.release.prerelease"
         with:
-          user: __token__
-          password: ${{ secrets.PYPI_API_TOKEN }}
+          repository_username: __token__
+          repository_password: ${{ secrets.PYPI_API_TOKEN }}
+          build_format: "sdist"
 
-  build_push:
-    name: Package docker image
-    runs-on: ubuntu-18.04
-    needs: ["checks", "release"]
-    if: "success('checks') && !failure('release')"
-    steps:
-      #- uses: actions/checkout@v2
-      - name: Login to GitHub Container Registry
-        uses: docker/login-action@v1
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      # build release image ----
-      - name: "Release: Build and push"
-        uses: docker/build-push-action@v2
-        if: ${{ github.event_name == 'release' &&  startsWith(github.event.release.tag_name, 'hub') }}
-        with:
-          push: true
-          tags: ghcr.io/${{github.repository}}:${{ github.event.release.tag_name }}
-
-      # build any image pushed on a branch starting with development ----
-      - name: "Development: Build and push"
-        uses: docker/build-push-action@v2
-        if: ${{ github.event_name != 'release' && startsWith(github.ref_name, 'development') }}
-        with:
-          push: true
-          tags: ghcr.io/${{github.repository}}:${{github.ref_name}}
+# This is disabled for now; I don't hate the image definition living here, but the CI/CD is overly complicated I think
+#  build_push:
+#    name: Package docker image
+#    runs-on: ubuntu-18.04
+#    needs:
+#      - lint
+#      - test
+#      - release
+#    if: "success('lint') && success('test') && !failure('release')"
+#    steps:
+#      - name: Login to GitHub Container Registry
+#        uses: docker/login-action@v1
+#        with:
+#          registry: ghcr.io
+#          username: ${{ github.actor }}
+#          password: ${{ secrets.GITHUB_TOKEN }}
+#
+#      # build release image ----
+#      - name: "Release: Build and push"
+#        uses: docker/build-push-action@v2
+#        if: ${{ github.event_name == 'release' &&  startsWith(github.event.release.tag_name, 'hub') }}
+#        with:
+#          push: true
+#          tags: ghcr.io/${{github.repository}}:${{ github.event.release.tag_name }}
+#
+#      # build any image pushed on a branch starting with development ----
+#      - name: "Development: Build and push"
+#        uses: docker/build-push-action@v2
+#        if: ${{ github.event_name != 'release' && startsWith(github.ref_name, 'development') }}
+#        with:
+#          push: true
+#          tags: ghcr.io/${{github.repository}}:${{github.ref_name}}

.pre-commit-config.yaml

@@ -3,6 +3,7 @@ repos:
     rev: v2.4.0
     hooks:
       - id: flake8
+        args: ["--ignore=E203,E501,W503"] # rely on black for these
         types:
           - python
       - id: trailing-whitespace
@@ -20,3 +21,8 @@ repos:
       - id: bandit
         args: ["-ll", "--skip=B108,B608,B701"]
         files: .py$
+  - repo: https://github.com/pycqa/isort
+    rev: 5.10.1
+    hooks:
+      - id: isort
+        args: ["--profile", "black", "--filter-files"]

calitp/__init__.py

@@ -1,6 +1,4 @@
 # flake8: noqa
 
-__version__ = "0.0.16"
-
-from .sql import get_table, write_table, query_sql, to_snakecase, get_engine
-from .storage import save_to_gcfs, read_gcfs
+from .sql import get_engine, get_table, query_sql, to_snakecase, write_table
+from .storage import read_gcfs, save_to_gcfs

calitp/__main__.py

@@ -1,7 +1,9 @@
-from calitp.protobuff import get_random_protobuff
 import datetime
+
 import typer
 
+from calitp.protobuff import get_random_protobuff
+
 app = typer.Typer()
 
 

calitp/config.py

@@ -1,10 +1,8 @@
 import os
 import warnings
-
 from contextlib import contextmanager
-from pathlib import Path
 from functools import wraps
-
+from pathlib import Path
 
 CALITP_BQ_MAX_BYTES = os.environ.get("CALITP_BQ_MAX_BYTES", 5_000_000_000)
 CALITP_BQ_LOCATION = os.environ.get("CALITP_BQ_LOCATION", "us-west2")

calitp/magics.py

@@ -1,8 +1,10 @@
 import json
+
+from IPython import get_ipython
 from IPython.core.magic import register_cell_magic
 from IPython.core.magic_arguments import argument, magic_arguments, parse_argstring
 from IPython.display import Markdown, display
-from IPython import get_ipython
+
 from calitp.sql import query_sql
 
 

calitp/protobuff.py

@@ -1,10 +1,12 @@
-from calitp.storage import get_fs
 import datetime
-from google.protobuf import json_format
-from google.transit import gtfs_realtime_pb2
 import json
 import random
 
+from google.protobuf import json_format
+from google.transit import gtfs_realtime_pb2
+
+from calitp.storage import get_fs
+
 
 def get_random_protobuff(glob, bucket="gtfs-data", date="", format="protobuff"):
     date = date.strip("*")  # ignore ending asterix

calitp/sql.py

@@ -1,18 +1,21 @@
 import os
-import pandas as pd
-
 from functools import singledispatch
-from sqlalchemy.sql.expression import Executable, ClauseElement
+
+import pandas as pd
+import yaml
+from jinja2 import Environment
+from sqlalchemy import MetaData, Table, create_engine, sql
 from sqlalchemy.ext.compiler import compiles
-from sqlalchemy import create_engine, Table, MetaData, sql
+from sqlalchemy.sql.expression import ClauseElement, Executable
 
 from .config import (
-    CALITP_BQ_MAX_BYTES,
     CALITP_BQ_LOCATION,
+    CALITP_BQ_MAX_BYTES,
     format_table_name,
     get_project_id,
     require_pipeline,
 )
+from .templates import user_defined_filters, user_defined_macros
 
 
 class CreateTableAs(Executable, ClauseElement):
@@ -113,10 +116,6 @@ def _write_table_df(sql_stmt, table_name, engine=None, replace=True):
 
 
 def query_sql(fname, write_as=None, replace=False, dry_run=False, as_df=True):
-    import yaml
-    from jinja2 import Environment
-    from .templates import user_defined_filters, user_defined_macros
-
     if fname.endswith(".yml") or fname.endswith(".yaml"):
         config = yaml.safe_load(open(fname))