Skip to content

Commit

Permalink
Update buildspec; Freeze protobuf and pyarrow (#129)
Browse files Browse the repository at this point in the history
* Update buildspec; Freeze protobuf and pyarrow

* Freeze protobuf to 3.20.1

* Freeze pyarrow to 0.16.0

* Freeze pyarrow to 1.0.0

* Use tox in buildspec; Correct pyarrow version to 1.0.0

* Make echo calls consistent

Co-authored-by: Mark Bunday <[email protected]>
  • Loading branch information
mabunday and Mark Bunday authored Aug 5, 2022
1 parent d8760d8 commit 645cfa7
Show file tree
Hide file tree
Showing 4 changed files with 36 additions and 19 deletions.
40 changes: 28 additions & 12 deletions ci/buildspec.yml
Original file line number Diff line number Diff line change
@@ -1,9 +1,4 @@
version: 0.2

env:
variables:
FRAMEWORK_VERSION: "0.23-1"

phases:
install:
runtime-versions:
Expand All @@ -13,28 +8,49 @@ phases:
commands:
- echo Logging in to Amazon ECR...
- $(aws ecr get-login --no-include-email --region $AWS_DEFAULT_REGION)
- echo Installing dependencies...
- curl -LO http://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh
- bash Miniconda3-latest-Linux-x86_64.sh -bfp /miniconda3
- export PATH=/miniconda3/bin:${PATH}
- conda install python=3.7
- conda update -y conda
- python3 -m pip install pip==20.1 # Pip dependency resolver in 20.2+ can't resolve some dependencies
- python3 -m pip install .[test]
build:
commands:
- echo Build started on `date`
- echo Building the Docker image...
- echo Building base container...
- docker login -u $dockerhub_username -p $dockerhub_password
- docker build -t sklearn-base:$FRAMEWORK_VERSION-cpu-py3 -f docker/$FRAMEWORK_VERSION/base/Dockerfile.cpu .
- pip install wheel setuptools
- python setup.py bdist_wheel
- echo Building final container...
- docker build -t preprod-sklearn:$FRAMEWORK_VERSION-cpu-py3 -f docker/$FRAMEWORK_VERSION/final/Dockerfile.cpu .
- docker tag preprod-sklearn:$FRAMEWORK_VERSION-cpu-py3 515193369038.dkr.ecr.us-west-2.amazonaws.com/sagemaker-scikit-learn:$FRAMEWORK_VERSION-cpu-py3
- echo Building test container...
- printf "FROM preprod-sklearn:$FRAMEWORK_VERSION-cpu-py3\nADD . /app\nWORKDIR /app\nRUN python3 -m pip install .[test]" > Dockerfile.test
- docker build -t test-sklearn -f Dockerfile.test .
- echo Running tox...
- docker run --rm --entrypoint /bin/bash test-sklearn -c "tox -e ALL"
- echo Running container tests...
- pytest test/integration --docker-base-name preprod-sklearn --tag $FRAMEWORK_VERSION-cpu-py3 --py-version 3 --framework-version $FRAMEWORK_VERSION
post_build:
commands:
- echo Build completed on `date`
- echo Pushing the Docker image...
- |
case $CODEBUILD_WEBHOOK_EVENT in
PUSH)
docker push 515193369038.dkr.ecr.us-west-2.amazonaws.com/sagemaker-scikit-learn:$FRAMEWORK_VERSION-cpu-py3 | grep -v -E "[0-9]{12}.dkr.ecr.\S+.amazonaws.com"
PULL_REQUEST_MERGED)
echo Logging in to Amazon ECR...
$(aws ecr get-login --no-include-email --region $AWS_DEFAULT_REGION)
echo Pushing the Docker image...
docker push $SM_ALPHA.dkr.ecr.us-west-2.amazonaws.com/sagemaker-scikit-learn:$FRAMEWORK_VERSION-cpu-py3 | grep -v -E "[0-9]{12}.dkr.ecr.\S+.amazonaws.com"
docker push $SM_ALPHA.dkr.ecr.us-west-2.amazonaws.com/sagemaker-scikit-learn:$FRAMEWORK_VERSION | grep -v -E "[0-9]{12}.dkr.ecr.\S+.amazonaws.com"
;;
PULL_REQUEST_MERGED | PULL_REQUEST_CREATED | PULL_REQUEST_UPDATED | PULL_REQUEST_REOPENED)
PULL_REQUEST_CREATED | PULL_REQUEST_UPDATED | PULL_REQUEST_REOPENED)
echo Logging in to Amazon ECR...
$(aws ecr get-login --no-include-email --region $AWS_DEFAULT_REGION)
echo Pushing the Docker image...
# pushes test tag for manual verification, requires cleanup in ECR every once in a while though
TEST_TAG=515193369038.dkr.ecr.us-west-2.amazonaws.com/sagemaker-scikit-learn:$FRAMEWORK_VERSION-cpu-py3-test
TEST_TAG=$SM_ALPHA.dkr.ecr.us-west-2.amazonaws.com/sagemaker-scikit-learn:${FRAMEWORK_VERSION}-cpu-py3-test
docker tag preprod-sklearn:$FRAMEWORK_VERSION-cpu-py3 ${TEST_TAG}
docker push ${TEST_TAG} | grep -v -E "[0-9]{12}.dkr.ecr.\S+.amazonaws.com"
;;
Expand Down
2 changes: 1 addition & 1 deletion docker/0.23-1/base/Dockerfile.cpu
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ ARG MINICONDA_VERSION=4.8.3
ARG CONDA_PY_VERSION=37
ARG CONDA_PKG_VERSION=4.9.0
ARG PYTHON_VERSION=3.7.10
ARG PYARROW_VERSION=3.0.0
ARG PYARROW_VERSION=1.0.0
ARG MLIO_VERSION=0.6.0

# Install python and other scikit-learn runtime dependencies
Expand Down
9 changes: 5 additions & 4 deletions requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -2,22 +2,23 @@ boto3==1.24.17
botocore==1.27.18
cryptography==35.0.0
Flask==1.1.1
itsdangerous==2.0.1
gunicorn==20.0.4
itsdangerous==2.0.1
jinja2==3.0.3
MarkupSafe==2.1.1
model-archiver==1.0.3
multi-model-server==1.1.1
numpy==1.19.2
pandas==1.1.3
protobuf==3.20.1
psutil==5.7.2
pyarrow==1.0.0
python-dateutil==2.8.1
sagemaker-inference==1.2.0
retrying==1.3.3
sagemaker-containers==2.8.6.post2
sagemaker-inference==1.2.0
sagemaker-training==4.0.1
scikit-learn==0.23.2
scipy==1.5.3
six==1.15.0
jinja2==3.0.3
MarkupSafe==2.1.1
Werkzeug==0.15.6
4 changes: 2 additions & 2 deletions tox.ini
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,8 @@ deps =
-r{toxinidir}/requirements.txt
-r{toxinidir}/test-requirements.txt
conda_deps=
pyarrow=3.0.0
mlio-py=0.5
pyarrow=1.0.0
mlio-py=0.6
conda_channels=
conda-forge
mlio
Expand Down

0 comments on commit 645cfa7

Please sign in to comment.