section 6.3: remove download dataset in tox file

.circleci/config.yml

@@ -0,0 +1,283 @@
+version: 2
+
+defaults: &defaults
+  docker:
+    - image: circleci/python:3.7.2
+  working_directory: ~/project
+
+prepare_venv: &prepare_venv
+  run:
+    name: Create venv
+    command: |
+      python3 -m venv venv
+      source venv/bin/activate
+      pip install --upgrade pip
+
+prepare_tox: &prepare_tox
+  run:
+    name: Install tox
+    command: |
+      sudo pip install --upgrade pip
+      pip install --user tox
+
+fetch_data: &fetch_data
+  run:
+    name: Set script permissions and fetch data
+    command: |
+      source venv/bin/activate
+      chmod +x ./scripts/fetch_kaggle_dataset.sh
+      ./scripts/fetch_kaggle_dataset.sh
+
+jobs:
+  test_regression_model_py36:
+    docker:
+      - image: circleci/python:3.6.9
+    working_directory: ~/project/packages/regression_model
+    steps:
+      - checkout:
+          path: ~/project
+      - run:
+          name: Run tests with Python 3.6
+          command: |
+            sudo pip install --upgrade pip
+            pip install --user tox
+            tox -e py36
+
+  test_regression_model_py37:
+    docker:
+      - image: circleci/python:3.7.6
+    working_directory: ~/project/packages/regression_model
+    steps:
+      - checkout:
+          path: ~/project
+      - run:
+          name: Run tests with Python 3.7
+          command: |
+            sudo pip install --upgrade pip
+            pip install --user tox
+            tox -e py37
+
+  test_regression_model_py38:
+    docker:
+      - image: circleci/python:3.8.0
+    working_directory: ~/project/packages/regression_model
+    steps:
+      - checkout:
+          path: ~/project
+      - run:
+          name: Run tests with Python 3.8
+          command: |
+            sudo pip install --upgrade pip
+            pip install --user tox
+            tox -e py38
+
+  test_ml_api_py36:
+    docker:
+      - image: circleci/python:3.6.9
+    working_directory: ~/project/packages/ml_api
+    steps:
+      - checkout:
+          path: ~/project
+      - run:
+          name: Run API tests with Python 3.6
+          command: |
+            sudo pip install --upgrade pip
+            pip install --user tox
+            tox -e py36
+
+  test_ml_api_py37:
+    docker:
+      - image: circleci/python:3.7.6
+    working_directory: ~/project/packages/ml_api
+    steps:
+      - checkout:
+          path: ~/project
+      - run:
+          name: Run API tests with Python 3.7
+          command: |
+            sudo pip install --upgrade pip
+            pip install --user tox
+            tox -e py37
+
+  test_ml_api_py38:
+    docker:
+      - image: circleci/python:3.8.1
+    working_directory: ~/project/packages/ml_api
+    steps:
+      - checkout:
+          path: ~/project
+      - run:
+          name: Run API tests with Python 3.8
+          command: |
+            sudo pip install --upgrade pip
+            pip install --user tox
+            tox -e py38
+
+  train_and_upload_regression_model:
+    <<: *defaults
+    steps:
+      - checkout
+      - *prepare_venv
+      - run: 
+          name: Install requirements
+          command: |
+            . venv/bin/activate
+            pip install -r packages/regression_model/requirements.txt
+      - *fetch_data
+      - run: 
+          name: Train model
+          command: |
+            . venv/bin/activate
+            PYTHONPATH=./packages/regression_model python3 packages/regression_model/regression_model/train_pipeline.py
+      - run: 
+          name: Publish model to Gemfury
+          command: |
+            . venv/bin/activate
+            chmod +x ./scripts/publish_model.sh
+            ./scripts/publish_model.sh ./packages/regression_model/
+
+  section_9_differential_tests:
+    <<: *defaults
+    steps:
+      - checkout
+      - *prepare_venv
+      - run:
+          name: Capturing previous model predictions
+          command: |
+            . venv/bin/activate
+            pip install -r packages/ml_api/diff_test_requirements.txt
+            PYTHONPATH=./packages/ml_api python3 packages/ml_api/tests/capture_model_predictions.py
+      - run:
+          name: Runnning differential tests
+          command: |
+            . venv/bin/activate
+            pip install -r packages/ml_api/requirements.txt
+            py.test -vv packages/ml_api/tests -m differential
+
+  section_10_deploy_to_heroku:
+    <<: *defaults
+    steps:
+      - checkout
+      - run:
+          name: Deploy to Heroku
+          command: |
+            git push https://heroku:$HEROKU_API_KEY@git.heroku.com/$HEROKU_APP_NAME.git master
+
+  section_11_build_and_push_to_heroku_docker:
+    <<: *defaults
+    steps:
+      - checkout
+      - setup_remote_docker:
+          docker_layer_caching: true
+      - run: docker login --username=$HEROKU_EMAIL --password=$HEROKU_API_KEY registry.heroku.com
+      - run:
+          name: Setup Heroku CLI
+          command: |
+            wget -qO- https://cli-assets.heroku.com/install-ubuntu.sh | sh
+      - run: 
+          name: Build and Push Image
+          command: |
+            make build-ml-api-heroku push-ml-api-heroku
+      - run: 
+          name: Release to Heroku
+          command: |
+            heroku container:release web --app $HEROKU_APP_NAME
+
+  section_12_publish_docker_image_to_aws:
+    <<: *defaults
+    working_directory: ~/project/packages/ml_models
+    steps:
+      - checkout
+      - setup_remote_docker
+      - run:
+          name: Publishing docker image to aws ECR
+          command: |
+            sudo pip install awscli
+            eval $(aws ecr get-login --no-include-email --region us-east-1)
+            make build-ml-api-aws tag-ml-api push-ml-api-aws
+            aws ecs update-service --cluster ml-api-cluster --service custom-service --task-definition  first-run-task-definition --force-new-deployment
+
+  section_13_train_and_upload_neural_network_model:
+    docker:
+      - image: circleci/python:3.6.4-stretch
+    working_directory: ~/project
+    steps:
+      - checkout
+      - *prepare_venv
+      - run: 
+          name: Install requirements
+          command: |
+            . venv/bin/activate
+            pip install -r packages/neural_network_model/requirements.txt
+      - run:
+          name: Fetch Training data - 2GB
+          command: |
+            . venv/bin/activate
+            chmod +x ./scripts/fetch_kaggle_large_dataset.sh
+            ./scripts/fetch_kaggle_large_dataset.sh
+      - run: 
+          name: Train model
+          command: |
+            . venv/bin/activate
+            PYTHONPATH=./packages/neural_network_model python3 packages/neural_network_model/neural_network_model/train_pipeline.py
+      - run: 
+          name: Publish model to Gemfury
+          command: |
+            . venv/bin/activate
+            chmod +x ./scripts/publish_model.sh
+            ./scripts/publish_model.sh ./packages/neural_network_model/
+
+workflows:
+  version: 2
+  test-all:
+    jobs:
+      - test_regression_model_py36
+      - test_regression_model_py37
+      - test_regression_model_py38
+      - test_ml_api_py36
+      - test_ml_api_py37
+      # - test_ml_api_py38 pending NN model update
+      - section_9_differential_tests
+      - train_and_upload_regression_model:
+          requires:
+            - test_regression_model_py36
+            - test_regression_model_py37
+            - test_regression_model_py38
+            - test_ml_api_py36
+            - test_ml_api_py37
+            - section_9_differential_tests
+          filters:
+            branches:
+              only:
+                - master
+      # - section_10_deploy_to_heroku:
+      #     requires:
+      #       - train_and_upload_regression_model
+      #     filters:
+      #       branches:
+      #         only:
+      #           - master
+#      - section_11_build_and_push_to_heroku_docker:
+#          requires:
+#            - train_and_upload_regression_model
+#          filters:
+#            branches:
+#              only:
+#                - master
+      # - section_12_publish_docker_image_to_aws:
+      #     requires:
+      #       - train_and_upload_regression_model
+      #     filters:
+      #       branches:
+      #         only:
+      #           - master
+#      - section_13_train_and_upload_neural_network_model:
+#          requires:
+#            - test_regression_model
+#            - test_ml_api
+#            - section_9_differential_tests
+            # - train_and_upload_regression_model
+          # filters:
+          #   branches:
+          #     only:
+          #       - master

.dockerignore

@@ -0,0 +1,9 @@
+jupyter_notebooks*
+*/env*
+*/venv*
+.circleci*
+packages/regression_model
+*.env
+*.log
+.git
+.gitignore

.gitignore

@@ -106,10 +106,30 @@ venv.bak/
 # pycharm
 .idea/
 
-# pickle files
-*.pkl
-
 # datafiles
 packages/regression_model/regression_model/datasets/*.csv
 packages/regression_model/regression_model/datasets/*.zip
 packages/regression_model/regression_model/datasets/*.txt
+train.csv
+test.csv
+data_description.txt
+house-prices-advanced-regression-techniques.zip
+sample_submission.csv
+test_data_predictions.csv
+v2-plant-seedlings-dataset/
+v2-plant-seedlings-dataset.zip
+
+# all logs
+logs/
+
+# trained models (will be created in CI)
+packages/regression_model/regression_model/trained_models/*.pkl
+packages/neural_network_model/neural_network_model/trained_models/*.pkl
+packages/neural_network_model/neural_network_model/trained_models/*.h5
+*.h5
+packages/neural_network_model/neural_network_model/datasets/training_data_reference.txt
+
+.DS_Store
+
+kaggle.json
+packages/ml_api/uploads/*

Dockerfile

@@ -0,0 +1,23 @@
+FROM python:3.6.4
+
+# Create the user that will run the app
+RUN adduser --disabled-password --gecos '' ml-api-user
+
+WORKDIR /opt/ml_api
+
+ARG PIP_EXTRA_INDEX_URL
+ENV FLASK_APP run.py
+
+# Install requirements, including from Gemfury
+ADD ./packages/ml_api /opt/ml_api/
+RUN pip install --upgrade pip
+RUN pip install -r /opt/ml_api/requirements.txt
+
+RUN chmod +x /opt/ml_api/run.sh
+RUN chown -R ml-api-user:ml-api-user ./
+
+USER ml-api-user
+
+EXPOSE 5000
+
+CMD ["bash", "./run.sh"]

LICENSE

@@ -0,0 +1,29 @@
+BSD 3-Clause License
+
+Copyright (c) 2019, Soledad Galli and Christopher Samiullah. Deployment of Machine Learning Models, online course.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+   contributors may be used to endorse or promote products derived from
+   this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.