From 349a9e5f30e02b6fff35ab19a74cf8aa909d1d7c Mon Sep 17 00:00:00 2001 From: ChristopherGS Date: Thu, 17 Jan 2019 13:15:36 -0800 Subject: [PATCH 01/24] Section 7.2 - REST API Skeleton --- packages/ml_api/__init__.py | 0 packages/ml_api/api/__init__.py | 0 packages/ml_api/api/app.py | 13 +++++++++++++ packages/ml_api/api/controller.py | 10 ++++++++++ packages/ml_api/requirements.txt | 6 ++++++ packages/ml_api/run.py | 7 +++++++ 6 files changed, 36 insertions(+) create mode 100644 packages/ml_api/__init__.py create mode 100644 packages/ml_api/api/__init__.py create mode 100644 packages/ml_api/api/app.py create mode 100644 packages/ml_api/api/controller.py create mode 100644 packages/ml_api/requirements.txt create mode 100644 packages/ml_api/run.py diff --git a/packages/ml_api/__init__.py b/packages/ml_api/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/packages/ml_api/api/__init__.py b/packages/ml_api/api/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/packages/ml_api/api/app.py b/packages/ml_api/api/app.py new file mode 100644 index 000000000..d4f88ca52 --- /dev/null +++ b/packages/ml_api/api/app.py @@ -0,0 +1,13 @@ +from flask import Flask + + +def create_app() -> Flask: + """Create a flask app instance.""" + + flask_app = Flask('ml_api') + + # import blueprints + from api.controller import prediction_app + flask_app.register_blueprint(prediction_app) + + return flask_app diff --git a/packages/ml_api/api/controller.py b/packages/ml_api/api/controller.py new file mode 100644 index 000000000..ccac65fd1 --- /dev/null +++ b/packages/ml_api/api/controller.py @@ -0,0 +1,10 @@ +from flask import Blueprint, request + + +prediction_app = Blueprint('prediction_app', __name__) + + +@prediction_app.route('/health', methods=['GET']) +def health(): + if request.method == 'GET': + return 'ok' diff --git a/packages/ml_api/requirements.txt b/packages/ml_api/requirements.txt new file mode 100644 index 000000000..03a61dde0 --- /dev/null +++ b/packages/ml_api/requirements.txt @@ -0,0 +1,6 @@ +# api +flask==1.0.2 + +# local regression_model package +# update with your local path +-e "C:\Users\chris\repos\deploying-machine-learning-models\packages\regression_model" \ No newline at end of file diff --git a/packages/ml_api/run.py b/packages/ml_api/run.py new file mode 100644 index 000000000..06fce5e27 --- /dev/null +++ b/packages/ml_api/run.py @@ -0,0 +1,7 @@ +from api.app import create_app + + +application = create_app() + +if __name__ == '__main__': + application.run() From 17948c1e3d851fc8d79bc604f17d8331a15ddbf7 Mon Sep 17 00:00:00 2001 From: ChristopherGS Date: Fri, 18 Jan 2019 13:16:07 -0800 Subject: [PATCH 02/24] Section 7.3 - Setup Config and Logging --- packages/ml_api/api/app.py | 9 +++- packages/ml_api/api/config.py | 64 ++++++++++++++++++++++++ packages/ml_api/api/controller.py | 5 ++ packages/ml_api/run.py | 5 +- packages/ml_api/{ => tests}/__init__.py | 0 packages/ml_api/tests/conftest.py | 18 +++++++ packages/ml_api/tests/test_controller.py | 6 +++ 7 files changed, 105 insertions(+), 2 deletions(-) create mode 100644 packages/ml_api/api/config.py rename packages/ml_api/{ => tests}/__init__.py (100%) create mode 100644 packages/ml_api/tests/conftest.py create mode 100644 packages/ml_api/tests/test_controller.py diff --git a/packages/ml_api/api/app.py b/packages/ml_api/api/app.py index d4f88ca52..40abdb55f 100644 --- a/packages/ml_api/api/app.py +++ b/packages/ml_api/api/app.py @@ -1,13 +1,20 @@ from flask import Flask +from api.config import get_logger -def create_app() -> Flask: + +_logger = get_logger(logger_name=__name__) + + +def create_app(*, config_object) -> Flask: """Create a flask app instance.""" flask_app = Flask('ml_api') + flask_app.config.from_object(config_object) # import blueprints from api.controller import prediction_app flask_app.register_blueprint(prediction_app) + _logger.debug('Application instance created') return flask_app diff --git a/packages/ml_api/api/config.py b/packages/ml_api/api/config.py new file mode 100644 index 000000000..62ed27c62 --- /dev/null +++ b/packages/ml_api/api/config.py @@ -0,0 +1,64 @@ +import logging +from logging.handlers import TimedRotatingFileHandler +import pathlib +import os +import sys + +PACKAGE_ROOT = pathlib.Path(__file__).resolve().parent.parent + +FORMATTER = logging.Formatter( + "%(asctime)s — %(name)s — %(levelname)s —" + "%(funcName)s:%(lineno)d — %(message)s") +LOG_DIR = PACKAGE_ROOT / 'logs' +LOG_DIR.mkdir(exist_ok=True) +LOG_FILE = LOG_DIR / 'ml_api.log' + + +def get_console_handler(): + console_handler = logging.StreamHandler(sys.stdout) + console_handler.setFormatter(FORMATTER) + return console_handler + + +def get_file_handler(): + file_handler = TimedRotatingFileHandler( + LOG_FILE, when='midnight') + file_handler.setFormatter(FORMATTER) + file_handler.setLevel(logging.WARNING) + return file_handler + + +def get_logger(*, logger_name): + """Get logger with prepared handlers.""" + + logger = logging.getLogger(logger_name) + + logger.setLevel(logging.DEBUG) + + logger.addHandler(get_console_handler()) + logger.addHandler(get_file_handler()) + logger.propagate = False + + return logger + + +class Config: + DEBUG = False + TESTING = False + CSRF_ENABLED = True + SECRET_KEY = 'this-really-needs-to-be-changed' + SERVER_PORT = 5000 + + +class ProductionConfig(Config): + DEBUG = False + SERVER_PORT = os.environ.get('PORT', 5000) + + +class DevelopmentConfig(Config): + DEVELOPMENT = True + DEBUG = True + + +class TestingConfig(Config): + TESTING = True diff --git a/packages/ml_api/api/controller.py b/packages/ml_api/api/controller.py index ccac65fd1..81a531581 100644 --- a/packages/ml_api/api/controller.py +++ b/packages/ml_api/api/controller.py @@ -1,5 +1,9 @@ from flask import Blueprint, request +from api.config import get_logger + +_logger = get_logger(logger_name=__name__) + prediction_app = Blueprint('prediction_app', __name__) @@ -7,4 +11,5 @@ @prediction_app.route('/health', methods=['GET']) def health(): if request.method == 'GET': + _logger.info('health status OK') return 'ok' diff --git a/packages/ml_api/run.py b/packages/ml_api/run.py index 06fce5e27..75a66dc7b 100644 --- a/packages/ml_api/run.py +++ b/packages/ml_api/run.py @@ -1,7 +1,10 @@ from api.app import create_app +from api.config import DevelopmentConfig -application = create_app() +application = create_app( + config_object=DevelopmentConfig) + if __name__ == '__main__': application.run() diff --git a/packages/ml_api/__init__.py b/packages/ml_api/tests/__init__.py similarity index 100% rename from packages/ml_api/__init__.py rename to packages/ml_api/tests/__init__.py diff --git a/packages/ml_api/tests/conftest.py b/packages/ml_api/tests/conftest.py new file mode 100644 index 000000000..3134a9b4d --- /dev/null +++ b/packages/ml_api/tests/conftest.py @@ -0,0 +1,18 @@ +import pytest + +from api.app import create_app +from api.config import TestingConfig + + +@pytest.fixture +def app(): + app = create_app(config_object=TestingConfig) + + with app.app_context(): + yield app + + +@pytest.fixture +def flask_test_client(app): + with app.test_client() as test_client: + yield test_client diff --git a/packages/ml_api/tests/test_controller.py b/packages/ml_api/tests/test_controller.py new file mode 100644 index 000000000..232704a61 --- /dev/null +++ b/packages/ml_api/tests/test_controller.py @@ -0,0 +1,6 @@ +def test_health_endpoint_returns_200(flask_test_client): + # When + response = flask_test_client.get('/health') + + # Then + assert response.status_code == 200 From b02a1560db3fa8b50cbb36cdd6522a2320f5b512 Mon Sep 17 00:00:00 2001 From: ChristopherGS Date: Sat, 19 Jan 2019 02:32:56 -0800 Subject: [PATCH 03/24] Section 7.4 - Prediction Endpoint --- packages/ml_api/api/controller.py | 19 ++++++++++++++- packages/ml_api/tests/test_controller.py | 30 ++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/packages/ml_api/api/controller.py b/packages/ml_api/api/controller.py index 81a531581..2654e99c6 100644 --- a/packages/ml_api/api/controller.py +++ b/packages/ml_api/api/controller.py @@ -1,4 +1,5 @@ -from flask import Blueprint, request +from flask import Blueprint, request, jsonify +from regression_model.predict import make_prediction from api.config import get_logger @@ -13,3 +14,19 @@ def health(): if request.method == 'GET': _logger.info('health status OK') return 'ok' + + +@prediction_app.route('/v1/predict/regression', methods=['POST']) +def predict(): + if request.method == 'POST': + json_data = request.get_json() + _logger.info(f'Inputs: {json_data}') + + result = make_prediction(input_data=json_data) + _logger.info(f'Outputs: {result}') + + predictions = result.get('predictions')[0] + version = result.get('version') + + return jsonify({'predictions': predictions, + 'version': version}) diff --git a/packages/ml_api/tests/test_controller.py b/packages/ml_api/tests/test_controller.py index 232704a61..3290f46dc 100644 --- a/packages/ml_api/tests/test_controller.py +++ b/packages/ml_api/tests/test_controller.py @@ -1,6 +1,36 @@ +from regression_model.config import config as model_config +from regression_model.processing.data_management import load_dataset +from regression_model import __version__ as _version + +import json +import math + + def test_health_endpoint_returns_200(flask_test_client): # When response = flask_test_client.get('/health') # Then assert response.status_code == 200 + + +def test_prediction_endpoint_returns_prediction(flask_test_client): + # Given + # Load the test data from the regression_model package + # This is important as it makes it harder for the test + # data versions to get confused by not spreading it + # across packages. + test_data = load_dataset(file_name=model_config.TESTING_DATA_FILE) + post_json = test_data[0:1].to_json(orient='records') + + # When + response = flask_test_client.post('/v1/predict/regression', + json=post_json) + + # Then + assert response.status_code == 200 + response_json = json.loads(response.data) + prediction = response_json['predictions'] + response_version = response_json['version'] + assert math.ceil(prediction) == 112476 + assert response_version == _version From 0d47e30867cf79775ff8efb872e0586204569004 Mon Sep 17 00:00:00 2001 From: ChristopherGS Date: Sat, 19 Jan 2019 04:08:54 -0800 Subject: [PATCH 04/24] Section 7.5 - API Versioning --- packages/ml_api/VERSION | 1 + packages/ml_api/api/__init__.py | 4 ++++ packages/ml_api/api/controller.py | 9 +++++++++ packages/ml_api/tests/test_controller.py | 13 +++++++++++++ 4 files changed, 27 insertions(+) create mode 100644 packages/ml_api/VERSION diff --git a/packages/ml_api/VERSION b/packages/ml_api/VERSION new file mode 100644 index 000000000..341cf11fa --- /dev/null +++ b/packages/ml_api/VERSION @@ -0,0 +1 @@ +0.2.0 \ No newline at end of file diff --git a/packages/ml_api/api/__init__.py b/packages/ml_api/api/__init__.py index e69de29bb..ad56c24c1 100644 --- a/packages/ml_api/api/__init__.py +++ b/packages/ml_api/api/__init__.py @@ -0,0 +1,4 @@ +from api.config import PACKAGE_ROOT + +with open(PACKAGE_ROOT / 'VERSION') as version_file: + __version__ = version_file.read().strip() diff --git a/packages/ml_api/api/controller.py b/packages/ml_api/api/controller.py index 2654e99c6..75e410e3e 100644 --- a/packages/ml_api/api/controller.py +++ b/packages/ml_api/api/controller.py @@ -1,7 +1,9 @@ from flask import Blueprint, request, jsonify from regression_model.predict import make_prediction +from regression_model import __version__ as _version from api.config import get_logger +from api import __version__ as api_version _logger = get_logger(logger_name=__name__) @@ -16,6 +18,13 @@ def health(): return 'ok' +@prediction_app.route('/version', methods=['GET']) +def version(): + if request.method == 'GET': + return jsonify({'model_version': _version, + 'api_version': api_version}) + + @prediction_app.route('/v1/predict/regression', methods=['POST']) def predict(): if request.method == 'POST': diff --git a/packages/ml_api/tests/test_controller.py b/packages/ml_api/tests/test_controller.py index 3290f46dc..24a4cfa22 100644 --- a/packages/ml_api/tests/test_controller.py +++ b/packages/ml_api/tests/test_controller.py @@ -5,6 +5,8 @@ import json import math +from api import __version__ as api_version + def test_health_endpoint_returns_200(flask_test_client): # When @@ -14,6 +16,17 @@ def test_health_endpoint_returns_200(flask_test_client): assert response.status_code == 200 +def test_version_endpoint_returns_version(flask_test_client): + # When + response = flask_test_client.get('/version') + + # Then + assert response.status_code == 200 + response_json = json.loads(response.data) + assert response_json['model_version'] == _version + assert response_json['api_version'] == api_version + + def test_prediction_endpoint_returns_prediction(flask_test_client): # Given # Load the test data from the regression_model package From cc2e92e92adb16203a9dfbb332b9e8bd7e9efc3f Mon Sep 17 00:00:00 2001 From: ChristopherGS Date: Sat, 19 Jan 2019 09:02:29 -0800 Subject: [PATCH 05/24] Section 7.6 - API Schema Validation --- packages/ml_api/api/config.py | 2 +- packages/ml_api/api/controller.py | 19 +- packages/ml_api/api/validation.py | 149 ++++++ packages/ml_api/requirements.txt | 3 + packages/ml_api/test_data_predictions.csv | 501 ++++++++++++++++++ packages/ml_api/tests/test_controller.py | 4 +- packages/ml_api/tests/test_validation.py | 26 + .../regression_model/predict.py | 2 +- .../regression_model/tests/test_predict.py | 4 +- 9 files changed, 699 insertions(+), 11 deletions(-) create mode 100644 packages/ml_api/api/validation.py create mode 100644 packages/ml_api/test_data_predictions.csv create mode 100644 packages/ml_api/tests/test_validation.py diff --git a/packages/ml_api/api/config.py b/packages/ml_api/api/config.py index 62ed27c62..028782c54 100644 --- a/packages/ml_api/api/config.py +++ b/packages/ml_api/api/config.py @@ -33,7 +33,7 @@ def get_logger(*, logger_name): logger = logging.getLogger(logger_name) - logger.setLevel(logging.DEBUG) + logger.setLevel(logging.INFO) logger.addHandler(get_console_handler()) logger.addHandler(get_file_handler()) diff --git a/packages/ml_api/api/controller.py b/packages/ml_api/api/controller.py index 75e410e3e..9b39aaac7 100644 --- a/packages/ml_api/api/controller.py +++ b/packages/ml_api/api/controller.py @@ -3,6 +3,7 @@ from regression_model import __version__ as _version from api.config import get_logger +from api.validation import validate_inputs from api import __version__ as api_version _logger = get_logger(logger_name=__name__) @@ -28,14 +29,22 @@ def version(): @prediction_app.route('/v1/predict/regression', methods=['POST']) def predict(): if request.method == 'POST': + # Step 1: Extract POST data from request body as JSON json_data = request.get_json() - _logger.info(f'Inputs: {json_data}') + _logger.debug(f'Inputs: {json_data}') - result = make_prediction(input_data=json_data) - _logger.info(f'Outputs: {result}') + # Step 2: Validate the input using marshmallow schema + input_data, errors = validate_inputs(input_data=json_data) - predictions = result.get('predictions')[0] + # Step 3: Model prediction + result = make_prediction(input_data=input_data) + _logger.debug(f'Outputs: {result}') + + # Step 4: Convert numpy ndarray to list + predictions = result.get('predictions').tolist() version = result.get('version') + # Step 5: Return the response as JSON return jsonify({'predictions': predictions, - 'version': version}) + 'version': version, + 'errors': errors}) diff --git a/packages/ml_api/api/validation.py b/packages/ml_api/api/validation.py new file mode 100644 index 000000000..84414a294 --- /dev/null +++ b/packages/ml_api/api/validation.py @@ -0,0 +1,149 @@ +from marshmallow import Schema, fields +from marshmallow import ValidationError + +import typing as t +import json + + +class InvalidInputError(Exception): + """Invalid model input.""" + + +SYNTAX_ERROR_FIELD_MAP = { + '1stFlrSF': 'FirstFlrSF', + '2ndFlrSF': 'SecondFlrSF', + '3SsnPorch': 'ThreeSsnPortch' +} + + +class HouseDataRequestSchema(Schema): + Alley = fields.Str(allow_none=True) + BedroomAbvGr = fields.Integer() + BldgType = fields.Str() + BsmtCond = fields.Str() + BsmtExposure = fields.Str(allow_none=True) + BsmtFinSF1 = fields.Float() + BsmtFinSF2 = fields.Float() + BsmtFinType1 = fields.Str() + BsmtFinType2 = fields.Str() + BsmtFullBath = fields.Float() + BsmtHalfBath = fields.Float() + BsmtQual = fields.Str(allow_none=True) + BsmtUnfSF = fields.Float() + CentralAir = fields.Str() + Condition1 = fields.Str() + Condition2 = fields.Str() + Electrical = fields.Str() + EnclosedPorch = fields.Integer() + ExterCond = fields.Str() + ExterQual = fields.Str() + Exterior1st = fields.Str() + Exterior2nd = fields.Str() + Fence = fields.Str(allow_none=True) + FireplaceQu = fields.Str(allow_none=True) + Fireplaces = fields.Integer() + Foundation = fields.Str() + FullBath = fields.Integer() + Functional = fields.Str() + GarageArea = fields.Float() + GarageCars = fields.Float() + GarageCond = fields.Str() + GarageFinish = fields.Str(allow_none=True) + GarageQual = fields.Str() + GarageType = fields.Str(allow_none=True) + GarageYrBlt = fields.Float() + GrLivArea = fields.Integer() + HalfBath = fields.Integer() + Heating = fields.Str() + HeatingQC = fields.Str() + HouseStyle = fields.Str() + Id = fields.Integer() + KitchenAbvGr = fields.Integer() + KitchenQual = fields.Str() + LandContour = fields.Str() + LandSlope = fields.Str() + LotArea = fields.Integer() + LotConfig = fields.Str() + LotFrontage = fields.Float(allow_none=True) + LotShape = fields.Str() + LowQualFinSF = fields.Integer() + MSSubClass = fields.Integer() + MSZoning = fields.Str() + MasVnrArea = fields.Float() + MasVnrType = fields.Str(allow_none=True) + MiscFeature = fields.Str(allow_none=True) + MiscVal = fields.Integer() + MoSold = fields.Integer() + Neighborhood = fields.Str() + OpenPorchSF = fields.Integer() + OverallCond = fields.Integer() + OverallQual = fields.Integer() + PavedDrive = fields.Str() + PoolArea = fields.Integer() + PoolQC = fields.Str(allow_none=True) + RoofMatl = fields.Str() + RoofStyle = fields.Str() + SaleCondition = fields.Str() + SaleType = fields.Str() + ScreenPorch = fields.Integer() + Street = fields.Str() + TotRmsAbvGrd = fields.Integer() + TotalBsmtSF = fields.Float() + Utilities = fields.Str() + WoodDeckSF = fields.Integer() + YearBuilt = fields.Integer() + YearRemodAdd = fields.Integer() + YrSold = fields.Integer() + FirstFlrSF = fields.Integer() + SecondFlrSF = fields.Integer() + ThreeSsnPortch = fields.Integer() + + +def _filter_error_rows(errors: dict, + validated_input: t.List[dict] + ) -> t.List[dict]: + """Remove input data rows with errors.""" + + indexes = errors.keys() + # delete them in reverse order so that you + # don't throw off the subsequent indexes. + for index in sorted(indexes, reverse=True): + del validated_input[index] + + return validated_input + + +def validate_inputs(input_data): + """Check prediction inputs against schema.""" + + # set many=True to allow passing in a list + schema = HouseDataRequestSchema(strict=True, many=True) + + # convert syntax error field names (beginning with numbers) + for dict in input_data: + for key, value in SYNTAX_ERROR_FIELD_MAP.items(): + dict[value] = dict[key] + del dict[key] + + errors = None + try: + schema.load(input_data) + except ValidationError as exc: + errors = exc.messages + + # convert syntax error field names back + # this is a hack - never name your data + # fields with numbers as the first letter. + for dict in input_data: + for key, value in SYNTAX_ERROR_FIELD_MAP.items(): + dict[key] = dict[value] + del dict[value] + + if errors: + validated_input = _filter_error_rows( + errors=errors, + validated_input=input_data) + else: + validated_input = input_data + + return validated_input, errors diff --git a/packages/ml_api/requirements.txt b/packages/ml_api/requirements.txt index 03a61dde0..0d24a7bbd 100644 --- a/packages/ml_api/requirements.txt +++ b/packages/ml_api/requirements.txt @@ -1,6 +1,9 @@ # api flask==1.0.2 +# schema validation +marshmallow==2.17.0 + # local regression_model package # update with your local path -e "C:\Users\chris\repos\deploying-machine-learning-models\packages\regression_model" \ No newline at end of file diff --git a/packages/ml_api/test_data_predictions.csv b/packages/ml_api/test_data_predictions.csv new file mode 100644 index 000000000..d1117a25b --- /dev/null +++ b/packages/ml_api/test_data_predictions.csv @@ -0,0 +1,501 @@ +,predictions,version +0,143988.30704997465,0.2.0 +1,116598.08159580332,0.2.0 +2,130128.90560814076,0.2.0 +3,113470.10675716968,0.2.0 +4,159022.48121448176,0.2.0 +5,139861.32732907546,0.2.0 +6,227118.89767805065,0.2.0 +7,91953.99400144782,0.2.0 +8,225573.26579772323,0.2.0 +9,125802.8602526304,0.2.0 +10,137481.49149643493,0.2.0 +11,124990.09839895074,0.2.0 +12,133270.15609091,0.2.0 +13,192143.4530280595,0.2.0 +14,123206.5594461486,0.2.0 +15,201801.77975634683,0.2.0 +16,198027.98470170778,0.2.0 +17,185664.94305866087,0.2.0 +18,146728.39264190392,0.2.0 +19,152443.1572738422,0.2.0 +20,197054.58979409203,0.2.0 +21,146781.9115319493,0.2.0 +22,138838.0050135225,0.2.0 +23,259997.45200360558,0.2.0 +24,220904.18524276977,0.2.0 +25,162760.6578114075,0.2.0 +26,81622.7760115488,0.2.0 +27,104671.50728326188,0.2.0 +28,129551.38264993431,0.2.0 +29,95446.01639989471,0.2.0 +30,129507.4444341237,0.2.0 +31,95477.93516568728,0.2.0 +32,129422.6043698834,0.2.0 +33,128062.38086640426,0.2.0 +34,123419.71922835958,0.2.0 +35,128318.94350485185,0.2.0 +36,207431.6698047325,0.2.0 +37,174685.92854135018,0.2.0 +38,204544.1513220886,0.2.0 +39,188046.15280301377,0.2.0 +40,182971.78532877663,0.2.0 +41,70097.27238622728,0.2.0 +42,110733.2059471847,0.2.0 +43,93994.92500037784,0.2.0 +44,252924.35745892464,0.2.0 +45,214641.99038515135,0.2.0 +46,154979.9669243978,0.2.0 +47,160810.80098181101,0.2.0 +48,230690.236786167,0.2.0 +49,196243.15614263792,0.2.0 +50,177792.5604951465,0.2.0 +51,150956.42632815256,0.2.0 +52,168211.15880784288,0.2.0 +53,158387.31855224012,0.2.0 +54,114339.5601018531,0.2.0 +55,90052.36198593948,0.2.0 +56,89964.45949954129,0.2.0 +57,98668.89304456668,0.2.0 +58,121518.86270978909,0.2.0 +59,134198.59781615838,0.2.0 +60,163434.02753944616,0.2.0 +61,135542.55508479764,0.2.0 +62,141825.43043982252,0.2.0 +63,227613.38755000453,0.2.0 +64,188761.60830094197,0.2.0 +65,116489.4563051063,0.2.0 +66,167327.47818717395,0.2.0 +67,183019.80781626955,0.2.0 +68,263704.159135985,0.2.0 +69,194109.36377179576,0.2.0 +70,300262.7532032975,0.2.0 +71,223004.09657281314,0.2.0 +72,229985.38944263826,0.2.0 +73,184172.20037350367,0.2.0 +74,188222.84233142118,0.2.0 +75,188097.29339417908,0.2.0 +76,172331.10498565168,0.2.0 +77,174886.6907641111,0.2.0 +78,201441.14534017237,0.2.0 +79,178852.47480584026,0.2.0 +80,225286.87493988863,0.2.0 +81,186618.03844702366,0.2.0 +82,253907.81542043414,0.2.0 +83,240359.90484464006,0.2.0 +84,238601.0921535284,0.2.0 +85,177935.77765021168,0.2.0 +86,162057.79394455065,0.2.0 +87,163514.64562596226,0.2.0 +88,133002.50357947565,0.2.0 +89,126285.82757075419,0.2.0 +90,114122.89197558099,0.2.0 +91,118965.43322308766,0.2.0 +92,107820.17501469971,0.2.0 +93,107672.41260124673,0.2.0 +94,161142.56666974662,0.2.0 +95,155175.112064241,0.2.0 +96,159626.62056220102,0.2.0 +97,159289.85166702382,0.2.0 +98,164753.43823200595,0.2.0 +99,130441.66184067688,0.2.0 +100,150115.21843697876,0.2.0 +101,363780.0225506806,0.2.0 +102,330017.780544809,0.2.0 +103,331883.3191102819,0.2.0 +104,406837.5511403465,0.2.0 +105,292997.10969063273,0.2.0 +106,306609.27632288035,0.2.0 +107,329626.60615839734,0.2.0 +108,311532.52238578524,0.2.0 +109,302589.7805774104,0.2.0 +110,313113.53389941505,0.2.0 +111,255492.2795391536,0.2.0 +112,348040.2630000232,0.2.0 +113,286215.77612206567,0.2.0 +114,257811.3774942191,0.2.0 +115,219056.33504400466,0.2.0 +116,221072.9009001751,0.2.0 +117,227272.5447635412,0.2.0 +118,389000.9584031945,0.2.0 +119,333081.2372066048,0.2.0 +120,301748.2795090072,0.2.0 +121,268886.605541231,0.2.0 +122,292214.7783535345,0.2.0 +123,218893.10534405566,0.2.0 +124,198679.87790616706,0.2.0 +125,198256.12319179106,0.2.0 +126,203810.58008877232,0.2.0 +127,200888.22351579432,0.2.0 +128,208173.15639542375,0.2.0 +129,208236.64492513813,0.2.0 +130,204263.56750308358,0.2.0 +131,194016.82016564548,0.2.0 +132,247220.62121392722,0.2.0 +133,186454.85767170336,0.2.0 +134,183808.3284633914,0.2.0 +135,184105.97903285234,0.2.0 +136,239209.89605894414,0.2.0 +137,184218.80235097196,0.2.0 +138,307821.6280329202,0.2.0 +139,309780.2215794851,0.2.0 +140,250051.75088695402,0.2.0 +141,264234.36472344183,0.2.0 +142,238517.39539507058,0.2.0 +143,253639.64599699862,0.2.0 +144,266777.25555390265,0.2.0 +145,249262.33173072065,0.2.0 +146,354687.6212203011,0.2.0 +147,211718.31772737036,0.2.0 +148,208112.29103266165,0.2.0 +149,269063.04990015837,0.2.0 +150,232554.7387626751,0.2.0 +151,267547.16223942576,0.2.0 +152,259496.4322217068,0.2.0 +153,254987.37388475015,0.2.0 +154,213297.22522688,0.2.0 +155,209521.4853124122,0.2.0 +156,168400.4848772304,0.2.0 +157,168269.52494463106,0.2.0 +158,138015.7063444789,0.2.0 +159,197692.7497359191,0.2.0 +160,210792.23068435694,0.2.0 +161,160895.21637656086,0.2.0 +162,129967.65699942572,0.2.0 +163,148887.7470968613,0.2.0 +164,189032.60710901304,0.2.0 +165,206354.3720483368,0.2.0 +166,170625.45360343822,0.2.0 +167,161155.2832590772,0.2.0 +168,177241.4857453312,0.2.0 +169,152617.9750132888,0.2.0 +170,164767.3082372813,0.2.0 +171,121689.0145099861,0.2.0 +172,114755.20351999925,0.2.0 +173,109385.54490451732,0.2.0 +174,115908.28531894127,0.2.0 +175,127297.15226141199,0.2.0 +176,111687.7144642378,0.2.0 +177,250341.40946203517,0.2.0 +178,231747.51470786144,0.2.0 +179,273940.75455758354,0.2.0 +180,223840.72800951728,0.2.0 +181,207683.72914446727,0.2.0 +182,185613.50839666792,0.2.0 +183,195932.25270587756,0.2.0 +184,248138.38057655803,0.2.0 +185,188290.29546011682,0.2.0 +186,210444.7210381098,0.2.0 +187,205928.18597414377,0.2.0 +188,210044.0320203481,0.2.0 +189,156787.38785618285,0.2.0 +190,149779.3462459088,0.2.0 +191,222254.2913941949,0.2.0 +192,117338.5782329264,0.2.0 +193,144956.37156722017,0.2.0 +194,190502.7599290919,0.2.0 +195,176058.9300745161,0.2.0 +196,113437.17520996452,0.2.0 +197,113005.87286210393,0.2.0 +198,148396.4974016323,0.2.0 +199,155111.51255427708,0.2.0 +200,160895.4088655705,0.2.0 +201,146811.64156366416,0.2.0 +202,161697.96498210484,0.2.0 +203,175408.29205737467,0.2.0 +204,119486.7853118973,0.2.0 +205,155735.2535739763,0.2.0 +206,161732.25789945782,0.2.0 +207,186302.28474718594,0.2.0 +208,126314.40090076534,0.2.0 +209,161489.29160402366,0.2.0 +210,142192.79730554653,0.2.0 +211,125295.79760954925,0.2.0 +212,133726.54674477206,0.2.0 +213,131402.58297528428,0.2.0 +214,147256.8448434014,0.2.0 +215,130042.3601888925,0.2.0 +216,126109.99661525768,0.2.0 +217,104028.06280588396,0.2.0 +218,139015.86204044707,0.2.0 +219,123915.67823516048,0.2.0 +220,178112.6718654715,0.2.0 +221,125873.4394256058,0.2.0 +222,94911.69337443665,0.2.0 +223,137426.63537243495,0.2.0 +224,110144.45586689096,0.2.0 +225,119424.4928970573,0.2.0 +226,149432.93149379385,0.2.0 +227,163081.24792773716,0.2.0 +228,72754.84825273752,0.2.0 +229,107008.00619034276,0.2.0 +230,97026.69480171583,0.2.0 +231,176624.72236581342,0.2.0 +232,136815.75834336376,0.2.0 +233,136527.98103527437,0.2.0 +234,149254.9171475344,0.2.0 +235,127404.15185928933,0.2.0 +236,150150.4110071018,0.2.0 +237,122947.21890337647,0.2.0 +238,123038.56391694587,0.2.0 +239,106055.04206900226,0.2.0 +240,133737.62620695255,0.2.0 +241,127761.33500718801,0.2.0 +242,148651.3511288533,0.2.0 +243,150394.04939898496,0.2.0 +244,137871.15589031755,0.2.0 +245,137889.2545253325,0.2.0 +246,135021.79176355613,0.2.0 +247,132212.93368155853,0.2.0 +248,132394.6589172383,0.2.0 +249,116451.46796853734,0.2.0 +250,132045.77239979545,0.2.0 +251,93828.92317256187,0.2.0 +252,98304.79957463636,0.2.0 +253,116592.62783055207,0.2.0 +254,98723.66631722648,0.2.0 +255,70121.22021310769,0.2.0 +256,97709.23487001589,0.2.0 +257,117883.99993469544,0.2.0 +258,145026.28625503322,0.2.0 +259,153912.57618886943,0.2.0 +260,93381.08729006874,0.2.0 +261,123495.69496267234,0.2.0 +262,151217.31007381002,0.2.0 +263,70925.4220942242,0.2.0 +264,134164.7860642941,0.2.0 +265,137115.50773650245,0.2.0 +266,112454.46885682318,0.2.0 +267,113576.35603796394,0.2.0 +268,126311.04816994928,0.2.0 +269,130853.87341430226,0.2.0 +270,134365.47254085648,0.2.0 +271,149331.816504544,0.2.0 +272,113846.4490674583,0.2.0 +273,127309.62370143532,0.2.0 +274,138936.11004121447,0.2.0 +275,126773.14110750334,0.2.0 +276,118674.20763474096,0.2.0 +277,94732.55765810968,0.2.0 +278,115042.27875631058,0.2.0 +279,97413.63757181565,0.2.0 +280,125103.21858739002,0.2.0 +281,127112.78156168538,0.2.0 +282,100712.28345775318,0.2.0 +283,123435.94852302536,0.2.0 +284,146777.37991798244,0.2.0 +285,141324.91303095603,0.2.0 +286,147015.62617541858,0.2.0 +287,182059.49685921244,0.2.0 +288,66635.70748853082,0.2.0 +289,113133.7345902136,0.2.0 +290,115399.86396709623,0.2.0 +291,142613.97712567318,0.2.0 +292,122675.88261778199,0.2.0 +293,128951.35723355877,0.2.0 +294,159633.68071362676,0.2.0 +295,163672.2859152473,0.2.0 +296,200101.77128067127,0.2.0 +297,166260.33914041193,0.2.0 +298,150329.84339014755,0.2.0 +299,140794.76572322496,0.2.0 +300,166102.833620058,0.2.0 +301,140183.19131161584,0.2.0 +302,257819.0508760762,0.2.0 +303,257819.0508760762,0.2.0 +304,257819.0508760762,0.2.0 +305,297489.40422482847,0.2.0 +306,288713.0465842733,0.2.0 +307,238840.80382128613,0.2.0 +308,264054.2118258276,0.2.0 +309,214038.27040784762,0.2.0 +310,216541.14163119273,0.2.0 +311,251482.14382697808,0.2.0 +312,201302.78506297944,0.2.0 +313,221418.6030263962,0.2.0 +314,143245.9627266626,0.2.0 +315,195099.27104358346,0.2.0 +316,194957.58888827328,0.2.0 +317,196553.0339968338,0.2.0 +318,209163.81006532238,0.2.0 +319,137593.75834543034,0.2.0 +320,139886.56269297737,0.2.0 +321,224462.0649769455,0.2.0 +322,249722.4606197197,0.2.0 +323,196221.2726508532,0.2.0 +324,200883.07978660773,0.2.0 +325,236876.5404898464,0.2.0 +326,265449.9719556491,0.2.0 +327,210031.52797804037,0.2.0 +328,250335.16327422266,0.2.0 +329,193702.5517580212,0.2.0 +330,113345.66683243777,0.2.0 +331,141908.87717126816,0.2.0 +332,98061.70102934526,0.2.0 +333,122961.05363435802,0.2.0 +334,117995.15041902235,0.2.0 +335,134068.9122846434,0.2.0 +336,122607.11339521343,0.2.0 +337,128632.12690453106,0.2.0 +338,130665.06200115388,0.2.0 +339,181867.81868509538,0.2.0 +340,172320.99427457084,0.2.0 +341,163115.13448378997,0.2.0 +342,142692.95549842576,0.2.0 +343,204336.63049215134,0.2.0 +344,151865.2725254776,0.2.0 +345,187999.9387459913,0.2.0 +346,153898.50002741258,0.2.0 +347,201370.60175011388,0.2.0 +348,136260.79769104172,0.2.0 +349,167661.378830941,0.2.0 +350,151900.7260108396,0.2.0 +351,203200.5976776774,0.2.0 +352,275987.18626456213,0.2.0 +353,131731.26809609786,0.2.0 +354,72685.59185678526,0.2.0 +355,264769.3677760745,0.2.0 +356,223505.75506482823,0.2.0 +357,140373.47418071458,0.2.0 +358,165740.37720853413,0.2.0 +359,153501.3958318297,0.2.0 +360,333345.8132030645,0.2.0 +361,284907.13582157245,0.2.0 +362,235976.61331734635,0.2.0 +363,237331.86536503406,0.2.0 +364,222571.43251950064,0.2.0 +365,330547.42125199316,0.2.0 +366,126425.36283381855,0.2.0 +367,150931.15863895716,0.2.0 +368,116973.81860226691,0.2.0 +369,147483.17081444428,0.2.0 +370,137775.93779758728,0.2.0 +371,136213.6538169831,0.2.0 +372,160855.09129555486,0.2.0 +373,180999.95456004038,0.2.0 +374,177875.4323401108,0.2.0 +375,183722.0684301858,0.2.0 +376,183394.03709605164,0.2.0 +377,167171.69796713692,0.2.0 +378,253008.1582497637,0.2.0 +379,208356.18546752,0.2.0 +380,184067.27386951286,0.2.0 +381,184525.57241064525,0.2.0 +382,234914.10484877022,0.2.0 +383,319321.39732491894,0.2.0 +384,329258.81904322456,0.2.0 +385,171807.44667235087,0.2.0 +386,300439.8001753106,0.2.0 +387,168715.42175203658,0.2.0 +388,224083.29347340713,0.2.0 +389,169027.4893700393,0.2.0 +390,219986.76456349975,0.2.0 +391,206599.36694968113,0.2.0 +392,168431.21773772905,0.2.0 +393,198938.11718684685,0.2.0 +394,137044.70162504562,0.2.0 +395,256489.3797086342,0.2.0 +396,169081.6811380493,0.2.0 +397,246159.3182317069,0.2.0 +398,146517.01285907425,0.2.0 +399,115488.93084257792,0.2.0 +400,124226.28849234067,0.2.0 +401,105765.49539858926,0.2.0 +402,105734.63795160982,0.2.0 +403,109307.7618847266,0.2.0 +404,153399.47012489414,0.2.0 +405,148098.79308079585,0.2.0 +406,256865.85340555105,0.2.0 +407,353705.2884855737,0.2.0 +408,339406.68729405693,0.2.0 +409,370934.7245862843,0.2.0 +410,412758.66452745936,0.2.0 +411,337318.9162127192,0.2.0 +412,292636.5292003634,0.2.0 +413,306738.89042618143,0.2.0 +414,395200.33469924616,0.2.0 +415,265420.90751885757,0.2.0 +416,304674.1881521481,0.2.0 +417,322466.11906014563,0.2.0 +418,309583.69640512683,0.2.0 +419,222251.71906371377,0.2.0 +420,305633.12114918296,0.2.0 +421,246068.43249597988,0.2.0 +422,237392.40028237563,0.2.0 +423,211279.01604200783,0.2.0 +424,228094.0196541859,0.2.0 +425,217362.23612708444,0.2.0 +426,212395.21391217507,0.2.0 +427,192157.327626266,0.2.0 +428,210131.93667451647,0.2.0 +429,218479.26431069477,0.2.0 +430,227732.65975321413,0.2.0 +431,207550.8611689138,0.2.0 +432,196406.28233478937,0.2.0 +433,215352.46117706495,0.2.0 +434,195390.69073167298,0.2.0 +435,268095.89486272854,0.2.0 +436,317322.5783410133,0.2.0 +437,292294.5209052129,0.2.0 +438,256214.48067033372,0.2.0 +439,289956.5518384693,0.2.0 +440,285699.6865787319,0.2.0 +441,238369.04431785582,0.2.0 +442,266162.84585317614,0.2.0 +443,276105.07384260837,0.2.0 +444,241944.78930174315,0.2.0 +445,212994.50831895912,0.2.0 +446,266502.50110652676,0.2.0 +447,203362.7111452237,0.2.0 +448,180227.73055119175,0.2.0 +449,188392.39553333411,0.2.0 +450,142481.50831170173,0.2.0 +451,174912.95802564104,0.2.0 +452,168060.24103720946,0.2.0 +453,170840.3065243665,0.2.0 +454,185335.0674102329,0.2.0 +455,175685.71835342573,0.2.0 +456,182131.57134249242,0.2.0 +457,127731.04705949678,0.2.0 +458,130944.89863769621,0.2.0 +459,105125.80701127343,0.2.0 +460,113673.41846707783,0.2.0 +461,171746.81645701104,0.2.0 +462,147544.47667904384,0.2.0 +463,266570.15210116236,0.2.0 +464,340483.4209594863,0.2.0 +465,193926.64894274823,0.2.0 +466,177273.1783748505,0.2.0 +467,188439.6899965548,0.2.0 +468,179646.3820244513,0.2.0 +469,277801.9107183519,0.2.0 +470,244750.34380769494,0.2.0 +471,264143.13027023565,0.2.0 +472,264084.9900022445,0.2.0 +473,190623.30283373612,0.2.0 +474,218303.47626378198,0.2.0 +475,209178.35576652727,0.2.0 +476,210247.40015571192,0.2.0 +477,305489.9014144604,0.2.0 +478,206548.65094650167,0.2.0 +479,260901.671279582,0.2.0 +480,234130.08563281858,0.2.0 +481,215084.1602052955,0.2.0 +482,162068.0157257143,0.2.0 +483,175403.3655499554,0.2.0 +484,188329.78909449733,0.2.0 +485,148772.6745077038,0.2.0 +486,135234.48910921262,0.2.0 +487,132981.35850945665,0.2.0 +488,142443.15434220844,0.2.0 +489,172322.6219487221,0.2.0 +490,114015.40802504608,0.2.0 +491,131679.82317114327,0.2.0 +492,140830.26421534023,0.2.0 +493,96630.01740632812,0.2.0 +494,146497.76662391485,0.2.0 +495,161384.411998765,0.2.0 +496,122294.75296565886,0.2.0 +497,187349.35839738324,0.2.0 +498,139773.34125411394,0.2.0 +499,151158.00827612064,0.2.0 diff --git a/packages/ml_api/tests/test_controller.py b/packages/ml_api/tests/test_controller.py index 24a4cfa22..cc0beb09f 100644 --- a/packages/ml_api/tests/test_controller.py +++ b/packages/ml_api/tests/test_controller.py @@ -38,12 +38,12 @@ def test_prediction_endpoint_returns_prediction(flask_test_client): # When response = flask_test_client.post('/v1/predict/regression', - json=post_json) + json=json.loads(post_json)) # Then assert response.status_code == 200 response_json = json.loads(response.data) prediction = response_json['predictions'] response_version = response_json['version'] - assert math.ceil(prediction) == 112476 + assert math.ceil(prediction[0]) == 112476 assert response_version == _version diff --git a/packages/ml_api/tests/test_validation.py b/packages/ml_api/tests/test_validation.py new file mode 100644 index 000000000..d34d86c72 --- /dev/null +++ b/packages/ml_api/tests/test_validation.py @@ -0,0 +1,26 @@ +import json + +from regression_model.config import config +from regression_model.processing.data_management import load_dataset + + +def test_prediction_endpoint_validation_200(flask_test_client): + # Given + # Load the test data from the regression_model package. + # This is important as it makes it harder for the test + # data versions to get confused by not spreading it + # across packages. + test_data = load_dataset(file_name=config.TESTING_DATA_FILE) + post_json = test_data.to_json(orient='records') + + # When + response = flask_test_client.post('/v1/predict/regression', + json=json.loads(post_json)) + + # Then + assert response.status_code == 200 + response_json = json.loads(response.data) + + # Check correct number of errors removed + assert len(response_json.get('predictions')) + len( + response_json.get('errors')) == len(test_data) diff --git a/packages/regression_model/regression_model/predict.py b/packages/regression_model/regression_model/predict.py index e28ed8a96..34a9d9c3d 100644 --- a/packages/regression_model/regression_model/predict.py +++ b/packages/regression_model/regression_model/predict.py @@ -18,7 +18,7 @@ def make_prediction(*, input_data) -> dict: """Make a prediction using the saved model pipeline.""" - data = pd.read_json(input_data) + data = pd.DataFrame(input_data) validated_data = validate_inputs(input_data=data) prediction = _price_pipe.predict(validated_data[config.FEATURES]) output = np.exp(prediction) diff --git a/packages/regression_model/tests/test_predict.py b/packages/regression_model/tests/test_predict.py index 0357307b7..8c06e5b78 100644 --- a/packages/regression_model/tests/test_predict.py +++ b/packages/regression_model/tests/test_predict.py @@ -7,7 +7,7 @@ def test_make_single_prediction(): # Given test_data = load_dataset(file_name='test.csv') - single_test_json = test_data[0:1].to_json(orient='records') + single_test_json = test_data[0:1] # When subject = make_prediction(input_data=single_test_json) @@ -22,7 +22,7 @@ def test_make_multiple_predictions(): # Given test_data = load_dataset(file_name='test.csv') original_data_length = len(test_data) - multiple_test_json = test_data.to_json(orient='records') + multiple_test_json = test_data # When subject = make_prediction(input_data=multiple_test_json) From 653581a0159d0e6f80748fb2983adb73d04d837f Mon Sep 17 00:00:00 2001 From: ChristopherGS Date: Sun, 20 Jan 2019 02:24:46 -0800 Subject: [PATCH 06/24] Section 8.3 - Setup to CircleCI Config --- .circleci/config.yml | 24 ++++++++++++++++++ .gitignore | 3 --- .../lasso_regression_output_v0.1.0.pkl | Bin 0 -> 4737 bytes packages/regression_model/requirements.txt | 3 +++ scripts/fetch_kaggle_dataset.sh | 3 +++ 5 files changed, 30 insertions(+), 3 deletions(-) create mode 100644 .circleci/config.yml create mode 100644 packages/regression_model/regression_model/trained_models/lasso_regression_output_v0.1.0.pkl create mode 100644 scripts/fetch_kaggle_dataset.sh diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 000000000..2a09aef7f --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,24 @@ +version: 2 +jobs: + test_regression_model: + working_directory: ~/project + docker: + - image: circleci/python:3.7.2 + steps: + - checkout + - run: + name: Runnning tests + command: | + virtualenv venv + . venv/bin/activate + pip install --upgrade pip + pip install -r packages/regression_model/requirements.txt + chmod +x ./scripts/fetch_kaggle_dataset.sh + ./scripts/fetch_kaggle_dataset.sh + py.test -vv packages/regression_model/tests + +workflows: + version: 2 + test-all: + jobs: + - test_regression_model diff --git a/.gitignore b/.gitignore index cda241356..526da42f5 100644 --- a/.gitignore +++ b/.gitignore @@ -106,9 +106,6 @@ venv.bak/ # pycharm .idea/ -# pickle files -*.pkl - # datafiles packages/regression_model/regression_model/datasets/*.csv packages/regression_model/regression_model/datasets/*.zip diff --git a/packages/regression_model/regression_model/trained_models/lasso_regression_output_v0.1.0.pkl b/packages/regression_model/regression_model/trained_models/lasso_regression_output_v0.1.0.pkl new file mode 100644 index 0000000000000000000000000000000000000000..c7c7b75b4cb0cf6a774b6f47b65f90812c04714d GIT binary patch literal 4737 zcmbUl3v?9Kbs+@8l8~qfiim(()~czfU{GW>zYRFqCD~2FBCg|P_hsM6?9ASo*?iEc zU?D=QXccSypjO3Hv?rci*fiN}J16CL3C1Udc06(O9s_pFrTOl`z7Fk)dR9AP_JumN%g&1*1aA6L6iD zEtb^{B`vFxlFJt?WW<2cTtf%Dk0sn8Dg1{)H}k5JuyU zV{9l%j>Qc+WkZqMRc6B26a+)bAi|=u*`^sSJ$VM>k|hYJF>_XPK~|wOS?Vq8yYsqP zFc_32#~~=9;I^uqX3Yf{PeK}GL(Vc1QchehAfjlB$)SRo?Y(eZ@;Kzx3OVLksJRv% ze+(AJeC|Q}PC)x6*l=Qqqe&AHe{wj?;ohLicEG^&#C zsZ8qztIQQtOOXvj?g^Ucec1@YWEUUISm>G{oRS=gayqJDiWm?_+yJM>VQNS$N{&G2 zJUA^Di(U3PKJ+IBEST0_>o^miSQw`F!i+eaZWhoDXOJ5#mdon~#@P=EoOujxXz{q) zym;1|mXeb#9o7CC`eUXIA@avr6pJO8jd6y#Q=Q4c*JVmIgLN>2X=yg7fC*JgsG#gtB+>?97qS4V8K0ek8uzV_Rp;;ptWAzV zgKKqF&DI)FN2^6@rbX}4pq^IsnJ(GLm=K{=LdW;Uw4i}jF+<5@7g(@>Rt<{U&I}6{ zQUDWjR}Y5hGFrqI({3Ws)fmeXN`^tBE0O3HLzGsvhTPSm8xW&al-#1iA`;S~6Agj!g)s3jhUxGTe{duUC9s#{|5!bB^y5GG;i8mlp&l~y%sPRm*vw9zWc z4=V+~Htg3BpI7ASdG%J{exToFo+LKntRGNfF&#-Ri%@BFH+QPKYz;(2!W3*V| z`mms=)(dgyf=ad8T62yx&B zS=Iz=Siz|!YmYkO{9^L$V<;wHR71XlHRMJcZo-nsFW~rQuERIehMV2|aLNzdA_ll1 zj^NeeNIqXI;$d-AfzP(#yCEJ&)M7*>5fwpHqd1D!i^cphaWroc$M9OQgf9}u@~9Z3 zd?R_cisQH_mhy#S8E+HE^E$Dd$HWSr5Rc;x;_v7Yi;>ZFYXT$9wo*It(9^$SAR zMby=Z;)qfl)I~PjIm`ohA-j$=Ii%5$Ms;YaY`8nb4Mc%BqQK|ca8HPL;nG6N0#bH5 z`;WI_Lx|f5W(d9p!OKxV5Blj^vADp?ZMc_o7&7fXq+5Y>*CX9^NVn3VtFYmISJzdD zUXAEpNBYG!{J_;=4T1q3Lh15AvK+Hd)w2>7Y|X^dg!K;|{L)nEz-o_OiWzLwvn@kh zL3ju=WTOoaJ10tCh)<$3WtM40R3j0AAJS3LU^FV7)drK{5o%X9QPs~FdS3G7!lPfh z{o&169dfp6%<0r*qbK6429KdGKeAynChIv)Tl6@+gHG3h_H7wz9tS%+r_amCmK=;K zT2$^%5M>ayqMRpecrrwKdMZk2HP+^*oF1hYeoPTb>-ipddJXwglT106#|BU`8PgE2&$O@=6-J+aUH~`JQU0Us`O)`8>nMDt-Mqf=`OBvlhbPa;m$40_J$b$jM~=RY{^;X}Y<>SBb$4(>aX48K2m~bW zA}@7f=Z)8Ic%4e+d6nnm5B4}s%^P$bX~avuY`C{`cyq9XI(L5`jMq%8aEgXEbScBk zG`3X1My!u|5Z=Nf@H-pcc6(h}&2t&{HVBq`38JG)orQ(nQgrxzgK!Wv!^f@qiX|x; z7V0$1TkwZ9l#Ci4m9nZVXYj|hR5GRiq5q$Iqv zhU5Kf@|<@+m{NN#G2ihZRD++1m)ew$Hc3j6wb}uzj@{s5N>Lox_r~6TZJXjyXWmWdAsny zBgx4w7Eqr!nF{=IG2x@UyugH3^*FK!gB zIQ2gt_I>)Wphfl%j0(Rb99mV~_uwJhD_R&V$0Zs3rXpqKyaa#ellfHqJTPs_>RtEh z0({sD|A@m!tDAqW!-mX82;G{AIISn{yL}JJH*JdIlS2$J@BuX3$Oei DhDwLD literal 0 HcmV?d00001 diff --git a/packages/regression_model/requirements.txt b/packages/regression_model/requirements.txt index 0f6f283d5..919b75917 100644 --- a/packages/regression_model/requirements.txt +++ b/packages/regression_model/requirements.txt @@ -14,3 +14,6 @@ pytest>=5.3.2,<6.0.0 # packaging setuptools>=41.4.0,<42.0.0 wheel>=0.33.6,<0.34.0 + +# fetching datasets +kaggle>=1.5.6,<1.6.0 diff --git a/scripts/fetch_kaggle_dataset.sh b/scripts/fetch_kaggle_dataset.sh new file mode 100644 index 000000000..455b9c970 --- /dev/null +++ b/scripts/fetch_kaggle_dataset.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +kaggle competitions download -c house-prices-advanced-regression-techniques -p packages/regression_model/regression_model/datasets/ \ No newline at end of file From a5c6752ae4c6b727aa33322d7b499bc4fa1fb2b5 Mon Sep 17 00:00:00 2001 From: ChristopherGS Date: Sun, 20 Jan 2019 05:21:11 -0800 Subject: [PATCH 07/24] Section 8.4 - Publishing the model in CI --- .circleci/config.yml | 41 +++++++++++++++++++++++++++++ packages/ml_api/requirements.txt | 7 ++--- scripts/publish_model.sh | 44 ++++++++++++++++++++++++++++++++ 3 files changed, 89 insertions(+), 3 deletions(-) create mode 100644 scripts/publish_model.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index 2a09aef7f..05bb7a242 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -17,8 +17,49 @@ jobs: ./scripts/fetch_kaggle_dataset.sh py.test -vv packages/regression_model/tests + test_ml_api: + working_directory: ~/project + docker: + - image: circleci/python:3.7.2 + steps: + - checkout + - run: + name: Runnning tests + command: | + virtualenv venv + . venv/bin/activate + pip install --upgrade pip + pip install -r packages/ml_api/requirements.txt + py.test -vv packages/ml_api/tests + + train_and_upload_regression_model: + working_directory: ~/project + docker: + - image: circleci/python:3.7.2 + steps: + - checkout + - run: + name: Setup env + command: | + virtualenv venv + . venv/bin/activate + pip install -r packages/regression_model/requirements.txt + - run: + name: Publish model + command: | + . venv/bin/activate + chmod +x ./scripts/fetch_kaggle_dataset.sh ./scripts/publish_model.sh + ./scripts/fetch_kaggle_dataset.sh + PYTHONPATH=./packages/regression_model python3 packages/regression_model/regression_model/train_pipeline.py + ./scripts/publish_model.sh ./packages/regression_model/ + workflows: version: 2 test-all: jobs: - test_regression_model + - test_ml_api + - train_and_upload_regression_model + - test_ml_api: + requires: + - train_and_upload_regression_model diff --git a/packages/ml_api/requirements.txt b/packages/ml_api/requirements.txt index 0d24a7bbd..13e82d198 100644 --- a/packages/ml_api/requirements.txt +++ b/packages/ml_api/requirements.txt @@ -1,9 +1,10 @@ +--extra-index-url=${PIP_EXTRA_INDEX_URL} + # api flask==1.0.2 # schema validation marshmallow==2.17.0 -# local regression_model package -# update with your local path --e "C:\Users\chris\repos\deploying-machine-learning-models\packages\regression_model" \ No newline at end of file +# Install from gemfury +regression-model==0.1.0 \ No newline at end of file diff --git a/scripts/publish_model.sh b/scripts/publish_model.sh new file mode 100644 index 000000000..9b29b7505 --- /dev/null +++ b/scripts/publish_model.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +# Building packages and uploading them to a Gemfury repository + +GEMFURY_URL=$PIP_EXTRA_INDEX_URL + +set -e + +DIRS="$@" +BASE_DIR=$(pwd) +SETUP="setup.py" + +warn() { + echo "$@" 1>&2 +} + +die() { + warn "$@" + exit 1 +} + +build() { + DIR="${1/%\//}" + echo "Checking directory $DIR" + cd "$BASE_DIR/$DIR" + [ ! -e $SETUP ] && warn "No $SETUP file, skipping" && return + PACKAGE_NAME=$(python $SETUP --fullname) + echo "Package $PACKAGE_NAME" + python "$SETUP" sdist bdist_wheel || die "Building package $PACKAGE_NAME failed" + for X in $(ls dist) + do + curl -F package=@"dist/$X" "$GEMFURY_URL" || die "Uploading package $PACKAGE_NAME failed on file dist/$X" + done +} + +if [ -n "$DIRS" ]; then + for dir in $DIRS; do + build $dir + done +else + ls -d */ | while read dir; do + build $dir + done +fi \ No newline at end of file From fa819c2bc9de6cfd4f06c3e5e7b12f01a88b4b10 Mon Sep 17 00:00:00 2001 From: ChristopherGS Date: Sun, 20 Jan 2019 06:29:23 -0800 Subject: [PATCH 08/24] Section 8.5 - Testing the CI Pipeline --- .circleci/config.yml | 82 ++++++++++++++++++++++++++++++++------------ 1 file changed, 60 insertions(+), 22 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 05bb7a242..d33ec5d85 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,28 +1,56 @@ version: 2 + +defaults: &defaults + docker: + - image: circleci/python:3.7.2 + working_directory: ~/project + +prepare_venv: &prepare_venv + run: + name: Create venv + command: | + python3 -m venv venv + source venv/bin/activate + pip install --upgrade pip + +fetch_data: &fetch_data + run: + name: Set script permissions and fetch data + command: | + source venv/bin/activate + chmod +x ./scripts/fetch_kaggle_dataset.sh + ./scripts/fetch_kaggle_dataset.sh + jobs: test_regression_model: - working_directory: ~/project - docker: - - image: circleci/python:3.7.2 + <<: *defaults steps: - checkout + - *prepare_venv - run: - name: Runnning tests + name: Install requirements command: | - virtualenv venv . venv/bin/activate - pip install --upgrade pip pip install -r packages/regression_model/requirements.txt - chmod +x ./scripts/fetch_kaggle_dataset.sh - ./scripts/fetch_kaggle_dataset.sh + - *fetch_data + - run: + name: Train model + command: | + . venv/bin/activate + PYTHONPATH=./packages/regression_model python3 packages/regression_model/regression_model/train_pipeline.py + - run: + name: Run tests + command: | + . venv/bin/activate py.test -vv packages/regression_model/tests test_ml_api: - working_directory: ~/project - docker: - - image: circleci/python:3.7.2 + <<: *defaults steps: - checkout + - restore_cache: + keys: + - py-deps-{{ checksum "packages/ml_api/requirements.txt" }} - run: name: Runnning tests command: | @@ -31,26 +59,32 @@ jobs: pip install --upgrade pip pip install -r packages/ml_api/requirements.txt py.test -vv packages/ml_api/tests + - save_cache: + key: py-deps-{{ checksum "packages/ml_api/requirements.txt" }} + paths: + - "/venv" train_and_upload_regression_model: - working_directory: ~/project - docker: - - image: circleci/python:3.7.2 + <<: *defaults steps: - checkout + - *prepare_venv - run: - name: Setup env + name: Install requirements command: | - virtualenv venv . venv/bin/activate pip install -r packages/regression_model/requirements.txt + - *fetch_data - run: - name: Publish model + name: Train model command: | . venv/bin/activate - chmod +x ./scripts/fetch_kaggle_dataset.sh ./scripts/publish_model.sh - ./scripts/fetch_kaggle_dataset.sh PYTHONPATH=./packages/regression_model python3 packages/regression_model/regression_model/train_pipeline.py + - run: + name: Publish model to Gemfury + command: | + . venv/bin/activate + chmod +x ./scripts/publish_model.sh ./scripts/publish_model.sh ./packages/regression_model/ workflows: @@ -59,7 +93,11 @@ workflows: jobs: - test_regression_model - test_ml_api - - train_and_upload_regression_model - - test_ml_api: + - train_and_upload_regression_model: requires: - - train_and_upload_regression_model + - test_regression_model + - test_ml_api + # filters: + # branches: + # only: + # - master From c267000079f464f132043767810f3968558392ee Mon Sep 17 00:00:00 2001 From: ChristopherGS Date: Sun, 20 Jan 2019 13:31:35 -0800 Subject: [PATCH 09/24] Section 9.2 - Setup Differential Tests --- .circleci/config.yml | 2 +- .../tests/differential_tests/__init__.py | 0 .../capture_model_predictions.py | 38 ++ .../differential_tests/test_differential.py | 47 ++ .../datasets/test_data_predictions.csv | 501 ++++++++++++++++++ 5 files changed, 587 insertions(+), 1 deletion(-) create mode 100644 packages/ml_api/tests/differential_tests/__init__.py create mode 100644 packages/ml_api/tests/differential_tests/capture_model_predictions.py create mode 100644 packages/ml_api/tests/differential_tests/test_differential.py create mode 100644 packages/regression_model/regression_model/datasets/test_data_predictions.csv diff --git a/.circleci/config.yml b/.circleci/config.yml index d33ec5d85..e880646bf 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -58,7 +58,7 @@ jobs: . venv/bin/activate pip install --upgrade pip pip install -r packages/ml_api/requirements.txt - py.test -vv packages/ml_api/tests + py.test -vv packages/ml_api/tests -m "not differential" - save_cache: key: py-deps-{{ checksum "packages/ml_api/requirements.txt" }} paths: diff --git a/packages/ml_api/tests/differential_tests/__init__.py b/packages/ml_api/tests/differential_tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/packages/ml_api/tests/differential_tests/capture_model_predictions.py b/packages/ml_api/tests/differential_tests/capture_model_predictions.py new file mode 100644 index 000000000..801c3bab5 --- /dev/null +++ b/packages/ml_api/tests/differential_tests/capture_model_predictions.py @@ -0,0 +1,38 @@ +""" +This script should only be run in CI. +Never run it locally or you will disrupt the +differential test versioning logic. +""" + +import pandas as pd + +from regression_model.predict import make_prediction +from regression_model.processing.data_management import load_dataset + +from api import config + + +def capture_predictions( + *, + save_file: str = 'test_data_predictions.csv'): + """Save the test data predictions to a CSV.""" + + test_data = load_dataset(file_name='test.csv') + + # we take a slice with no input validation issues + multiple_test_json = test_data[99:600] + + predictions = make_prediction(input_data=multiple_test_json) + + # save predictions for the test dataset + predictions_df = pd.DataFrame(predictions) + + # hack here to save the file to the regression model + # package of the repo, not the installed package + predictions_df.to_csv( + f'{config.PACKAGE_ROOT.parent}/' + f'regression_model/regression_model/datasets/{save_file}') + + +if __name__ == '__main__': + capture_predictions() diff --git a/packages/ml_api/tests/differential_tests/test_differential.py b/packages/ml_api/tests/differential_tests/test_differential.py new file mode 100644 index 000000000..b755570db --- /dev/null +++ b/packages/ml_api/tests/differential_tests/test_differential.py @@ -0,0 +1,47 @@ +import math + +import pytest + +from regression_model.config import config +from regression_model.predict import make_prediction +from regression_model.processing.data_management import load_dataset + + +@pytest.mark.differential +def test_model_prediction_differential( + *, + save_file='test_data_predictions.csv'): + """ + This test compares the prediction result similarity of + the current model with the previous model's results. + """ + # Given + previous_model_df = load_dataset(file_name='test_data_predictions.csv') + previous_model_predictions = previous_model_df.predictions.values + test_data = load_dataset(file_name='test.csv') + multiple_test_json = test_data[99:600] + + # When + response = make_prediction(input_data=multiple_test_json) + current_model_predictions = response.get('predictions') + + # Then + # diff the current model vs. the old model + assert len(previous_model_predictions) == len( + current_model_predictions) + + # Perform the differential test + for previous_value, current_value in zip( + previous_model_predictions, current_model_predictions): + + # convert numpy float64 to Python float. + previous_value = previous_value.item() + current_value = current_value.item() + + # rel_tol is the relative tolerance – it is the maximum allowed + # difference between a and b, relative to the larger absolute + # value of a or b. For example, to set a tolerance of 5%, pass + # rel_tol=0.05. + assert math.isclose(previous_value, + current_value, + rel_tol=config.ACCEPTABLE_MODEL_DIFFERENCE) diff --git a/packages/regression_model/regression_model/datasets/test_data_predictions.csv b/packages/regression_model/regression_model/datasets/test_data_predictions.csv new file mode 100644 index 000000000..4fda7985c --- /dev/null +++ b/packages/regression_model/regression_model/datasets/test_data_predictions.csv @@ -0,0 +1,501 @@ +,predictions,version +0,143988.30704997465,0.1.0 +1,116598.08159580332,0.1.0 +2,130128.90560814076,0.1.0 +3,113470.10675716968,0.1.0 +4,159022.48121448176,0.1.0 +5,139861.32732907546,0.1.0 +6,227118.89767805065,0.1.0 +7,91953.99400144782,0.1.0 +8,225573.26579772323,0.1.0 +9,125802.8602526304,0.1.0 +10,137481.49149643493,0.1.0 +11,124990.09839895074,0.1.0 +12,133270.15609091,0.1.0 +13,192143.4530280595,0.1.0 +14,123206.5594461486,0.1.0 +15,201801.77975634683,0.1.0 +16,198027.98470170778,0.1.0 +17,185664.94305866087,0.1.0 +18,146728.39264190392,0.1.0 +19,152443.1572738422,0.1.0 +20,197054.58979409203,0.1.0 +21,146781.9115319493,0.1.0 +22,138838.0050135225,0.1.0 +23,259997.45200360558,0.1.0 +24,220904.18524276977,0.1.0 +25,162760.6578114075,0.1.0 +26,81622.7760115488,0.1.0 +27,104671.50728326188,0.1.0 +28,129551.38264993431,0.1.0 +29,95446.01639989471,0.1.0 +30,129507.4444341237,0.1.0 +31,95477.93516568728,0.1.0 +32,129422.6043698834,0.1.0 +33,128062.38086640426,0.1.0 +34,123419.71922835958,0.1.0 +35,128318.94350485185,0.1.0 +36,207431.6698047325,0.1.0 +37,174685.92854135018,0.1.0 +38,204544.1513220886,0.1.0 +39,188046.15280301377,0.1.0 +40,182971.78532877663,0.1.0 +41,70097.27238622728,0.1.0 +42,110733.2059471847,0.1.0 +43,93994.92500037784,0.1.0 +44,252924.35745892464,0.1.0 +45,214641.99038515135,0.1.0 +46,154979.9669243978,0.1.0 +47,160810.80098181101,0.1.0 +48,230690.236786167,0.1.0 +49,196243.15614263792,0.1.0 +50,177792.5604951465,0.1.0 +51,150956.42632815256,0.1.0 +52,168211.15880784288,0.1.0 +53,158387.31855224012,0.1.0 +54,114339.5601018531,0.1.0 +55,90052.36198593948,0.1.0 +56,89964.45949954129,0.1.0 +57,98668.89304456668,0.1.0 +58,121518.86270978909,0.1.0 +59,134198.59781615838,0.1.0 +60,163434.02753944616,0.1.0 +61,135542.55508479764,0.1.0 +62,141825.43043982252,0.1.0 +63,227613.38755000453,0.1.0 +64,188761.60830094197,0.1.0 +65,116489.4563051063,0.1.0 +66,167327.47818717395,0.1.0 +67,183019.80781626955,0.1.0 +68,263704.159135985,0.1.0 +69,194109.36377179576,0.1.0 +70,300262.7532032975,0.1.0 +71,223004.09657281314,0.1.0 +72,229985.38944263826,0.1.0 +73,184172.20037350367,0.1.0 +74,188222.84233142118,0.1.0 +75,188097.29339417908,0.1.0 +76,172331.10498565168,0.1.0 +77,174886.6907641111,0.1.0 +78,201441.14534017237,0.1.0 +79,178852.47480584026,0.1.0 +80,225286.87493988863,0.1.0 +81,186618.03844702366,0.1.0 +82,253907.81542043414,0.1.0 +83,240359.90484464006,0.1.0 +84,238601.0921535284,0.1.0 +85,177935.77765021168,0.1.0 +86,162057.79394455065,0.1.0 +87,163514.64562596226,0.1.0 +88,133002.50357947565,0.1.0 +89,126285.82757075419,0.1.0 +90,114122.89197558099,0.1.0 +91,118965.43322308766,0.1.0 +92,107820.17501469971,0.1.0 +93,107672.41260124673,0.1.0 +94,161142.56666974662,0.1.0 +95,155175.112064241,0.1.0 +96,159626.62056220102,0.1.0 +97,159289.85166702382,0.1.0 +98,164753.43823200595,0.1.0 +99,130441.66184067688,0.1.0 +100,150115.21843697876,0.1.0 +101,363780.0225506806,0.1.0 +102,330017.780544809,0.1.0 +103,331883.3191102819,0.1.0 +104,406837.5511403465,0.1.0 +105,292997.10969063273,0.1.0 +106,306609.27632288035,0.1.0 +107,329626.60615839734,0.1.0 +108,311532.52238578524,0.1.0 +109,302589.7805774104,0.1.0 +110,313113.53389941505,0.1.0 +111,255492.2795391536,0.1.0 +112,348040.2630000232,0.1.0 +113,286215.77612206567,0.1.0 +114,257811.3774942191,0.1.0 +115,219056.33504400466,0.1.0 +116,221072.9009001751,0.1.0 +117,227272.5447635412,0.1.0 +118,389000.9584031945,0.1.0 +119,333081.2372066048,0.1.0 +120,301748.2795090072,0.1.0 +121,268886.605541231,0.1.0 +122,292214.7783535345,0.1.0 +123,218893.10534405566,0.1.0 +124,198679.87790616706,0.1.0 +125,198256.12319179106,0.1.0 +126,203810.58008877232,0.1.0 +127,200888.22351579432,0.1.0 +128,208173.15639542375,0.1.0 +129,208236.64492513813,0.1.0 +130,204263.56750308358,0.1.0 +131,194016.82016564548,0.1.0 +132,247220.62121392722,0.1.0 +133,186454.85767170336,0.1.0 +134,183808.3284633914,0.1.0 +135,184105.97903285234,0.1.0 +136,239209.89605894414,0.1.0 +137,184218.80235097196,0.1.0 +138,307821.6280329202,0.1.0 +139,309780.2215794851,0.1.0 +140,250051.75088695402,0.1.0 +141,264234.36472344183,0.1.0 +142,238517.39539507058,0.1.0 +143,253639.64599699862,0.1.0 +144,266777.25555390265,0.1.0 +145,249262.33173072065,0.1.0 +146,354687.6212203011,0.1.0 +147,211718.31772737036,0.1.0 +148,208112.29103266165,0.1.0 +149,269063.04990015837,0.1.0 +150,232554.7387626751,0.1.0 +151,267547.16223942576,0.1.0 +152,259496.4322217068,0.1.0 +153,254987.37388475015,0.1.0 +154,213297.22522688,0.1.0 +155,209521.4853124122,0.1.0 +156,168400.4848772304,0.1.0 +157,168269.52494463106,0.1.0 +158,138015.7063444789,0.1.0 +159,197692.7497359191,0.1.0 +160,210792.23068435694,0.1.0 +161,160895.21637656086,0.1.0 +162,129967.65699942572,0.1.0 +163,148887.7470968613,0.1.0 +164,189032.60710901304,0.1.0 +165,206354.3720483368,0.1.0 +166,170625.45360343822,0.1.0 +167,161155.2832590772,0.1.0 +168,177241.4857453312,0.1.0 +169,152617.9750132888,0.1.0 +170,164767.3082372813,0.1.0 +171,121689.0145099861,0.1.0 +172,114755.20351999925,0.1.0 +173,109385.54490451732,0.1.0 +174,115908.28531894127,0.1.0 +175,127297.15226141199,0.1.0 +176,111687.7144642378,0.1.0 +177,250341.40946203517,0.1.0 +178,231747.51470786144,0.1.0 +179,273940.75455758354,0.1.0 +180,223840.72800951728,0.1.0 +181,207683.72914446727,0.1.0 +182,185613.50839666792,0.1.0 +183,195932.25270587756,0.1.0 +184,248138.38057655803,0.1.0 +185,188290.29546011682,0.1.0 +186,210444.7210381098,0.1.0 +187,205928.18597414377,0.1.0 +188,210044.0320203481,0.1.0 +189,156787.38785618285,0.1.0 +190,149779.3462459088,0.1.0 +191,222254.2913941949,0.1.0 +192,117338.5782329264,0.1.0 +193,144956.37156722017,0.1.0 +194,190502.7599290919,0.1.0 +195,176058.9300745161,0.1.0 +196,113437.17520996452,0.1.0 +197,113005.87286210393,0.1.0 +198,148396.4974016323,0.1.0 +199,155111.51255427708,0.1.0 +200,160895.4088655705,0.1.0 +201,146811.64156366416,0.1.0 +202,161697.96498210484,0.1.0 +203,175408.29205737467,0.1.0 +204,119486.7853118973,0.1.0 +205,155735.2535739763,0.1.0 +206,161732.25789945782,0.1.0 +207,186302.28474718594,0.1.0 +208,126314.40090076534,0.1.0 +209,161489.29160402366,0.1.0 +210,142192.79730554653,0.1.0 +211,125295.79760954925,0.1.0 +212,133726.54674477206,0.1.0 +213,131402.58297528428,0.1.0 +214,147256.8448434014,0.1.0 +215,130042.3601888925,0.1.0 +216,126109.99661525768,0.1.0 +217,104028.06280588396,0.1.0 +218,139015.86204044707,0.1.0 +219,123915.67823516048,0.1.0 +220,178112.6718654715,0.1.0 +221,125873.4394256058,0.1.0 +222,94911.69337443665,0.1.0 +223,137426.63537243495,0.1.0 +224,110144.45586689096,0.1.0 +225,119424.4928970573,0.1.0 +226,149432.93149379385,0.1.0 +227,163081.24792773716,0.1.0 +228,72754.84825273752,0.1.0 +229,107008.00619034276,0.1.0 +230,97026.69480171583,0.1.0 +231,176624.72236581342,0.1.0 +232,136815.75834336376,0.1.0 +233,136527.98103527437,0.1.0 +234,149254.9171475344,0.1.0 +235,127404.15185928933,0.1.0 +236,150150.4110071018,0.1.0 +237,122947.21890337647,0.1.0 +238,123038.56391694587,0.1.0 +239,106055.04206900226,0.1.0 +240,133737.62620695255,0.1.0 +241,127761.33500718801,0.1.0 +242,148651.3511288533,0.1.0 +243,150394.04939898496,0.1.0 +244,137871.15589031755,0.1.0 +245,137889.2545253325,0.1.0 +246,135021.79176355613,0.1.0 +247,132212.93368155853,0.1.0 +248,132394.6589172383,0.1.0 +249,116451.46796853734,0.1.0 +250,132045.77239979545,0.1.0 +251,93828.92317256187,0.1.0 +252,98304.79957463636,0.1.0 +253,116592.62783055207,0.1.0 +254,98723.66631722648,0.1.0 +255,70121.22021310769,0.1.0 +256,97709.23487001589,0.1.0 +257,117883.99993469544,0.1.0 +258,145026.28625503322,0.1.0 +259,153912.57618886943,0.1.0 +260,93381.08729006874,0.1.0 +261,123495.69496267234,0.1.0 +262,151217.31007381002,0.1.0 +263,70925.4220942242,0.1.0 +264,134164.7860642941,0.1.0 +265,137115.50773650245,0.1.0 +266,112454.46885682318,0.1.0 +267,113576.35603796394,0.1.0 +268,126311.04816994928,0.1.0 +269,130853.87341430226,0.1.0 +270,134365.47254085648,0.1.0 +271,149331.816504544,0.1.0 +272,113846.4490674583,0.1.0 +273,127309.62370143532,0.1.0 +274,138936.11004121447,0.1.0 +275,126773.14110750334,0.1.0 +276,118674.20763474096,0.1.0 +277,94732.55765810968,0.1.0 +278,115042.27875631058,0.1.0 +279,97413.63757181565,0.1.0 +280,125103.21858739002,0.1.0 +281,127112.78156168538,0.1.0 +282,100712.28345775318,0.1.0 +283,123435.94852302536,0.1.0 +284,146777.37991798244,0.1.0 +285,141324.91303095603,0.1.0 +286,147015.62617541858,0.1.0 +287,182059.49685921244,0.1.0 +288,66635.70748853082,0.1.0 +289,113133.7345902136,0.1.0 +290,115399.86396709623,0.1.0 +291,142613.97712567318,0.1.0 +292,122675.88261778199,0.1.0 +293,128951.35723355877,0.1.0 +294,159633.68071362676,0.1.0 +295,163672.2859152473,0.1.0 +296,200101.77128067127,0.1.0 +297,166260.33914041193,0.1.0 +298,150329.84339014755,0.1.0 +299,140794.76572322496,0.1.0 +300,166102.833620058,0.1.0 +301,140183.19131161584,0.1.0 +302,257819.0508760762,0.1.0 +303,257819.0508760762,0.1.0 +304,257819.0508760762,0.1.0 +305,297489.40422482847,0.1.0 +306,288713.0465842733,0.1.0 +307,238840.80382128613,0.1.0 +308,264054.2118258276,0.1.0 +309,214038.27040784762,0.1.0 +310,216541.14163119273,0.1.0 +311,251482.14382697808,0.1.0 +312,201302.78506297944,0.1.0 +313,221418.6030263962,0.1.0 +314,143245.9627266626,0.1.0 +315,195099.27104358346,0.1.0 +316,194957.58888827328,0.1.0 +317,196553.0339968338,0.1.0 +318,209163.81006532238,0.1.0 +319,137593.75834543034,0.1.0 +320,139886.56269297737,0.1.0 +321,224462.0649769455,0.1.0 +322,249722.4606197197,0.1.0 +323,196221.2726508532,0.1.0 +324,200883.07978660773,0.1.0 +325,236876.5404898464,0.1.0 +326,265449.9719556491,0.1.0 +327,210031.52797804037,0.1.0 +328,250335.16327422266,0.1.0 +329,193702.5517580212,0.1.0 +330,113345.66683243777,0.1.0 +331,141908.87717126816,0.1.0 +332,98061.70102934526,0.1.0 +333,122961.05363435802,0.1.0 +334,117995.15041902235,0.1.0 +335,134068.9122846434,0.1.0 +336,122607.11339521343,0.1.0 +337,128632.12690453106,0.1.0 +338,130665.06200115388,0.1.0 +339,181867.81868509538,0.1.0 +340,172320.99427457084,0.1.0 +341,163115.13448378997,0.1.0 +342,142692.95549842576,0.1.0 +343,204336.63049215134,0.1.0 +344,151865.2725254776,0.1.0 +345,187999.9387459913,0.1.0 +346,153898.50002741258,0.1.0 +347,201370.60175011388,0.1.0 +348,136260.79769104172,0.1.0 +349,167661.378830941,0.1.0 +350,151900.7260108396,0.1.0 +351,203200.5976776774,0.1.0 +352,275987.18626456213,0.1.0 +353,131731.26809609786,0.1.0 +354,72685.59185678526,0.1.0 +355,264769.3677760745,0.1.0 +356,223505.75506482823,0.1.0 +357,140373.47418071458,0.1.0 +358,165740.37720853413,0.1.0 +359,153501.3958318297,0.1.0 +360,333345.8132030645,0.1.0 +361,284907.13582157245,0.1.0 +362,235976.61331734635,0.1.0 +363,237331.86536503406,0.1.0 +364,222571.43251950064,0.1.0 +365,330547.42125199316,0.1.0 +366,126425.36283381855,0.1.0 +367,150931.15863895716,0.1.0 +368,116973.81860226691,0.1.0 +369,147483.17081444428,0.1.0 +370,137775.93779758728,0.1.0 +371,136213.6538169831,0.1.0 +372,160855.09129555486,0.1.0 +373,180999.95456004038,0.1.0 +374,177875.4323401108,0.1.0 +375,183722.0684301858,0.1.0 +376,183394.03709605164,0.1.0 +377,167171.69796713692,0.1.0 +378,253008.1582497637,0.1.0 +379,208356.18546752,0.1.0 +380,184067.27386951286,0.1.0 +381,184525.57241064525,0.1.0 +382,234914.10484877022,0.1.0 +383,319321.39732491894,0.1.0 +384,329258.81904322456,0.1.0 +385,171807.44667235087,0.1.0 +386,300439.8001753106,0.1.0 +387,168715.42175203658,0.1.0 +388,224083.29347340713,0.1.0 +389,169027.4893700393,0.1.0 +390,219986.76456349975,0.1.0 +391,206599.36694968113,0.1.0 +392,168431.21773772905,0.1.0 +393,198938.11718684685,0.1.0 +394,137044.70162504562,0.1.0 +395,256489.3797086342,0.1.0 +396,169081.6811380493,0.1.0 +397,246159.3182317069,0.1.0 +398,146517.01285907425,0.1.0 +399,115488.93084257792,0.1.0 +400,124226.28849234067,0.1.0 +401,105765.49539858926,0.1.0 +402,105734.63795160982,0.1.0 +403,109307.7618847266,0.1.0 +404,153399.47012489414,0.1.0 +405,148098.79308079585,0.1.0 +406,256865.85340555105,0.1.0 +407,353705.2884855737,0.1.0 +408,339406.68729405693,0.1.0 +409,370934.7245862843,0.1.0 +410,412758.66452745936,0.1.0 +411,337318.9162127192,0.1.0 +412,292636.5292003634,0.1.0 +413,306738.89042618143,0.1.0 +414,395200.33469924616,0.1.0 +415,265420.90751885757,0.1.0 +416,304674.1881521481,0.1.0 +417,322466.11906014563,0.1.0 +418,309583.69640512683,0.1.0 +419,222251.71906371377,0.1.0 +420,305633.12114918296,0.1.0 +421,246068.43249597988,0.1.0 +422,237392.40028237563,0.1.0 +423,211279.01604200783,0.1.0 +424,228094.0196541859,0.1.0 +425,217362.23612708444,0.1.0 +426,212395.21391217507,0.1.0 +427,192157.327626266,0.1.0 +428,210131.93667451647,0.1.0 +429,218479.26431069477,0.1.0 +430,227732.65975321413,0.1.0 +431,207550.8611689138,0.1.0 +432,196406.28233478937,0.1.0 +433,215352.46117706495,0.1.0 +434,195390.69073167298,0.1.0 +435,268095.89486272854,0.1.0 +436,317322.5783410133,0.1.0 +437,292294.5209052129,0.1.0 +438,256214.48067033372,0.1.0 +439,289956.5518384693,0.1.0 +440,285699.6865787319,0.1.0 +441,238369.04431785582,0.1.0 +442,266162.84585317614,0.1.0 +443,276105.07384260837,0.1.0 +444,241944.78930174315,0.1.0 +445,212994.50831895912,0.1.0 +446,266502.50110652676,0.1.0 +447,203362.7111452237,0.1.0 +448,180227.73055119175,0.1.0 +449,188392.39553333411,0.1.0 +450,142481.50831170173,0.1.0 +451,174912.95802564104,0.1.0 +452,168060.24103720946,0.1.0 +453,170840.3065243665,0.1.0 +454,185335.0674102329,0.1.0 +455,175685.71835342573,0.1.0 +456,182131.57134249242,0.1.0 +457,127731.04705949678,0.1.0 +458,130944.89863769621,0.1.0 +459,105125.80701127343,0.1.0 +460,113673.41846707783,0.1.0 +461,171746.81645701104,0.1.0 +462,147544.47667904384,0.1.0 +463,266570.15210116236,0.1.0 +464,340483.4209594863,0.1.0 +465,193926.64894274823,0.1.0 +466,177273.1783748505,0.1.0 +467,188439.6899965548,0.1.0 +468,179646.3820244513,0.1.0 +469,277801.9107183519,0.1.0 +470,244750.34380769494,0.1.0 +471,264143.13027023565,0.1.0 +472,264084.9900022445,0.1.0 +473,190623.30283373612,0.1.0 +474,218303.47626378198,0.1.0 +475,209178.35576652727,0.1.0 +476,210247.40015571192,0.1.0 +477,305489.9014144604,0.1.0 +478,206548.65094650167,0.1.0 +479,260901.671279582,0.1.0 +480,234130.08563281858,0.1.0 +481,215084.1602052955,0.1.0 +482,162068.0157257143,0.1.0 +483,175403.3655499554,0.1.0 +484,188329.78909449733,0.1.0 +485,148772.6745077038,0.1.0 +486,135234.48910921262,0.1.0 +487,132981.35850945665,0.1.0 +488,142443.15434220844,0.1.0 +489,172322.6219487221,0.1.0 +490,114015.40802504608,0.1.0 +491,131679.82317114327,0.1.0 +492,140830.26421534023,0.1.0 +493,96630.01740632812,0.1.0 +494,146497.76662391485,0.1.0 +495,161384.411998765,0.1.0 +496,122294.75296565886,0.1.0 +497,187349.35839738324,0.1.0 +498,139773.34125411394,0.1.0 +499,151158.00827612064,0.1.0 From e8f6dc411c14a629160b7d3868db14d139d91eb2 Mon Sep 17 00:00:00 2001 From: ChristopherGS Date: Mon, 21 Jan 2019 12:40:13 -0800 Subject: [PATCH 10/24] Section 9.3 - Differential Tests in CI Part 1 --- .circleci/config.yml | 22 +- .gitignore | 9 + packages/ml_api/diff_test_requirements.txt | 10 + .../capture_model_predictions.py | 13 +- .../differential_tests/test_differential.py | 27 +- .../regression_model/regression_model/VERSION | 2 +- .../datasets/test_data_predictions.csv | 501 ------------------ .../regression_model/predict.py | 15 +- .../processing/data_management.py | 12 +- .../lasso_regression_output_v0.1.0.pkl | Bin 4737 -> 0 bytes .../regression_model/tests/test_predict.py | 8 +- 11 files changed, 88 insertions(+), 531 deletions(-) create mode 100644 packages/ml_api/diff_test_requirements.txt rename packages/ml_api/tests/{differential_tests => }/capture_model_predictions.py (68%) delete mode 100644 packages/regression_model/regression_model/datasets/test_data_predictions.csv delete mode 100644 packages/regression_model/regression_model/trained_models/lasso_regression_output_v0.1.0.pkl diff --git a/.circleci/config.yml b/.circleci/config.yml index e880646bf..5108aaa3d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -54,7 +54,7 @@ jobs: - run: name: Runnning tests command: | - virtualenv venv + python3 -m venv venv . venv/bin/activate pip install --upgrade pip pip install -r packages/ml_api/requirements.txt @@ -87,16 +87,36 @@ jobs: chmod +x ./scripts/publish_model.sh ./scripts/publish_model.sh ./packages/regression_model/ + section_9_differential_tests: + <<: *defaults + steps: + - checkout + - *prepare_venv + - run: + name: Capturing previous model predictions + command: | + . venv/bin/activate + pip install -r packages/ml_api/diff_test_requirements.txt + PYTHONPATH=./packages/ml_api python3 packages/ml_api/tests/capture_model_predictions.py + - run: + name: Runnning differential tests + command: | + . venv/bin/activate + pip install -r packages/ml_api/requirements.txt + py.test -vv packages/ml_api/tests -m differential + workflows: version: 2 test-all: jobs: - test_regression_model - test_ml_api + - section_9_differential_tests - train_and_upload_regression_model: requires: - test_regression_model - test_ml_api + - section_9_differential_tests # filters: # branches: # only: diff --git a/.gitignore b/.gitignore index 526da42f5..134143492 100644 --- a/.gitignore +++ b/.gitignore @@ -110,3 +110,12 @@ venv.bak/ packages/regression_model/regression_model/datasets/*.csv packages/regression_model/regression_model/datasets/*.zip packages/regression_model/regression_model/datasets/*.txt +train.csv +test.csv +test_data_predictions.csv + +# all logs +logs/ + +# trained models (will be created in CI) +packages/regression_model/regression_model/trained_models/*.pkl diff --git a/packages/ml_api/diff_test_requirements.txt b/packages/ml_api/diff_test_requirements.txt new file mode 100644 index 000000000..7ac05fce6 --- /dev/null +++ b/packages/ml_api/diff_test_requirements.txt @@ -0,0 +1,10 @@ +--extra-index-url=${PIP_EXTRA_INDEX_URL} + +# api +flask==1.0.2 + +# schema validation +marshmallow==2.17.0 + +# Set this to the previous model version +regression-model==0.1.0 \ No newline at end of file diff --git a/packages/ml_api/tests/differential_tests/capture_model_predictions.py b/packages/ml_api/tests/capture_model_predictions.py similarity index 68% rename from packages/ml_api/tests/differential_tests/capture_model_predictions.py rename to packages/ml_api/tests/capture_model_predictions.py index 801c3bab5..19a71142a 100644 --- a/packages/ml_api/tests/differential_tests/capture_model_predictions.py +++ b/packages/ml_api/tests/capture_model_predictions.py @@ -12,26 +12,23 @@ from api import config -def capture_predictions( - *, - save_file: str = 'test_data_predictions.csv'): +def capture_predictions() -> None: """Save the test data predictions to a CSV.""" + save_file = 'test_data_predictions.csv' test_data = load_dataset(file_name='test.csv') # we take a slice with no input validation issues - multiple_test_json = test_data[99:600] + multiple_test_input = test_data[99:600] - predictions = make_prediction(input_data=multiple_test_json) + predictions = make_prediction(input_data=multiple_test_input) # save predictions for the test dataset predictions_df = pd.DataFrame(predictions) # hack here to save the file to the regression model # package of the repo, not the installed package - predictions_df.to_csv( - f'{config.PACKAGE_ROOT.parent}/' - f'regression_model/regression_model/datasets/{save_file}') + predictions_df.to_csv(f'{config.PACKAGE_ROOT}/{save_file}') if __name__ == '__main__': diff --git a/packages/ml_api/tests/differential_tests/test_differential.py b/packages/ml_api/tests/differential_tests/test_differential.py index b755570db..2675d1840 100644 --- a/packages/ml_api/tests/differential_tests/test_differential.py +++ b/packages/ml_api/tests/differential_tests/test_differential.py @@ -1,29 +1,36 @@ import math -import pytest - -from regression_model.config import config +from regression_model.config import config as model_config from regression_model.predict import make_prediction from regression_model.processing.data_management import load_dataset +import pandas as pd +import pytest + +from api import config + +@pytest.mark.skip @pytest.mark.differential def test_model_prediction_differential( *, - save_file='test_data_predictions.csv'): + save_file: str = 'test_data_predictions.csv'): """ This test compares the prediction result similarity of the current model with the previous model's results. """ + # Given - previous_model_df = load_dataset(file_name='test_data_predictions.csv') + # Load the saved previous model predictions + previous_model_df = pd.read_csv(f'{config.PACKAGE_ROOT}/{save_file}') previous_model_predictions = previous_model_df.predictions.values - test_data = load_dataset(file_name='test.csv') - multiple_test_json = test_data[99:600] + + test_data = load_dataset(file_name=f'{save_file}') + multiple_test_input = test_data[99:600] # When - response = make_prediction(input_data=multiple_test_json) - current_model_predictions = response.get('predictions') + current_result = make_prediction(input_data=multiple_test_input) + current_model_predictions = current_result.get('predictions') # Then # diff the current model vs. the old model @@ -44,4 +51,4 @@ def test_model_prediction_differential( # rel_tol=0.05. assert math.isclose(previous_value, current_value, - rel_tol=config.ACCEPTABLE_MODEL_DIFFERENCE) + rel_tol=model_config.ACCEPTABLE_MODEL_DIFFERENCE) diff --git a/packages/regression_model/regression_model/VERSION b/packages/regression_model/regression_model/VERSION index 6c6aa7cb0..afaf360d3 100644 --- a/packages/regression_model/regression_model/VERSION +++ b/packages/regression_model/regression_model/VERSION @@ -1 +1 @@ -0.1.0 \ No newline at end of file +1.0.0 \ No newline at end of file diff --git a/packages/regression_model/regression_model/datasets/test_data_predictions.csv b/packages/regression_model/regression_model/datasets/test_data_predictions.csv deleted file mode 100644 index 4fda7985c..000000000 --- a/packages/regression_model/regression_model/datasets/test_data_predictions.csv +++ /dev/null @@ -1,501 +0,0 @@ -,predictions,version -0,143988.30704997465,0.1.0 -1,116598.08159580332,0.1.0 -2,130128.90560814076,0.1.0 -3,113470.10675716968,0.1.0 -4,159022.48121448176,0.1.0 -5,139861.32732907546,0.1.0 -6,227118.89767805065,0.1.0 -7,91953.99400144782,0.1.0 -8,225573.26579772323,0.1.0 -9,125802.8602526304,0.1.0 -10,137481.49149643493,0.1.0 -11,124990.09839895074,0.1.0 -12,133270.15609091,0.1.0 -13,192143.4530280595,0.1.0 -14,123206.5594461486,0.1.0 -15,201801.77975634683,0.1.0 -16,198027.98470170778,0.1.0 -17,185664.94305866087,0.1.0 -18,146728.39264190392,0.1.0 -19,152443.1572738422,0.1.0 -20,197054.58979409203,0.1.0 -21,146781.9115319493,0.1.0 -22,138838.0050135225,0.1.0 -23,259997.45200360558,0.1.0 -24,220904.18524276977,0.1.0 -25,162760.6578114075,0.1.0 -26,81622.7760115488,0.1.0 -27,104671.50728326188,0.1.0 -28,129551.38264993431,0.1.0 -29,95446.01639989471,0.1.0 -30,129507.4444341237,0.1.0 -31,95477.93516568728,0.1.0 -32,129422.6043698834,0.1.0 -33,128062.38086640426,0.1.0 -34,123419.71922835958,0.1.0 -35,128318.94350485185,0.1.0 -36,207431.6698047325,0.1.0 -37,174685.92854135018,0.1.0 -38,204544.1513220886,0.1.0 -39,188046.15280301377,0.1.0 -40,182971.78532877663,0.1.0 -41,70097.27238622728,0.1.0 -42,110733.2059471847,0.1.0 -43,93994.92500037784,0.1.0 -44,252924.35745892464,0.1.0 -45,214641.99038515135,0.1.0 -46,154979.9669243978,0.1.0 -47,160810.80098181101,0.1.0 -48,230690.236786167,0.1.0 -49,196243.15614263792,0.1.0 -50,177792.5604951465,0.1.0 -51,150956.42632815256,0.1.0 -52,168211.15880784288,0.1.0 -53,158387.31855224012,0.1.0 -54,114339.5601018531,0.1.0 -55,90052.36198593948,0.1.0 -56,89964.45949954129,0.1.0 -57,98668.89304456668,0.1.0 -58,121518.86270978909,0.1.0 -59,134198.59781615838,0.1.0 -60,163434.02753944616,0.1.0 -61,135542.55508479764,0.1.0 -62,141825.43043982252,0.1.0 -63,227613.38755000453,0.1.0 -64,188761.60830094197,0.1.0 -65,116489.4563051063,0.1.0 -66,167327.47818717395,0.1.0 -67,183019.80781626955,0.1.0 -68,263704.159135985,0.1.0 -69,194109.36377179576,0.1.0 -70,300262.7532032975,0.1.0 -71,223004.09657281314,0.1.0 -72,229985.38944263826,0.1.0 -73,184172.20037350367,0.1.0 -74,188222.84233142118,0.1.0 -75,188097.29339417908,0.1.0 -76,172331.10498565168,0.1.0 -77,174886.6907641111,0.1.0 -78,201441.14534017237,0.1.0 -79,178852.47480584026,0.1.0 -80,225286.87493988863,0.1.0 -81,186618.03844702366,0.1.0 -82,253907.81542043414,0.1.0 -83,240359.90484464006,0.1.0 -84,238601.0921535284,0.1.0 -85,177935.77765021168,0.1.0 -86,162057.79394455065,0.1.0 -87,163514.64562596226,0.1.0 -88,133002.50357947565,0.1.0 -89,126285.82757075419,0.1.0 -90,114122.89197558099,0.1.0 -91,118965.43322308766,0.1.0 -92,107820.17501469971,0.1.0 -93,107672.41260124673,0.1.0 -94,161142.56666974662,0.1.0 -95,155175.112064241,0.1.0 -96,159626.62056220102,0.1.0 -97,159289.85166702382,0.1.0 -98,164753.43823200595,0.1.0 -99,130441.66184067688,0.1.0 -100,150115.21843697876,0.1.0 -101,363780.0225506806,0.1.0 -102,330017.780544809,0.1.0 -103,331883.3191102819,0.1.0 -104,406837.5511403465,0.1.0 -105,292997.10969063273,0.1.0 -106,306609.27632288035,0.1.0 -107,329626.60615839734,0.1.0 -108,311532.52238578524,0.1.0 -109,302589.7805774104,0.1.0 -110,313113.53389941505,0.1.0 -111,255492.2795391536,0.1.0 -112,348040.2630000232,0.1.0 -113,286215.77612206567,0.1.0 -114,257811.3774942191,0.1.0 -115,219056.33504400466,0.1.0 -116,221072.9009001751,0.1.0 -117,227272.5447635412,0.1.0 -118,389000.9584031945,0.1.0 -119,333081.2372066048,0.1.0 -120,301748.2795090072,0.1.0 -121,268886.605541231,0.1.0 -122,292214.7783535345,0.1.0 -123,218893.10534405566,0.1.0 -124,198679.87790616706,0.1.0 -125,198256.12319179106,0.1.0 -126,203810.58008877232,0.1.0 -127,200888.22351579432,0.1.0 -128,208173.15639542375,0.1.0 -129,208236.64492513813,0.1.0 -130,204263.56750308358,0.1.0 -131,194016.82016564548,0.1.0 -132,247220.62121392722,0.1.0 -133,186454.85767170336,0.1.0 -134,183808.3284633914,0.1.0 -135,184105.97903285234,0.1.0 -136,239209.89605894414,0.1.0 -137,184218.80235097196,0.1.0 -138,307821.6280329202,0.1.0 -139,309780.2215794851,0.1.0 -140,250051.75088695402,0.1.0 -141,264234.36472344183,0.1.0 -142,238517.39539507058,0.1.0 -143,253639.64599699862,0.1.0 -144,266777.25555390265,0.1.0 -145,249262.33173072065,0.1.0 -146,354687.6212203011,0.1.0 -147,211718.31772737036,0.1.0 -148,208112.29103266165,0.1.0 -149,269063.04990015837,0.1.0 -150,232554.7387626751,0.1.0 -151,267547.16223942576,0.1.0 -152,259496.4322217068,0.1.0 -153,254987.37388475015,0.1.0 -154,213297.22522688,0.1.0 -155,209521.4853124122,0.1.0 -156,168400.4848772304,0.1.0 -157,168269.52494463106,0.1.0 -158,138015.7063444789,0.1.0 -159,197692.7497359191,0.1.0 -160,210792.23068435694,0.1.0 -161,160895.21637656086,0.1.0 -162,129967.65699942572,0.1.0 -163,148887.7470968613,0.1.0 -164,189032.60710901304,0.1.0 -165,206354.3720483368,0.1.0 -166,170625.45360343822,0.1.0 -167,161155.2832590772,0.1.0 -168,177241.4857453312,0.1.0 -169,152617.9750132888,0.1.0 -170,164767.3082372813,0.1.0 -171,121689.0145099861,0.1.0 -172,114755.20351999925,0.1.0 -173,109385.54490451732,0.1.0 -174,115908.28531894127,0.1.0 -175,127297.15226141199,0.1.0 -176,111687.7144642378,0.1.0 -177,250341.40946203517,0.1.0 -178,231747.51470786144,0.1.0 -179,273940.75455758354,0.1.0 -180,223840.72800951728,0.1.0 -181,207683.72914446727,0.1.0 -182,185613.50839666792,0.1.0 -183,195932.25270587756,0.1.0 -184,248138.38057655803,0.1.0 -185,188290.29546011682,0.1.0 -186,210444.7210381098,0.1.0 -187,205928.18597414377,0.1.0 -188,210044.0320203481,0.1.0 -189,156787.38785618285,0.1.0 -190,149779.3462459088,0.1.0 -191,222254.2913941949,0.1.0 -192,117338.5782329264,0.1.0 -193,144956.37156722017,0.1.0 -194,190502.7599290919,0.1.0 -195,176058.9300745161,0.1.0 -196,113437.17520996452,0.1.0 -197,113005.87286210393,0.1.0 -198,148396.4974016323,0.1.0 -199,155111.51255427708,0.1.0 -200,160895.4088655705,0.1.0 -201,146811.64156366416,0.1.0 -202,161697.96498210484,0.1.0 -203,175408.29205737467,0.1.0 -204,119486.7853118973,0.1.0 -205,155735.2535739763,0.1.0 -206,161732.25789945782,0.1.0 -207,186302.28474718594,0.1.0 -208,126314.40090076534,0.1.0 -209,161489.29160402366,0.1.0 -210,142192.79730554653,0.1.0 -211,125295.79760954925,0.1.0 -212,133726.54674477206,0.1.0 -213,131402.58297528428,0.1.0 -214,147256.8448434014,0.1.0 -215,130042.3601888925,0.1.0 -216,126109.99661525768,0.1.0 -217,104028.06280588396,0.1.0 -218,139015.86204044707,0.1.0 -219,123915.67823516048,0.1.0 -220,178112.6718654715,0.1.0 -221,125873.4394256058,0.1.0 -222,94911.69337443665,0.1.0 -223,137426.63537243495,0.1.0 -224,110144.45586689096,0.1.0 -225,119424.4928970573,0.1.0 -226,149432.93149379385,0.1.0 -227,163081.24792773716,0.1.0 -228,72754.84825273752,0.1.0 -229,107008.00619034276,0.1.0 -230,97026.69480171583,0.1.0 -231,176624.72236581342,0.1.0 -232,136815.75834336376,0.1.0 -233,136527.98103527437,0.1.0 -234,149254.9171475344,0.1.0 -235,127404.15185928933,0.1.0 -236,150150.4110071018,0.1.0 -237,122947.21890337647,0.1.0 -238,123038.56391694587,0.1.0 -239,106055.04206900226,0.1.0 -240,133737.62620695255,0.1.0 -241,127761.33500718801,0.1.0 -242,148651.3511288533,0.1.0 -243,150394.04939898496,0.1.0 -244,137871.15589031755,0.1.0 -245,137889.2545253325,0.1.0 -246,135021.79176355613,0.1.0 -247,132212.93368155853,0.1.0 -248,132394.6589172383,0.1.0 -249,116451.46796853734,0.1.0 -250,132045.77239979545,0.1.0 -251,93828.92317256187,0.1.0 -252,98304.79957463636,0.1.0 -253,116592.62783055207,0.1.0 -254,98723.66631722648,0.1.0 -255,70121.22021310769,0.1.0 -256,97709.23487001589,0.1.0 -257,117883.99993469544,0.1.0 -258,145026.28625503322,0.1.0 -259,153912.57618886943,0.1.0 -260,93381.08729006874,0.1.0 -261,123495.69496267234,0.1.0 -262,151217.31007381002,0.1.0 -263,70925.4220942242,0.1.0 -264,134164.7860642941,0.1.0 -265,137115.50773650245,0.1.0 -266,112454.46885682318,0.1.0 -267,113576.35603796394,0.1.0 -268,126311.04816994928,0.1.0 -269,130853.87341430226,0.1.0 -270,134365.47254085648,0.1.0 -271,149331.816504544,0.1.0 -272,113846.4490674583,0.1.0 -273,127309.62370143532,0.1.0 -274,138936.11004121447,0.1.0 -275,126773.14110750334,0.1.0 -276,118674.20763474096,0.1.0 -277,94732.55765810968,0.1.0 -278,115042.27875631058,0.1.0 -279,97413.63757181565,0.1.0 -280,125103.21858739002,0.1.0 -281,127112.78156168538,0.1.0 -282,100712.28345775318,0.1.0 -283,123435.94852302536,0.1.0 -284,146777.37991798244,0.1.0 -285,141324.91303095603,0.1.0 -286,147015.62617541858,0.1.0 -287,182059.49685921244,0.1.0 -288,66635.70748853082,0.1.0 -289,113133.7345902136,0.1.0 -290,115399.86396709623,0.1.0 -291,142613.97712567318,0.1.0 -292,122675.88261778199,0.1.0 -293,128951.35723355877,0.1.0 -294,159633.68071362676,0.1.0 -295,163672.2859152473,0.1.0 -296,200101.77128067127,0.1.0 -297,166260.33914041193,0.1.0 -298,150329.84339014755,0.1.0 -299,140794.76572322496,0.1.0 -300,166102.833620058,0.1.0 -301,140183.19131161584,0.1.0 -302,257819.0508760762,0.1.0 -303,257819.0508760762,0.1.0 -304,257819.0508760762,0.1.0 -305,297489.40422482847,0.1.0 -306,288713.0465842733,0.1.0 -307,238840.80382128613,0.1.0 -308,264054.2118258276,0.1.0 -309,214038.27040784762,0.1.0 -310,216541.14163119273,0.1.0 -311,251482.14382697808,0.1.0 -312,201302.78506297944,0.1.0 -313,221418.6030263962,0.1.0 -314,143245.9627266626,0.1.0 -315,195099.27104358346,0.1.0 -316,194957.58888827328,0.1.0 -317,196553.0339968338,0.1.0 -318,209163.81006532238,0.1.0 -319,137593.75834543034,0.1.0 -320,139886.56269297737,0.1.0 -321,224462.0649769455,0.1.0 -322,249722.4606197197,0.1.0 -323,196221.2726508532,0.1.0 -324,200883.07978660773,0.1.0 -325,236876.5404898464,0.1.0 -326,265449.9719556491,0.1.0 -327,210031.52797804037,0.1.0 -328,250335.16327422266,0.1.0 -329,193702.5517580212,0.1.0 -330,113345.66683243777,0.1.0 -331,141908.87717126816,0.1.0 -332,98061.70102934526,0.1.0 -333,122961.05363435802,0.1.0 -334,117995.15041902235,0.1.0 -335,134068.9122846434,0.1.0 -336,122607.11339521343,0.1.0 -337,128632.12690453106,0.1.0 -338,130665.06200115388,0.1.0 -339,181867.81868509538,0.1.0 -340,172320.99427457084,0.1.0 -341,163115.13448378997,0.1.0 -342,142692.95549842576,0.1.0 -343,204336.63049215134,0.1.0 -344,151865.2725254776,0.1.0 -345,187999.9387459913,0.1.0 -346,153898.50002741258,0.1.0 -347,201370.60175011388,0.1.0 -348,136260.79769104172,0.1.0 -349,167661.378830941,0.1.0 -350,151900.7260108396,0.1.0 -351,203200.5976776774,0.1.0 -352,275987.18626456213,0.1.0 -353,131731.26809609786,0.1.0 -354,72685.59185678526,0.1.0 -355,264769.3677760745,0.1.0 -356,223505.75506482823,0.1.0 -357,140373.47418071458,0.1.0 -358,165740.37720853413,0.1.0 -359,153501.3958318297,0.1.0 -360,333345.8132030645,0.1.0 -361,284907.13582157245,0.1.0 -362,235976.61331734635,0.1.0 -363,237331.86536503406,0.1.0 -364,222571.43251950064,0.1.0 -365,330547.42125199316,0.1.0 -366,126425.36283381855,0.1.0 -367,150931.15863895716,0.1.0 -368,116973.81860226691,0.1.0 -369,147483.17081444428,0.1.0 -370,137775.93779758728,0.1.0 -371,136213.6538169831,0.1.0 -372,160855.09129555486,0.1.0 -373,180999.95456004038,0.1.0 -374,177875.4323401108,0.1.0 -375,183722.0684301858,0.1.0 -376,183394.03709605164,0.1.0 -377,167171.69796713692,0.1.0 -378,253008.1582497637,0.1.0 -379,208356.18546752,0.1.0 -380,184067.27386951286,0.1.0 -381,184525.57241064525,0.1.0 -382,234914.10484877022,0.1.0 -383,319321.39732491894,0.1.0 -384,329258.81904322456,0.1.0 -385,171807.44667235087,0.1.0 -386,300439.8001753106,0.1.0 -387,168715.42175203658,0.1.0 -388,224083.29347340713,0.1.0 -389,169027.4893700393,0.1.0 -390,219986.76456349975,0.1.0 -391,206599.36694968113,0.1.0 -392,168431.21773772905,0.1.0 -393,198938.11718684685,0.1.0 -394,137044.70162504562,0.1.0 -395,256489.3797086342,0.1.0 -396,169081.6811380493,0.1.0 -397,246159.3182317069,0.1.0 -398,146517.01285907425,0.1.0 -399,115488.93084257792,0.1.0 -400,124226.28849234067,0.1.0 -401,105765.49539858926,0.1.0 -402,105734.63795160982,0.1.0 -403,109307.7618847266,0.1.0 -404,153399.47012489414,0.1.0 -405,148098.79308079585,0.1.0 -406,256865.85340555105,0.1.0 -407,353705.2884855737,0.1.0 -408,339406.68729405693,0.1.0 -409,370934.7245862843,0.1.0 -410,412758.66452745936,0.1.0 -411,337318.9162127192,0.1.0 -412,292636.5292003634,0.1.0 -413,306738.89042618143,0.1.0 -414,395200.33469924616,0.1.0 -415,265420.90751885757,0.1.0 -416,304674.1881521481,0.1.0 -417,322466.11906014563,0.1.0 -418,309583.69640512683,0.1.0 -419,222251.71906371377,0.1.0 -420,305633.12114918296,0.1.0 -421,246068.43249597988,0.1.0 -422,237392.40028237563,0.1.0 -423,211279.01604200783,0.1.0 -424,228094.0196541859,0.1.0 -425,217362.23612708444,0.1.0 -426,212395.21391217507,0.1.0 -427,192157.327626266,0.1.0 -428,210131.93667451647,0.1.0 -429,218479.26431069477,0.1.0 -430,227732.65975321413,0.1.0 -431,207550.8611689138,0.1.0 -432,196406.28233478937,0.1.0 -433,215352.46117706495,0.1.0 -434,195390.69073167298,0.1.0 -435,268095.89486272854,0.1.0 -436,317322.5783410133,0.1.0 -437,292294.5209052129,0.1.0 -438,256214.48067033372,0.1.0 -439,289956.5518384693,0.1.0 -440,285699.6865787319,0.1.0 -441,238369.04431785582,0.1.0 -442,266162.84585317614,0.1.0 -443,276105.07384260837,0.1.0 -444,241944.78930174315,0.1.0 -445,212994.50831895912,0.1.0 -446,266502.50110652676,0.1.0 -447,203362.7111452237,0.1.0 -448,180227.73055119175,0.1.0 -449,188392.39553333411,0.1.0 -450,142481.50831170173,0.1.0 -451,174912.95802564104,0.1.0 -452,168060.24103720946,0.1.0 -453,170840.3065243665,0.1.0 -454,185335.0674102329,0.1.0 -455,175685.71835342573,0.1.0 -456,182131.57134249242,0.1.0 -457,127731.04705949678,0.1.0 -458,130944.89863769621,0.1.0 -459,105125.80701127343,0.1.0 -460,113673.41846707783,0.1.0 -461,171746.81645701104,0.1.0 -462,147544.47667904384,0.1.0 -463,266570.15210116236,0.1.0 -464,340483.4209594863,0.1.0 -465,193926.64894274823,0.1.0 -466,177273.1783748505,0.1.0 -467,188439.6899965548,0.1.0 -468,179646.3820244513,0.1.0 -469,277801.9107183519,0.1.0 -470,244750.34380769494,0.1.0 -471,264143.13027023565,0.1.0 -472,264084.9900022445,0.1.0 -473,190623.30283373612,0.1.0 -474,218303.47626378198,0.1.0 -475,209178.35576652727,0.1.0 -476,210247.40015571192,0.1.0 -477,305489.9014144604,0.1.0 -478,206548.65094650167,0.1.0 -479,260901.671279582,0.1.0 -480,234130.08563281858,0.1.0 -481,215084.1602052955,0.1.0 -482,162068.0157257143,0.1.0 -483,175403.3655499554,0.1.0 -484,188329.78909449733,0.1.0 -485,148772.6745077038,0.1.0 -486,135234.48910921262,0.1.0 -487,132981.35850945665,0.1.0 -488,142443.15434220844,0.1.0 -489,172322.6219487221,0.1.0 -490,114015.40802504608,0.1.0 -491,131679.82317114327,0.1.0 -492,140830.26421534023,0.1.0 -493,96630.01740632812,0.1.0 -494,146497.76662391485,0.1.0 -495,161384.411998765,0.1.0 -496,122294.75296565886,0.1.0 -497,187349.35839738324,0.1.0 -498,139773.34125411394,0.1.0 -499,151158.00827612064,0.1.0 diff --git a/packages/regression_model/regression_model/predict.py b/packages/regression_model/regression_model/predict.py index 34a9d9c3d..7e4ed3d67 100644 --- a/packages/regression_model/regression_model/predict.py +++ b/packages/regression_model/regression_model/predict.py @@ -7,6 +7,7 @@ from regression_model import __version__ as _version import logging +import typing as t _logger = logging.getLogger(__name__) @@ -15,12 +16,22 @@ _price_pipe = load_pipeline(file_name=pipeline_file_name) -def make_prediction(*, input_data) -> dict: - """Make a prediction using the saved model pipeline.""" +def make_prediction(*, input_data: t.Union[pd.DataFrame, dict], + ) -> dict: + """Make a prediction using a saved model pipeline. + + Args: + input_data: Array of model prediction inputs. + + Returns: + Predictions for each input row, as well as the model version. + """ data = pd.DataFrame(input_data) validated_data = validate_inputs(input_data=data) + prediction = _price_pipe.predict(validated_data[config.FEATURES]) + output = np.exp(prediction) results = {"predictions": output, "version": _version} diff --git a/packages/regression_model/regression_model/processing/data_management.py b/packages/regression_model/regression_model/processing/data_management.py index 388412a7d..0357e1219 100644 --- a/packages/regression_model/regression_model/processing/data_management.py +++ b/packages/regression_model/regression_model/processing/data_management.py @@ -6,6 +6,7 @@ from regression_model import __version__ as _version import logging +import typing as t _logger = logging.getLogger(__name__) @@ -28,7 +29,7 @@ def save_pipeline(*, pipeline_to_persist) -> None: save_file_name = f"{config.PIPELINE_SAVE_FILE}{_version}.pkl" save_path = config.TRAINED_MODEL_DIR / save_file_name - remove_old_pipelines(files_to_keep=save_file_name) + remove_old_pipelines(files_to_keep=[save_file_name]) joblib.dump(pipeline_to_persist, save_path) _logger.info(f"saved pipeline: {save_file_name}") @@ -41,14 +42,17 @@ def load_pipeline(*, file_name: str) -> Pipeline: return trained_model -def remove_old_pipelines(*, files_to_keep) -> None: +def remove_old_pipelines(*, files_to_keep: t.List[str]) -> None: """ Remove old model pipelines. + This is to ensure there is a simple one-to-one mapping between the package version and the model version to be imported and used by other applications. + However, we do also include the immediate previous + pipeline version for differential testing purposes. """ - + do_not_delete = files_to_keep + ['__init__.py'] for model_file in config.TRAINED_MODEL_DIR.iterdir(): - if model_file.name not in [files_to_keep, "__init__.py"]: + if model_file.name not in do_not_delete: model_file.unlink() diff --git a/packages/regression_model/regression_model/trained_models/lasso_regression_output_v0.1.0.pkl b/packages/regression_model/regression_model/trained_models/lasso_regression_output_v0.1.0.pkl deleted file mode 100644 index c7c7b75b4cb0cf6a774b6f47b65f90812c04714d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4737 zcmbUl3v?9Kbs+@8l8~qfiim(()~czfU{GW>zYRFqCD~2FBCg|P_hsM6?9ASo*?iEc zU?D=QXccSypjO3Hv?rci*fiN}J16CL3C1Udc06(O9s_pFrTOl`z7Fk)dR9AP_JumN%g&1*1aA6L6iD zEtb^{B`vFxlFJt?WW<2cTtf%Dk0sn8Dg1{)H}k5JuyU zV{9l%j>Qc+WkZqMRc6B26a+)bAi|=u*`^sSJ$VM>k|hYJF>_XPK~|wOS?Vq8yYsqP zFc_32#~~=9;I^uqX3Yf{PeK}GL(Vc1QchehAfjlB$)SRo?Y(eZ@;Kzx3OVLksJRv% ze+(AJeC|Q}PC)x6*l=Qqqe&AHe{wj?;ohLicEG^&#C zsZ8qztIQQtOOXvj?g^Ucec1@YWEUUISm>G{oRS=gayqJDiWm?_+yJM>VQNS$N{&G2 zJUA^Di(U3PKJ+IBEST0_>o^miSQw`F!i+eaZWhoDXOJ5#mdon~#@P=EoOujxXz{q) zym;1|mXeb#9o7CC`eUXIA@avr6pJO8jd6y#Q=Q4c*JVmIgLN>2X=yg7fC*JgsG#gtB+>?97qS4V8K0ek8uzV_Rp;;ptWAzV zgKKqF&DI)FN2^6@rbX}4pq^IsnJ(GLm=K{=LdW;Uw4i}jF+<5@7g(@>Rt<{U&I}6{ zQUDWjR}Y5hGFrqI({3Ws)fmeXN`^tBE0O3HLzGsvhTPSm8xW&al-#1iA`;S~6Agj!g)s3jhUxGTe{duUC9s#{|5!bB^y5GG;i8mlp&l~y%sPRm*vw9zWc z4=V+~Htg3BpI7ASdG%J{exToFo+LKntRGNfF&#-Ri%@BFH+QPKYz;(2!W3*V| z`mms=)(dgyf=ad8T62yx&B zS=Iz=Siz|!YmYkO{9^L$V<;wHR71XlHRMJcZo-nsFW~rQuERIehMV2|aLNzdA_ll1 zj^NeeNIqXI;$d-AfzP(#yCEJ&)M7*>5fwpHqd1D!i^cphaWroc$M9OQgf9}u@~9Z3 zd?R_cisQH_mhy#S8E+HE^E$Dd$HWSr5Rc;x;_v7Yi;>ZFYXT$9wo*It(9^$SAR zMby=Z;)qfl)I~PjIm`ohA-j$=Ii%5$Ms;YaY`8nb4Mc%BqQK|ca8HPL;nG6N0#bH5 z`;WI_Lx|f5W(d9p!OKxV5Blj^vADp?ZMc_o7&7fXq+5Y>*CX9^NVn3VtFYmISJzdD zUXAEpNBYG!{J_;=4T1q3Lh15AvK+Hd)w2>7Y|X^dg!K;|{L)nEz-o_OiWzLwvn@kh zL3ju=WTOoaJ10tCh)<$3WtM40R3j0AAJS3LU^FV7)drK{5o%X9QPs~FdS3G7!lPfh z{o&169dfp6%<0r*qbK6429KdGKeAynChIv)Tl6@+gHG3h_H7wz9tS%+r_amCmK=;K zT2$^%5M>ayqMRpecrrwKdMZk2HP+^*oF1hYeoPTb>-ipddJXwglT106#|BU`8PgE2&$O@=6-J+aUH~`JQU0Us`O)`8>nMDt-Mqf=`OBvlhbPa;m$40_J$b$jM~=RY{^;X}Y<>SBb$4(>aX48K2m~bW zA}@7f=Z)8Ic%4e+d6nnm5B4}s%^P$bX~avuY`C{`cyq9XI(L5`jMq%8aEgXEbScBk zG`3X1My!u|5Z=Nf@H-pcc6(h}&2t&{HVBq`38JG)orQ(nQgrxzgK!Wv!^f@qiX|x; z7V0$1TkwZ9l#Ci4m9nZVXYj|hR5GRiq5q$Iqv zhU5Kf@|<@+m{NN#G2ihZRD++1m)ew$Hc3j6wb}uzj@{s5N>Lox_r~6TZJXjyXWmWdAsny zBgx4w7Eqr!nF{=IG2x@UyugH3^*FK!gB zIQ2gt_I>)Wphfl%j0(Rb99mV~_uwJhD_R&V$0Zs3rXpqKyaa#ellfHqJTPs_>RtEh z0({sD|A@m!tDAqW!-mX82;G{AIISn{yL}JJH*JdIlS2$J@BuX3$Oei DhDwLD diff --git a/packages/regression_model/tests/test_predict.py b/packages/regression_model/tests/test_predict.py index 8c06e5b78..3c7147f89 100644 --- a/packages/regression_model/tests/test_predict.py +++ b/packages/regression_model/tests/test_predict.py @@ -7,10 +7,10 @@ def test_make_single_prediction(): # Given test_data = load_dataset(file_name='test.csv') - single_test_json = test_data[0:1] + single_test_input = test_data[0:1] # When - subject = make_prediction(input_data=single_test_json) + subject = make_prediction(input_data=single_test_input) # Then assert subject is not None @@ -22,10 +22,10 @@ def test_make_multiple_predictions(): # Given test_data = load_dataset(file_name='test.csv') original_data_length = len(test_data) - multiple_test_json = test_data + multiple_test_input = test_data # When - subject = make_prediction(input_data=multiple_test_json) + subject = make_prediction(input_data=multiple_test_input) # Then assert subject is not None From 6cd3645cbf7e42e62355fe98569d485126ef0802 Mon Sep 17 00:00:00 2001 From: ChristopherGS Date: Fri, 25 Jan 2019 05:51:19 -0800 Subject: [PATCH 11/24] Section 9.4 - Differential Tests in CI Part 2 --- .circleci/config.yml | 8 ++++---- packages/ml_api/VERSION | 2 +- packages/ml_api/requirements.txt | 2 +- .../ml_api/tests/differential_tests/test_differential.py | 3 +-- 4 files changed, 7 insertions(+), 8 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5108aaa3d..bc490b4a3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -117,7 +117,7 @@ workflows: - test_regression_model - test_ml_api - section_9_differential_tests - # filters: - # branches: - # only: - # - master + filters: + branches: + only: + - master diff --git a/packages/ml_api/VERSION b/packages/ml_api/VERSION index 341cf11fa..7dff5b892 100644 --- a/packages/ml_api/VERSION +++ b/packages/ml_api/VERSION @@ -1 +1 @@ -0.2.0 \ No newline at end of file +0.2.1 \ No newline at end of file diff --git a/packages/ml_api/requirements.txt b/packages/ml_api/requirements.txt index 13e82d198..a21d786b7 100644 --- a/packages/ml_api/requirements.txt +++ b/packages/ml_api/requirements.txt @@ -7,4 +7,4 @@ flask==1.0.2 marshmallow==2.17.0 # Install from gemfury -regression-model==0.1.0 \ No newline at end of file +regression-model==1.0.0 \ No newline at end of file diff --git a/packages/ml_api/tests/differential_tests/test_differential.py b/packages/ml_api/tests/differential_tests/test_differential.py index 2675d1840..acabf724d 100644 --- a/packages/ml_api/tests/differential_tests/test_differential.py +++ b/packages/ml_api/tests/differential_tests/test_differential.py @@ -10,7 +10,6 @@ from api import config -@pytest.mark.skip @pytest.mark.differential def test_model_prediction_differential( *, @@ -25,7 +24,7 @@ def test_model_prediction_differential( previous_model_df = pd.read_csv(f'{config.PACKAGE_ROOT}/{save_file}') previous_model_predictions = previous_model_df.predictions.values - test_data = load_dataset(file_name=f'{save_file}') + test_data = load_dataset(file_name=model_config.TESTING_DATA_FILE) multiple_test_input = test_data[99:600] # When From 87215a1455727ffa1730c61cc61690ef542837dd Mon Sep 17 00:00:00 2001 From: ChristopherGS Date: Fri, 25 Jan 2019 07:05:31 -0800 Subject: [PATCH 12/24] Section 10.3 - Heroku Deployment Config --- Procfile | 1 + jupyter_notebooks/requirements.txt | 4 ++++ packages/ml_api/requirements.txt | 5 ++++- packages/ml_api/run.py | 4 ++-- requirements.txt | 1 + 5 files changed, 12 insertions(+), 3 deletions(-) create mode 100644 Procfile create mode 100644 jupyter_notebooks/requirements.txt create mode 100644 requirements.txt diff --git a/Procfile b/Procfile new file mode 100644 index 000000000..2d349992c --- /dev/null +++ b/Procfile @@ -0,0 +1 @@ +web: gunicorn --pythonpath packages/ml_api --access-logfile - --error-logfile - run:application \ No newline at end of file diff --git a/jupyter_notebooks/requirements.txt b/jupyter_notebooks/requirements.txt new file mode 100644 index 000000000..bf6868546 --- /dev/null +++ b/jupyter_notebooks/requirements.txt @@ -0,0 +1,4 @@ +jupyter==1.0.0 +matplotlib==3.0.2 +pandas==0.23.4 +scikit-learn==0.20.2 diff --git a/packages/ml_api/requirements.txt b/packages/ml_api/requirements.txt index a21d786b7..57d269ea5 100644 --- a/packages/ml_api/requirements.txt +++ b/packages/ml_api/requirements.txt @@ -7,4 +7,7 @@ flask==1.0.2 marshmallow==2.17.0 # Install from gemfury -regression-model==1.0.0 \ No newline at end of file +regression-model==1.0.0 + +# Deployment +gunicorn==19.9.0 \ No newline at end of file diff --git a/packages/ml_api/run.py b/packages/ml_api/run.py index 75a66dc7b..7f60a072a 100644 --- a/packages/ml_api/run.py +++ b/packages/ml_api/run.py @@ -1,9 +1,9 @@ from api.app import create_app -from api.config import DevelopmentConfig +from api.config import DevelopmentConfig, ProductionConfig application = create_app( - config_object=DevelopmentConfig) + config_object=ProductionConfig) if __name__ == '__main__': diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 000000000..e391ca79f --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +-r packages/ml_api/requirements.txt From 07ca7ea23b36b67c915e6688d260f146658c1f03 Mon Sep 17 00:00:00 2001 From: ChristopherGS Date: Fri, 25 Jan 2019 09:20:46 -0800 Subject: [PATCH 13/24] Section 10.4 - Test Deployed API --- scripts/input_test.json | 82 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 scripts/input_test.json diff --git a/scripts/input_test.json b/scripts/input_test.json new file mode 100644 index 000000000..bee61b12a --- /dev/null +++ b/scripts/input_test.json @@ -0,0 +1,82 @@ +[{ + "Id": 1461, + "MSSubClass": 20, + "MSZoning": "RH", + "LotFrontage": 80.0, + "LotArea": 11622, + "Street": "Pave", + "Alley": null, + "LotShape": "Reg", + "LandContour": "Lvl", + "Utilities": "AllPub", + "LotConfig": "Inside", + "LandSlope": "Gtl", + "Neighborhood": "NAmes", + "Condition1": "Feedr", + "Condition2": "Norm", + "BldgType": "1Fam", + "HouseStyle": "1Story", + "OverallQual": 5, + "OverallCond": 6, + "YearBuilt": 1961, + "YearRemodAdd": 1961, + "RoofStyle": "Gable", + "RoofMatl": "CompShg", + "Exterior1st": "VinylSd", + "Exterior2nd": "VinylSd", + "MasVnrType": "None", + "MasVnrArea": 0.0, + "ExterQual": "TA", + "ExterCond": "TA", + "Foundation": "CBlock", + "BsmtQual": "TA", + "BsmtCond": "TA", + "BsmtExposure": "No", + "BsmtFinType1": "Rec", + "BsmtFinSF1": 468.0, + "BsmtFinType2": "LwQ", + "BsmtFinSF2": 144.0, + "BsmtUnfSF": 270.0, + "TotalBsmtSF": 882.0, + "Heating": "GasA", + "HeatingQC": "TA", + "CentralAir": "Y", + "Electrical": "SBrkr", + "1stFlrSF": 896, + "2ndFlrSF": 0, + "LowQualFinSF": 0, + "GrLivArea": 896, + "BsmtFullBath": 0.0, + "BsmtHalfBath": 0.0, + "FullBath": 1, + "HalfBath": 0, + "BedroomAbvGr": 2, + "KitchenAbvGr": 1, + "KitchenQual": "TA", + "TotRmsAbvGrd": 5, + "Functional": "Typ", + "Fireplaces": 0, + "FireplaceQu": null, + "GarageType": "Attchd", + "GarageYrBlt": 1961.0, + "GarageFinish": "Unf", + "GarageCars": 1.0, + "GarageArea": 730.0, + "GarageQual": "TA", + "GarageCond": "TA", + "PavedDrive": "Y", + "WoodDeckSF": 140, + "OpenPorchSF": 0, + "EnclosedPorch": 0, + "3SsnPorch": 0, + "ScreenPorch": 120, + "PoolArea": 0, + "PoolQC": null, + "Fence": "MnPrv", + "MiscFeature": null, + "MiscVal": 0, + "MoSold": 6, + "YrSold": 2010, + "SaleType": "WD", + "SaleCondition": "Normal" +}] \ No newline at end of file From bd602b23a80e184babf0f1abb6c8222f4356549a Mon Sep 17 00:00:00 2001 From: ChristopherGS Date: Fri, 25 Jan 2019 09:42:11 -0800 Subject: [PATCH 14/24] Section 10.5 - Deploy to Heroku with CircleCI --- .circleci/config.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index bc490b4a3..b0bb051ee 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -105,6 +105,15 @@ jobs: pip install -r packages/ml_api/requirements.txt py.test -vv packages/ml_api/tests -m differential + section_10_deploy_to_heroku: + <<: *defaults + steps: + - checkout + - run: + name: Deploy to Heroku + command: | + git push https://heroku:$HEROKU_API_KEY@git.heroku.com/$HEROKU_APP_NAME.git master + workflows: version: 2 test-all: @@ -121,3 +130,10 @@ workflows: branches: only: - master + - section_10_deploy_to_heroku: + requires: + - train_and_upload_regression_model + filters: + branches: + only: + - master From 3f25370a1b464014419ac49a1e4eeee6b19cbe09 Mon Sep 17 00:00:00 2001 From: ChristopherGS Date: Sat, 26 Jan 2019 05:25:26 -0800 Subject: [PATCH 15/24] Section 11.3 - Dockerfile Setup --- .dockerignore | 9 +++++++++ Dockerfile | 23 +++++++++++++++++++++++ packages/ml_api/run.sh | 3 +++ 3 files changed, 35 insertions(+) create mode 100644 .dockerignore create mode 100644 Dockerfile create mode 100644 packages/ml_api/run.sh diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..b9e54fcad --- /dev/null +++ b/.dockerignore @@ -0,0 +1,9 @@ +jupyter_notebooks* +*/env* +*/venv* +.circleci* +packages/regression_model +*.env +*.log +.git +.gitignore \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..bbba25c1a --- /dev/null +++ b/Dockerfile @@ -0,0 +1,23 @@ +FROM python:3.6.4 + +# Create the user that will run the app +RUN adduser --disabled-password --gecos '' ml-api-user + +WORKDIR /opt/ml_api + +ARG PIP_EXTRA_INDEX_URL +ENV FLASK_APP run.py + +# Install requirements, including from Gemfury +ADD ./packages/ml_api /opt/ml_api/ +RUN pip install --upgrade pip +RUN pip install -r /opt/ml_api/requirements.txt + +RUN chmod +x /opt/ml_api/run.sh +RUN chown -R ml-api-user:ml-api-user ./ + +USER ml-api-user + +EXPOSE 5000 + +CMD ["bash", "./run.sh"] \ No newline at end of file diff --git a/packages/ml_api/run.sh b/packages/ml_api/run.sh new file mode 100644 index 000000000..45fba354b --- /dev/null +++ b/packages/ml_api/run.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +export IS_DEBUG=${DEBUG:-false} +exec gunicorn -b :${PORT:-5000} --access-logfile - --error-logfile - run:application \ No newline at end of file From fbea252bfd5a82678dc6f9fcad7ef1218a74d231 Mon Sep 17 00:00:00 2001 From: ChristopherGS Date: Sat, 26 Jan 2019 10:39:46 -0800 Subject: [PATCH 16/24] Section 11.5 - Using Docker with Heroku --- .circleci/config.yml | 29 ++++++++++++++++++++++++++++- Makefile | 9 +++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) create mode 100644 Makefile diff --git a/.circleci/config.yml b/.circleci/config.yml index b0bb051ee..55a978023 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -113,6 +113,26 @@ jobs: name: Deploy to Heroku command: | git push https://heroku:$HEROKU_API_KEY@git.heroku.com/$HEROKU_APP_NAME.git master + + section_11_build_and_push_to_heroku_docker: + <<: *defaults + steps: + - checkout + - setup_remote_docker: + docker_layer_caching: true + - run: docker login --username=$HEROKU_EMAIL --password=$HEROKU_API_KEY registry.heroku.com + - run: + name: Setup Heroku CLI + command: | + wget -qO- https://cli-assets.heroku.com/install-ubuntu.sh | sh + - run: + name: Build and Push Image + command: | + make build-ml-api-heroku push-ml-api-heroku + - run: + name: Release to Heroku + command: | + heroku container:release web --app $HEROKU_APP_NAME workflows: version: 2 @@ -130,7 +150,14 @@ workflows: branches: only: - master - - section_10_deploy_to_heroku: + # - section_10_deploy_to_heroku: + # requires: + # - train_and_upload_regression_model + # filters: + # branches: + # only: + # - master + - section_11_build_and_push_to_heroku_docker: requires: - train_and_upload_regression_model filters: diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..54dd9bed6 --- /dev/null +++ b/Makefile @@ -0,0 +1,9 @@ +NAME=udemy-ml-api +COMMIT_ID=$(shell git rev-parse HEAD) + + +build-ml-api-heroku: + docker build --build-arg PIP_EXTRA_INDEX_URL=${PIP_EXTRA_INDEX_URL} -t registry.heroku.com/$(NAME)/web:$(COMMIT_ID) . + +push-ml-api-heroku: + docker push registry.heroku.com/${HEROKU_APP_NAME}/web:$(COMMIT_ID) From 6f56a463b951a436f4683c9b1cf7cd5c8ceda196 Mon Sep 17 00:00:00 2001 From: ChristopherGS Date: Sat, 26 Jan 2019 23:44:08 -0800 Subject: [PATCH 17/24] Section 12.14 - Deploying to ECS via CI Pipeline --- .circleci/config.yml | 21 +++++++++++++++++++++ Makefile | 9 +++++++++ packages/ml_api/api/config.py | 3 ++- packages/ml_api/run.sh | 2 +- 4 files changed, 33 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 55a978023..73a890de7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -134,6 +134,20 @@ jobs: command: | heroku container:release web --app $HEROKU_APP_NAME + section_12_publish_docker_image_to_aws: + <<: *defaults + working_directory: ~/project/packages/ml_models + steps: + - checkout + - setup_remote_docker + - run: + name: Publishing docker image to aws ECR + command: | + sudo pip install awscli + eval $(aws ecr get-login --no-include-email --region us-east-1) + make build-ml-api-aws tag-ml-api push-ml-api-aws + aws ecs update-service --cluster ml-api-cluster --service custom-service --task-definition first-run-task-definition --force-new-deployment + workflows: version: 2 test-all: @@ -164,3 +178,10 @@ workflows: branches: only: - master + # - section_12_publish_docker_image_to_aws: + # requires: + # - train_and_upload_regression_model + # filters: + # branches: + # only: + # - master diff --git a/Makefile b/Makefile index 54dd9bed6..7fe16bef3 100644 --- a/Makefile +++ b/Makefile @@ -7,3 +7,12 @@ build-ml-api-heroku: push-ml-api-heroku: docker push registry.heroku.com/${HEROKU_APP_NAME}/web:$(COMMIT_ID) + +build-ml-api-aws: + docker build --build-arg PIP_EXTRA_INDEX_URL=${PIP_EXTRA_INDEX_URL} -t $(NAME):$(COMMIT_ID) . + +push-ml-api-aws: + docker push ${AWS_ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com/$(NAME):$(COMMIT_ID) + +tag-ml-api: + docker tag $(NAME):$(COMMIT_ID) ${AWS_ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com/$(NAME):$(COMMIT_ID) diff --git a/packages/ml_api/api/config.py b/packages/ml_api/api/config.py index 028782c54..c4284a4cc 100644 --- a/packages/ml_api/api/config.py +++ b/packages/ml_api/api/config.py @@ -52,7 +52,8 @@ class Config: class ProductionConfig(Config): DEBUG = False - SERVER_PORT = os.environ.get('PORT', 5000) + SERVER_ADDRESS: os.environ.get('SERVER_ADDRESS', '0.0.0.0') + SERVER_PORT: os.environ.get('SERVER_PORT', '5000') class DevelopmentConfig(Config): diff --git a/packages/ml_api/run.sh b/packages/ml_api/run.sh index 45fba354b..f579e6b1a 100644 --- a/packages/ml_api/run.sh +++ b/packages/ml_api/run.sh @@ -1,3 +1,3 @@ #!/usr/bin/env bash export IS_DEBUG=${DEBUG:-false} -exec gunicorn -b :${PORT:-5000} --access-logfile - --error-logfile - run:application \ No newline at end of file +exec gunicorn --bind 0.0.0.0:5000 --access-logfile - --error-logfile - run:application \ No newline at end of file From a314f0573c5d46d21d5e506a7c41cd31ba36cc75 Mon Sep 17 00:00:00 2001 From: Christopher Samiullah Date: Sun, 10 Feb 2019 12:42:56 +0000 Subject: [PATCH 18/24] Section 13.8 - Publish Neural Network Model --- .circleci/config.yml | 42 +++++- .gitignore | 10 ++ jupyter_notebooks/requirements.txt | 6 +- packages/neural_network_model/MANIFEST.in | 17 +++ packages/neural_network_model/config.yml | 4 + .../neural_network_model/VERSION | 1 + .../neural_network_model/__init__.py | 7 + .../neural_network_model/config/__init__.py | 0 .../neural_network_model/config/config.py | 38 +++++ .../neural_network_model/datasets/__init__.py | 0 .../datasets/test_data/Black-grass/1.png | Bin 0 -> 29128 bytes .../datasets/test_data/Charlock/1.png | Bin 0 -> 40902 bytes .../datasets/test_data/__init__.py | 0 .../neural_network_model/model.py | 79 +++++++++++ .../neural_network_model/pipeline.py | 10 ++ .../neural_network_model/predict.py | 67 +++++++++ .../processing/__init__.py | 0 .../processing/data_management.py | 130 ++++++++++++++++++ .../neural_network_model/processing/errors.py | 6 + .../processing/preprocessors.py | 50 +++++++ .../neural_network_model/train_pipeline.py | 27 ++++ .../trained_models/__init__.py | 0 .../neural_network_model/requirements.txt | 18 +++ packages/neural_network_model/setup.py | 79 +++++++++++ .../neural_network_model/tests/__init__.py | 0 .../neural_network_model/tests/conftest.py | 20 +++ .../tests/test_predict.py | 17 +++ scripts/fetch_kaggle_large_dataset.sh | 11 ++ scripts/publish_model.sh | 2 +- 29 files changed, 638 insertions(+), 3 deletions(-) create mode 100644 packages/neural_network_model/MANIFEST.in create mode 100644 packages/neural_network_model/config.yml create mode 100644 packages/neural_network_model/neural_network_model/VERSION create mode 100644 packages/neural_network_model/neural_network_model/__init__.py create mode 100644 packages/neural_network_model/neural_network_model/config/__init__.py create mode 100644 packages/neural_network_model/neural_network_model/config/config.py create mode 100644 packages/neural_network_model/neural_network_model/datasets/__init__.py create mode 100644 packages/neural_network_model/neural_network_model/datasets/test_data/Black-grass/1.png create mode 100644 packages/neural_network_model/neural_network_model/datasets/test_data/Charlock/1.png create mode 100644 packages/neural_network_model/neural_network_model/datasets/test_data/__init__.py create mode 100644 packages/neural_network_model/neural_network_model/model.py create mode 100644 packages/neural_network_model/neural_network_model/pipeline.py create mode 100644 packages/neural_network_model/neural_network_model/predict.py create mode 100644 packages/neural_network_model/neural_network_model/processing/__init__.py create mode 100644 packages/neural_network_model/neural_network_model/processing/data_management.py create mode 100644 packages/neural_network_model/neural_network_model/processing/errors.py create mode 100644 packages/neural_network_model/neural_network_model/processing/preprocessors.py create mode 100644 packages/neural_network_model/neural_network_model/train_pipeline.py create mode 100644 packages/neural_network_model/neural_network_model/trained_models/__init__.py create mode 100644 packages/neural_network_model/requirements.txt create mode 100644 packages/neural_network_model/setup.py create mode 100644 packages/neural_network_model/tests/__init__.py create mode 100644 packages/neural_network_model/tests/conftest.py create mode 100644 packages/neural_network_model/tests/test_predict.py create mode 100755 scripts/fetch_kaggle_large_dataset.sh mode change 100644 => 100755 scripts/publish_model.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index 73a890de7..284be6e2c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -113,7 +113,7 @@ jobs: name: Deploy to Heroku command: | git push https://heroku:$HEROKU_API_KEY@git.heroku.com/$HEROKU_APP_NAME.git master - + section_11_build_and_push_to_heroku_docker: <<: *defaults steps: @@ -148,6 +148,36 @@ jobs: make build-ml-api-aws tag-ml-api push-ml-api-aws aws ecs update-service --cluster ml-api-cluster --service custom-service --task-definition first-run-task-definition --force-new-deployment + section_13_train_and_upload_neural_network_model: + docker: + - image: circleci/python:3.6.4-stretch + working_directory: ~/project + steps: + - checkout + - *prepare_venv + - run: + name: Install requirements + command: | + . venv/bin/activate + pip install -r packages/neural_network_model/requirements.txt + - run: + name: Fetch Training data - 2GB + command: | + . venv/bin/activate + chmod +x ./scripts/fetch_kaggle_large_dataset.sh + ./scripts/fetch_kaggle_large_dataset.sh + - run: + name: Train model + command: | + . venv/bin/activate + PYTHONPATH=./packages/neural_network_model python3 packages/neural_network_model/neural_network_model/train_pipeline.py + - run: + name: Publish model to Gemfury + command: | + . venv/bin/activate + chmod +x ./scripts/publish_model.sh + ./scripts/publish_model.sh ./packages/neural_network_model/ + workflows: version: 2 test-all: @@ -185,3 +215,13 @@ workflows: # branches: # only: # - master + - section_13_train_and_upload_neural_network_model: + requires: + - test_regression_model + - test_ml_api + - section_9_differential_tests + # - train_and_upload_regression_model + # filters: + # branches: + # only: + # - master diff --git a/.gitignore b/.gitignore index 134143492..1c5cde46d 100644 --- a/.gitignore +++ b/.gitignore @@ -113,9 +113,19 @@ packages/regression_model/regression_model/datasets/*.txt train.csv test.csv test_data_predictions.csv +v2-plant-seedlings-dataset/ +v2-plant-seedlings-dataset.zip # all logs logs/ # trained models (will be created in CI) packages/regression_model/regression_model/trained_models/*.pkl +packages/neural_network_model/neural_network_model/trained_models/*.pkl +packages/neural_network_model/neural_network_model/trained_models/*.h5 +*.h5 +packages/neural_network_model/neural_network_model/datasets/training_data_reference.txt + +.DS_Store + +kaggle.json diff --git a/jupyter_notebooks/requirements.txt b/jupyter_notebooks/requirements.txt index bf6868546..aa8ad9311 100644 --- a/jupyter_notebooks/requirements.txt +++ b/jupyter_notebooks/requirements.txt @@ -1,4 +1,8 @@ jupyter==1.0.0 matplotlib==3.0.2 pandas==0.23.4 -scikit-learn==0.20.2 +numpy==1.13.3 +scikit-learn==0.19.0 +Keras==2.1.3 +opencv-python==4.0.0.21 +h5py==2.9.0 diff --git a/packages/neural_network_model/MANIFEST.in b/packages/neural_network_model/MANIFEST.in new file mode 100644 index 000000000..f9aca5b03 --- /dev/null +++ b/packages/neural_network_model/MANIFEST.in @@ -0,0 +1,17 @@ +include *.txt +include *.md +include *.cfg +include *.pkl +recursive-include ./neural_network_model/*.py + +include neural_network_model/trained_models/*.pkl +include neural_network_model/trained_models/*.h5 +include neural_network_model/VERSION +include neural_network_model/datasets/test_data/Black-grass/1.png +include neural_network_model/datasets/test_data/Charlock/1.png + +include ./requirements.txt +exclude *.log + +recursive-exclude * __pycache__ +recursive-exclude * *.py[co] \ No newline at end of file diff --git a/packages/neural_network_model/config.yml b/packages/neural_network_model/config.yml new file mode 100644 index 000000000..a939e5708 --- /dev/null +++ b/packages/neural_network_model/config.yml @@ -0,0 +1,4 @@ +MODEL_NAME: ${MODEL_NAME:cnn_model} +PIPELINE_NAME: ${PIPELINE_NAME:cnn_pipe} +CLASSES_PATH: ${CLASSES_PATH:False} +IMAGE_SIZE: $(IMAGE_SIZE:150} diff --git a/packages/neural_network_model/neural_network_model/VERSION b/packages/neural_network_model/neural_network_model/VERSION new file mode 100644 index 000000000..6c6aa7cb0 --- /dev/null +++ b/packages/neural_network_model/neural_network_model/VERSION @@ -0,0 +1 @@ +0.1.0 \ No newline at end of file diff --git a/packages/neural_network_model/neural_network_model/__init__.py b/packages/neural_network_model/neural_network_model/__init__.py new file mode 100644 index 000000000..b6c968d56 --- /dev/null +++ b/packages/neural_network_model/neural_network_model/__init__.py @@ -0,0 +1,7 @@ +import os + +from neural_network_model.config import config + + +with open(os.path.join(config.PACKAGE_ROOT, 'VERSION')) as version_file: + __version__ = version_file.read().strip() diff --git a/packages/neural_network_model/neural_network_model/config/__init__.py b/packages/neural_network_model/neural_network_model/config/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/packages/neural_network_model/neural_network_model/config/config.py b/packages/neural_network_model/neural_network_model/config/config.py new file mode 100644 index 000000000..4d8b173d7 --- /dev/null +++ b/packages/neural_network_model/neural_network_model/config/config.py @@ -0,0 +1,38 @@ +# The Keras model loading function does not play well with +# Pathlib at the moment, so we are using the old os module +# style + +import os + +PWD = os.path.dirname(os.path.abspath(__file__)) +PACKAGE_ROOT = os.path.abspath(os.path.join(PWD, '..')) +DATASET_DIR = os.path.join(PACKAGE_ROOT, 'datasets') +TRAINED_MODEL_DIR = os.path.join(PACKAGE_ROOT, 'trained_models') +DATA_FOLDER = os.path.join(DATASET_DIR, 'v2-plant-seedlings-dataset') + +# MODEL PERSISTING +MODEL_NAME = 'cnn_model' +PIPELINE_NAME = 'cnn_pipe' +CLASSES_NAME = 'classes' +ENCODER_NAME = 'encoder' + +# MODEL FITTING +IMAGE_SIZE = 150 # 50 for testing, 150 for final model +BATCH_SIZE = 10 +EPOCHS = int(os.environ.get('EPOCHS', 1)) # 1 for testing, 10 for final model + + +with open(os.path.join(PACKAGE_ROOT, 'VERSION')) as version_file: + _version = version_file.read().strip() + +MODEL_FILE_NAME = f'{MODEL_NAME}_{_version}.h5' +MODEL_PATH = os.path.join(TRAINED_MODEL_DIR, MODEL_FILE_NAME) + +PIPELINE_FILE_NAME = f'{PIPELINE_NAME}_{_version}.pkl' +PIPELINE_PATH = os.path.join(TRAINED_MODEL_DIR, PIPELINE_FILE_NAME) + +CLASSES_FILE_NAME = f'{CLASSES_NAME}_{_version}.pkl' +CLASSES_PATH = os.path.join(TRAINED_MODEL_DIR, CLASSES_FILE_NAME) + +ENCODER_FILE_NAME = f'{ENCODER_NAME}_{_version}.pkl' +ENCODER_PATH = os.path.join(TRAINED_MODEL_DIR, ENCODER_FILE_NAME) diff --git a/packages/neural_network_model/neural_network_model/datasets/__init__.py b/packages/neural_network_model/neural_network_model/datasets/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/packages/neural_network_model/neural_network_model/datasets/test_data/Black-grass/1.png b/packages/neural_network_model/neural_network_model/datasets/test_data/Black-grass/1.png new file mode 100644 index 0000000000000000000000000000000000000000..c4a76e4072d12ead9188ba960acb0d8fae358af3 GIT binary patch literal 29128 zcmXt9X*iVM-yi!}$_x@^8AC$KGTF-BU@&&dntg<a6umdH$NX!{`Uas&)>(Z zISC3BrxAvJpB@|>9(O)$6+YdR$*&xZ=&sLy)m%|Z8W?~BvLNma zo{s9VNBtV7Q@>C1h%=cbrzdBJy{ErcGCQT`I;((k@mT{T%}$$L`N5Ne(brpZk-K$D znZrn>t1InGzPqnwUOV=ZR1QzZN$?a<}*rQoaUh#MH z?{D`LScxinq>fuBUAw&=B29w)DA(@CK3}+o5n>AWDVgH*kSr~T&(?vBoyGLbjLhEb zg1CGZD}-xKM#j;R8r|Ey)aqm50|#jnI<7XwNwSt*@#V*ZiJ$VPA17oJ81Kne)S zIQI0!MvIhHrC@8wS5YW&ECg&)qp}p^A=&~@q2`vcbM`}P8Ja$aHzw&)JGoxC&t4; zz+4FG0w)0c0&UEzg>nPMV}CU8@=Mkr#GKZ6e#b=lcC)gB2}B|*W=;qMM7~`Lr6Z1Q zO>w`%BD$}~x4NqhLO?*A7=1kSt-7MOCrp$+uB*fY28F^tHH*UV&^{=d1Ii1I>nlPG z@=3FC0-#)A?LxSrBlYBBoK~Lyo=k~Gn+}}|_>(S@NaSCyrh|Xz-+Fh%hCVts@WSH_(RD)CuDYkLs4etWQB&jHb}{Kl!CeNB{wc zJzP-FAVIS3iywOa#LV51tIC`MaWc>j#FTdcN@KeeMGHb>}?+f<>eO zKb4i8*BEzvf=8+OGnJVWHBxm=SZw=UFisJg#G+}FDftW$YGK8zt1IZ}rd#1PIxv}k zUKDft2>5{Lyr^onD6oYdC$NHuvysW)BDjt(?{riBFZ*nKpHd|a{v3@s`K4ub1+3g97lSg!wS!E{iFXaW&~10jdgWHZx%ubIY>;0EO0e`R8%|8PGH*-ez#n~;Ucx1gex zchQTBi!-C*(z!$-sVhQoBpg)znjb%R1^fkJ%4Wuh!}h~Oq$#U^V!C2a{si~#7V{8^ zj0Py^8@;RoO;f`p%WWbUyrSH^p-dlt{4|di^MvJET-eEfbcj(T!eN|rySNrKxFetzWA(uQGiK-RLW*uIxC>BMpcV3RDs`S&lRn_z_Yc4j0< z4vS-jLkM<>58O^}XtJir9WLf;x?#G;JaGVd)_8z(np!-$pfx1Tb^Fy_%d(#AF9Iv3 zC}>)MhXe6lW;6iXI9>vhfa1ctYwKhn`|bfsSCeT%8QLbit-Hwuz1?pCg*Jv@u!z+B z?I|@?>ElK?qdYEKlNkQ9O%6lM2@|rg$yni|3WDK}x}%T&4F$)}^d5Efa^p{w2bI_O z_>GAxA*N;NC+)r6Q74xeT{iPQ9C&MjR+pzR*HjnoS#t67UlH0NiVO@Oq0-q9)J3Wv zP8N?Wv|0T{N+V?}!ybN~g3z^J%_4R@X4fqIxeUN&5g~5nJf)sFxP{>bc(BF}qUtt1 zeb3`)2Rom~jwNMyU<)}b2?3Y)rdJ>9$Cb(0iDj%n#Cdn3Sqr2Bi-}^qYJ;jfckvLY zD1t!55hZLei2lm1$N^Q|4{E7zfIv7Nj~1{>BoeTK%dI(ai5VyLu?KCj7o0jfl8NqN z5%2mlMTTQfh0m4>hhk#S{x0mQ$DH*a?+=s`-^i`YSvJ4p2J0LP44rIGuP+aYSc$^1 z*CyBq&}>Es&RRzrM(4~tOp3gW_Q(^X=T?PH$h%LmeUHQ9oO#2cIJPBZ90q4)+b%p* zU*|L9y=%CeD&Y}y-YAsriJ3vtnQBPZS%fKLq&AbF7Ky(YK=1l@@@_nqj~|cMj}xF$ z0x0hjkX9m>xS$DDATzhxeDB0d`Ha&)2EP~E35o4Zk2ZI9#71LPNnK9}9D^1cj;A?Jv;&u4=$FvJJ1@L5w?`Pa)T3_sT)>0&+G;* zwu}dRt_d#2!{=$*1B%I? zy&~ah1}sd<>+=O6rne0Jj6xu4&DCUXy?3Hs3h77;&IXR*o5Oarh(qDhBXqJB2bR-; zf&QqNe>*QHPQs3kV}4?#A#l=j)q>0FN{ZajY<{?Z!BCImYwu+`&|~8g#Ubo3slah# z{Q(jZ%|TC??aw%Ia&f+xN`kL--lD8d+M$(vCL`w&N)TKiD*_G@`;>ph~p#CTx|C#U=KaNXgG%Kf$31 zIQf^=SIpo5IiK3CPBjgm8AI z+x1P#Vfm%+9?lR}C#dr{kUr(l`wr*aINE<8k}D<>yc4Z1D0?L4}2aFz2 zs}^m++|kzao!}u+&{WCE&5a(xi!S0>HH|{=o zZ1kU6BCdUfWFg#X6?GBjygqXUA*kw?Uu-pqvQX#Ex>wj%@lrs99sFvR(R4H!cOQKj z48gGrmC0Tozg4IuS4V9*SW-_iyLtmUtV+C$(lzMDloDXK%0kz z4Z59T;#_kzN2Vt)*yBNg{_ySd<$$uYP&rt*iX^NEm;DYo+M2+X3zvmKFDQYXoeSRA z%;kbLd35A6L}Bbpe+!|ihctIe%EM=_C9atWmm<_q06eLK;W``3H@B;}%oV^jJzt8u zfF85E`^AK+I|1!hHkZURA=+V}M>Y=%Ty18yRvVWbcWE(`zC3SlveR)4+ekc@dRpEu zx*%y`HRI#HoYJ1a;DOc;(gmmJbF~cvp4{MK*L3S6bF=Q(a=wmb#hBQeH_3W%5#3ORuV8%mWIO+G)%b(Nc>jW2 z{rz_@g;0uPzHJn_!(qhNKS9^PrV&>NWu8z!TBOfCxTW9iA(6iP(Sx$Le2^-vYzw*5 z>jCRk{zbc`TMt~m3?Z_SY!l6C8_$I3xt@9s0>IOf%uZ|07t16x{yP$2aJ4Mgzy}XC$d$nHj{0=0O?&mY3bs6<<>Heu)=(5Pfyk(>4r@AS zUE28oliF+asiV3(ETm&o>JT-+3AnBD-hVWd8`O}KRVfvq)iN!LOGzv!Aew~2CUOk2 z@@HPx1QR3mO|2EKNAAp_QQiqjAr_&{D}vOs|m%7m_C~%}Gwi zP7JPVF<6y}H!7oijYLaZZe70wU(R4``bm3O%gUW93&v(Kf#&VJeKv&EqitnofUm9f z4Y+)}>68cGOThq(iv)@)!ziu+WN3Hm|{)D=zp1dUsgx$0SzzUt;s&!sx3Y7$~jP{HCr`z zx;L9Y3H&=ddHe^M-|3zcmg*|EOLM$3RW0m4vCLhycJCf(bSSP*6q|6%gSb5>T>tpe zz@xA#K4p7wB^1@(6$zy@@>t`Lq1@jfY&`M^umf4=@};LEQ65b9KaokPjL>a|7Ar$@ z0WM7<_`}7xim~$7I(aOUq_IggKPqK!FF9=(InAsIxBf!HV&6$i*Q$HSrNsU_s-&F= z0#VmiqnE~vY#%(OPaM#OgSA+V$4RaN9_CB|TOk}~R$Q!2E_JvOxYv9V!mz1pBxS&A88@O}c6zD9Z1kN3O= zhu*l@cs~(q*97IIbCWk?tvAxCYAg4i8P;$;gpyDg?&qZ+$b}~^TzI(k??$4(5)`Fw z*C00(kR~Ucja>~4ms(p|z>Q1a`&u!kHh!HU-XQl6b>?#BFRnr^of=(C^BPJ_ZAw%W zCryH36HlctTzkG{z*>aD+)09bYl@|V_ALwwT%#cVlV+mhdfEv4*IT6ao?Zim&X!YJukIeJLSiYt>-d$Z_A>!*M*ncHgq@ zzpNN;+`gR7Q=EwuD{#+0vhaej8wwiF|<(8WS67llSz9(P)4R z#?`RfwzW5uZ6K=M^5BU*k1m&$5Relz<~hxCnPCHE>N)@T3b@(}PL^ZKBz+$Ku4nZ- zV&RCp1zM{uU1|Uyu#Dp14hTkI=uvP2T
4dontyGOOkPeltb_Q#uuYjd_4g&DYA1;V&VTvmYaaM-Be%(34;fF+SzV)Rtiz#t> zd=*xz=#;y}Dr!Z>K(7Do#v2kSwAfrc&GPWMvvJX|{Q~#_D?<1)GR7Bdj5YfHhr}X-n)VJkT4hY#LKGN}3 zH+P=;)lx8B5U1ZPqH2OPx{kIO)%{P3#N~@fsrj#Nc|r|xeM>s{3eqO}g%*u);snWe?B9Q`q&7dQXPQZ?l7Fx5Ds2hL%_@ z-WiQr3+SD2&Db~z3@+$=+b`B~3cE$2U52|6$2QXOWLR5O`x1y@bLSt&qU-Cvkn_Pb)WpXa8pa52Yb8YTTN)UpGB$*{1(GkyArhzVS%Kmi`@EH&hrw0mZ%)Db z$JPm`Ta7yO;0puNjOL$65KT)M60Tavx3QjbdAFu!;5wazlwyS)3X`0=8St}C$+M2l zzB3B9Bsm&tQW?O}@fi1Gt-zuumciF~T?>EV!x#_7(V-=evFPk`5&=Y6T<{2t%w zt~dmSyAI!=g$o%5k$spd&(Pf;Fqoqxs zdD{c(j5x3W)0;AW*ibdjRPHkVZey^0axlFPFEc4Xm-gKn;|N7EIMD`@c%@%*KM?mO}EZqBd=PrtFx_&D=;)hFTr|G z^#utE8pmpy4|YCY;q$dKuS|=r$O>tS%%bUkRK88~*GQ!2ay)h^M zrY0Ou5B9dVD4elJkzGb?Y$j{+AF~=k?SB+C7)zR1CURs_ke32N(XUp1Rq46c==#j} z-w5#4Q!v>vI}_X?+K8#f!LzZ6GSQN|!6*UOP8(gwpZ2AK##;*uDKRO-zaSM*yDZ{Q zk~VS*HK#>lO%c1gH(PlZQFgQLW>9y%31wuX4uNp#S2?3?L}TH5@%HAs_uO>6^{RGs zYob@6jb?JT;z%#~kvv1DOh$sAk$CWd2|p&~w#IUdOC}^u5ZB%<`7_AV|Ggluip{OL zXrIR+Hm@dh@#pZP=!O+zP4An~OxN9-neF4F(@Ki~2}7B}wu>&I4DT~RtG^`dw|tGB z!f^=9$#TZ%@JUSX(qhh|uG+bk9ntfMOhCPMlWQ#dLFVOLh1Iv1Q zqHv6cuOUASdzbvA=|3~F;2srk$|urh95tG}cB3L`?9S!Dt*ynQ@wC*T*+dK7o;w#5d zPsz5sTkf=jb&Z?5yU}*q;R~(b{!QGA%?siCSIbgt__-_*Z!?pzC3NMSE0xZGO1|#wfRy-h<26$CzQpwMcCWC%Oi@ z51l=i5bf-4>BxC_sxa*cX>gt>dB;`9S#)+K%jylDQxDgrv2lDc#{+sf!y;Jb&+bpK zmPPW`he0uMpNR0zR`RKRgNmVX@#ebC9=8ZvL>r}@|4T@cQu?@9uC01=bd zg5La(u$IA|hDm56S$EJI=~1`&MU?k`P`FU?*+81e7RT3w*M*h4Yf%{g-BK4;n!?8R# z(EVHB37^%s2=2M#sj0OB`N&;`wdZrgBwN3sZPXAS&A;haZGtGN`(|I1Kr*HN8Kpcl z(SYl}?U1w6ba(s*aXGYcKW1VY8?ssZVzj3Q#vuBp?mpA6`r55KJoD@D#tEh4F132vU%bXo2 z%X#eD(GLiyR|=M>N6$QyHk|Nx>m+)ecdk_O2ogP7mKIb{I;JL2?XxmKfc4?RDfL3j zF~C>p&XcxwKE~!Z(LvA!2b*67Kzi|Fb^ZY)|0hd3Z(Jdo3xUCZ9JV+*NmmmE;`Mm1 z9bdgTE_Q2`3GM`XC5y1^>%Nci-tY8M|JIhTYAY&MG?-0N;UIQ}WqT4Sl-gi0V0N?j5cAaIu*Z@`E3Y~4r(+RuaGo*bJBAl>CHLi& z4CAs!Cx_%Zi@`SR0{n^0DYNxgia3>juLpUZNCKgo==&K$L-7Ly^hP%_l&CNRMBf-s z#!?!&xKk5keOk$Oc2H3KY9QUhf{%~gQtQSPX%ZQ1CsRJctT}fjK2dVs>Izq>nv3yW z!W9Udr}DKqHoL6aWn8}&nERnoUxTOv zHhorjFw$!C+@BcNu(Jo0_KoPXp0}Q|;_9>(1&KTyh6uE8^7{WQ01T}zdr;z>k`i@iTHu*US;ED1DRi4G#6S zjSpLU_IvghDXl(B(Py-+A;WjZH)d~UfnreF+@?~|%Uz;n~-+;13}!U-yf zF<0Y~sgyD?uBLMy#bj1r#!+m$>2gem$s(Y)fQfUs%|o`Z#F)O|rJ074_;M4vR+Y z607a+_1TPwauptU?5gyFQ@A90Q721% zArVqqhib1da(iQX9Hb;N_T7_v-lq9)1prLtc-9}i79gBoS?Lq`M+NJ-aBI@;Z%_{2 zX6&A07sU+rQ^Cj7MrxkcSZ0UmI@(cEUE^7(?1?$Gq4h=6-SBq()HxcTIC^K%jy+Q) zxi!Qq028gT|2~J;uvPjbcUj1I;>zP#KKGt6dCMZH*|lzkHTi2sC`A}AulNnfk3q&% zv>&!3Nq=!^(O6N$rMKH>X368cYt`_qtBa?KYdy<;l*79tg_Pdv)D_4doeF4Le0D@< z3W4l3bb;1&@sZMAfO8~tf(87vHD8z@ zxO5E4WM8qQZw#qt(7?Qmr3Tg?=!y6&Ea57PlFx>x+_{D!)Zp`=NhJ!}bmzsnaKv&} z;46-#bha$!1?QZdES<>EwhvAYW&-BdPJ5OdedY%PuF|4OFMUB0>T-Re_nMzh-4mV` z%6t{T*6Gz;%v*$ThhoB;;N5g)QL=N6B?BCTt>N(-lgO(3aOv;0h$%t*#Wi zEi&|GC1!qc7vR1Y5s$f zVY0$ibX2u>)TCgE*FXU^>Z_HKXIH(-7V7Mw9-qtdH{!6=lh(R|nh3sl!a#qDZ!mT4 zEm9-nq<`~EC##EAfyB7gagXzJ{(L{NqrS%mmFES&jse-+9XZm@M^x6&zogu%FDlf!PiT*n^ z6`U5`Asy||azp+Sg`4vG_)zERBa}H@*PGpY#@#N@a5sPZPf&NvTh-_v6rE$KIwc>= zm0`&JD~bl%bs)UULSm{nb~s#MDe2o@q2yNxz~-`WnhGob24tM8%5dOC!pt*Pm}rtB zzpKof`%RW$&o!fsmCz-ap~_?VK2F0C%67CvTdQ=j@6G@RZ=QH`Fcr8WhBgdyXzG*3 z=PIBIezJ-R?jQuH;l0}{Z~VEY*Evl-aMXPM+jZD-;Ipt^b!cy?!g~ERLLuiK<{7YE zxU01o*QxVF#qpuKCsDbbi=N$9emI4u6g%cs zZtYc$?p1CdJAbL)3VGqq@alF3fKN=JAS{o!i91Qo5YxgPbjr*r)Vcd#1KD2FZD2o} z&w#l_MKp&fdJn55+k}?T6Qi4M-K90&o=zl`S7Q&!3j!Q$&1AMpc*X5YU}qPlJmaW0 z>xC~9?r7==0yOlv@nVM?Y zZPtM}J=wihU)D!c2)!gW8J|lcR?c}GZO9{y(}}bBys^p~-WMK}{*1(G5(I_NOG^uD z4NHTwLk$#x>33+O+l80IJ~G&IgEbT(!hv3^P0i@xZOfp)&iv9wRh@k*7Yq8=3teXk ziXPjgz^0EXVY%S~$px9N^NT&jIq%#LUHdz8`FyVRMNQSixv+jw#J2D7JfAzY6vCZ{AMYLUawzKWQC znz|hUh2FWY4kNzL9ch1j8H~!qI-x#^pZ(rFJ5NtIl7Cr}&1Jag64p2Fu&fh8n^r<1 z8ybY$lxSBY?62de-&_I#%q8B0llLC3RlGdlE@ucU$pU6SD~TV zQ1!9EvN_eaii&DOa#6=y`7*n^e$F|>33fs7fYWj?rKiOOP^NIdE}9Ld#2G3K6WyvH zxG=5}c6jxt<~h*c4rZt`GE~aUGBYLArsZ|FPr95(+dTozqsMCOoK51%UqY3dpM(i!1d zt-)Q-X08d{$p{xfYy4cDp}dZbI5{4gSr&5M8mf%3Y26YZk-k0nKV-0v{k6|Uek}CQ zC>`N8*dWYA>Fb|wjIsvzO+MqwmXx6+{&nlsgqky6dR03QF9`W`dc2Butlw`jJa=p# zyx|UbCz{R(3z=@E7!Ek^DXwqF#L|A9II2ewDFdfy-qoK*4=Wz!@}wjI z;9;FZg}KPShoP+Pi8CA=n~nUCg|ug#DpiWFbRW)->C7RR_YX$dp@|)@8y(1-SwNdT zA*hQGN*p@%SR(`JZ`YAOym)T+&&Uw*#hMvl8Am@EnO&;g+ul1qCPg3bHvL`g#AQoy z=Uf=@^mkU@5O#41>zvrHGQ*N14}|u#G`aC9pnTyXxGg62b4!M$F(c;#ccT$;hmK%} zJoo5c)q|}Aliq*v!Ge2Zt|c)Z-{NrcS;ekO7D6jNC2~6}2O($2|2Aa)tx!jg-8!3l zuo7?yqVT_rLJ9OQ>DS$?H?uA#s8{~kq7e{q8_ESJl;3hiyT z(|vAkAT6DoCf~%PK9SaVZgp`mRaIoPR&pXbmwdD5ZN`#hhi2DLPF6=>ue?$5*Z9U4 z)ODopA-X+N9?ZK+l5FH{__NWq+o3yY`L__w>K4Hm4|T9xrrRP z;@H2-p>3g-MC5N8IXiJYjn~|;dEznJnO)OdAI$_WpO#X6$JK{don^hdgE90H_iCn{eT=_#f*H%(;(V$HeS(V!7B3jp4yrK@w&OwT7*_>pA1t9a z>Riq$(N~(%D=Qy>W@V?wkZyG+a>-3n*aa(FJ6gtvuxS{g5=c zVsHhH9qz1+df44htSWeG`Rre1@0lMx*IKD_c4P3<^U0$jBEB`=tM6euQU{+Eih>?` zxW&xx%J>e&L^cJGR2qA}R2LR!6!|$_#A^z)6>JZ!mz)hVVI!v=8q1R8ybaXm9}6f5 z@vQ$V&Qu|tQ=z%j!#kORjGuI+2i8yC5$_jS6egdqc)Siymj8#c-N&Un`-}-k}U`>R18}Z-s`Y{ehE=iwS|LU5& z+Jp2}gYTWb3Wg+U=#}!yEO^=fcErL=7?H`ACc%x$Tgdrdj!GL>@wXl1pVy{MndyBc z5zy@>CU*zNqUU%0hGOm>o&;@paB1A4zH{Z`)gL~0w2c6Z`pbLj>ZB=#>`So@Aw|3P@NgmnfZli$S?3oF<;I@9?W1EOj zb1RrI^!nH3X)&vXoBu9#FQ;5JUKq>H-eEesQ$AD^5eASK?OPbLJh*6YzXxZcy z<{4E^VnsBL(EP!coeLEK-|TaDu18+O&(;qT41Ll%3~vtbJ&N`fIu3hVkkO;@_;9~= zRYglo5cv)R{M4qmbU3wPnSY-=qGQg8Mb1~%jLU@p+x#_$HB8{P{qvS>w61jDJBD_E zfK8-&))B`qIjT?OF72?rd#s1s&^T$0Pw_ewRZ#-=^IJ+m*xh$)KrVp0yJHf8ZZ7fn zSA%?oHDaVzSs$c|ccXKgho4a%L6WPDK^b-=KNI|^u1GQ9CtlWLguS}ec%?w z_3*+xL&C*B&D>g(6Wk_lYpz~%bfm{WX{TYjmUVV&*zwFn?A$1|j z%9zp6AtBmib=C;?F6giahW32=>bXC}?cJ>9n5ECN^Us7H3j;Nk-PzfxmZBesi7j}k zSO14^=T@4|ch;q>gP;q~7n7@rUsg_A{W3diw>CI1TN`e7&=kVPhu>ipMuuSWr&K5J z)gbljXNfsfZp;AM7@yXuw|m!6M9i6$+BUO9p7N|VG=}_s$No|MPJRTEy`L_DXgPgi z&9(erhNk!_HTFz6Q-MblUd+oM^-NJNllwm{=@v5C%Wf<1TZGgozbW?FT@f~Vcb=lG ziV)5To+la6KQm9;DFVLSbeJsAU{5N-f!FCG6!Hza)=_Q}xm zhJ2fC-9B8TjYE%2Z0iW&&%e-R!RfQgKX*Rs+W7hMm91w>(JgZ~hx;(KS`4D))>`;h z%GN=R;!uoXl9P{*jY&%jkE^b5xdN2A%!dM(YHIIzqY1!pCt$@k1?k5);s*;Ui@4SH z6VRIdHJPi#b#d`vPJglG*fZ+chDu^m90+xnPtt_tVW%>kp(N1MTj$$(9jX~Rc;6M3 z!AuMCv(tM##}Qxg=%&2!SGc--=g@9x=T29~xq3&t@4X83v58|nf^>1e#f|vC{QEz)Qwgbvmh|teQhANLPg^Qx_(5F}eyLypS2*}r9x4VT*J8jo6 z&fjX?y>p%UF{R&9Z)xcCQ5|@>v9|oaWO+-{z|Hmjj?*SL?VCe}&dgxxG`MeGF-)v} zG_ZF`S9lmHo$PdId1Z_Pr`9*0C7FXs2>X83eDlF{CEH#!o$Bmtma#-8^eMOf^@HYz zGX_Uc8_=*Q%anj09Hh^X3OoxvX8IVgU37Ii1udWQW4_mhsiRtR)m z(*BV)VJtU1`@%OA(hc7izaHXwN2HXTl3RWeY``n9>RIop3)M=BYOu4})tv4}E-apKsrrB+;brWo=xBb4QNPyw?h zKm?4cO|-3cIG&tajZ!a?k(uEpV5a&O|2od_7oe!vz{j@lV1BZJ@$CI#!VEyQnqb4U@Vfu`E-A-w?+41APp6=G@M? zLT4No_q+4wWIs;t`YkofG~JBj!=49^n>JO8>WP1Y(KEzve**8Ivw0FlB{Xu=g4a~s z_lsXYu1^b>W!G`jX9hHWHXjmjI}D3TXSunh;Mf{uw+2z3SF6pH(m~$4IPrPv+MKPS z#e`g3MrHqAn6b&Gh$cV|oa+Vm_O5FQ==P-y(M}FxXd#hjb-TMSm-uSNi{8=SCRK!@ zO2{mayL+jphcR~@>V-|#Ikz^-bIJI3{V0}S4AuHyosrrYY4_dRk$M9v$9p;^-hPeV zT=(n<*I0S~((*ate|#qQSLg2sF3nqbPn5WDvbe}i@eD0L=YorvTWDw5{$_O{xVBgG zsEiK)c9JmWTZk1-{HB=L_g`f+IR}l?S!zIUFVsLX70AuR^*x#nRFP35H}r~QItsKO zor^>4@7n`}Hm)zLC^(ECnSdaqADRh6lf2HJbvG-gHloxV)P1Ql+lO~rX)C7}oS8XV zSe`R6%2gR-+IJnh7KLeTt0#^1GwvO?wlSJtKhbvTccvB=ir6PMYOVn7Q`1$3qH6Gg zrmV}fCViar_v7|$m?B(~K>s$G%JOy70qr1ANSF_(JYZmgN zjl)My!as?m|6@QyZ%qBUYZY8m04^oA*ajP;{_SYN+2~70*7{m*jYf3D%`fHh(Od2*DGn^y$iA2zIBDEx3$5N7L^_H+}cP%&{tCvM;Nq27f| z9~55Y&@gd%nfWQQPQlaAx07lN2SP|TkKAioZerY=%-zy_>*=(xqY?C9RSvo;(`?97 z$#G8{`0&nRc>~{}agoSOOOgFISNr6alP&Z3va4J|G{u*Jx66H~p8f7G%nfc#<{iw_ z)}td;g?2M3Z9^wp0+}j1kGkIPyr6}6Fr#FDf+6-pr^c2TZ^m*hT*qhg^}-{Lq;cBI zWSzN1FG+xNg|Y}1(16f47EUF6DRgxh2RiS~B1{J=pxMb24VAI6x(^6Me#NF9zv$gN zd-fj#9pqq973e(oF@{ag?zho2ubSxb)>@WS8PjP@dMd~j&-L|_+GT6lW!-|)W??yP zV6MvaWcP62fXe0?saT`wN%i4hEw8(?*mz)Hps7r2p{G?+{v4Z?9MEPpY45Y)QnB&Q9CwN(%;4G1*-qK_06-5<{2HIW-V{)*@U~j0Lz|K7QnRS5fETdjSg}PY*JCu4lDpwxpG~G$3&hm)9HB4`|f1G zT_p_FdZR|Ze-;qroG%x6!89;>^lfK3p62dfsHBv%e0g%)*Z^1`*X-hn%H=;`t8lD(&aa|5<9r$=-VP9Ql|EEbHdJ@fr8hx=J+&T zHXaKlHwYa|3*yf=@YgW|y=;?MsK-@Ac|4H_Z*}SY6n#wgp|)#;|)>#!yeMOuQpCei)6^M810{Y(N@* zEVRZ8$Zk4Q{%Sy2-$-*4{9L>{Q_>G$^Q^uQr*hpN@~v>B&)ls>{pH=>v%(-z*%$OR z8va>Ig9=FIkQGBr{-w2*yl%z;~|*kG%<_!fwF@nCRZS?k7m&Rmua za8Xd7$%wj%rUex+K)G5}k6{rc-wKVt;R8pXbM{5-<+TQ`VX~nTB%U@>jW6Mlai@o;V(~#X5o2;{mvn}DT zD$=7mlA;PiCNtb(`n0fXoM}&z#a6y;m|IL+icfCD>vap3k&1uvMzJ-f5oR7&my;9V zl~dtr4%S|EhIKd4ADrk15g(s~0p$8&;b5oDBAqIbyumBxogWh4`Rr#u2^Q-*&Zr*> za5VB{@^!Mf_^=zAr2;;CbnBAGn%#q`0*f+r9oL#bXkqQXY6jpr>>d}wKMGP*(jL(? zzjP7| zxJv7`*%rN+{WX$Wp&7tsNk}N7g0uOS$!mC(u~_ZfcX+y>0jCO;&I0{v&GxrZgy@Xu z@^$rs$X-k|ww6xnXUuRe&}DJr?C%$Rj0@DEXBX^#@DTe+G)@F#Kei@T6DPOk zpVD~C_4x4bh_#)kkrd!D6$%(JrkZ4;XgZZUx9i-dI;Pfe}%Awl8`?V{7F2t+e9Afzc${2qLfl9km6h%bDbj8276b(qR@{QJnAat4(DpT;$$BLaDt%DK|qF= zW&zi}`p@ve1)MOl>jQ%2KtmN~Cy-FR;-kM!UwOgW*!QCI?Fb=Wj#2}5%&gKvD10Dp zbd?^6CW#%B)JYt&_w)Cf4n5Ai@A>0Aw)q{-?WxLqH$_uM zx&*)*PeM{PgNVpw8_+@&Jw3O#_5BFqcyzB1t?TAx`1pTMyPh!4B*ZZ$+)cT|OcVGH z_APHP)d7gJ%5n~5?KeXhJBf(}`EzA~tt6+cGh4;Om@o1k!nD5v@LRg_iA{gT=OOsp0r22ok*MoR z;%G6;mVH!SU*F>j3y-8qY1#y!$5ANrY_Ht}8uoo@ebH4)r_{rGI6f``V0N!Q;-~qe2BMyN9S5Cp?;mI3_Np>HhZIp zO)!~>aR4jF0{c5`ti^PlgMMhc;i;`rW&F!PNR~idWNsbD|7QUL5MnL96yKf2NC2Xm zL{Cs*HbJD`lMR^6xyH$KR9_YaTIL%jt(;$v|N2P7>Z$}2Yo|oF!4_f z>_9X?*7g4aEiuy0c+8mJk9kA{Ba(-m-hqsiX52UVBnXH=u9lt(h!B!|m>59F3{FNc z$u-|n1Wt_Rq52?6fCYh}Iw$}T3v+3$3?1fXt^fv<%axOq(|Uc}-Yu-G;%@E=fEXmL z282Ry>w47qMi97rP=JqMDy~svWvE%G8A_9PUc*_ZQ zTG1hNj8SV9GlOTzlpiUOaK|G|dyGtubxH^$KM#bFXoN#ZfJ~hVamL;8n6qR$(BTP> ziRWK)jB@m&h&q)6&3;zB!V!r z!r*9XN>;Kn0FesUxA$)%g$R+s!IpKo-EUK;&cpZWS-@(pY7xkZyTAyP(rRssgl1m` z9#%&H%0jdwMqrWi@TdDAjcClG8bp+Ysd$P6$FspBn~Q`HhxNyg!$P=gUYj6*5a*aV zc+RuOVSYz)4;FDxj6j$gGcD_ah-w=9;SGMdevEJc7p_6|IPW?@fT@o8@R*2EYeT{b zour^}6k=ROOeV_BxY{8JqN(LoOLSZDGHf_9etnoKHrvhA*2Ay z(4ZZ0HbCY8G#G&lMUw81>DmCYX#fgo$SFwSbAPxSvuso4fBfS=%CaPcA0fE#-~Q{r z`}D&X3|2E6k`+Gx zLen!dVj|(TE?G88NKB|YS@Mt6)3j-j z*MBfGupq$iozqeZIOy;q6oJGw)Ot@kTmhoU2s0BviO|4s93GBDQrh;LZs@P;Za(XzxvmI z`|-=?-uo)$mw)=_|NdY8^G`p1-cO5^76G%5?fv%c=b!icdtwfmIX5wE4O^?1%jYj& z%v7Z2qKyE;sXOt^7T?qi(p3@x2xE+8DW#UoNGI^;u-(pixc%J$Fc6sqJ)ilJIAD@) zBh4SvJv=OF&OoM?LzB@*ZfA+V26vltSx+prEsGTHebb?BX`>H{pbYqd?|sXjKFI!F zOv#-CVQk4ig}kgNDv+kCYw7?4qG#U_BG~Kz5Me|OvuCHA%*j1!UBUqhgc&%1ht0uK z!O3S35jyv|x8)>%_=}&)(yn+V_x=9<(;t4geS2Lmm-WKT1q1rtE&S#4Cx9S8-RIcG z<86bYpa1IR$3OfbzwZ&AVj?{EnIwn9bL!*IxEdg^5HXcn0w4*r@m-6JOvEQ4+B{&6 znFHB5rNwY1$1d_VkCqZM77mRXwP~6F%rWF5awl=ucsHHTBPmU zHpT!1lIm(U2A7IK7~}-5fz0i6MkHzLi3$?{16*FNkGI!fe*XFL>C@@*au}?HsFmgA zDntzU`sc4wi8_4v>BpBJzkK}RGZi_^-p?squ3~Cdq&)W#NYj2E#M{!~$)kj0z>3Ea zKdE3@LK_E_5eZFINaj!=@_;!NkwvPGT=m#pX7BlFs%oG6alO{bgd%laPHkDj&2%DT zY8?67HFIG}`lFO;M*#xmcAc7fS`q-CH5*verfCE@fFS{rTghm8Br`{FRm-UG z^U=anf^*BFGDM5m_dQ?AzVE${x-8()+R{qB-|i551V+%*mj(%8VyXcgz<^;ky*P2W zE3%0cck`H~wR3ArTYvedU-J^OWvyxifM(q(0wNVc0-_ScR1nbJrPU=MB8*w~9P9V7 zJs#V(0TVG^Uaqt6@853yzMGnJ`SO>4dH(cr{q&)oRy>L?i}XH9E%=<@9r2d{ zRh6Ui$yg4Gh?yfEfym4sCBd4B6u{M5tpGqsPQ*lg3}O=Ez7JPBa4V%80I-N;LWKttr{gev!r|bUA{R=jNMLhjS2YvQF(@Z1+2r-S zG6-?zQc{JkpD!P;wY6j$nt7q4d^x>qX)sGcnvj4a8)frR zPBTQ3!`_G4Kc&-|DY^bR@nggUE~fcCp>AXEh&eV1VIBYn1&$C1sYs}8UvD06u5xN5 z;yzBFJ}zrR^imokh9z^j_R$?2k8WX)fJ5hg&Li_^U}hdpz^VJzjLjmNYN@UF$(%u2 z+_u|kU6aD?L5D&> z9@U9BTbf>9-+ue^FR#CT)2X~JKmFa`T)up|yj+%LA)<)j?7KzG15OH^!Xhm8U2!;` z6#!x8+)69|A2xQW#og8Ha71Amr-7N>^pIR3ghfqtOmN6aQd<`2WADQv@3bM8hJ@~+ zI!L50ba}btaXpVPbBrX~3rp{#ElXsH`cp13_fpI3BXb1ndfx8u!jj}a%yQ?0`{|_? zJS-J)Ih_!ZnA>u8Af2|^PRw(vc8kQdU|Oz3&6%*&b~;^jBrBC zzau~?6^LqU5b(Hfzy9f$Uw;0xto8F>|KY<=pRb=@*3;^yJ|>xyKhyk&U-amUb8`$N zpo2<*N9;Hc4k;Msnn#DB4vFaS%0iZQr?vPf%n;@z3)xig!$=$fkH^Ep4jZ|M18^;7 zW;V?YFpvol@@Gd_%Hv5-2#DHhDJ31-0TFHhP(&a?MAGo9V-_x{I@iRW!tv#xXWK3z zo=&H130juZx}GkV3nC%0nr(ZZX33ZaqLi;P3{qOF%PBpr%$%)`d2I0r@i0OP04BU` z8=I%5umn)2v68CW{L6k*jihTiDG!3fWFI9epoLk<>bEa z2vMZEd+$4vMEL#oZU(j0EFW~YQkt;1S*^9S(oSnxDiFcJxXJG;g#!XgD~yOb=G^yw zzxy0N{`BL^raRq?3oc6x6$T7n|z;Re=Bcgo3guJc|4}6?^L}I{~ z^|bdrUj=uU$_NqQS&YF{+?1HZr|NcjS>2)c>23-BcAfhiN17f>X*I>FFbj7SUY2F5 zrXnZrU%zrKKIXhV_V?SmEY}~uTt8gP=`{MF!YsTtwNiedCAf4?nsZ z7V+of2~`Ym^*H$N0Rc=0sRW6i+A1KxQPTI_NtX{5(_Kw@xG=ZV=>(L|0D%sVaj9kK z2+yVb`F|tKK9CV&1P|I|(=A%1-Q_UAwSeEx7G7boVs(;W3$x+*)j1{}A6+iaPlY=I^7w^> z7&`mx6XV#8%F@EUKW?ox6hU`E%JT&xnj5p2ALRpcsxWgc-(3-Ip5klO2e_MDp7&-weKC1iGI;eY}4D*v_Y{{{q7En~r2+GE?-^Lkm&vrnlN zYbk5XSsNTv=$xY^c>XZAaN+Nfhn$zH*|Hu&HR1Cz`+P|0Y%EfgWw_%1;xC3yVHU>A<+Pqp%evY#M;{QE>-nd@{mX5R*Z14k z+f9nJ+NzZ0!}V|e&A+;S`M`z$g=H}gL*-%kg>Bz2<(ylHNYR*p*lPP;&G58zCz0^` z0gy6F>)lf^Vku+HQcEdSb!Kmo1N5!iwB#TaDQ52B8dju??8-DwcxFUA?l(jtk!$-f z`Y5H&G1WBV7^T!tpI>foud@Rs{1m}d5hwyP-O$I5L{#M95<<>|j>VNoT*v5}(73<< zQpHMPaH~>;nP_dbEz9=y>)YrP!dq(xE=P^ZaW~JO+5xA({Qt6C{?KA+An*Lqs06wIaUDa-^Vqktk}S(=#vp{dCMcK}bH zc%)6vTdharINJw^4~qyQ5(k7x#+R(s8tz0?q=ZNGS(u=faNqV_m=NLi^*2E15Unl! zem||P@4axNgX?291I+<(-}m(00Yo3Yl%iwgT3V!xF_>#+KnNKTL6~bfCQ<~*FnR}o zA_V~4O~W8yf85Wj)%saD$NwJIr7TWy`t(xkTBL@_$xJX-<>~ni&PIJTq zna4T|28YEF({TE{Uq01qKz?3Mi2IeMbayTk5hs9?zd+OyC4e zD1;pXA=dNhcp1Y1srUY162_Wec|ZVg)!LSPToDccOtkNh$L;;Jp1`g5jfjA#lv?WI zW*9Jg2X_Q;Jcc84j?n;Cf!}uNSE`ZA@)hd!OIFzMl8X zPk;GGbq6zM@dyTb1{|InQNYs|oDT!%)Iv1evNP@I){Ia!Nbw>I6af4vlYpsKYdH?V z5YOxVQH;fe;09rc@f^)W@)yF@4X7;v@_6j-`tifd$`TNH|IZVm&Pklpjdk?BEejE9 z1hF6>7HOvyk!!0y&E`zVwQfs@O*O$w3TKDbo^PheR(`8+d z#RJXb{dQOT@#7DlPp#%El+NDVHFDc}dU%e1`1|G`(6@~l=a>P}T5~^|u?{lw6aPlc zBwSmA@HCC&fB1Mj@@Gec`vD#a(|DSSe2(+^Tbc`G;0842t07SU&z29%>fdG%kZQ3MZAP}kUKE~9kkH>btTnI5+F?1>% z=?rV_?1ya(Sy-5KX#jA}_7&eVk5)N(H1bW}YeoAh~7Y(c0o_bKg-ovU)8I z?-6r!ou(dK3Lvo*)jfc-7%Kw8Lp7NL;HGZoIw7#NqPmBHnhV#dN+QJUVLDYLSc=@g zy}kYRwYGBp{IYy}DVOtdI-5t5bw~;lUOo$nyxnde+i5@?nMFj$EK;fzp92s?WE}9D z)7`>Mkd^bYTi@Z>FQ=VJK;(%r|WWAZ*TX#)&BMhAh+9AYNaYj!iYx=JqXu| z0Nb|b%Z{mo0Yt#|coddnF=PfrRo%Du+qbvJ`(u0idb)nhwv=bU29r}_W?AJnr=}Hl z^uZzs$OuKm3$~)O_e3gNn3R&l2XK)0UtR?XAWAJn6oAZP7EDY+W@a8Ntkc)?X??kr zRvjWoJl{?0lr&_D>}t*PwVDY7;M>FFbod^d+I%)F9XSPH{PVIDm z+~H|w^l)MdKnowL_H-^1Be;J4;fL#|7b$#sxn}Dv5zg7S+vX0Y7C;`bo-fPkG*t^r zsqHAPh9i!N@LWRN_kEAVAcgli@Avz@ZxQU^RB6s$L@FmU9KsBsLXc_n7;Rafw+ooF z*Af7dYehh~D{+$B>A0hCtK$L&7$-8|dXJ;0|;09b0v6WF>g=gZ~#>61E0 zDa^+n^!zK-G%sdxoTZ;OV1$&U)HK4+=hIQpeDo;+zyT`DDX;r+j{$&WBIG0CB5=%7 z0}zM~0UJe-`ItpAncMktx_*3Fmd1pS`^|KijZ$bmHPr`1=(JjDnt|T$H$6Ud4FF_8 zW?`*w(W1P?_Ym=eS5tl_H7I_4az7Qg`9+9d)5 z2uOgWwi9tp84r*$Wa(kyE-Vyq#E9H&4q304+v~47?DpI1=Rf@9I*BQq4@?lizVE6k zLg=xc&+GNN_OYB#nSjkz>v%o@#1sGyQCLhB4h$v`lesd{$fMc;m*t31v^Z^Z1(6g*8~-}5}eA^@4#g<~fZMpXbWLansZWj&wI*?5kG0CHLv8}oENefjc- zZ@<0;(fjLLJ5^>OX2>O(C_}3V;AV(aiITv$Kl=Uc-uoC+4aook$OBy+z=?S7ldqEH zZm0F!3QG~>r@g(klyvhN7>MinI>rEi!lh_vP~T?$Ynu}Z%+@*-uWXSA}l>n%t!C4I!C`hZY+d!@Mz66SJ!}WH$a>^vzBD`T^QH3efapH2p!!T z5$?Kgk6bNMZW_ItX>Ij||shM*}z|lt^vv2+O{^-N^uJ^~-x*FmzxNm(- zb&pz0Tk2_PUw-&l5;7BV+!G+pN`P7F;{k_K5D}#{A_5F^MS#&e9ytZCtw#92xMqQwC3~Bh&$W)( z=N!RUi>zy1)^<9r;YN)3yzL6JM@ZrIyzbkM7`^xX@nEL9uBDa$G&5pJD|bNXoO>U) zw_D%4TBwC{Dc5RarV*l*QiN+M6uwA&`t<3CAAVpAVT8l-L5ScUgq&jW93vbMNlM94 zr-w1~dO4fB&UvuhBY=PbMV5B`;Y+`71n_vfWACoIzungJDHDFnvR*!XXs1ezQcAOxGEFgFo&zg|vlscmgY&}vK92Q%e&g#v;A7yR*$KaM_UAFtoOK5p+t z#EF29EWV8bv?ar{qcThmeyK~$^Xm$ z_P@8*fLUr8z32E3B23lBbTtBa-0o?Q5@8X}9#29_rb13$1LFJB_#UWw3fR9ptz32V zJqrj#xGfD36O~6yc=Y-OBuambnKkCeTL|uU&u+!Gt`2}n5l!560JxcZ2up5kYpXEF+W2x_ zu9vgOG5Z>$Q_^{kKs9(ac^sB8I*c-W?DPmoK;fucf4n^ym{ItD{dfOiT~`)v>&mqx zN-@W`V@@JgQ#X74`fZLG5v|pASy&{0;9r;Fn`NO-<6PuD5{0t%7Nu5&PPDMgAv zlJbR1squTGeMBCy$FwoPEcd8G2R-_#lk|iU=?=hs-;lW zaCbA~V4gK|Ep^u;GQz{`d-n6OKY%#bRn^bq+#ko=_xIkPJ=vy_e2&CKxGZaE=TnmX zaiCj5NQ_xvyIyih4tBX*L?q|p?h!(FmjD0}KuJVFR8g}=jeL-xE)8k-5>Y24QqPlKJ??A8OFP50=4 z<7m-1&;Rt#|MJUEzm$|}$+_g~b=$7n^~=lk^-H;|oU^&H5P~OR8!EYMU%sSTNhJK! z^r^^kNJ--1q31yt^Og1yQ_Q|Ua?UKwEK}Tr(M{v{^lbxxn+@&OhqhB&b5ml9g*Q|K z;RlXO?w>(F)5K~~mZt7Nj3hcC1<=vf_2SP~$UeFw5Oc1pnt%W0?c@F9*v}};0A?Uc zwWL~eEibRHz4t)BSR%$fXkzy<1fi6?tTkuB=>ukn(bJSfmUU}mm|0Xpgtaf^rIHf3 zg#v_#&|$Q5oQ;6^AO7wi-hTS&%a@mJyC6ZzSxP3Blyg~@%eJLjOx10myD$@{-~Z(w z^0FA<6S#)Yv>#?xIqbNoTr=dKmP6CkK_3I`gMD`B2%;vMW}qOwbZh%>vp}k>VEssvzM|+UGlOL zO9;t>L=M$@J6kDPXM)vat7C(lBC^qmDek^F8>4Q8X_ln}0y9mK?ii6)JWP?$p0W*a z`&6B%sky7_{0a7rnq90n!R5dz1dO;QAt z$6ew3{q8oR?=}=${cNDB=OOV{18~z^N~x8ZYgu}0L?k&6RbqCeNaGilk~1?CLgbc4 z(6758KxEeIXye?4OlR$C@3~ymOj2q?IV!}!DW~PSZLeS7fBNY-Z?5|BetZ4HBnv*L zwP1I0DO1f0o+Qm+zDW~M@BKLTZQJGv=57G|JRIY~3q@=zGm%J=Tma)W1K>7#XWqpKkRY$sMwflx1-Ko}0bB2slbffMB74pOQcinsFJHd=T+8wH z9*Vsn6oNRIBmzv8r9K11!i=cXY#sV3JTkz9pWuQaK$0|4W=WJXaVGNK zP1VpH0dp!t_2~(jkXgyK_dZ(xc>BKJ9%in6xchmw@%^!E+i{%t+bty)CW$^J!uUf| z&L1D&THi0%*@zNa;_LP6``gF;_Apg2bhVuGzVE_B%(rj9lplZR82lTwuE*ZnIc$9r z_{2<blYLcSmMDZ6 zI+*$B!wr#za{Bt?Yq&DP(8I)i=ySyXRgc@p$dZ`FOv7d3?jnqcwUqPh5idADkKb;v z;6&s^QH>LgpE@*zhLJWowD+#D7muz_2mKgBXDkGYyLR>d*c~iym$PYWAB2XWOjzr( zTvi87DM?PZTWft^)+{+Wkt0f45YcSBe7&Y5N#yo+ALcp+#VYQem&Ip3BBP_5wl@0N zObtOo44iT{wRWCnASuTn3hoYm*|xKvpyRRc0KdM*fs~Uh*G*zo+~03M|M&k$NlOwS zYGY_?VRK=mb3gBIx3;(K%hw+l0d0;d5pKfCz6yBA=caBqDI9@Mtl|t5+D0vjO6HHk|c>f%v7?Pnxp^g_1;4 zCV}%ji3=j!e*R_s@&X_>oaM5t>yLl_7l1vEF|%-k3f>Kgqw0`yz&^_mDW=Z(=c;r$YEM*CC#=hSn0-xODpo)U6 z5Cnsx9IZ=EWvP$bP5zh2UAB#C&C8P3_5N|&KOP@Hy*)m5PPuH?x?MyhGI&G1{uC2V z(GEJ8YLr6;^75o7BC&j;3X<4##x=%25aB+23VO_0goaA2NJ+FYxQZlm7m;>$MjEO% zR7A$;(P%6wNm4W9)UA(HN^i~7>h&6d8<$J%=b1|}SA+pbfG#O8{~7Ta%W*B`@wkyY z6RQs?7u)?HgrigIt2Zi|?9cTU< zAO}e3%n4%}RMUP6Ars`1qGR)Hx?LASSZe+D^LK0(@nh%XzP!BVoW5MI>ykM!;6$AR zSOUN*>E-46{rydc5~7XaI^5LN+%0;pbaX<}G0eP=foSskKmPS~tC!0{gyrSBzWzA6 z{?GsZ^wa;n<(xSsB&shjzx(lrAAWo-%fd+#3qMCU`iwK22pj|y7DrR4_i;WPa_kQW zmz?7UrO6!r22MqP+EF1~5=h8QIj0BA%yM_=y)&_cEz7cP7e4sg_n&hqEJCXNenv7d z6PA)6DZ$L$cPykh_Gx6pqh7Lki;S!(|IYyY&aM{Ija~fi8)DXOk)h!#;BzT z6Sw`izqyhv%Ox-6<%d^h%!zBsB7#hu5&+av+i~oV`x5X$%&$R90uZ1PfsW@)OQtF% zMbHT{%d)J5;27^@sOzOJm)`p5!yRfZQj)rpaLO6ng5e>A*^2#KRz%nX&x_LG=e_W$ zOKZIWz(i4rrbPr`sKS)XhBCxsUP6pxjR(`yS%5;^dyl9v5$URGgG2)NMM4%XwYVO@ zoKq?}-Dh+W09e;11f`^xA6}QsC3@h;h#Jg7C~nqUi`y#(4Fouj2NQEI6TMMAF=-^2l;!4nBvJEtw3g@}Zc!a%^u-p3H(CPEJRJddE*``L*|5^J|4Qc7v9 z4eP|TtQQ?eTFPa+etrEq`W<}4aYO_)=g0jXvTr1rfBN%ks;cQgv{KUkIEn0fU2VYS z%NMDI2wz@aVim!l7r`x5jEI=Tdv||^c7_-hFy*`-JByfG?>(h-|M+0$c-+l~fICyn zwX^vpTpTb{!7-K_CUWzfgg!4WA;dd&egzyRPwEaxC_Gzy-625f$Nj#&T;Vrk-g&l# zWkfAd%z6InSP>#DL}ZqfOOuwe4V`*Yo2Bh=>^q5wNGB#VCQg$2*_gSMTI*?U?Y?uW z=XvIw5HaN>LT(5Sl4?}qKK2n|9i=P|05eE~v!uu4W;PIEhR&hqM59Q9QP+YZwXT3D z%%dF$0uHS;aoG1q)c;UQ{M1VH)?>mwjuVl3YcR0XW%ORx3p1VPVLIZpAa`H{2Vj{& z$upxN7A1&;-pL(6c(>9NRl^gQpJ0aGxbd7o}D>b$KK=4S8B#EX0CV+U4nt760 zu^vPbGoyu043J6t@Mpyf5}DbQV3@07X}dLtfUUjgIET86CIFR&pUQFem23X{ciK&N>bl;2y;+ zZV;NLxN)9mX&P-XllxdMuchXpL8y*eYOM{M!5$J+Ynj|UN5){PMpXBI**P+%Z%BnLG^Ks7^CH8_r4N3S`@bjr-3M&{^jtTmT% z*|y_sfYe$;1Y!n<#3hi6-kYisr+D!gcw$Bj@K8+^z;!4mVd7wY&vq_Li5e*JCtMX| zLmc!o=4oyxb9sF7F*8aYeS*vZNPK>7pFAl(b-dvYL2w(R?YnrUl$@B5TklWHBZPY- zX716aH2|I(5&&2)7uC^Q8|~EI-4qOibp-^X8lvx1lDgI@xwc^=Y_R5sY9}Nc>S{G_ z{k$V0xE{x&mZgsq0Exqb3Z+zV=&i+!is&h&F&a7$VoK@thaVN@?4UYE?_T4_Vyz2D z7!6ATG&2u;-MzK5lro~2b@X^+n5wb5SxPBMB>+(CHe{%ZB=}AJs+szHx+m@*7n~58 z`8-cavsPv{R7?`Y16vE(+tU>`KeKo;6N?W$AA4RA1m*!cy)5u6=>vdBIi<(rPR~6l z=Zr*SP_A{q-`$h=qld&sV#y+-~WJ$JW?8-8Op^|NchLB`mqQ;UzSq4!PVr*F_dkJG1 zrtFNN5@VOf5@Y%K{_(r6bI$wExz2k%=X&4gxu5&K-$d(MCg-^>axpP6oi{Uu*q)BX z|GPO@PM@`A_?M@{*~ey9(6e)#Y&?or5J&$in3yD(%piuar}?nfz6@!z44K`h0JugF zrneYh`#iUqGla)gPvmU=ADc6Gkj>wzaBR(|yfQooSYH%!!>30<;aTU`K@cV&P$uX0nAsS<~>KZ&ayQa6kKWAdoc{qtBe?k@8@J1yV#XW{Goq$q@wJ zLrcHIv8Y=x#o@EC9*G=gEYjsk1~ow3qC-&{O9oW34+s@d5~L}TE$z3748~lv15YPi ztCycgDb6On!ebGAJX7e>F6KYya?fS@*Wv?gq+Vk3|{j{iVG7U64ZH%)eJ*fM~>27X!a!O{PQ$Ad4sr+g1eN$~(cnbeHN zM)N-pPlv9Nt`Hhk28Ll!hy>4=Ct_kf3y!Q> zFaZo$S+zBpnEK?IurfT%SE5t%^Q6lmwD*R*KqUJ;e_NZBED|{)STl)a9#}xTIygDyNXx~=MO|He+*49SBuENv4n;0R$cB*3?G0RgzQ1VZ zzsX&#AIUy2lQlNj6}jVCgG4$B_}KTU8G8U|!H`u7&w&0tQbuQOJ#Dsn9n{Og0d-xh3XE8ojB?*?yP)NK+b~fCF*Wy7kubV-XTCD@d;D=t`S|r)SxUQl<9u z%7G^R!BOfUDUV!xmP4UZS%mZcmOX$80)c~!;ZW|6GfKv2Cayx-$e{YU7%bG94R2gr zR%5DOSM^J8RhhYIg~^b4M!7CY1|)$m#j_-uf(I>sRv6~Ih&qnjZ-&05!;$s@kerK& z!s(Vw6}jC-2J*0!;7US{U}=mzC?lpUiwnUj$^Yz2lA0U89V7{!X_WMt1mXN$+jdpc z5Jl-R7a-%^xIR`9l5l)^Dc>b|LfP*7xfNsrgB0mndzF=LRV4|$%t_10DkjsRp{*X9ZtUg{R%|=}L^@f;~X=zzm zlBsW{W9IAVhq;*43%OW4gq^9@5)LOWwKlSY?cnZ@@sJX%A(J7Dnco)1%Ku9A3_pxT zK%*BydG$8e=sq%F7hQ$7@C_)4%%2v7!xo15Ed>(4b0Uke=U$7HUj1^kW4B48Z;g>a z@0F7qvINv_fvc5jVKQK*#zuMRgZsr;PY99&l`lt1Dn@oEk%Zr?=V{numl{#>yv6wU zAwh5fn=*3^-^#m#ja;W2I6&0;UvrKEVt%GN%!>bAzl0p4!$qmV4>x#qj; zqPn|hiqOZmPdJVqn2O)tI@2l%l$2OT${Rn&wuIY9zV>2~w->+swv5vtFy&ST+ykD; z@i~xiKarUw*t4dV30GSv2Lgc)`phL6$HH3`N#!46;0wr#)lC*9)D*MLPGeed>}BTI z{z-fdU?#zOp1S}d3f$Uddb7g_BQvWr40n)sZrM? zsUEZ;7339=5@QWfEjpWawQhf4@g}g7YMeTLxb~q8pu}$jv@I32sX%|ZI6q?&jTCjy zSG(n*E;a9AjLd04=QOM@WZ4*{$G6n@Z2$L-njQRxtb}K3)&5?uyfH|^==BH#aT}Sg z`0Bza=8mMvL(#WBZrZ#*j*r{$#@oRy$zk+`{asNL%?1=jLI>v{19@ls_xPZ;Di6Df z*{cy}LSPXk*DgG3GI`$MS(R?Yp9Yd7h9>ziSzo@H)KH_mXRn-^|D}7 z047{;3pUg|WSSHwd1Y}JmoyDFv}bW>k?WZJRe+1vI{cA*vB{sQZnpn%#Ik@#NFI%W z>HX(&nU~bG(pk&|Jy=|2%khp=WT^wSA=>fb!Au7pxj04sz?I{wv(@hLPJJ-D;KHlM ze^&80a|xV-F&z0^^_&4*#(Q>%F0E2uidb7Y6P_5XFY?piVgm> zgr;*GV2wIb&WR8g6S#W+4M)V|7wCU6zjK&)L9AYH3U35Ic}2>9Sp=%XJK}QzK04>3 z)4MOecQBq4C(_qvI@Zam2y*Aty!A~OsvWJS>edd8JR;(Mn?FI2*gTH@Cc`eI?&xq&Xv|&zG52x!-(+_E@{KgQ^=B6Zm z3kf5SqP#LwxMxYz%_2N(;TbO0!=pL3%ot)15H~jua0`sP&uMMoVSOb`oHmQuqfxTw z_ZPMf+dlf<40xRal&^>KeJ9$%V9HmRkpI1_JpQZ8@KMFI!EV%I0%i$$NxzqHx$$L4 z@xGDP*em`6jm&AJbScG&o&R#))Kn07*D%De@KD_+tcpo8cvM%f2hvzPMd|kw#^c?a zv;SV5)K;8JaFRf{Tk->WG|#-jwp|dY1o1F=iKpbVu%50aPPk2Fe{joX27P8Pc&bO( z$2c9WcDv+O83zcTzFY?`U)JsR*?snD9P^A5Qqp4RVUOfVSF?DAkG)a{YF=i27`A-P zZR3;ANP~$ixMycq%b1A-%3$$WEOG+AdTs@;;3)9Y-`%5c9c>-?YtLei8~#{ogI|>z z@#0o+g0Cp*!NbQF;Z6UNtD7+&_}|uZa+LS_44lVSm+kgIoKRduz|Pn8gScQCU84+8{~)r>a>FF}$T zR0aap25Q$ZAFyT9{JM1@FxbNJJ~Q-6?fpPwIUmHu9365Z8l5tWm5EPGnx+*-g>JM6 zi2(fcTz~Rw!aT*)+913Kqc>H4T*K14i4fLqVRu|&|JAy(uXjFOczr?FT0iZixYd9) z=uBG0zSzH;*8Cu2r2Pi%ID&B;aWX9Kf}zdT&D!1av8@$`9KGFQY?Um=@IX_6pRdx5 z+BfG$Q!?6~B-)g^G221~5)2xe@c1xiRM3n-eOj!QLMojX{KA9y__`{5XY(Y@UFc9@FtA7 z^lrMc1&`gPlep&W4T?N)Lo4&TsS8ieKp~df49Wz#Flx6EHa5R?>objcp}Qr%8*f{~ z`5BRu5Y}1DIlpof+c%Z>VMzzxgcyU)Jj6W)*Z^vgLiYpyhGR{6N-sFYtZD`&RRU_L z1hFs||RK6$+1U?Hp zCxg`BjN^j*+nTGTD&CX;Iq*Fk;2NWA4b>5a0SHcXzB@$qsto4?x*#c=ao0t^a`kBU zz&u8M;qsdWE*ATp?^c(^eGn;rht|5#EKlv-%N0raz^<(b(7%J_V)a^foo+eJY{1=E z1$nE!qnge8R)B>2n_dE!`O>UQvZitewv~@R7+)W=!kKCBku1W@9ZBF>05-TsLHTX830x-`hIN- zDx6C8@(@xpxHAEe;{XA}`X96>_RsG}ZUyhOOmY2Mpnt&QazCZNHBT(f4YSJ}&p-LL zwY_$uN4rPcSo}SP0}YSk*@2GQAhf%J5i_lolt`*y_R+Jl%pEi4#e0RnZ=(AH38Czy zHWSUd$hq=iQk_bAjY)KCB<5eMUwM-KbEaQ|B;boA<$>n-KVlM2XqWCe|P>YdFi(x_lPN zv=JkqR8TF7`9N!Kw*AK3@zayzLbZ0j>%pwg@*DfsuCx58=}4FPD139whQ<6{yjx>g zDHenFoPt=J(t#EZ;A`(vmOwsk{FFut`W-Yk@$F_yZu$OpksCLYRJi?`2@`v7qsonz za&s_l=7_G^L2^%n+M!neK5bWA)435-c1vq*LnZ+Pz_J;{1XFv2apvEg@*=I_*~Kqs zvx^gPaV7WFazIs2#+LE-ZJ;x4j*H616)2e>65znbKX}`w8C90u3$gw4%Qar}K6FkF{e`_$_Ume?Uvd6GJe~y>sPaT0`k%{>&;sJ|tlZ z^i*%Wcz@N`-Zt1HFa@p4008(=tAUK|+BToTEH+D#b1WQ`+7xa(zDE!q-2+Lk@8y@c zJak@4fLvGLQf_Hn33)wQZ1m6BMsT*GQ2On{FL+#Da#~*nti>8EL9DsGxK;_@(kci! z(OtK(+3}9gW$@pSU1l{Hml=+_Uk-5JeHv1@*bh%j0=W&UatvMgviSAR8(O4b#=P#} zYXfEpVh)ZApOa3XQnx#gP#cPjJ~{ZeTR49xwlAULoOVA`SInRFj;6IKJ|_vYRFs8A zTY2>1R$z}_$)ANdtH;EbN~)+9VM4*G>-f*2`(sVJJ+X%-k@lJFZb+e>W>`s)e zqm4)z9`xL(2BuV35?{Q+_XPbJCZ`b$hCx|jdo)IKB@1K|=Ty`(&QWU#v7}`vtscyX z!?6etQ+okw9tMEW4$e@2a~TFBMFzSFlwn>So=NJHd2?HlO7#CCsk~COu%bR9Z8wtgO!2TBHDaoQoA4Ogv zuGWdF>Gv}R6dwgp6~2$6`}qw$8^$451Jf${f#CN^G5k=--S{U>t93_~;f(%P4%v`j z7lJjUsFA#X%0Ap$Ka#4vqG_3EYK$*y@hTePzbOC#+FDsvhseZ{Y%?oJM)t;wHTvz5 zN0j5kVPbfV7c_vywWbkBcRPXA8mC~c_IkzeuQ+PJW95BgXG|%Fm+eU6M z^A#H#zr2d7^iG*&P?l)z9eLWLUDHwdj37DB6sPIH*qPq(>7qCAWu%eEc<3p%pf)-q zhuWgxpPIu8u<6_5zn}3w%X#Z{?O$-)Ci#W6L9P|O8T56AW|vor7A!eow>MNHFr<55 z^(Kbx#FXCV0jgEJ<)T0=Kk+r<6nI`SEmfGA7^dw8Oyy_f)T>&^t_Xm@$Ckndfoj-k!qer+s? z=Q9a&~Oah~>J2BKXle7)gFO;0}K<#g7~{^meME$EM7!y&l!FLx*^$sE7wbrY^$jwVi(*d86q%Z4v4t~tEv3KLI`gE*&88J4g;Rg^p>E|XkS|W$=B0?*8P?TW z591Uyft<3sj8`45-uL>NpNup>txq_rNe{U6nh$6ArKMivA{XjAjJfyN4Z9}yh{qA( zdL57rFWqy+?dfc4Y#C;6_vp6lR}cl3);fjVkpO1I9z)Jf>IWYeytH4sVMHaKU*;Xu z5F;gi#_L8{iW(CNrAEi(Rw#L|X2{sdrCqqfh&7KLh#E?C9@GV{+Gc&)ynPDA0nS49 zo<%vvi}Zcgw+3C0LeRI?@&IrFz>1x6HU3UkCFqOon_2fHCiSxaU@&Op-QA-KC2P*U z(5SIHG5xuB_uLJu(pKuPo1E|UAc=p!k`|McAqq&smO>z?JTH%w92MHAg1&Ula^%q@?;oS0qb_ZU2HQ%y*%{iwc_?8*rpUtw2H2Y<*#?^jHG(eqVXBY| z5CSTId#2AiX!-g8=<>;ku?kt6F47#WT|LdUMgDMGM4c8Mh#A^V8?U;kM z#rZup8d557xfpI+T*Y#y zn%-luz{~U#N9Ql-$Ak-uPXaQ`+Q+W4d*R)p{N|C&xxeSN3=x)nc3 zg5+5$@Mh6YzS2_ctecn*s>#aho+u`XFp+*l`b}hKBA8m-mm&t&P@Ey%us6KVV;tXf z^M~|%&;h4G_3vSB=YfaMr5To*QKk=AORoH9Ss7*9Zs8#qTgqz-wdwPI&cUW`MP8Lc z+hl6k{3bQR#TX0BoS zL&K#39ca0kDw_^2*+8H|+JYPRU^VIGF5M;cxlsZxJ_#-W?tW-|ZE^Q*))>h#syMJW zQvXpx_f@ItGq-U|4n64AW=Ld`2;Q+1D+{EmS|s9*dvKzI)^Adpy?_b@WQ)!vR6HWGU!x ztqUH_-ZhqR%Lg#$8R~$5Ox|5bg{P(C15!l^(Q9yf@&*=C?gEmR4E4 z*I=R82&_~sonXjcp&(5HrzXBotMB4*MOzq@;*GJY0U#{G=EN}7KYFz+1+pK^_2zrZf|JpvcIbPL4fH`gUyO2F!ucc=PhnIoWOR&;v?e6h z2Gn7B;DL|B%cj0=q^?Yj<%{4}2Fo4g)f_Spdq1UX7?YKPWwdXGxfJJgZTO zC^iyiOYij(tb031l_rWJO9^{K>Bpp*853@eH(ivIzZ4g|E!$jsgtFQ@0}tvrgGR0{ zya$m{oCw}uua4f|4%8RN(Dn$`9VY=TRKLCOmP-ATLr(cyN*wq<5qfhSk#imT?a@b1 zB83i@KaeQ{>vUCpiy-&%{*fM%|>? z@$vadu-zE{H0BMdj1^yMl+*{RL$0uBrxKeVGC@hCu=pw9$6kpuB^w0%^xkD_umBi! z{##JIkpYz|#YD#{IZ=2*Z`DJxH9)dvL44Tl+>2cqsK2orxIt9FkVd!%OShJNd_q+VeUqu``@eq|W6ia=go8pih)sovUL%{<{UtTcy?T5kDl)&asb2u$A)OqxK|5j09T%cYf3NSr2cr*m zvQH(^*7ljf%HGDSObem7@$B=_XqCpOX2yUIfemDheg6R%p|^K*beHXPy=@fgw;gVW z_RM35;FBBehg%_Y)4p_%mwH3Qkr-s$#WWk;bMLKfexhD-Hw16D*E<`Z&CxC~?E^8B zE-r3QlWbkgn(^4Yn@19dh9KhdwJmW9kA5Z$S&<6zm=+XDtZ`!AGHF@I~7MAT0PUAs9hD1b+4F)*-*rAhs9rG6IbA4cw{!?**`z zjzk<$vi((6_vVfdCdknT-4lZZI2D=+q=iPsluR*RjqLL%#9p%)AJvk}8|t0kI{eGn zj6NPHtZdLO=lt%*F9~cb{2s1<+eK5LZO$KqdbeF+2$AH{4z0&FRd02alt9kk_t>5F zVu3@l8lb5jA92d(wA!evl1yLF-womPY>Hfz(n|9~fS1}J?Z#PNZ9x~r!ho&LHcrr< zML^OVHeG8;AkfqD2(ar)* z|L9<#y5snOroTU2(td*$^LzfB-GQu0N0JY6$U!^uh9oF3T33~B(Vfj6<#;Zz?9xaA zz9J~4QCYRnh3lDc`dJaA@nTf#ZG-EP@Q^~C4bk92_e}6bd(wDT=#Lp8L4?c?REtS| zAolFW^th+S_cGa^H;)rWM=hX_Q;+Z6)#c;yS40;pTR1OTz6-SjgI@~cn=8s&%1cXA zIq2gN*hf-e6Nu*+nB7f+dO(=!%g7iTZCVLecYqj)M*)l& z_e46A=@r^VH_8mhT4g@BSt&#z2)7IMz$+^=-5|6N^TG0miJf+3iqq$y=u`jzE48@& zpY~2|bHr4+PbaKg-mbYI)O$?WoxP{zP|SU3Z+Niezv48ymf62yoI2<&ROE)(N9`*H zaRarfT3)$le>AHAd;3e3zG&Zyh+iYE>4xAe2M0Mnsm6#BM^J1fsqa`SEu*A&f_Ww4 zL9k3y^(z5P1FeI<`j$PJ{btD{k*LZVe1qrR@iLwqyr%y^U<;KNe31nLiOO-nhI+;$ ze6B?+LtIrBukh`8I+x=MiAx_!)fG)fg|&^LZ<7E5^ONGI3T014D7HzdJn=_TvpPt! zw&Ie6hLMDO~vPK+UXftX3C_M zNpT_4LBlGV#~^7bw{5jlK4=1oW={a6=+^z`#*^J=h(^QN-m6G*KVD!humQ`GlAZbuo^e}*z9dS)U2S( zZsg{Y>fQul>t!gc#u1Ds@~H6%lvsFnA*7RR8Vom zpk#L(>FtgjFBIm9z`a@D0!3^9BU=EYcO-S+k&H`#PtBh*1Is`*gWBw`#>jfNBWhCV z)?cGGi2F-5LUAM!$ph06Vby9N^`hdJ={;ipwx)*tWg=f%Fa>652{hh}jLXJJo1<)ZjJ7UM%BAE(~QJ}(V_e}ShYOKO%#sV7*4cA$o* zuU6Z;EJmIeNB&^bJ$0nyKx5K8Jk@=k->Z7(&`UD^yv!m1u+t=nxDKGT(`11bzEgHp zmn173XD@=stYGgiV6)C@hklu2JfXO8hm0hyIs3wkyPmFLP7W_Ku*6U++E7tMe42n- zsvVbn)olDyq@_ij3eBiwMr$7EK9obU%zxlP`uQ!`x%QWq81}<*v5+UX0N)b-cf9a- z@K>7c$C<}sSt2e|;|x@|ccVv2KWu5(J{FmxP-)`H>sASQZyxzh9zoeP|Iyesr28Ky zdOVZts#daJU$NR;F?C()udWt*P~4NU3M?e&wKqb@_uh{vG3jpppvgBRQvf0>xYv6>?5#$j;HPH8v`lLe?TmB{wP58{bJNrRY=Zzd)z130`>`#nl{5xLH{~fPT4jpR z51!Mt;%n=lP)7{^P3X6)P>|z8^oYpF8{&-S)4307E&JX-?KZWl-27l0zeb+NbR3~V zLlOR(mQd>k<@KGB4=ZwbM*$JTp}vgeFcr!o1z5E)-La{b`aPuAGbXQ5Tu&#t+$&pr3WW=g|z=Njb3bbyB zG%>BzPe2~+oAT84s1?k$AFMZ4tpgc{O5r4j6b70ns*C0U|{lTn$7Jxn#{=vDq!# zm!cE>1_rdkU%FvtZY#96w>D#wQlnB%`_mgir|4#JrjOfpkiR*}qo_2W*i@B=Pg*wN z0^kMREuRGh`lXFrJW=ZoS=V85Qz~G@PCb*8N0)sr%}Jbke3|VEypZq?f_p0V0bhnM z%cZ~yS^vzPZt_BZjKBXT8yc^1vY+kJ=o00l8uE$kaZsxiU=KhcGRDSt_jlS)0dZSk z7=W1>)}Y}$P1&4l4K-#RIOQZK=fq)#;Eu{KuK{YeMwyaH*le_@kQp9yrzr%UEK28M zZx96Ftw@V0U$&to5vWD@&kZbHu=Oy{pQ0%noDCgb7WL(5>sTfarELOyc7tR^s%!^6vo~hWA+%sRDD>% zmx#N5%4V!mC==O~wh-gsZ1xiTP>rzVj2y2v>0)BVl^}TK8tB>)XTyRC=R~ODJHlF- z8$4oTbGpeV@AoJ4PGhCc$Z4gUqd78jyxHTpwR*`qlSO^_PMr2l3mnkezRVONN)AH! zZ|I%+6q=L8CNUW@5<()}0<6ZK|QUmK2V{&h(y0SCoWcbWfT z#{o8b(-#|@?*+dxI~%@fm#Mw`zR=5e12m`Z8;=zu zs?8FK>xLG-mABd6)+V>M&ahb%mgbLxB2!a9b6Gp>o(l{H0bg*0o&V0bt|1qJsuv%bk9wjrlFerUxHSGFOk``RHy;X&OGV6M|dQ0x3 zd-~E6!xZ&O!8jHn0Q!Nm4&T-`W&K2Mg&HR9ka4w!tg_(&LE0DUjnu{5+jeF(jCp&R zn-;x#z5y{rSr0*YEF&*C1is@RQ}uAnv_hPv4ZSe_?vP*Z6j#**(8mAZcW!R{!5vH z9DA=a(1BA5R`DXEIl>$~av8}SHy zkz$-$UN3(p>FOo-q28rQPp+!p8FA;5=egcO7_C%45N{1cdNuOPXl|-DwsxkElw!bp zXwH92f_AaA)ddcmbjpYMxF0Q?GXsZ7zYQI#_C7M)zK{-9egz=Dg-5< zKZ(2<9Tj!_uoggne8#;6_2E1!EIRs<@9wEgS1_+7t{*K%e$nl%9XIsZb=Q>iO#&^A zy@ZD6Rw7bqPQhUvHws#mXP_h1Pf2s%CY|d+utV|6=MbNKH8hTH?&xvJ^ z?+7A(M}D%PT*HG03tb9Hhqn#&{sb;sBaF*l(rxxHT9(T56W?AB2ID)+`)!g*X% z0Lb5Z#E-7zYFgtuqrL5D;Vz-Xd2X}iirkm+z&i1{j+1}$7wT%}#{POAdZ=yuiNAf8 zgXaRDU$MS|rW4fW&Af!$o`l;4O=lb)?<>~HyFB0^k?owpbB;&s)n_XLT0H~a~u03SK8-pVhPuCkCjQ#5ta{edsYh#Mf;nsMX^Npv(% zZ5$8W^R#fBshhk2e}n9NonEP_a!qKd&3E|;Dy4e!G#=uKoaM~#T?%2HLOeRXj_x+7 zO}O-F6W3LIhqfXbE1!WCGHmx6+o+8URL7h(AYh|?TsM`75pVBl*Zk8wcR<^xsuhy; z!#AZ7VRC@}6z$s!_eAG3N<1%T3>Q^u>>b&!S;nG&)(u6-+IufJ*rR%3nGO9`1%!3` zP`7l%xyXH!CdW72MpF3Jd+Y$C8E%7-^G}~1?Gl^jX2F}J$1l%`bUkq9%hEC9>;f)N zG3<5io?`-%e9|)o&PKJgom;&~voJn?%1~aLA|cz}JNZ_QcnW=!1!l%u1LhZp`b8!; z!;^V=svFp?1~^=+P}ilIeVs-MCGDT)1AxV)v&!EtK9RKo&Ji|3%^au> z(B4Da9j@}J`+f-=@t2AcO+zt*4nW>7UD)`*TqXhA+ug!LazD9xK0Va?iEHCOZ(i{d zoRXL`NrgFGE;V`yBMYOq8+K6`QrWKl?+Tn``vF*4q*k`Ac+_}(RTd9y zwR;{^wh(h(nLR{6nw@!B?k`HDqCY;U4DYHH!A4T5c+!)sEhe4p6J3W*wax$!cekR% zE;wfpv`(#S@5i-%hLL>9!=z@=_hA^3pdVuy(pQ{ z$7L_;`rTB{Ekx>(H7GvgoTRG!e0CorA}aCGoKrmCP1x6GrVlGgOv1~)PQ?%J5>1}B zc(0m%d}Q?`N=E2+XaVC0l?jrx{RI6Hp+^p#ot^Xbm7m=C`-4S>rIZk{&VTI%hX$3D zRUfi*-3JgDZ{lA)P|5SXOdC0!qR{%B&WC7~Xz7UQk%+7_5GX8cr033mVJyJFG8p$o znS>9@;v4UI9l%-lej=s_w<6k}!TFbiM6m&`n+Cpj;pgltV%L1)0id=h$03HIWZ zZi2Gg)S4^Kuw$1)bqh6|y~~9}@3}4(d>Vx=?3yM!iV6CX+{RWo6`rgDH?e%umCDXC zoHd?7BqI4VCt_wpPz!NP0J644gLQ8J5H5fvEm=X|1}! zMqsLG`#XQsoPF=l&dm-ug;d^!nrK#8k3o!!lnD!sDLEr;9GQ??&Nh-uB0-g&BSA*g zVF%r3i?uf;*xbNyYYf0K2bdz#8L=e->x@?{aEMR7X*YJu*{W{9cq?jxO&Yz_J~wd@ z%NbvqtBSKH$n%jy*J@q5z7TseF0|utCHwBIlbNtee(mt5vR`IA z0Cv^*h}5b2hS_*F{!GVvSK}WHn0iyCmKTv)goT5na1LQxR)-sSw4 z2oKghGo$to&hZqShUlL9grd4*7%&^t;yGC>^-<}z^SM@DLur7GB_KU%$w`g_PbE2) zcCz;W`$7^F#F~ltfw=mCjjJXYQ+TD&;ebM)&3r;$OPL=4ka_u8+13bhlTYAOOJ0fUXqaXBVpYnS6KB{@yO<@&pB`0!`6v*SLi?hKH0~z`nPtF#m9s*eb8Y(z=u;9 z^GU7l#hjl%ssE0un2USD-IbStFJOZ#X!VanTZdr}{FRm>^;6|?prfFkXFgj&FMKC{ z5A)gB8t1IxalYJ=$Lo6?RRIz|7kzV8^JtQ7_vpSazs(lyHH(S>{y6Qgc$t}_EP-Z! z>S3U^$-|I0F=YLoecHztwC#@z@B2TMK9+q9-E={jJVrD@x;lO5D*{VqcWZ9# z3H)qZM}lv_2*36fGh!0s!-|+iIiB;>c}z*~CDCdJWI0fkxaZg$L`Es$e0r}(n#psh z6JnG8>vg<8g|W2*P26@2iagq%-#zwQqy_K>P5G+m%nni|fS@1s!u>DfmF-!)wR-T( zjyfBR%Vq#=qEk?RmA4+*t6*S;xb>-=#}>W%2L=!f-a7!|bQVon(r0G%((dyXQSK~n z_(F|i-e@5Z4R^|OMJ&52$IdnLj^vd7={V?jAh4m)QXuy0GnP40H##{(b@d0w$%WCP zDro+us^)hw=9h}DafW;O%e_@Q(@C28TJ zFGNX}iC=`a|82Ys*H*=$;QpvH?*Wi1YgRbl#SWA-7HOAyn3DO^Dm?u}ru)ATSs7VE zH}`sYyxQ{Wo+NPh%2-=Fp+ylj!kzlDR7>I1)PQ6NPuke%JoyCFidJXa(;X-A< zwluh<(#6$*^I==Hr-nAg@mf}%8Q)d2bYejJ-0bV6erL2&*AKU4=kn3KJ4jY@kc(k@ zqEKkU0QrX!H`iz27DoJJz#V(z#;rQ9PNlVp!nuQW;R_+E8a6Ujn(Hxy#=)&p0rB+v zw%!1hE|Gxz!pIhrw>Y21$ZDP!tvMZ62^nwrf~pfT;~Wv0=sr=8g@f8h--RHz)&_MXS(-9UcH*PR;tIb1qHsmNLpLZND`f_-}oLJ@P|$jyyXI zJ+2px$&PQ9k259eGlZ;>>EROJGP0||us-o5m4D#<#NX+}LF zlxmr+GCXXChVuLud-b`Ulf={n(6#C-c}@34ZI#BUh-voaEW^$y zo_;FjR~mz54?4Z1Dg#7HfS|QLoKdQ}0?(+@+IFa;QXpo$@e!dZr|RiJFNX^bSRwK1 zr6=EXX@`GB>)}UJ0p)Y~`%H0YY@R`aC&_(^hmup@DPnm~wA zeD0%1R}vD;-NywVM4r3N3rFci>JFhO97R@0{Nv!b1m9I4uOKm?9z}n&PB>K`8%rv8 z=E>sGQ`eeC&!?I~c6xaMq{ul2UCsQutM=uf$@d|EA`@kwJ5e2$r4OKS-h4FmJh?ASCI|V3$QpO)oCP?jhs4W?_HxNi`D;#p+oYsC z(=nS5Sk!KR(tgIQli5Tk1IR5s@O&zwtTL9{b?AzOjg19K<67oxMOVRhGltx^`uS(; zH*G2}Bc`NQ3Wf=c>n+`M+60{}-e`&N^WQVuI^No%Z%LHQ9(WU@TGWC_X(#h1haKa> z*Cw>c9@F}IkLMigS@`BN^Gz*Bkp2e|QE!OzPmd#XJ5-}ttBcHeNER&@Jl-*fzSPxN zLS#1CFPTgZ&>Z3~eY&dEYbghUnddh4|L!Br|6?(204o3z!WYg83{L!*VY*&pspXXBSK~O=BB91POIshMY|}<+^pWoKfZLeNHN;+Pg*{SWM=eQq?-(1I5pnGQtTp|lSwT9Q!wQ@*a>r2Z!`)#;5^&=<08xS-!@|LM~o@{AV z*q7$|hs_#JK@sdmNiUJZc;6+8oVA>{AVh!gDsxo0z9J8zu%3<}j=&`u>6&UHi`t1Xgi+CMKZ zLu9UiF4ZxWY6)d#iLll-b=oKBk1Yr;TZOo?0n(*1-=zM(7vSF1cC~;6xZ1#jkbVCZ z9Nti4-I?oSC$Pv^zw21;lBM!GfRocxz1Ga#4fKBi$Ury0jP!c=QbY+5;tX)=JjOa! ztPqH=74BwMbzp%sUx_4<$bh8QYg>XPU-_62mT=qpO-M2j9_a`tqOG@grXHTmO{HJ* z4G@uJCXom=0>9QCWWsDDs&V!Nh39x2--DO)3Ne?J8o`zn#I$XjZ~;AFW^JwCZn`yX zDop#fQBspu`vsMy{J;O(zx%Ss&z~_DMX*TQx4vyuqa7ej7FGz_kMIBW{g3}Rp8uHZ zE4lgPYteeb<{nEUD6%iYldx*rwtxKo*Z1Fk6XrNh ze>}(Mr;k$sq%o&3BJAgnfBgCVzsC9b9G?#Nbi52^Mbi>-Mks4Fr$z=*b-$BH1Hddr zuoPCQx+O5zjEr&<5}3gr27tJ9^RKIYeRCm7RUyi{qD07WLJF4^02$u2@wHw%Qz92~ zby1|b4GLClk{gF|2!jDI(Z~D8``h~!`~@Up%)?^^dAoONl2490 z;hD)0T-Td^{mH7WuHk&yoa1m`mqsQs6X*G5cHm{Tsfv#pD1dxD7Y;GVf-LTcT%a8D z%nWm{ES*wYFwuED%T`hHHvl3k|M&m%Z_IML?Z5y2`#R4x#&J9&ucUMlHeYV@cz*ry z_y2f)eLv5i%a0W!(unyS&!T0DKB8-R^=dU_(FpgjZNIHGvtp1$5+IUDH#ZR}MQI5( zT5Cm%(Uq-La2OSno7YX9$P^YTdiS-LqEOGU@O8-wGTg_d3`VKDiP2#SQ<5)iqUh;J-c<6%P**(TD&Ot1g@FK}7Ds%p{VLcvTBzWEFUd z2eu7({Q7(xpZ9O?x3`-#9pn7*;~$T&f1Kw}%L%DPfrz+o`z6E#s_Qe; z>jVMd;Q=gn^P;_zMTEsFhul$7p4CT)gvDS_Wi1q}VoPqfk8?b5;l0-aK*U!PW${Je zxt_|}xS2n-wPnlGiAZ&;m~it+V5GnqPMkfe`ho@QH3tLetrs_b$vB9#i#8_mLA1iEF9S1}%aWh3WURQ3p>(>GcS%h9_O@0LwK&IljGwptO+?*> zN{?KbP7t9MxvKZRHDwW>bq2OF3x~%V0~c-(kx6lt{n<46@BjWkDjz37ec${2Mq1Ga z{Pp?c^XHHAczk~T`274n=VSTdSamW1C@BFEGEo`1S4#{)#7U*oAQ6r9>cU!U^xh-G zY%&+motgge{$XpB&_!6P9!R7U@v?;qD!5?5iyos33K!LDhY|2?yOq_d5<#*`5z>h{ zVhZ2Wh@w$63Kib0qDo%+q&ll=BVdv$m645fXOaY!1f7!Dcg z&iVZM8plbH_m6L8et&zf8gBQHfBX0USrUI1X-@7M14WC8=<>-IUl&b#ku59|S2i{SQ~;BRFyWutIjYh8`Z?x0g&}ecXj{LTTfOy{ z&DOeydm0fFQi$56D6Nw^Dl|DN-q!81WTq!zK0Q{Z157~(;piOdmcm;^O6zM5t}e8z z7MoDoSQTn$wQ>j12wT4wLz0)aL^54jGn1H;5LdxML1|fc?q%^L zN?c7f?p`S!?i0ueBVlGH=HBlB%tyV&ieiqS3~~!5Sz{ev4_|K6!gGvK5=YfG&H+IF z$N&6)GyiF6i3F*LD43+RZ-4nqQhq%@=lYudG)xkTG`A}?rlzo}?knyof!6mihp?85 zfG_Yw38fkmCVeL1s@G&@V)vN|^L4p`MLbsN;uxfmS2a4(%SPJ!j)byiIFq~# z8wAk4y{@>A~1 z(!pLgVrg4TLKA7rr?&m_kJK50RO!XiinPoSZC+!%XsHv;&qA5#WuUw^8KR~=vI03d z4Pl5CU*CV8U(d?>HnWKIh&ksR!`+zm;?J2Y)EOX4Lr6B+NqCOKkx6bjj`^jVmZphC zk!e0v>p+HxF1yhGYSgR4ch1p!4|f)wX31bt5qU+IM0k)CG{(Jc?u+A6xggWi*ECQz zZ2}4RWKPQ3l)7}FfZJ@X$65hKNzIZeeVxznhP+ZcD5?D5>3#w4LohrG1BW! z;zW9y3OiN}z5o%pQ2n*6_igJpTca=`WP%uARIEDCbT8plO{pG6#EHV36&c|r^QAE# zqT2-wBF$z2TupXdhkRjaQsn(*enNnR5Q#jG@%-_ek9jp!A?IA$UrCxsZCxsBtnN8C z4~q)P2*3KoYTREo0^YWkh(HyI*6rDtb=jIW17KEj2YA^OmI`LnrbJ}6B9d7OT{m}P zs)*s>igA>pag5wy3r#yf+B z6yfQP$TbdRvM>V#v{p3;bi3Vq-->fq!NhA#fD8WxU^WX- zD(<^3wXIom#svz|#V8B+Iv?@U^Dmpl?bbQ3(uFj_uW+pNlIE>-E*pjJ5oeMoxySIB zV;*abxrRry-bw^*YmMV%V4Wwo@+JYbh7w4Kx7I2`Z=7e^avMmeORUd%$yH16WUcL$ zsSnV7|3F!`SjQZhmtI}95%HKaiI>~3>EY%znsGTN!o${F^UO?PtuJY7=4HT3GmA`? zJxP~)Br5aIe9d*vIakq{#Kc-kQDmmOBvF8Z5uV}1D9K`_@75e>ZgVcG_vEDzhc-zH z5slXyV$8$CB0PK*ZT4%5EX2sxh%h8k@~@DJT2qnmm~Nwo69}Hb^e&GW$PS|b9~iI&ddo|9m`;fq$H(AK~bTlqbexBpKNI-uC-#+xOO5 zS@#gWZ!fJM(ev?zDCatJ0#_Z#D-Nu#$C_i!r`zbNW;5I>#jS4lCfj{`d(Ec>UtZ}) z;f7b^LE%T$WVyV23#gb&;U=utrUE7ikuVg=OiD>^hQLL?fwgs1m|V5rWEne@NFqFe zBr27Khn0;k0&6Ur7FJrlW@dF}dR=MCS511J=M$Migme}VB@tE8p2tL{T>^Nzm?6>^ zj7rFonYY4lGowrbxWMLX9Sirz;}hWC?krlfEOV<}rVa@LZQBhX50A72M0%zd6lXSj zzb6ndaIz3MJcfw$zU}+1Yv22=sWuT|gsuCne|x{93SFjIU9Y&5#t0+>?zYyMVIs65 zY9MhpW+@>tD48XepXcMY-2rklZXKCh+beuXL@X9EJlt}TR;#JFo4G4XcqDLDlpy8G zDb5<$v*f*T9jarF2!}}7?(>DgTn^R&tj zXB%O@7CcngY!V~Xz*JF?HPtQ~BCVvGnNaDywd43n7mz{~6jQP1B=idmXVT0>IFa1; zZwX9WfmChv$P&S93ak5I6*GGtKR(|6#lz}%Ha8C=LepJ!tIp7UzpY`y%`CyW?K`DB z{D;L#4@S7<)>_Wd2&$y@w24YAyM8-iN-~>nX6sDyfL)df60R^L)(AH*?-Re>-XgBk zUWollaLokV7BVdsG3P&Hi>Vf^>K=@e2@nB67gbc1S4c``aTHQkoGvjlC|SgP5@}w2 zL#RZ5Ch@iKQmWf>B3{#l6xXVjk=gr}k#jt?Z6Hn%0(B=xIB`ar`4rKxB)+H-WD=N3 zBbIOr_rqqj`+8J+O2;d0*v+!cQR%g|MW&z(f4qDYv{Hjexg^8=98W?vZQJ%{Ze?LH zpQ@@*y6ToDkOZFVAaJIM_TX}RYcBsB-=AOKZ*Sj(K)xs-$c%OA#jYyP$O1uYUT|M# zGkirFkhSoE_&lFm--)>QTPAF+tc(p@4{kkb7qt%%QIj4>;Y-U=y7?@)MELpqy6tyV zkyR#FLu0~xi3&3JzDJgssKDWs;Xscldu$@Zd?G`ng&Q+h1~y1C-OQMKW;L2!qZczj zpPyUb+*cqyLZvge^Z-1tK+$?9VNc75s+1vsSz(%?IYKnjnX^A7+ zzL6)~rPke;OsT5MAv?V|s7xjhkyQsYlew8qv*9t#*HwX$LEzpuciZ~iBQ865_;DP! z{q5J4j&K$!;rA8MktmS4u&1lYmvYY8w>>DsE2Ow!^O`dR3AgD1X0CdY*0(YVCqhM4 z+3h5dvSf1MYyJu=RO#*hZHzI;AaF*GaT2qzL@aIP#7Aa`i2H&Ed4x}H6{gk${<>@B zS}C)#>2=OCk$t<(d5VazErW#cr+j`{s zRqzP_>46ZgCTCPi8xs+Ua0_#oDb4 z)^dc;bwsR$3ljhf3lC;uVh9pgHfr($(SCcohLi}AmprNbLd;{H6+uy3EaKHI4{Uw6 z+CvI6a!dEd^1`8rxoK;qV5$GTqHQBw`<}km99!Q-`dlMD7#i@1Nh#BwK7*+#YbIHy z_I}&np3g6;l*~kgFPmT@aQNWJEUFip2#e|I+Bz;gkec4eY+KYLL?rm9&V4R+O~e5>k9}bWfnqYE=FSrkbHUD z7{U-PE$aLG8`ACVCSt;NfBV?BCc>?2?PD*oMyV68+0xcLBbM9vb&xjgrCJFO4`Wd& zmYP_#5sR=8Qne9ZVl-i?F_@{+!7?+B^J~nLi0U7zmm*zl(XBklh0v)sBJx<*%$wnE ziOP79dX4qI5#Y8m!pnUXaZS$QWjof&W(#BDudhGrW8m7UFvlQ}+3;9N5wMauEE`pM zlpv9|ZEwG(@zP*{uxym$$J{Cuu(cbJW=6R+!<@jk{XHU!7!_?Lj6h=F?&Vs=FH+G> zL{*w_%gi~?y7LRWW!Bn;l$GMwdy#|%3BEduQ-$B&-#$Lx-ro1yt@SRwE0;7PU;KVy zpqH6FeXUVNF)9K?g$Gu$id3xuw_jq1=ku%58AQZm72fM!i#wV$m3jm~MEiCxEbNNu z0X5NZALrBEnl>V;VAM!2Zs20UrIp&e{2Z-q)eKVzGZQ!6Tf1dRTpd1Ja@ z33q!wJ_%RiH32G85n+67D_5uqKuxy_cvRV$^m4mE5>)>hz!!oGUbZ6h`ezBuLYI92 zs;o-4+ilzT_xF#I(P#-!-Cb3#<5zguYk!QzLd=R}pJ%v5CXo>9T!UDMxiY`{wv~L0 z`SMbzsM|6#)mFhgo?gj`7XVxwA99Rw=eND@wdn##3T>LO090DFkrzu;qi>`!Rr}`^ zK-t^w990Iu1oNff3t|hj8kjRtnm7_;WCjZp=ytoiFA%h~aO?b@M1W+Hdn!}$Hy)vF z6S-7-B4Rc&L%F|xac+9EF!SWs^O{#~#V-sFW(!1_4$5;puKPf@!Z*tuXYM8It?d^` zB4yU_S#Or*W^O7yknWLk?LEYFo(E+xqBUhnraX_Y5`*Xk6`;_5&MJX?ZT=pXaP#R| zniGVrWoO^+S^cssQGvIB#nKlBbb&2It>C~)@vJad{3_dU0JrUajdhpNS6;d;LPDjL zg@A~NDbts&QiLf>`0`kdH6t=$%Rr6MlvMQw&+x^>t<^c{?xT`#$_P=`Ln4fEun;q4 zmeaEZWrD^$xBab(HVA}^jAwwxJofDl1d}vrW(!QXSYSd)>Cz)iSykJw^Rsl>nNi!K zsuF6|yjtr6QM1KuzmIt&OhqdsWUeVJOj>{#Ny`cb6?3x;A%-JjEkX1~7J>9}JbJ&~ zw%Z&-MK~QkB%#D^=*@7^G1py2JvCZR?SyS68`ZMr79KAY5Z#VQ6*tfl{;u z(!y#^NaR>Y=^D;?)cxzeN^qX(z4a<-FRO5-Q|mw^!hPQMk81`-cIiw)K?x?=&4PKW z?&d^f_xjlZM9d7st9_w}D>Ez5ib#nivbt@l6vd^g;|ky^kDGfPulM~U!i1ZOMtFe+ zOu{VY4k{Dt#e6TH#r#H8L|;9xBv(|Usq}wd@9^GlbB$vh!c0(iFF9LgHrW9HwJ*`D zc3O&Yne(i|EChtv5$TiQoH3qXVM|}ZELBiYA|-;=hAPB-yS=S=S}qWf&MHhAzOEP> zATx_46ygXoTetmfuCwIE$ML+~ZWe3BimYlcFfprU#`4jm*GjG2K*)3zc_oOHCAelo zW@8))<}VWIK$hG?rBU!p>cUA;q6Xpq zQluv_!_oWZ;icnvJb!GxG0WcGP!D7AwX7aapk<4a8#hbyFrij_{y3juL&XTW&QFRE z?g}Q9NdRw+49b}O%Bq}=AxG287h{>(*b*}`HN%sAg1(QjQ z>|P^%Y21jB=?Sg`w20N(hV;l_BJ+VvQI$HeIfOOcYj;;xfp|KTs_b3gGKe5HotfPH zw!fFa{9II-rAo^w(`}0ISf8NI zsz?^?NO)MLGq<7;ky*gC`2tZv8PeLAPgbGJl@g9IXHP`n z;$Z<0xQeElXat0qQ}*p+)3>gB7{)mTN?BD+#o0<5|( zA6wreR=w9nnuoV-t2s1$i3pc?i^biT1aM`Iw2b=A<+AJrXV93_&A;6~=JRx06lUjh zZ}4#lWx&JMvY}(Gvgj2Vx6DM?z545`HbRR0*4E7na5aX27#U5sNc>`LvtEi3EUF?N zZtkpFgIZB@Ou~s|ClVrx*Dff+1if$oq=zq`EZo|jVf0G=U=|@&>6A>=MEAe_>;L|K z`}X$Z195+^G$bNW(|W1YR-D|AJhCh&Ot zx$Qf%l;kwi3r4RQ6GY05aHU~7yeJ%DzVC1M+sAPp<-(>{CX7(?D<+B~Gkj*{Q^X=l zVHV$LD#HV@Taij(?LtKkfm2s}Q2_;J#j!lDF9QfcWUV+o1g{-PohfSIve zyCA*R@V4zlQfPCe6RT)9k3@PpmvpPVIW^v2>MT)~0G5{nsxZ3{(Yekhok2iISwxuO ztKK)^s}L)?Qu&jFlWF->De-wQv9LDX_I5*XZ{M1}-QWKD@$uW&&nJ=9M*elvT5c4@7#9*P`FB4j+zOK~jnnQT2 z^E~`&NfDBO5wZKoh~xP3uoCUIwgKe65k+x0yZGDCI zy$e}-2FkRU3A05>D*akLGV$_}obzN=6?M0_+x>a`EbTPC_Dhu1J{^Ub<090rlG}ny z)9A`fP4iK6sc^4*AC4~ft$+LdU;ej`Z+~m8OWTNBF_e7H|$t~Aq&>n-xlr(6jE>1HTz0A0g~r~uBSplG@i%70a* z94zQv-@kqQ?YH0l+qb{`ZgZ+`s$=|!SF6g+8_)e*_vTN)0{mG zSE%G7X~HsXNh?Gl%twsDEUoQVhIiiyt3(1k73^U8N-E{2Q-;T!@=>yAZ<&8Ln zIssL8jYL5ZeeCt!n3!4nHXe^JOc3T59!><JtC1VQb-F_xG`#+@YzYl$=p%UEXTcDoTe zix9KuP%|$w(aeg*u75;Sw2l4k<6|Uuvk%qb>%pA2abIh_*srYYl!C(&#K=rvbI$oe zl+uU0H|;Mi>Gf$dv!Llt8O}n?T(019Ow|s?45~ee8&_=}XXDm+|Md2|&!2yL+uycr zUv~9JFtduLs1vD3WVr&aDCCN0B@ogQlSLuC$RKqYAtRF(7kf)yXfYv~wYs{^pKeQc zMUpaaV>`}g_+l1N&h;QLi!47%0Zfs{=+o)MlMV5J9SRImR zyS;Ds&yV?wuj&3IKv^PIrZwHeqaNVL^Oru}2rO5}xaq6k4VlbRZOi6USZh()`f#5_ zAi1K`2owlrV(C=VZw3`=GY`I@6=rHgIF7H8AyQ6&P!)g^wb~?* zkLNFQeN=K?c!)M)RjxvV%SL22S=w=YY-5kGN(>Md)$ZYR-O~lG)o-KaLav|{SS+DL z3`UA3IYAKxl6uS&CGn`FFH!eX!@tvlX>O=2iXv7My$^5`*~EvfXV-n%GvF$0ej@Um zPi?o`{_gcdBbaoIy)f!!R-UWKK$c}Uv`AcEOsU#mzNt&Bt<;#se5weOR+=ia2&na~ z%f63K`|b0mAAYyr-vCijQT=M3USw{C4WySS5tr1n{z-!LW|`>St9L5h;Z@0KX+*TV zzEC4$*<`SCYpPYMll>Rsawf?68ryTof}d(g?}SC_#PUow-H&oDW`$ncSjk&2`SR zI`ASdJ7J|Pg%dLg%f8)@cF#C7j)PY+OW$IZzTVBBJB? zrXmDVQ4lk+$GoO-7*Ro%42s$pn6&~oDIubb$F}{r-GBP@(_e1)cd!U)L}+!5S6lf@ zOT$-m%w?~xLy{!GFhmd>RzPxQLLOM3001BWNklN+S2WMqP77&GQNh2^EDzjhT5uW#j*;0DpF z^@RrRFs8~nlviBuB}4!b1+#}UcCquA$2PX~@Z;$Fn|}NC*K%6dmQ_*$mNH8DX!_~) zK0n^(^H<3pe$w^-K<_(JBEmh9PDHJ>t3ppSGHf28a%nS=!n$wonT8Bdf&$DyCMCZR z)(U$DqL6lGDo$5r2x;q|b^HAG<6rKdej;t!dI|2#yjF~BceoC7E`p(=OFaCT2Xt-w zW?`g45VPkcvSWBGVm4o;183rG+~@hASM`X6i)hp4VHNwGh)nD4Q(iUl6{e#s5^m;8 zm_-H%8D7I+UcD^U$3aPzPSA02#8GDet9Y))BNG{JaYee4Q%>>~5`qAil$(jZw}E7S ze0kh_Gd?b%*Hx`+xuESAs=B9Yn&AVrwE|g`LmmlR^-uabN7MS)#|C$m=#VHe%ia;=!Tl9{7!L~RyE*Mxix z08DzG2Z79{Y8M?5>vDd9dfUw^RiFKG$RaZ%gEQ7LVd+xx#b&;el9`1XF?+0Q&@Ih9 zYsB(cs$IC9>#IsnT_qy{|M*W`8xyUyNW{aXH%EZ8(%lJ8aNqzTh*_&%pju4A-9#FL%LZy@Rf`(2GTV#k zD)J4-`RJ|Vm4HcHx}|VGRa&{y*LwE8k2bDPJa?kj$0up`{r=MrKmJ)Cd+)o?RbW8^ zYpq(ruHfoRoOKOm?#?Qn@%F=~Gd|`zZf|eLc_34@CDMH2(%KP$JzS*$b4%|QN4pP!#v`V+B7R6 zg8&L5Ppq*C>!6U{n&%O@%K5$Zn|YX9#A0M`Ei?9g!{wtqN8inkZM?nRKATSvdf!VB zmy~t=G;Yj{j1nhAEGA`c!oBymx3}MZ{`C9%{m0R^bgW}iA}lI-^L##wA$|$bEB7py z*_!7VEh&$0jl|V^xK?gxvT5b7U5u~6avk45Eqgd&*R%Z z_C)OC?VRToGoMi@*+96jxFnsG`K`V0MCpFgm1~tuJjQ*l2T?ZdSKvVLvO(c zNo5pgUCX2mrmk{ta=&e#?)!V!@%;9vt*W<97NTrz3Co)21^ysQl2#suwhfD!R6}^S zt)DilG`nWGdfv?9Vays4%v$-h6@lP!)>7wwr47_Z%&mp5ZNF#wTF=%7D4UFkFgpuK zcb~WYK7A!BK^I<8`WYFfeP1@~N}K1min3?oy!=`0(*&CzW)H(kik8#vB}HI4mSZ0NE#N>@c5b96Q6TDM zj_``bCzh?5LD%}KS@F>z!vaR!B>{w;5xN@r?C>#4SifP9ICTV^tGA3H#x z)YeO7%FIn$Z-a@1G~6@M+a}zUw{Q`@3Eh}Bpy|j5wc1 zM*RAxpa1xWe;!?1(`t>13=4-4Nlt%QJ}u7`Q=3G|82}=^sy{DbIWeoy%eO})%;)(S zeUy~B&faCG^xk^UD#wDC^*oW4HzQf`ILL`WmFtlfmsPP?Qzk09P=vL~ zfD???^ME0aGiR*F!%o}AMnqEWQ%r=bFSd*s5%D5Hq7tq%41|n4&r`IqHf_C!`?hb- z$1@_oXSuu;RaloOT>ZoM^h6o#{@b z_1uY=se6UNa3ABd5NrYHgvK-|8-*|{Yy0%JE9>Lq*|gdiU~BFBxQxn7=bYy5;o+X} zgn7>76=~C-B)9-2#FdM6iN7dvF`7y@TUA|JZ@_JT%S&+(6`0Zoo1Z;JRy>HBF z_UwH`Sol)yfE7Z@;7DW2gkM1|$UqQfZ^O)(>MaQmR}tk#A!%X92_r57yP>~-f+Nf> z;F*Ao>(~ryIy^$InzamfWF)*6fe2IK+6m=v{`y~s`#dLoH`!kLQhIq!jG0AMR9jPk zOOlM_401~X0|FA}PTb70gsGX>w|9atH$i8##=}=IWa~HOO}HVIXmovJY|J<07T&^G z(1klglv~q{z*lAzS8rL0NM@eL5#erAgow$^&f^KB*^blxravS%xj;6h7%QR3h^NO>4-Bj@& zX?UHQuW$}9n896krJgJO^qhbZ1ySUux6ho|_8T*o3rBh*x|j|EmA5`KG6ESH4!9^M z7BLXz%aKUGY^bWG=Z)eJ^I}XT-0%`ILV}M+Aqwz@(~BQi4jm!4(bx2u#UB`sw}ing-DQc60YKU@>3Q^>*LFR%S4B zeJF`I!bN%c5jZ`b%(dl zLYcaCs4Vy7tC6>yUlsoZP`W1*33*J^Wl4J4>TNeOM21iKyTAQAVMc_z3(FY2zL7Ay zEhefa^xEMvGblZxA_Fcm9F$2Jl);ojp`4ONVJXnYo1h~kozo>v5eo0{kv=GTnA)On zL1xpFL{kE-neEsrEb-0>Ygg-vMwa(KYE1#aH=b~32 zb`4_0%ETfnQlbP(P()-vxaFn2=kmi;VdiDM5g}sFtLm>vPvu5Pm1eexF8>%1Ns*vF zHnW3CfXIwilotq~a)3~>1d|gx#~Jf61wB?WxW+0?$udt9VAP5m5d`=2yKnbVW*mtfEg-5!J3K4?v-vHA`mfDsgq${!%45QKzNKow*o_5Ap%MhBB*cx!{3Xj7CiKseI(+xGY( z;m%h{8phVmPs`~!1C~gZY9T{8+^W1KVwFQ*MCumHGRcD7UA184(t%ydH}^%Qku2MU zCsUHjT8I0Q>D#ygXicTc$a5XXRi`Gr0dQN)qT1Y}c7 zu!v9AAP^?AJ0c-WKsh6+y0BPFK?~QHLG4_ElUcIwfrO&U&~^F7)jw*tX-*i zRNGqT%OC*{CvYEIwZPU9O+Y2-sPt0msW$heGMJ*0{d<3_NCZ)dN-ZD;(&KEh^9#T$ zfwOyAGQ_80Ia`tg5uB;SSxb3=gEk4|?scyclc%@dh`4D#kCT~_n1S>0aUQm9dzjgm zbKLemHg_vxO`FsjqTA?hUP}QoBI~b0b1jOfE?fdClSqUuLqKaSB|eYiZNKHqGFYGX zLjBiVW3;O1F2c$qSxYF#jx0{v%zIa63a>LHjAe|jsuf8Gy4I8mah&Io=@p@V#X2H0 zUG?foAf{NJ6hK-SmlLycFsY1KC6TL%zB+sL5{?p|=h@r9%XXRJh|DmNT8qWPns#>! zGgVPh_p?IDL_0Bi0eqPZG-3j*AL9~HttL)BS1;KGpkCqwk!IM z9D`os2w<%V#D04}k7rVD_Z^9FPfH?NbD4V|yR?ysxu&~ELVxP?R{i0@g}l z5{bqvs>F;6*81)YEd>N$Bl_4k7Vk%3V?5D^>pEW$Y@V5ENNeaw*yhszL^8CF8Oz5mj0^i}^K-6{SXO z09QSoyLp8E^}qQ$)D)hXX7fDKFNUL}1JxvX(bev2%`7s?qvAIbS>a{vDhkiYZ1+w? z8No#RCh=;u%Rpo{ZD7)-HGhfl^ZDpwP! zKmGv$Wf55k<^D5E@S`TqIGxBHvk zKYhArazF@Smc=h#nyw7M+^f9`fO{qFx|Hu|Qm4n7w#>B9ARtyX#x7=I zQM}gu(OX2k>07Ny%bb{1bsJk|2#Ypu+CbELZ@n{DXh)>yr?*eP{_*FmG*7ppNr+rXAIMx_mZmqCI@M zuio49_$GXJi{yleu((;k?Sg4Zb=YzautiHD;zqjZex3(|@AuEo=hNmin;GWey6u@# z@9A7t^8`!F%iDjYe$*mLP0N%9q1*e9Uw-++w_ktx_=Yh2_P6`zAAWOxdsl58FTr=X za~t%M6xPKosumuR%v_g$!Nou>yON0Oy~FGCR36IudKEGE3`Yb-+FEO_-aB3vH2}-X zn1d`Xg;|+fV?tKsGq_y2G_j_`VLl~^8NWX5?O{nkNn`*wT( z{CqwDPN0;sRt;$-WUJ7FD=D$ul@#gqxdBnRULu^BdTYn|0hOjB=VdAuZO8fO9^1Gx zN!JaT5wXGvnCmD@QDwaQG;tcT(TXr@oGFG3f2MfkH+Dpcjh z)gmjRjVS@)+WjwRzMzkS>F^RBTr^7MeqOx`?z^{4exXnSYNUrsjLRISADOUK-?$F&5M zP%a~xN#tL?KY;r9qDo$wsNU8Zrd!)aFXY+Xidc4!F~-NoH|;%-$2?ErR{;HYmZ>K!z}I8Kjkni|NnnXDDZjS$eS& zWdQbY$E?Xt?J~n}fC z^>@hhutk*8d-|jdN~@zWeTBiiosZM=DKpC^ z!m2X%TN_<%P7sSI1K~a+szZ`W-s>QOM=aPCd2;>Y(o677q$=rg&dJw|gVtK@_yAF5 zn0T09Atz==L>`$dN+Dm1W*yoU*jOJmm8xH$t&h@Yc}$V3kWX8iXIxR)?)36-xGj-k zzL1Qxta~-n+^Rjs!<&c~@`Es6%a?E`s(hl%%ybuN*Y@GT#K;(|%EGFKRgg2?Kv z0OX1I!aEcdC*NK$FxAwh7gWA(kyX}#Y(!aREyk$#@v2$}9(8McV3x4MA~ zAMyyKMR?Bhuyu-}?_1w?5?59+XxsMUu$e`h_We#m`t8eCfJ9#^3+9Da#U@?{ z6^Mu_2_S8{)(k>iFkfNLAizDpv#Th`<8^<(mZd5z;q(w@2O_{ss6uBTkd`3@eA4%; znQ)xvZG2IFAEo|q{N!+6`Rd#oFyx( z*4&+m!s}JJXh6y!qGbUl5jG1VC`6QzNWe1~9EBoYI~&roy!J1I7i-RRH!ol~%+0Js zLL{tYL?Gp6X+WOyNMvOz0F;?3jVPy`!ozNEU)07%3h1;5rjEGer60w};v23c6XwOXIj;xwW zNifNXd=waNYh`9rr3_!^^XnfRbKHI$+uQr6AGZCzZMTTfCLZB#9^S^Lt?57ifB!^W z8eZ$I(5F%wlrigqOV6aG~2XmGHl!YZSrS;|OHMt|QV(lc&15Pg* zst&QRAmVdA>4gvxP(qprPp@87s5H-Hrt~Xf7MVnl@)suDO?ZeXK!Ah?2W7x)QlhnH zK1rKWi&%3$h%mx{q5TiKW+E-e!KO) zy|zbcZIm9eUgOMkTbcq3^Ho+v@o;x*O_A|@JeYWlQ95TX_lb)6^m(PIo~u%hh^5u~ zR3NqpC!+UWiN-Y#WaikmT0mJEF-PVYLz#Q;y|;>AZN2x_gym5QPW3FyU(~y(oW~Qd zs_*mpKt|0U->H$hESDYNLTNK1VoqVrbU^6Z$Z?(z5q-P=@a4n&p0 ze9q(BetQqER11n!g1FK1^e?|YzdeuSt=wxOdB5L2|MZvl_fKQ%ecM=tRZHAawUq$Y zoFYPU@lTKEQqKtYrbS?6tzaI`L}T0BJsIIwodlTbnU(c2RF0B*M(Nn>n2Hg|vj^lpW(WuE5=qPNeV`WR7npvOAT zFTeb0Y^_?OiFx#G+ELp+i+IkYh=SS(K)QP&w)Fy*tl&7F1-${VFeYK5G4|)c0{_8WTt8lJAHW-0j}d)i+Hsg?J04$%aoe{ae*3#mKm54g-&#{Ush|u2Ze^8N9a;dcHkt$?)&UwG> zMd!69%%sw5@UHMSW)`N5WTdY}#J1L_PjAo1$8kL8d4B%k2L;j$$ZubMZK`4Gm!JRm z+u#3o+wbSe^{zyku!=O#HJ7nc8M{$rwsruK?zWh@f-Hq~#j71w8UVNDzCbzcXwtg& zDxbGl=Xs8C6Xs(bq#C}OHU`f1Xd)oY^D+A7VXD3IZG1VS0U)T`Dbm-PECiNwp66-w z0s)zYxusK3y65BL`FyVDVW;`>HToF)`2An~)!V1{+x=GSr7(wE=_z_^C9b_fTUg~n zOY86@eXU;Mo<$YZgl2B4LRAC^N}`C^T1UnECXyb@EFImH5iir_0|)sMmRnp`~o6 z%5ssh^-HyZEEnhf?G|n&=Ba#&O6C#aZQqzVJlunb-GUNLn)wV1WnH=Wn#C#&In17! zsWMO%8&?n4WgleXIgd&;RFzA=ksc9a+u4MW#h{R|*}5!L<>yA2FJ=}QMDUn3cdRuf zhtIW6%fOlofHq}f56>XWv$xUOU9_9ooMr)YX5Gg=%t9F?@%eYZ`TXM#z3bJ7<58r< z<+?<=IVo?q+qZAu9*>7=CldEiX~{C@)UYZVA+V%7F{z6C)i$KsuH!vYMeC`mHewW( z36Pg8 z3yWb8iPl6{(l;>r0018{NklRAdW5s;ae8L1plM-&=Q&&7o08jXTf5zF%ot-k&x6X-nlGk+$UQ`_%UD=@ z-{zco(SxwL(p?E8B^D!M70L_`Vo=ik?d?3zk_aKuw2T*EWe?wP`^U$lwM)5BoH5GG z0$fjmC?Zs^z|mU6h_CUuwV~XYR76{o*7Sbg=Q%y%0e&8V;qiE;1p&Ru>+E3`8QUO` zTM)5X5Ysu&+uOb1Id>~^uWrFuXK%ODzJv!6HywdE&u?SgfE1R|w|Ywxaghc1eg?6s zs%WLXmi8?woTJPf)%VV_+_-9cGw1y1eJ5t7ERI;Xx#xX*o3<>D44TV&-$B~?Mx;nb z#9VotM^aN25ZUso&TL~NQ39m(zO`+?G0SyRB0+S!-ya{}>I0x6M2y7a;{!zVJP0hz zW^2ur5w#&})&0;lfI)zIv1CefUe ztC+ozwMZn27n*A>TOwkSQd?Z2HB}Kp_=+nzkI2kx#;>=r;v8D*b?RR5Y?%{!7v_Xe zrk&=7pT8U*$K$b{_xtufHW5t=GT6C#A8T60Os%zE<^eBRM7S4`sM@#fe$H>WGR=X^ z%D~7B)m}eqyhitJ?4p|RWxnlq5Ws9@$09*yL{?{2poCD#m3)|iLYjz$D>;?Ioht?s z5SGlWxT<>9-QxN9xZhdF4$wJgNe0{_EHp`Ec+9cy`~97(*Nda=Q2#<`2*tR_r$MYe=+LVRN{o~t1Uphn(VeF$e^zwI^ zSuR{8ynan6<%P=fx$H>b{r3Lx{CdnsZ=;~{0uf$wA_#20h%XC4`08zACQugEQI$Jp zD}1VKaZYB*3en9fvX!uyd1ORudS$Q^GA!G;-*=YQxBd3^X66KMt&uQajZ>7Fb52|$ zmrEa3!>p(ZAligD+}AqiwEO$pnkQFim53J6goQ+&b0Ra$N(WKYk=N>#r7b6S7m-RP zC4sxu6xBz^H7i1xiMd4SxWv3dL~BmnHdWd7jZi^L>Q`_P(&BhLfA=?k{&)ZGZ-4(6 zfA;IwFX!|5hkyFVfBeUPz-gJOqi^qTdCuo?))>JmbFB=gEI$K5Dg zi@A?#Of2CMkme_o5~WHD#K+@{M{NCVjBU;{(km9M$~%e?&x{JWa9b*ksHzjAG#(Wt zSmCL@CgO_vUbYwwh;6$AaEpxSjhJL?{qgu#y!GSb(fiO^rYTuC8QgTOSyJ61AAK!m z3V3zlY3p0R)d0JVeV&s}CMFNB+#jm$e|TGgHLO z=A1NoDYr#6JP_9+m~rhn#a5cxoM#`Mn97Pawjs=_QU*+Iy5IK*`~6@2t~GfckAL@X z|2qIAiH^_Pr=bI5%t+7lhy=A>xYz|7s5EmU5kfWbiBO9f9xlugj;l)`aUq0@K+-hQ z0Al7Ud9Phy%~>}Qa}kG}$Z+>q5jJB9^A(PHjrZwpy|oK*?1#h?rale_J=+kU?hQzF4g#P!GKB{#|p zC8EeAXU9VKMTC|`w}%{j-mT{rEe!wObnmXe_N-uY6brJE56F$D7co(K<75@BX( ztpRAQHPtz1?@gK#|NG_Z%P)WW;isSe$N%Yn`Q2aq z<FU;gxSltD=(BUI)2JR%~@lnY}?5+-*_rVvF(B;X_x<+~S6q{sAF!b4Qk zBd$(+K?aIkin!3=%(aeM{)HuxqOy(qoX@omB6QyB_p7ZDGtuR*sO0R7w6%zoIc!=c zoyRjBDg}B&TW?)IKE70!YhflLm8cs`s7R-4?|sc?l`0IRsMGdP-=MSpAOHT}|NQfxy!EXh zeutPzN2q6>itfN655E%T{Zm+9-3)gPT-ReRwsS7o43`kDpT7h=|^| z+qSRsbhFIbKRbm}f{4t#u3%-Ma~{V$@?X|E6zy;B!W@}lZk~m6tG0S6Y9YDZZ|8YL zMp~BExW0bdwqKuLR8&<7aP!L~mOxT(T|}6%ZKL=0>C-y^5$US?=)&}TJP0X{U;gR8 z&d26=_CNuTFA`c|PJtWq1T zwpRG~*I#~CX;;KQ(X@}_@zMGSfGfCwNX);wx891FW8up6g0poMv@$xQ%a z!To-V`2}D8_@Dmcf4=|p=`a56U;9#9Gu&0QsoJqJP^_niu`)p0KGvExXDrL<97L8J z%E7Q?P9iZ(ko2+p}G( zBQkvSKDM##dmpX#CST?pgpF9v<8-^mlmHM}&JH4I;V_pZF9CwGwtFLx+jkux#!$;_8M}lW5md81U53@l$LG$3Yv`DBsgA4Qovt;H09)0qNzAu>(*x1WFe35{eQ!I9~6&DQ#HoKe}tOvzXHex^O17253;N{|+5;3TXlg(Mgt zab$+IHY&Q8R8_m#1O;=ekW|9|-_)6H$B`pR)K?%PlcZGDGj;l5=Kp`L7fEDh1OQ*= z!6Vh9SLyIjxgZ>FX4eQIAfhq5M?e6AGBfeAslu&w3nKzTjVNDGm5XE=axfS`1VmoZ zT>yf-ZCmEtZrgUhSr|kj(U@n%x)K@*-@gC(d_EbOh3mf=;lQAq;>#EIn#Dj_w%g5r z{2~wOn=Lc<2=GXNtgM*8D-R8b5IN^m(OP{Iy->!4SDGHt!s*(v6EWw?M1aU;4v}_U z&|%IYqS~}>Hf8atW2A5>B3#GR8bejOwuoS)IfjUcNZ+>dw*L6}0~p)AfB);xcH5@g z<2XJa4-coLIsC%M6DH(24sD9WVHRcx#os^z6l6)p1ax2&ui{kz1ZdJC43b#-vaX z!prGhSOoX_e#6T!OWU@??`L6VJ#B?KF)f3TTd(_W;Xi*?h!=DOcLw(T~@ znE=#!xML)f26zGx@wVT*pmG)&U~~C0RVB~0)^Qxd%A#h*%vrjsML?O8D^?o8!nT&V zGjrQ}geL$K6SKLSd24Nq2>>cmYj&F!?jj;}@??0Wh=q$|(Ob*NwM;}rui;iz_ia-V zWkJYo+iF2}&aw4A=UIq_Oi)=OT*8c|0?ckXgjnLMB^w<`a<;bBwe0KB6tj z69|C5z8-|&%fcfp)>w0#aCPovZ>?$piRdcf6qWtf#+aEIZiqPN=)Dv3oU`>-X2st= zz-s|y29Uy%QIz!rDE=>16KVMsFSBXI4?G~V_YOt*0W4Lc^t~DhhMkEQg-XS6= zGwAmA4iIy$rVSCN2NDy|nv;P5Nkkyw;yy$~xTtQjmn`cZKYsjZO@(EiCq!L*tacKG z4r8HWHF&t0N2DOqI7fNw=~9Y%ng8~Euap19#*6&Dxy)3N*4i4gXdg}F`hPRmI=!;I zZbrn6q|8LvR3YKQ`vC!o)26i6 zvu^o?RS!>LIln$JAu=jlihxX<344D#j?d43|J1$R`=3p26GngstjLv!0AW^5 zD6Hjj!RtrK@{?G4dy8}rBP5J$q!gXALE^9z0?jM~B0K;{M4Lh;ZM}eF2+1`%CrHz6 z+ih9q!^RYrN?k!-Lr`zJka20}X0Cf9rfutmg{~%uwN`x|Gs7{SkH>eF z<&KcI+nrSdurSRca-P$xz@jEkL_kp036%{YcjwlISBfbSz~-owP=pf_G54lG*tU+u zMH?w@t=U-v1PNIzfFxk14&gvooqJ?1n@B>WX1^e(CZa2`j~9^r1* zw-(?e8~|<$0L?uUGRZ8?bB<|%YCe=uSh!#zy0uJ4#`=Oz&f6{z9XWTt*Y>f^w&C=0D*)GO9J?EnzDq4utIqlUSWLzU@k)+G7*7b z?EwJYUAQ32wJL8P-@f(U_k9zh^y+DrH7D__2iC2L*qpXj>kR>znGhex0d?YBZrE!o ztg^9;uvOf7YORGE6DRL8dt;R8AcbG3T5L8>}QC1%XVU0CQqO z%$!j$uRzXZOM@T?0ZR-_OYr1myhy@cxprax{QB~F#G2=rL9pHTw{PEw5i%(+G)KM4 z^Eik^h{L_UPW$Z^8Rs~W`0MKn2zqasK%yQ&go%j^HGHL6IE3dKYff7;mL-c2Q{CCz zOt#(=fU8jYbzI47@tF|>)eX}bx z^zr^C%m^vUEbQiME|w4lMn0cUx$SnTc)8}h-|wHle!+Ed&xE|BSpY)5-{0nVjBz4S zghlwaZRgXye1-soXlu2%otpqe5|~@1np^7;!H8iq6MfkWd`XO@3!6yC#OlhW@Z~Pj zkYr7B3k*yP^Z<+M4{$dmlE#IzA&+n0zuorkYh`9afM|pPQe!57c^tiUL@xEcGJ5Wg z^(G+_(wuY7alhZbK4T2M-QN%q8A&)HfP#e9vPiIL=i_;v!yOX{Nf7ZokD@VzFG0Yq zcaK28i)se7FhL^DfYN~Xw|$Jc-}Z5i$Rws>7Bka1#!Ksky!yvDF2^fJX4^Uwi>jzF zQES?Jdr1OmYqchGE@En0WQ(ni{e(P^W8e0F{o|i+-@YLh{RoN$ThDGHM&zjV3jiUm zHQg6LGC@ER;quA)2AD)hA~8q!B2-}>Zfj0=6K(Sxd*5H>ff>sQ7y=?)Rgr*#88Q|= z9uHeC+?ZNA`f^5;qR(g#$iTq#W!Ad#?FR4x3AN741eV4`5KbAl+b!J9W>u|M-vSFE zOL)#X-5rrc+nlSm*S-Y{XHm!rgt+yMVEy>@@9`MN^Ds{!P6$j3Hv%Y}oTvzRoX1&F zTM@~C(i;l4RF@uMs&0&GzCgnLc6&S?W=llIc{WwRase|$nmY=!(DKMkW#0E45k!TB zk?`&P&1~-bejZOmU?y#bOw4?HyVoa|RS-#bVU|Dt_FvOXw2_B4uvL}3Nkn6exGFwL zq=bqEnap&7nKApmp)e2>HjkMD5F*awnHg)G(i%i|Z55Zg%9`DY>Eign3k8NifN%Tz zqB+l%U>V7PBCYRxArKKSJwlp_wyLEDNEQV^tIZe!CJUi2Q|vIt=g(g(io}3ax)BkV zrO7=q5(#at%1(>0_CDuG&vVVspPzQF^YbyDb1n}^yp*tHk?QT+_B}GBi6(-|Wn2_( z|KV~WY}@8ma9+4l+DIZg<{;wRZ6_pb3IGsEgu)Ww%AAajsjB>H(rK-!YCWdsn!=<_ z-4$jc0B4G5{Q@3-?_FCTW0cKGv{ed+h_tnNjtYMeQSbeHmXrg+VrFH*cQ^M17=ZEi z{iAig`eHrtKf+&>KPLeqjG5DGF#vF3Q#0I%iVq-HZ=yL1+5*J1IUo=qblZ?wn?9e5 z7>Ssh03@sK+dj_IY!yU+tH&|3Z+*>?1d&N1_m7XY){h_metZAcTR+Dr@_sF}d>H_! z%Jb`J{Q-ElxGvqc?0h_a{o`Nf=aZ=g(u$>9156-b=G*Po^hSJfEw9SQDr=ytJ`o}z zBF=eAIsFL};599KMwr_eBZ

mL3S`W<&&xiIpR#)>;#3O`B?O%EFf%PH47D+^t-H zL|k(qQtSJ9o)8d)@Q3k%Vw2LMroFdvbUs;UY=}4{*Sct^k1q=W!727mz}@ z?2CxVB5RI}C~*@C6;OqG5a|X`kiOPc50~g>#I(!`XQ(Ox#;6JK_Wl6?=Xut&YwN644Kcg9TM*;d=coCynC{lM z+jtJ2h9H@EJkOtBKi|Lq5qN5o-nadJZ>_ONK>A#Ae}8`&Lh2z*k-5y62mnB$6GC7@15&O07$GE~n~&q{`_3xmQ%eA4(xQSfGomunHAWRt4Szc#s;CG6RP^!n zWV%L`+DU~cE5jwiCsp$RRl{@et30l%BQrA6%YabsZEd^0>B_fRhj3A03;UV-)z2XW&()JIoF&gaw0M0SYw9Sc|H@;tFoSuMMc}T-|uf9 zgOQYn5JN)z{`-KeV!Ym3x(C1-gIOr4E-A(FTegt-3iT6-w96BcQ9p=S ziNM{Fn3>wP*Hj;o=Q&DcVB*#qU=}r+kiu4&UEu^~h_q!}-|J_KsINtk!VFZ7vxy0e z1VrIcZDZyR=EQ&(5BR{cQ6dDUZrXE0NG7?zeca#Pd*2XYjFFiHD1ZL-7rc(agsfuA zIzltU-RnK8*K66jY_pJc0W)(m4}gsP`121l^9Z;ylUdxw+rEqN)|(2q3OfL5(=x~J ze79w_w~xqGzISF8_B9dk*UujaC?XcYSXu_C>ZID|K!zg(K$=@#aR~MmMCj=iyf~hZ zfba+=7E!2+oyQu((F*qOZiq-E#Mipp=Kx47Q|Jn0LL@Us5?@A0%hq@%8zY33IJC6%oyd^Eh6T2OxrQ`L_`%uiNT1;$B1Bb$l_>GIMvhd}zo- zYp(iDUJ0&3(=Cq|stthZy_1r}rS~SntRnh~9s?51w*(f6fO8B{Wnqsv&arR(*RNmi zAK!>EJhr``bJQx|RyQM*)}(DWX)Q4Wf`nNbK#_r~)wgtdIG&Gfy9qZy zN~~xKLDt((WnyG$NMqTGFv4mYSy3fnNU?maQ4?iTEem?A6}D`gxBX8yyOLoLt#_h| zc!s$Z30athSbp0)Ag@RSu^?Rz)LO!LCaH4MLOUP=6D7$7A0i5mh-4OCuSiis9qBct z6Mzc$t+l3bwdX)@&DL6~y|u_R_iysu%v-4qg+R6a2tTh3kaHqZ&lX$@K_;UC6k3Wc+MYr4iI3MRYt9^Woal5?(RGy@- zHA?yDtcb!AA`KErL|X$wq(la3*XP*R!UQ1-^N0!z5wbwZqn=hBI}wp7EGrfu^xNBc z9By$O2NEVyBGwbc)^syMR1qofbGX}zxw_~)j>{>K`SI;r-}?Q2uSPNz4PTHMmj4Ia uq>6FiNj;YU000bhMObu0Z*6U5Zgc=gQ#vv$9rbMh0000 dict: + """Make multiple predictions using the saved model pipeline. + + Currently, this function is primarily for testing purposes, + allowing us to pass in a directory of images for running + bulk predictions. + + Args: + images_df: Pandas series of images + + Returns + Dictionary with both raw predictions and their classifications. + """ + + _logger.info(f'received input df: {images_df}') + + predictions = KERAS_PIPELINE.predict(images_df) + readable_predictions = ENCODER.encoder.inverse_transform(predictions) + + _logger.info(f'Made predictions: {predictions}' + f' with model version: {_version}') + + return dict(predictions=predictions, + readable_predictions=readable_predictions, + version=_version) diff --git a/packages/neural_network_model/neural_network_model/processing/__init__.py b/packages/neural_network_model/neural_network_model/processing/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/packages/neural_network_model/neural_network_model/processing/data_management.py b/packages/neural_network_model/neural_network_model/processing/data_management.py new file mode 100644 index 000000000..675362ca0 --- /dev/null +++ b/packages/neural_network_model/neural_network_model/processing/data_management.py @@ -0,0 +1,130 @@ +import logging +import os +import typing as t +from glob import glob +from pathlib import Path + +import pandas as pd +from keras.models import load_model +from keras.wrappers.scikit_learn import KerasClassifier +from sklearn.externals import joblib +from sklearn.model_selection import train_test_split +from sklearn.pipeline import Pipeline +from sklearn.preprocessing import LabelEncoder + +from neural_network_model import model as m +from neural_network_model.config import config + +_logger = logging.getLogger(__name__) + + +def load_single_image(data_folder: str, filename: str) -> pd.DataFrame: + """Makes dataframe with image path and target.""" + + image_df = [] + + # search for specific image in directory + for image_path in glob(os.path.join(data_folder, f'{filename}')): + tmp = pd.DataFrame([image_path, 'unknown']).T + image_df.append(tmp) + + # concatenate the final df + images_df = pd.concat(image_df, axis=0, ignore_index=True) + images_df.columns = ['image', 'target'] + + return images_df + + +def load_image_paths(data_folder: str) -> pd.DataFrame: + """Makes dataframe with image path and target.""" + + images_df = [] + + # navigate within each folder + for class_folder_name in os.listdir(data_folder): + class_folder_path = os.path.join(data_folder, class_folder_name) + + # collect every image path + for image_path in glob(os.path.join(class_folder_path, "*.png")): + tmp = pd.DataFrame([image_path, class_folder_name]).T + images_df.append(tmp) + + # concatenate the final df + images_df = pd.concat(images_df, axis=0, ignore_index=True) + images_df.columns = ['image', 'target'] + + return images_df + + +def get_train_test_target(df: pd.DataFrame): + """Split a dataset into train and test segments.""" + + X_train, X_test, y_train, y_test = train_test_split(df['image'], + df['target'], + test_size=0.20, + random_state=101) + + X_train.reset_index(drop=True, inplace=True) + X_test.reset_index(drop=True, inplace=True) + + y_train.reset_index(drop=True, inplace=True) + y_test.reset_index(drop=True, inplace=True) + + return X_train, X_test, y_train, y_test + + +def save_pipeline_keras(model) -> None: + """Persist keras model to disk.""" + + joblib.dump(model.named_steps['dataset'], config.PIPELINE_PATH) + joblib.dump(model.named_steps['cnn_model'].classes_, config.CLASSES_PATH) + model.named_steps['cnn_model'].model.save(str(config.MODEL_PATH)) + + remove_old_pipelines( + files_to_keep=[config.MODEL_FILE_NAME, config.ENCODER_FILE_NAME, + config.PIPELINE_FILE_NAME, config.CLASSES_FILE_NAME]) + + +def load_pipeline_keras() -> Pipeline: + """Load a Keras Pipeline from disk.""" + + dataset = joblib.load(config.PIPELINE_PATH) + + build_model = lambda: load_model(config.MODEL_PATH) + + classifier = KerasClassifier(build_fn=build_model, + batch_size=config.BATCH_SIZE, + validation_split=10, + epochs=config.EPOCHS, + verbose=2, + callbacks=m.callbacks_list, + # image_size = config.IMAGE_SIZE + ) + + classifier.classes_ = joblib.load(config.CLASSES_PATH) + classifier.model = build_model() + + return Pipeline([ + ('dataset', dataset), + ('cnn_model', classifier) + ]) + + +def load_encoder() -> LabelEncoder: + encoder = joblib.load(config.ENCODER_PATH) + + return encoder + + +def remove_old_pipelines(*, files_to_keep: t.List[str]) -> None: + """ + Remove old model pipelines, models, encoders and classes. + + This is to ensure there is a simple one-to-one + mapping between the package version and the model + version to be imported and used by other applications. + """ + do_not_delete = files_to_keep + ['__init__.py'] + for model_file in Path(config.TRAINED_MODEL_DIR).iterdir(): + if model_file.name not in do_not_delete: + model_file.unlink() diff --git a/packages/neural_network_model/neural_network_model/processing/errors.py b/packages/neural_network_model/neural_network_model/processing/errors.py new file mode 100644 index 000000000..b92425437 --- /dev/null +++ b/packages/neural_network_model/neural_network_model/processing/errors.py @@ -0,0 +1,6 @@ +class BaseError(Exception): + """Base package error.""" + + +class InvalidModelInputError(BaseError): + """Model input contains an error.""" diff --git a/packages/neural_network_model/neural_network_model/processing/preprocessors.py b/packages/neural_network_model/neural_network_model/processing/preprocessors.py new file mode 100644 index 000000000..37f813c19 --- /dev/null +++ b/packages/neural_network_model/neural_network_model/processing/preprocessors.py @@ -0,0 +1,50 @@ +import numpy as np +import cv2 +from keras.utils import np_utils +from sklearn.preprocessing import LabelEncoder +from sklearn.base import BaseEstimator, TransformerMixin + + +class TargetEncoder(BaseEstimator, TransformerMixin): + + def __init__(self, encoder=LabelEncoder()): + self.encoder = encoder + + def fit(self, X, y=None): + # note that x is the target in this case + self.encoder.fit(X) + return self + + def transform(self, X): + X = X.copy() + X = np_utils.to_categorical(self.encoder.transform(X)) + return X + + +def _im_resize(df, n, image_size): + im = cv2.imread(df[n]) + im = cv2.resize(im, (image_size, image_size)) + return im + + +class CreateDataset(BaseEstimator, TransformerMixin): + + def __init__(self, image_size=50): + self.image_size = image_size + + def fit(self, X, y=None): + return self + + def transform(self, X): + X = X.copy() + tmp = np.zeros((len(X), + self.image_size, + self.image_size, 3), dtype='float32') + + for n in range(0, len(X)): + im = _im_resize(X, n, self.image_size) + tmp[n] = im + + print('Dataset Images shape: {} size: {:,}'.format( + tmp.shape, tmp.size)) + return tmp diff --git a/packages/neural_network_model/neural_network_model/train_pipeline.py b/packages/neural_network_model/neural_network_model/train_pipeline.py new file mode 100644 index 000000000..13110b145 --- /dev/null +++ b/packages/neural_network_model/neural_network_model/train_pipeline.py @@ -0,0 +1,27 @@ +from sklearn.externals import joblib + +from neural_network_model import pipeline as pipe +from neural_network_model.config import config +from neural_network_model.processing import data_management as dm +from neural_network_model.processing import preprocessors as pp + + +def run_training(save_result: bool = True): + """Train a Convolutional Neural Network.""" + + images_df = dm.load_image_paths(config.DATA_FOLDER) + X_train, X_test, y_train, y_test = dm.get_train_test_target(images_df) + + enc = pp.TargetEncoder() + enc.fit(y_train) + y_train = enc.transform(y_train) + + pipe.pipe.fit(X_train, y_train) + + if save_result: + joblib.dump(enc, config.ENCODER_PATH) + dm.save_pipeline_keras(pipe.pipe) + + +if __name__ == '__main__': + run_training(save_result=True) diff --git a/packages/neural_network_model/neural_network_model/trained_models/__init__.py b/packages/neural_network_model/neural_network_model/trained_models/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/packages/neural_network_model/requirements.txt b/packages/neural_network_model/requirements.txt new file mode 100644 index 000000000..ffac1feac --- /dev/null +++ b/packages/neural_network_model/requirements.txt @@ -0,0 +1,18 @@ +# production requirements +pandas==0.23.4 +numpy==1.13.3 +scikit-learn==0.19.0 +Keras==2.1.3 +opencv-python==4.0.0.21 +h5py==2.9.0 +Theano==0.9.0 + +# packaging +setuptools==40.6.3 +wheel==0.32.3 + +# testing requirements +pytest==4.0.2 + +# fetching datasets +kaggle==1.5.1.1 \ No newline at end of file diff --git a/packages/neural_network_model/setup.py b/packages/neural_network_model/setup.py new file mode 100644 index 000000000..dd4e4d6a6 --- /dev/null +++ b/packages/neural_network_model/setup.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import io +import os +from pathlib import Path + +from setuptools import find_packages, setup + + +# Package meta-data. +NAME = 'neural_network_model' +DESCRIPTION = 'Train and deploy neural network model.' +URL = 'your github project' +EMAIL = 'your_email@email.com' +AUTHOR = 'Your name' +REQUIRES_PYTHON = '>=3.6.0' + + +# What packages are required for this module to be executed? +def list_reqs(fname='requirements.txt'): + with open(fname) as fd: + return fd.read().splitlines() + + +# The rest you shouldn't have to touch too much :) +# ------------------------------------------------ +# Except, perhaps the License and Trove Classifiers! +# If you do change the License, remember to change the +# Trove Classifier for that! + +here = os.path.abspath(os.path.dirname(__file__)) + +# Import the README and use it as the long-description. +# Note: this will only work if 'README.md' is present in your MANIFEST.in file! +try: + with io.open(os.path.join(here, 'README.md'), encoding='utf-8') as f: + long_description = '\n' + f.read() +except FileNotFoundError: + long_description = DESCRIPTION + + +# Load the package's __version__.py module as a dictionary. +ROOT_DIR = Path(__file__).resolve().parent +PACKAGE_DIR = ROOT_DIR / NAME +about = {} +with open(PACKAGE_DIR / 'VERSION') as f: + _version = f.read().strip() + about['__version__'] = _version + + +# Where the magic happens: +setup( + name=NAME, + version=about['__version__'], + description=DESCRIPTION, + long_description=long_description, + long_description_content_type='text/markdown', + author=AUTHOR, + author_email=EMAIL, + python_requires=REQUIRES_PYTHON, + url=URL, + packages=find_packages(exclude=('tests',)), + package_data={'neural_network_model': ['VERSION']}, + install_requires=list_reqs(), + extras_require={}, + include_package_data=True, + license='MIT', + classifiers=[ + # Trove classifiers + # Full list: https://pypi.python.org/pypi?%3Aaction=list_classifiers + 'License :: OSI Approved :: MIT License', + 'Programming Language :: Python', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: Implementation :: CPython', + 'Programming Language :: Python :: Implementation :: PyPy' + ], +) diff --git a/packages/neural_network_model/tests/__init__.py b/packages/neural_network_model/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/packages/neural_network_model/tests/conftest.py b/packages/neural_network_model/tests/conftest.py new file mode 100644 index 000000000..90aa8aa79 --- /dev/null +++ b/packages/neural_network_model/tests/conftest.py @@ -0,0 +1,20 @@ +import pytest +import os + +from neural_network_model.config import config + + +@pytest.fixture +def black_grass_dir(): + test_data_dir = os.path.join(config.DATASET_DIR, 'test_data') + black_grass_dir = os.path.join(test_data_dir, 'Black-grass') + + return black_grass_dir + + +@pytest.fixture +def charlock_dir(): + test_data_dir = os.path.join(config.DATASET_DIR, 'test_data') + charlock_dir = os.path.join(test_data_dir, 'Charlock') + + return charlock_dir diff --git a/packages/neural_network_model/tests/test_predict.py b/packages/neural_network_model/tests/test_predict.py new file mode 100644 index 000000000..020fba5ab --- /dev/null +++ b/packages/neural_network_model/tests/test_predict.py @@ -0,0 +1,17 @@ +from neural_network_model import __version__ as _version +from neural_network_model.predict import (make_single_prediction) + + +def test_make_prediction_on_sample(charlock_dir): + # Given + filename = '1.png' + expected_classification = 'Charlock' + + # When + results = make_single_prediction(image_directory=charlock_dir, + image_name=filename) + + # Then + assert results['predictions'] is not None + assert results['readable_predictions'][0] == expected_classification + assert results['version'] == _version diff --git a/scripts/fetch_kaggle_large_dataset.sh b/scripts/fetch_kaggle_large_dataset.sh new file mode 100755 index 000000000..e83841e99 --- /dev/null +++ b/scripts/fetch_kaggle_large_dataset.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +TRAINING_DATA_URL="vbookshelf/v2-plant-seedlings-dataset" +NOW=$(date) + +kaggle datasets download -d $TRAINING_DATA_URL -p packages/neural_network_model/neural_network_model/datasets/ && \ +unzip packages/neural_network_model/neural_network_model/datasets/v2-plant-seedlings-dataset.zip -d packages/neural_network_model/neural_network_model/datasets/v2-plant-seedlings-dataset && \ +echo $TRAINING_DATA_URL 'retrieved on:' $NOW > packages/neural_network_model/neural_network_model/datasets/training_data_reference.txt && \ +mkdir -p "./packages/neural_network_model/neural_network_model/datasets/v2-plant-seedlings-dataset/Shepherds Purse" && \ +mv -v "./packages/neural_network_model/neural_network_model/datasets/v2-plant-seedlings-dataset/Shepherd’s Purse/"* "./packages/neural_network_model/neural_network_model/datasets/v2-plant-seedlings-dataset/Shepherds Purse" +rm -rf "./packages/neural_network_model/neural_network_model/datasets/v2-plant-seedlings-dataset/Shepherd’s Purse" \ No newline at end of file diff --git a/scripts/publish_model.sh b/scripts/publish_model.sh old mode 100644 new mode 100755 index 9b29b7505..9a1cad78a --- a/scripts/publish_model.sh +++ b/scripts/publish_model.sh @@ -2,7 +2,7 @@ # Building packages and uploading them to a Gemfury repository -GEMFURY_URL=$PIP_EXTRA_INDEX_URL +GEMFURY_URL=$GEMFURY_PUSH_URL set -e From dcecdf410ce68b5a83c4d13776df4d6d2e8a6e42 Mon Sep 17 00:00:00 2001 From: Christopher Samiullah Date: Sun, 10 Feb 2019 19:38:56 +0000 Subject: [PATCH 19/24] Section 13.9 - Update API with classification endpoint --- .gitignore | 1 + packages/ml_api/api/config.py | 5 +++ packages/ml_api/api/controller.py | 43 ++++++++++++++++++++++-- packages/ml_api/api/validation.py | 10 ++++-- packages/ml_api/requirements.txt | 1 + packages/ml_api/tests/test_controller.py | 38 ++++++++++++++++++--- 6 files changed, 90 insertions(+), 8 deletions(-) diff --git a/.gitignore b/.gitignore index 1c5cde46d..845b524e0 100644 --- a/.gitignore +++ b/.gitignore @@ -129,3 +129,4 @@ packages/neural_network_model/neural_network_model/datasets/training_data_refere .DS_Store kaggle.json +packages/ml_api/uploads/* diff --git a/packages/ml_api/api/config.py b/packages/ml_api/api/config.py index c4284a4cc..3ca849c99 100644 --- a/packages/ml_api/api/config.py +++ b/packages/ml_api/api/config.py @@ -12,6 +12,10 @@ LOG_DIR = PACKAGE_ROOT / 'logs' LOG_DIR.mkdir(exist_ok=True) LOG_FILE = LOG_DIR / 'ml_api.log' +UPLOAD_FOLDER = PACKAGE_ROOT / 'uploads' +UPLOAD_FOLDER.mkdir(exist_ok=True) + +ALLOWED_EXTENSIONS = set(['png', 'jpg', 'jpeg']) def get_console_handler(): @@ -48,6 +52,7 @@ class Config: CSRF_ENABLED = True SECRET_KEY = 'this-really-needs-to-be-changed' SERVER_PORT = 5000 + UPLOAD_FOLDER = UPLOAD_FOLDER class ProductionConfig(Config): diff --git a/packages/ml_api/api/controller.py b/packages/ml_api/api/controller.py index 9b39aaac7..4e683b2dc 100644 --- a/packages/ml_api/api/controller.py +++ b/packages/ml_api/api/controller.py @@ -1,9 +1,12 @@ from flask import Blueprint, request, jsonify from regression_model.predict import make_prediction from regression_model import __version__ as _version +from neural_network_model.predict import make_single_prediction +import os +from werkzeug.utils import secure_filename -from api.config import get_logger -from api.validation import validate_inputs +from api.config import get_logger, UPLOAD_FOLDER +from api.validation import validate_inputs, allowed_file from api import __version__ as api_version _logger = get_logger(logger_name=__name__) @@ -48,3 +51,39 @@ def predict(): return jsonify({'predictions': predictions, 'version': version, 'errors': errors}) + + +@prediction_app.route('/predict/classifier', methods=['POST']) +def predict_image(): + if request.method == 'POST': + # Step 1: check if the post request has the file part + if 'file' not in request.files: + return jsonify('No file found'), 400 + + file = request.files['file'] + + # Step 2: Basic file extension validation + if file and allowed_file(file.filename): + filename = secure_filename(file.filename) + + # Step 3: Save the file + # Note, in production, this would require careful + # validation, management and clean up. + file.save(os.path.join(UPLOAD_FOLDER, filename)) + + _logger.debug(f'Inputs: {filename}') + + # Step 4: perform prediction + result = make_single_prediction( + image_name=filename, + image_directory=UPLOAD_FOLDER) + + _logger.debug(f'Outputs: {result}') + + readable_predictions = result.get('readable_predictions') + version = result.get('version') + + # Step 5: Return the response as JSON + return jsonify( + {'readable_predictions': readable_predictions[0], + 'version': version}) diff --git a/packages/ml_api/api/validation.py b/packages/ml_api/api/validation.py index 84414a294..c143263a4 100644 --- a/packages/ml_api/api/validation.py +++ b/packages/ml_api/api/validation.py @@ -1,8 +1,9 @@ +import typing as t + from marshmallow import Schema, fields from marshmallow import ValidationError -import typing as t -import json +from api import config class InvalidInputError(Exception): @@ -147,3 +148,8 @@ def validate_inputs(input_data): validated_input = input_data return validated_input, errors + + +def allowed_file(filename): + return '.' in filename and \ + filename.rsplit('.', 1)[1].lower() in config.ALLOWED_EXTENSIONS diff --git a/packages/ml_api/requirements.txt b/packages/ml_api/requirements.txt index 57d269ea5..ac0e2e21b 100644 --- a/packages/ml_api/requirements.txt +++ b/packages/ml_api/requirements.txt @@ -8,6 +8,7 @@ marshmallow==2.17.0 # Install from gemfury regression-model==1.0.0 +neural_network_model==0.1.1 # Deployment gunicorn==19.9.0 \ No newline at end of file diff --git a/packages/ml_api/tests/test_controller.py b/packages/ml_api/tests/test_controller.py index cc0beb09f..e45179b14 100644 --- a/packages/ml_api/tests/test_controller.py +++ b/packages/ml_api/tests/test_controller.py @@ -1,9 +1,12 @@ -from regression_model.config import config as model_config -from regression_model.processing.data_management import load_dataset -from regression_model import __version__ as _version - +import io import json import math +import os + +from neural_network_model.config import config as ccn_config +from regression_model import __version__ as _version +from regression_model.config import config as model_config +from regression_model.processing.data_management import load_dataset from api import __version__ as api_version @@ -47,3 +50,30 @@ def test_prediction_endpoint_returns_prediction(flask_test_client): response_version = response_json['version'] assert math.ceil(prediction[0]) == 112476 assert response_version == _version + + +def test_classifier_endpoint_returns_prediction(flask_test_client): + # Given + # Load the test data from the neural_network_model package + # This is important as it makes it harder for the test + # data versions to get confused by not spreading it + # across packages. + data_dir = os.path.abspath(os.path.join(ccn_config.DATA_FOLDER, os.pardir)) + test_dir = os.path.join(data_dir, 'test_data') + black_grass_dir = os.path.join(test_dir, 'Black-grass') + black_grass_image = os.path.join(black_grass_dir, '1.png') + with open(black_grass_image, "rb") as image_file: + file_bytes = image_file.read() + data = dict( + file=(io.BytesIO(bytearray(file_bytes)), "1.png"), + ) + + # When + response = flask_test_client.post('/predict/classifier', + content_type='multipart/form-data', + data=data) + + # Then + assert response.status_code == 200 + response_json = json.loads(response.data) + assert response_json['readable_predictions'] From 16b5efa926feaf82b09b56fa51a75d624602abcb Mon Sep 17 00:00:00 2001 From: Soledad Galli Date: Thu, 9 May 2019 19:49:50 +0100 Subject: [PATCH 20/24] Create LICENSE (#39) --- LICENSE | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 000000000..f02d80abc --- /dev/null +++ b/LICENSE @@ -0,0 +1,29 @@ +BSD 3-Clause License + +Copyright (c) 2019, Soledad Galli and Christopher Samiullah. Deployment of Machine Learning Models, online course. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. From 6f35d7a0a6f58e39bf15502a0de3719fbe949e01 Mon Sep 17 00:00:00 2001 From: Soledad Galli Date: Sun, 12 May 2019 00:04:11 +0100 Subject: [PATCH 21/24] Logo and readme (#40) * Update README.md --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 692a61f79..7fbf80b75 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,4 @@ -# Deploying Machine Learning Models -For the documentation, visit the course on Udemy. +# Deployment of Machine Learning Models +Accompanying repo for the online course Deployment of Machine Learning Models. + +For the documentation, visit the [course on Udemy](https://www.udemy.com/deployment-of-machine-learning-models/?couponCode=TIDREPO). From 7605297a20efaadf3e680de4a0b925e2a63b0cb5 Mon Sep 17 00:00:00 2001 From: Soledad Galli Date: Wed, 29 May 2019 22:56:00 +0100 Subject: [PATCH 22/24] Update Jupyter Notebooks (#59) --- jupyter_notebooks/requirements.txt | 8 -------- 1 file changed, 8 deletions(-) delete mode 100644 jupyter_notebooks/requirements.txt diff --git a/jupyter_notebooks/requirements.txt b/jupyter_notebooks/requirements.txt deleted file mode 100644 index aa8ad9311..000000000 --- a/jupyter_notebooks/requirements.txt +++ /dev/null @@ -1,8 +0,0 @@ -jupyter==1.0.0 -matplotlib==3.0.2 -pandas==0.23.4 -numpy==1.13.3 -scikit-learn==0.19.0 -Keras==2.1.3 -opencv-python==4.0.0.21 -h5py==2.9.0 From 7d9f27ccdda0690ca7c384ae3961c0a4e1da1f91 Mon Sep 17 00:00:00 2001 From: Christopher Samiullah Date: Sun, 15 Mar 2020 18:20:30 +0000 Subject: [PATCH 23/24] Update 2020 part 1 --- .circleci/config.yml | 148 ++++++++++++------ .gitignore | 3 + packages/ml_api/VERSION | 2 +- packages/ml_api/diff_test_requirements.txt | 7 +- packages/ml_api/requirements.txt | 4 +- packages/ml_api/tox.ini | 32 ++++ .../regression_model/regression_model/VERSION | 2 +- packages/regression_model/setup.py | 14 +- packages/regression_model/tox.ini | 42 ++--- scripts/fetch_kaggle_dataset.sh | 0 10 files changed, 169 insertions(+), 85 deletions(-) create mode 100644 packages/ml_api/tox.ini mode change 100644 => 100755 scripts/fetch_kaggle_dataset.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index 284be6e2c..e37101be7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -13,6 +13,13 @@ prepare_venv: &prepare_venv source venv/bin/activate pip install --upgrade pip +prepare_tox: &prepare_tox + run: + name: Install tox + command: | + sudo pip install --upgrade pip + pip install --user tox + fetch_data: &fetch_data run: name: Set script permissions and fetch data @@ -22,47 +29,89 @@ fetch_data: &fetch_data ./scripts/fetch_kaggle_dataset.sh jobs: - test_regression_model: - <<: *defaults + test_regression_model_py36: + docker: + - image: circleci/python:3.6.9 + working_directory: ~/project/packages/regression_model steps: - - checkout - - *prepare_venv + - checkout: + path: ~/project - run: - name: Install requirements + name: Run tests with Python 3.6 command: | - . venv/bin/activate - pip install -r packages/regression_model/requirements.txt - - *fetch_data + sudo pip install --upgrade pip + pip install --user tox + tox -e py36 + + test_regression_model_py37: + docker: + - image: circleci/python:3.7.6 + working_directory: ~/project/packages/regression_model + steps: + - checkout: + path: ~/project - run: - name: Train model + name: Run tests with Python 3.7 command: | - . venv/bin/activate - PYTHONPATH=./packages/regression_model python3 packages/regression_model/regression_model/train_pipeline.py + sudo pip install --upgrade pip + pip install --user tox + tox -e py37 + + test_regression_model_py38: + docker: + - image: circleci/python:3.8.0 + working_directory: ~/project/packages/regression_model + steps: + - checkout: + path: ~/project - run: - name: Run tests + name: Run tests with Python 3.8 command: | - . venv/bin/activate - py.test -vv packages/regression_model/tests + sudo pip install --upgrade pip + pip install --user tox + tox -e py38 - test_ml_api: - <<: *defaults + test_ml_api_py36: + docker: + - image: circleci/python:3.6.9 + working_directory: ~/project/packages/ml_api steps: - - checkout - - restore_cache: - keys: - - py-deps-{{ checksum "packages/ml_api/requirements.txt" }} + - checkout: + path: ~/project - run: - name: Runnning tests + name: Run API tests with Python 3.6 command: | - python3 -m venv venv - . venv/bin/activate - pip install --upgrade pip - pip install -r packages/ml_api/requirements.txt - py.test -vv packages/ml_api/tests -m "not differential" - - save_cache: - key: py-deps-{{ checksum "packages/ml_api/requirements.txt" }} - paths: - - "/venv" + sudo pip install --upgrade pip + pip install --user tox + tox -e py36 + + test_ml_api_py37: + docker: + - image: circleci/python:3.7.6 + working_directory: ~/project/packages/ml_api + steps: + - checkout: + path: ~/project + - run: + name: Run API tests with Python 3.7 + command: | + sudo pip install --upgrade pip + pip install --user tox + tox -e py37 + + test_ml_api_py38: + docker: + - image: circleci/python:3.8.1 + working_directory: ~/project/packages/ml_api + steps: + - checkout: + path: ~/project + - run: + name: Run API tests with Python 3.8 + command: | + sudo pip install --upgrade pip + pip install --user tox + tox -e py38 train_and_upload_regression_model: <<: *defaults @@ -182,13 +231,20 @@ workflows: version: 2 test-all: jobs: - - test_regression_model - - test_ml_api + - test_regression_model_py36 + - test_regression_model_py37 + - test_regression_model_py38 + - test_ml_api_py36 + - test_ml_api_py37 + # - test_ml_api_py38 pending NN model update - section_9_differential_tests - train_and_upload_regression_model: requires: - - test_regression_model - - test_ml_api + - test_regression_model_py36 + - test_regression_model_py37 + - test_regression_model_py38 + - test_ml_api_py36 + - test_ml_api_py37 - section_9_differential_tests filters: branches: @@ -201,13 +257,13 @@ workflows: # branches: # only: # - master - - section_11_build_and_push_to_heroku_docker: - requires: - - train_and_upload_regression_model - filters: - branches: - only: - - master +# - section_11_build_and_push_to_heroku_docker: +# requires: +# - train_and_upload_regression_model +# filters: +# branches: +# only: +# - master # - section_12_publish_docker_image_to_aws: # requires: # - train_and_upload_regression_model @@ -215,11 +271,11 @@ workflows: # branches: # only: # - master - - section_13_train_and_upload_neural_network_model: - requires: - - test_regression_model - - test_ml_api - - section_9_differential_tests +# - section_13_train_and_upload_neural_network_model: +# requires: +# - test_regression_model +# - test_ml_api +# - section_9_differential_tests # - train_and_upload_regression_model # filters: # branches: diff --git a/.gitignore b/.gitignore index 845b524e0..29988fc6f 100644 --- a/.gitignore +++ b/.gitignore @@ -112,6 +112,9 @@ packages/regression_model/regression_model/datasets/*.zip packages/regression_model/regression_model/datasets/*.txt train.csv test.csv +data_description.txt +house-prices-advanced-regression-techniques.zip +sample_submission.csv test_data_predictions.csv v2-plant-seedlings-dataset/ v2-plant-seedlings-dataset.zip diff --git a/packages/ml_api/VERSION b/packages/ml_api/VERSION index 7dff5b892..9325c3ccd 100644 --- a/packages/ml_api/VERSION +++ b/packages/ml_api/VERSION @@ -1 +1 @@ -0.2.1 \ No newline at end of file +0.3.0 \ No newline at end of file diff --git a/packages/ml_api/diff_test_requirements.txt b/packages/ml_api/diff_test_requirements.txt index 7ac05fce6..37ebe9b56 100644 --- a/packages/ml_api/diff_test_requirements.txt +++ b/packages/ml_api/diff_test_requirements.txt @@ -1,10 +1,13 @@ --extra-index-url=${PIP_EXTRA_INDEX_URL} # api -flask==1.0.2 +flask>=1.1.1,<1.2.0 # schema validation marshmallow==2.17.0 # Set this to the previous model version -regression-model==0.1.0 \ No newline at end of file +regression-model==2.0.19 + +# temporarily necessary as we update sklearn +joblib>=0.14.1,<0.15.0 \ No newline at end of file diff --git a/packages/ml_api/requirements.txt b/packages/ml_api/requirements.txt index ac0e2e21b..39a8feec1 100644 --- a/packages/ml_api/requirements.txt +++ b/packages/ml_api/requirements.txt @@ -1,13 +1,13 @@ --extra-index-url=${PIP_EXTRA_INDEX_URL} # api -flask==1.0.2 +flask>=1.1.1,<1.2.0 # schema validation marshmallow==2.17.0 # Install from gemfury -regression-model==1.0.0 +regression-model==2.0.20 neural_network_model==0.1.1 # Deployment diff --git a/packages/ml_api/tox.ini b/packages/ml_api/tox.ini new file mode 100644 index 000000000..50e82033a --- /dev/null +++ b/packages/ml_api/tox.ini @@ -0,0 +1,32 @@ +[tox] +envlist = py36, py37, py38 +skipsdist = True + + +[testenv] +install_command = pip install --pre {opts} {packages} +deps = + -rrequirements.txt + +passenv = + PIP_EXTRA_INDEX_URL + KERAS_BACKEND + +setenv = + PYTHONPATH=. + +commands = + pytest \ + -s \ + -v \ + -m "not differential" \ + {posargs:tests} + + +# content of pytest.ini +[pytest] +markers = + integration: mark a test as an integration test. + differential: mark a test as a differential test. +filterwarnings = + ignore::DeprecationWarning \ No newline at end of file diff --git a/packages/regression_model/regression_model/VERSION b/packages/regression_model/regression_model/VERSION index afaf360d3..a30e84ffa 100644 --- a/packages/regression_model/regression_model/VERSION +++ b/packages/regression_model/regression_model/VERSION @@ -1 +1 @@ -1.0.0 \ No newline at end of file +2.0.20 \ No newline at end of file diff --git a/packages/regression_model/setup.py b/packages/regression_model/setup.py index 5200fe1e5..264c47805 100644 --- a/packages/regression_model/setup.py +++ b/packages/regression_model/setup.py @@ -10,14 +10,14 @@ # Package meta-data. NAME = 'regression_model' -DESCRIPTION = 'Train and deploy regression model.' -URL = 'your github project' -EMAIL = 'your_email@email.com' -AUTHOR = 'Your name' +DESCRIPTION = 'Regression model for using in the Train In Data online course "Deployment of Machine Learning Models".' +URL = 'https://github.com/trainindata/deploying-machine-learning-models' +EMAIL = 'christopher.samiullah@protonmail.com' +AUTHOR = 'ChristopherGS' REQUIRES_PYTHON = '>=3.6.0' -# What packages are required for this module to be executed? +# Packages that are required for this module to be executed def list_reqs(fname='requirements.txt'): with open(fname) as fd: return fd.read().splitlines() @@ -42,7 +42,7 @@ def list_reqs(fname='requirements.txt'): # Load the package's __version__.py module as a dictionary. ROOT_DIR = Path(__file__).resolve().parent -PACKAGE_DIR = ROOT_DIR / NAME +PACKAGE_DIR = ROOT_DIR / 'regression_model' about = {} with open(PACKAGE_DIR / 'VERSION') as f: _version = f.read().strip() @@ -65,7 +65,7 @@ def list_reqs(fname='requirements.txt'): install_requires=list_reqs(), extras_require={}, include_package_data=True, - license='MIT', + license='BSD 3', classifiers=[ # Trove classifiers # Full list: https://pypi.python.org/pypi?%3Aaction=list_classifiers diff --git a/packages/regression_model/tox.ini b/packages/regression_model/tox.ini index 7fdec534b..ed418416f 100644 --- a/packages/regression_model/tox.ini +++ b/packages/regression_model/tox.ini @@ -1,35 +1,25 @@ -# Tox is a generic virtualenv management and test command line tool. Its goal is to -# standardize testing in Python. We will be using it extensively in this course. - -# Using Tox we can (on multiple operating systems): -# + Eliminate PYTHONPATH challenges when running scripts/tests -# + Eliminate virtualenv setup confusion -# + Streamline steps such as model training, model publishing - [tox] -envlist = regression_model -skipsdist = True +envlist = py36, py37, py38 + [testenv] -install_command = pip install {opts} {packages} +install_command = pip install --pre {opts} {packages} +whitelist_externals = unzip deps = - -rrequirements.txt - -setenv = - PYTHONPATH=. + -rrequirements.txt -commands = - python regression_model/train_pipeline.py - pytest tests/ - - -[testenv:install_locally] -deps = - {[testenv]deps} +passenv = + KAGGLE_USERNAME + KAGGLE_KEY setenv = - PYTHONPATH=. + PYTHONPATH=. commands = - python regression_model/train_pipeline.py - python setup.py sdist bdist_wheel + kaggle competitions download -c house-prices-advanced-regression-techniques -p regression_model/datasets/ + unzip -o regression_model/datasets/house-prices-advanced-regression-techniques.zip -d regression_model/datasets + python regression_model/train_pipeline.py + pytest \ + -s \ + -v \ + {posargs:tests} diff --git a/scripts/fetch_kaggle_dataset.sh b/scripts/fetch_kaggle_dataset.sh old mode 100644 new mode 100755 From e7b1bddb17a73557bfcc8dfef2321d228f4775c1 Mon Sep 17 00:00:00 2001 From: Samira Alizadehzoj Date: Thu, 12 Nov 2020 21:51:28 -0500 Subject: [PATCH 24/24] section 6.3: remove download dataset in tox file --- packages/regression_model/tox.ini | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) mode change 100644 => 100755 packages/regression_model/tox.ini diff --git a/packages/regression_model/tox.ini b/packages/regression_model/tox.ini old mode 100644 new mode 100755 index ed418416f..7dde476d4 --- a/packages/regression_model/tox.ini +++ b/packages/regression_model/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py36, py37, py38 +envlist = py38 [testenv] @@ -16,8 +16,8 @@ setenv = PYTHONPATH=. commands = - kaggle competitions download -c house-prices-advanced-regression-techniques -p regression_model/datasets/ - unzip -o regression_model/datasets/house-prices-advanced-regression-techniques.zip -d regression_model/datasets + # kaggle competitions download -c house-prices-advanced-regression-techniques -p regression_model/datasets/ + # unzip -o regression_model/datasets/house-prices-advanced-regression-techniques.zip -d regression_model/datasets python regression_model/train_pipeline.py pytest \ -s \