From 8168884b7dcbbe0230850e47ebfababb79659770 Mon Sep 17 00:00:00 2001 From: Sandra Chung Date: Wed, 20 Sep 2017 18:43:07 -0600 Subject: [PATCH 1/4] swapped psql modules for easy deploy to lambda --- .gitignore | 1 + README.md | 8 ++++++++ connect.py | 8 ++++---- requirements.txt | 5 +++-- 4 files changed, 16 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index c23a545..8fcd359 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ ve +venv settings.py __pycache__ *.pyc \ No newline at end of file diff --git a/README.md b/README.md index 58ba4cb..70d9fec 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,14 @@ This Python 3.x script tries to connect to databases and sends a Slack alert wit * Create a settings file and fill it out: `cp settings.py.example settings.py` * Run script with `python check_dbs.py` +## Deploy to AWS Lambda with zappa + +* Check out the zappa branch +* `pip install zappa==0.44.3` +* set up your AWS creds that allow you to create s3 buckets +* `zappa init` and answer the questions to create your zappa_settings.json file. Modular path to function is `check_dbs.main` +* `zappa deploy production` or whatever the name of your stage is + ## TODO: * Add support for email and SMS alerts. \ No newline at end of file diff --git a/connect.py b/connect.py index 2263e6c..f70ce71 100644 --- a/connect.py +++ b/connect.py @@ -1,4 +1,4 @@ -import psycopg2 +import pg8000 import sys import settings import pymysql.cursors @@ -8,14 +8,14 @@ def connect_psql(psql): try: - con = psycopg2.connect(dbname = psql['db'], + con = pg8000.connect(database = psql['db'], host = psql['host'], port = psql['port'], user = psql['user'], password = psql['pwd']) con.close() except: - message = "{} connection error: {}".format(psql['name'],sys.exc_info()) + message = "db-status {} connection error: {}".format(psql['name'],sys.exc_info()) if settings.slack_alerts: post_to_slack(message) @@ -30,7 +30,7 @@ def connect_mysql(mysql): cursorclass=pymysql.cursors.DictCursor) con.close() except: - message = "{} connection error: {}".format(mysql['name'],sys.exc_info()) + message = "db-status {} connection error: {}".format(mysql['name'],sys.exc_info()) if settings.slack_alerts: post_to_slack(message) diff --git a/requirements.txt b/requirements.txt index 625776d..2b9d7c6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ -psycopg2==2.7.1 -PyMySQL==0.7.11 \ No newline at end of file +PyMySQL==0.7.11 +zappa==0.44.3 +pg8000==1.11.0 \ No newline at end of file From 0f0df55307dfc1fe117048e6da10d070924606dc Mon Sep 17 00:00:00 2001 From: Sandra Chung Date: Thu, 21 Sep 2017 08:20:11 -0600 Subject: [PATCH 2/4] now there's a lambda deploy option --- README.md | 14 +++++++------ connect.py | 42 +++++++++++++++++++++++++------------ requirements-lambda.txt | 2 ++ requirements-nolambda.txt | 2 ++ requirements.txt | 3 +-- settings.py.example | 3 ++- zappa_settings.json.example | 15 +++++++++++++ 7 files changed, 59 insertions(+), 22 deletions(-) create mode 100644 requirements-lambda.txt create mode 100644 requirements-nolambda.txt create mode 100644 zappa_settings.json.example diff --git a/README.md b/README.md index 70d9fec..290b51b 100644 --- a/README.md +++ b/README.md @@ -1,19 +1,21 @@ -This Python 3.x script tries to connect to databases and sends a Slack alert with error text if it fails. +This Python 3.6 script tries to connect to databases and sends a Slack alert with error text if it fails. Note: Python3.6 is compatible with AWS Lambda deployment. If you're not deploying to AWS Lambda, any other verson of Python3 will do. ## Setup -* Create a Python 3.x virtualenv: `virtualenv -p /path/to/python3 venv` +* Create a Python 3.6 virtualenv: `virtualenv -p /path/to/python3.6 venv` * Activate the virtualenv: `source venv/bin/activate` -* Install requirements: `pip install -r requirements.txt` -* Create a settings file and fill it out: `cp settings.py.example settings.py` +* Create a settings file and fill it out: `cp settings.py.example settings.py`. +* If you want to deploy the script to lambda, set `lambda_deploy = True` and `pip install -r requirements-lambda.txt` +* Else, leave `lambda_deploy = False` and `pip install -r requirements-nolambda.txt` * Run script with `python check_dbs.py` ## Deploy to AWS Lambda with zappa +* in settings.py, set `lambda_deploy = True` * Check out the zappa branch * `pip install zappa==0.44.3` -* set up your AWS creds that allow you to create s3 buckets -* `zappa init` and answer the questions to create your zappa_settings.json file. Modular path to function is `check_dbs.main` +* copy zappa_settings.json.example to zappa_settings.json and fill it out +* set up your AWS creds that allow you to create s3 buckets, and add your target s3 bucket to zappa_settings.json * `zappa deploy production` or whatever the name of your stage is ## TODO: diff --git a/connect.py b/connect.py index f70ce71..18845cf 100644 --- a/connect.py +++ b/connect.py @@ -1,23 +1,39 @@ -import pg8000 import sys import settings import pymysql.cursors import requests import json - +if settings.lambda_deploy: + import pg8000 +else: + import psycopg2 def connect_psql(psql): - try: - con = pg8000.connect(database = psql['db'], - host = psql['host'], - port = psql['port'], - user = psql['user'], - password = psql['pwd']) - con.close() - except: - message = "db-status {} connection error: {}".format(psql['name'],sys.exc_info()) - if settings.slack_alerts: - post_to_slack(message) + if settings.lambda_deploy: + try: + con = pg8000.connect(database = psql['db'], + host = psql['host'], + port = psql['port'], + user = psql['user'], + password = psql['pwd']) + con.close() + + except: + message = "db-status {} connection error: {}".format(psql['name'],sys.exc_info()) + if settings.slack_alerts: + post_to_slack(message) + else: + try: + con = psycopg2.connect(dbname = psql['db'], + host = psql['host'], + port = psql['port'], + user = psql['user'], + password = psql['pwd']) + con.close() + except: + message = "db-status {} connection error: {}".format(psql['name'],sys.exc_info()) + if settings.slack_alerts: + post_to_slack(message) def connect_mysql(mysql): diff --git a/requirements-lambda.txt b/requirements-lambda.txt new file mode 100644 index 0000000..fef1e71 --- /dev/null +++ b/requirements-lambda.txt @@ -0,0 +1,2 @@ +-r requirements.txt +pg8000==1.11.0 \ No newline at end of file diff --git a/requirements-nolambda.txt b/requirements-nolambda.txt new file mode 100644 index 0000000..d03acf5 --- /dev/null +++ b/requirements-nolambda.txt @@ -0,0 +1,2 @@ +-r requirements.txt +psycopg2==2.7.1 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 2b9d7c6..4bc4e36 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,2 @@ PyMySQL==0.7.11 -zappa==0.44.3 -pg8000==1.11.0 \ No newline at end of file +zappa==0.44.3 \ No newline at end of file diff --git a/settings.py.example b/settings.py.example index da0ae44..45a87d8 100644 --- a/settings.py.example +++ b/settings.py.example @@ -19,5 +19,6 @@ databases = { } } -slack_alerts = False +lambda_deploy = False +slack_alerts = True slack_webhook_url = 'set this up at https://yourslackteam.slack.com/apps/search?q=webhooks' \ No newline at end of file diff --git a/zappa_settings.json.example b/zappa_settings.json.example new file mode 100644 index 0000000..777e97d --- /dev/null +++ b/zappa_settings.json.example @@ -0,0 +1,15 @@ +{ + "production": { + "app_function": "check_dbs.main", + "aws_region": "us-west-1", + "profile_name": "default", + "project_name": "db-status", + "runtime": "python3.6", + "s3_bucket": "", + "keep_warm": false, + "events": [{ + "function": "check_dbs.main", // The function to execute + "expression": "rate(15 minutes)" // When to execute it (in cron or rate format) + }], + } +} \ No newline at end of file From 401c371430b2e50c2a70981956f301974d5faaec Mon Sep 17 00:00:00 2001 From: Sandra Chung Date: Thu, 21 Sep 2017 08:23:10 -0600 Subject: [PATCH 3/4] updated requirements and readme --- README.md | 2 -- requirements-lambda.txt | 1 + requirements.txt | 3 +-- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 290b51b..268b3ed 100644 --- a/README.md +++ b/README.md @@ -12,8 +12,6 @@ This Python 3.6 script tries to connect to databases and sends a Slack alert wit ## Deploy to AWS Lambda with zappa * in settings.py, set `lambda_deploy = True` -* Check out the zappa branch -* `pip install zappa==0.44.3` * copy zappa_settings.json.example to zappa_settings.json and fill it out * set up your AWS creds that allow you to create s3 buckets, and add your target s3 bucket to zappa_settings.json * `zappa deploy production` or whatever the name of your stage is diff --git a/requirements-lambda.txt b/requirements-lambda.txt index fef1e71..49a4040 100644 --- a/requirements-lambda.txt +++ b/requirements-lambda.txt @@ -1,2 +1,3 @@ -r requirements.txt +zappa==0.44.3 pg8000==1.11.0 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 4bc4e36..0ab10b1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1 @@ -PyMySQL==0.7.11 -zappa==0.44.3 \ No newline at end of file +PyMySQL==0.7.11 \ No newline at end of file From 2b7232bdcb28430f12212ee263526f3c80aba57f Mon Sep 17 00:00:00 2001 From: Sandra Chung Date: Thu, 21 Sep 2017 08:30:43 -0600 Subject: [PATCH 4/4] refactored alerts --- check_dbs.py | 2 +- connect.py | 28 +++++++++++----------------- 2 files changed, 12 insertions(+), 18 deletions(-) diff --git a/check_dbs.py b/check_dbs.py index 8b3c908..162506d 100644 --- a/check_dbs.py +++ b/check_dbs.py @@ -1,4 +1,4 @@ -from connect import connect_psql, connect_mysql, post_to_slack +from connect import connect_psql, connect_mysql import settings def main(): diff --git a/connect.py b/connect.py index 18845cf..a0eef5b 100644 --- a/connect.py +++ b/connect.py @@ -8,6 +8,13 @@ else: import psycopg2 +def slack_alert(name, slack_webhook_url = settings.slack_webhook_url): + message = "db-status {} connection error: {}".format(name,sys.exc_info()) + text = json.dumps({ 'text' : message }) + print(text) + if settings.slack_alerts: + r = requests.post(slack_webhook_url, data = text) + def connect_psql(psql): if settings.lambda_deploy: try: @@ -17,11 +24,9 @@ def connect_psql(psql): user = psql['user'], password = psql['pwd']) con.close() - except: - message = "db-status {} connection error: {}".format(psql['name'],sys.exc_info()) - if settings.slack_alerts: - post_to_slack(message) + slack_alert(psql['name']) + else: try: con = psycopg2.connect(dbname = psql['db'], @@ -31,10 +36,7 @@ def connect_psql(psql): password = psql['pwd']) con.close() except: - message = "db-status {} connection error: {}".format(psql['name'],sys.exc_info()) - if settings.slack_alerts: - post_to_slack(message) - + slack_alert(psql['name']) def connect_mysql(mysql): try: @@ -46,12 +48,4 @@ def connect_mysql(mysql): cursorclass=pymysql.cursors.DictCursor) con.close() except: - message = "db-status {} connection error: {}".format(mysql['name'],sys.exc_info()) - if settings.slack_alerts: - post_to_slack(message) - - -def post_to_slack(message, slack_webhook_url=settings.slack_webhook_url): - text = json.dumps({ 'text' : message }) - print(text) - r = requests.post(slack_webhook_url, data = text) \ No newline at end of file + slack_alert(mysql['name']) \ No newline at end of file