From a162bfada6deebfc5def2304c7be9560828f25a6 Mon Sep 17 00:00:00 2001 From: rachelannelise Date: Fri, 18 Dec 2020 12:29:20 -0500 Subject: [PATCH 01/35] updating required packages --- requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index 0be5471..9708069 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ boto3>=1.4.4,<1.17 botocore>=1.5.0,<1.20 -google-api-core>=1.16.0,<1.24.0 -protobuf>=3.6.0,<3.15 +google-api-core<2dev,>=1.21.0 +protobuf<3.12,>=3.6.0 google-api-python-client psycopg2cffi-compat six @@ -10,4 +10,4 @@ xlsxwriter tabulate oauth2client pytest -pyyaml<=5.1 \ No newline at end of file +pyyaml<=5.1 From 58c13ddef9d55f34af77a0f01368bd3d03a06e42 Mon Sep 17 00:00:00 2001 From: rachelannelise Date: Fri, 18 Dec 2020 12:38:44 -0500 Subject: [PATCH 02/35] upgrading requirements --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 9708069..e9e6651 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ boto3>=1.4.4,<1.17 botocore>=1.5.0,<1.20 google-api-core<2dev,>=1.21.0 -protobuf<3.12,>=3.6.0 +protobuf>=3.12.0 google-api-python-client psycopg2cffi-compat six From f993218613cdff27dc9a678f8c135bd5c4322749 Mon Sep 17 00:00:00 2001 From: rachelannelise Date: Fri, 18 Dec 2020 13:33:07 -0500 Subject: [PATCH 03/35] reorg for package consistency --- dbt/__init__.py | 1 - dbt/redshift/__init__.py | 3 --- dbt/requirements.txt | 4 ---- parsely_raw_data/__init__.py | 5 +++-- parsely_raw_data/dbt/__init__.py | 7 ++++++ .../redshift/Parsely_DPL_DBT_etl_model.png | Bin .../dbt}/redshift/README.md | 0 parsely_raw_data/dbt/redshift/__init__.py | 20 ++++++++++++++++++ .../dbt}/redshift/dbt_project.yml | 0 .../dbt}/redshift/models/base/calendar.sql | 0 .../models/base/parsely_all_events.sql | 0 .../models/base/parsely_base_events.sql | 0 .../base/parsely_parent_pageview_keys.sql | 0 .../base/parsely_parent_videostart_keys.sql | 0 .../redshift/models/base/parsely_rawdata.sql | 0 .../parsely_pageview_behavior_workflow.sql | 0 .../parsely_pageview_engagedtime.sql | 0 .../parsely_pageview_hanging_engagedtime.sql | 0 .../parsely_videoview_behavior_workflow.sql | 0 .../parsely_videoview_engagedtime.sql | 0 .../parsely_videoview_hanging_engagedtime.sql | 0 .../models/campaigns/parsely_campaigns.sql | 0 .../models/content/parsely_post_content.sql | 0 .../models/content/parsely_video_content.sql | 0 .../models/custom/parsely_custom_events.sql | 0 .../excluded_events/parsely_bot_traffic.sql | 0 .../models/excluded_events/parsely_errors.sql | 0 .../pageviews/parsely_incoming_pageviews.sql | 0 .../parsely_pageviews_sessionized.sql | 0 .../sessions/parsely_entry_exit_urls.sql | 0 .../sessions/parsely_incoming_sessions.sql | 0 .../models/sessions/parsely_sessions.sql | 0 .../models/users/parsely_incoming_users.sql | 0 .../redshift/models/users/parsely_users.sql | 0 .../parsely_incoming_videoviews.sql | 0 .../parsely_videoviews_sessionized.sql | 0 .../dbt}/redshift/redshift_etl.py | 4 ++-- .../dbt}/redshift/run_parsely_dpl.sh | 0 .../dbt/redshift/settings/__init__.py | 3 +++ .../dbt}/redshift/settings/default.py | 0 .../dbt}/redshift/settings/default.py.schema | 0 .../redshift/settings/merge_settings_yaml.py | 2 +- requirements.txt | 11 ++++++---- setup.py | 2 +- 44 files changed, 44 insertions(+), 18 deletions(-) delete mode 100644 dbt/__init__.py delete mode 100644 dbt/redshift/__init__.py delete mode 100644 dbt/requirements.txt create mode 100644 parsely_raw_data/dbt/__init__.py rename {dbt => parsely_raw_data/dbt}/redshift/Parsely_DPL_DBT_etl_model.png (100%) rename {dbt => parsely_raw_data/dbt}/redshift/README.md (100%) create mode 100644 parsely_raw_data/dbt/redshift/__init__.py rename {dbt => parsely_raw_data/dbt}/redshift/dbt_project.yml (100%) rename {dbt => parsely_raw_data/dbt}/redshift/models/base/calendar.sql (100%) rename {dbt => parsely_raw_data/dbt}/redshift/models/base/parsely_all_events.sql (100%) rename {dbt => parsely_raw_data/dbt}/redshift/models/base/parsely_base_events.sql (100%) rename {dbt => parsely_raw_data/dbt}/redshift/models/base/parsely_parent_pageview_keys.sql (100%) rename {dbt => parsely_raw_data/dbt}/redshift/models/base/parsely_parent_videostart_keys.sql (100%) rename {dbt => parsely_raw_data/dbt}/redshift/models/base/parsely_rawdata.sql (100%) rename {dbt => parsely_raw_data/dbt}/redshift/models/behavior_workflow/parsely_pageview_behavior_workflow.sql (100%) rename {dbt => parsely_raw_data/dbt}/redshift/models/behavior_workflow/parsely_pageview_engagedtime.sql (100%) rename {dbt => parsely_raw_data/dbt}/redshift/models/behavior_workflow/parsely_pageview_hanging_engagedtime.sql (100%) rename {dbt => parsely_raw_data/dbt}/redshift/models/behavior_workflow/parsely_videoview_behavior_workflow.sql (100%) rename {dbt => parsely_raw_data/dbt}/redshift/models/behavior_workflow/parsely_videoview_engagedtime.sql (100%) rename {dbt => parsely_raw_data/dbt}/redshift/models/behavior_workflow/parsely_videoview_hanging_engagedtime.sql (100%) rename {dbt => parsely_raw_data/dbt}/redshift/models/campaigns/parsely_campaigns.sql (100%) rename {dbt => parsely_raw_data/dbt}/redshift/models/content/parsely_post_content.sql (100%) rename {dbt => parsely_raw_data/dbt}/redshift/models/content/parsely_video_content.sql (100%) rename {dbt => parsely_raw_data/dbt}/redshift/models/custom/parsely_custom_events.sql (100%) rename {dbt => parsely_raw_data/dbt}/redshift/models/excluded_events/parsely_bot_traffic.sql (100%) rename {dbt => parsely_raw_data/dbt}/redshift/models/excluded_events/parsely_errors.sql (100%) rename {dbt => parsely_raw_data/dbt}/redshift/models/pageviews/parsely_incoming_pageviews.sql (100%) rename {dbt => parsely_raw_data/dbt}/redshift/models/pageviews/parsely_pageviews_sessionized.sql (100%) rename {dbt => parsely_raw_data/dbt}/redshift/models/sessions/parsely_entry_exit_urls.sql (100%) rename {dbt => parsely_raw_data/dbt}/redshift/models/sessions/parsely_incoming_sessions.sql (100%) rename {dbt => parsely_raw_data/dbt}/redshift/models/sessions/parsely_sessions.sql (100%) rename {dbt => parsely_raw_data/dbt}/redshift/models/users/parsely_incoming_users.sql (100%) rename {dbt => parsely_raw_data/dbt}/redshift/models/users/parsely_users.sql (100%) rename {dbt => parsely_raw_data/dbt}/redshift/models/videoviews/parsely_incoming_videoviews.sql (100%) rename {dbt => parsely_raw_data/dbt}/redshift/models/videoviews/parsely_videoviews_sessionized.sql (100%) rename {dbt => parsely_raw_data/dbt}/redshift/redshift_etl.py (98%) rename {dbt => parsely_raw_data/dbt}/redshift/run_parsely_dpl.sh (100%) create mode 100644 parsely_raw_data/dbt/redshift/settings/__init__.py rename {dbt => parsely_raw_data/dbt}/redshift/settings/default.py (100%) rename {dbt => parsely_raw_data/dbt}/redshift/settings/default.py.schema (100%) rename {dbt => parsely_raw_data/dbt}/redshift/settings/merge_settings_yaml.py (97%) diff --git a/dbt/__init__.py b/dbt/__init__.py deleted file mode 100644 index c396168..0000000 --- a/dbt/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from __future__ import absolute_import diff --git a/dbt/redshift/__init__.py b/dbt/redshift/__init__.py deleted file mode 100644 index c7b1bc9..0000000 --- a/dbt/redshift/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from __future__ import absolute_import - -from .redshift_etl import migrate_from_s3_by_day diff --git a/dbt/requirements.txt b/dbt/requirements.txt deleted file mode 100644 index 5815d69..0000000 --- a/dbt/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -dbt==0.17.2 -python-dateutil==2.8.1 -PyYAML==5.3.1 -sqlalchemy==1.3.20 \ No newline at end of file diff --git a/parsely_raw_data/__init__.py b/parsely_raw_data/__init__.py index cb6daab..935436b 100644 --- a/parsely_raw_data/__init__.py +++ b/parsely_raw_data/__init__.py @@ -14,9 +14,9 @@ limitations under the License. """ -__version__ = "2.4.0" +__version__ = "2.4.1" -from . import bigquery, docgen, redshift, s3, samples, schema, stream, utils +from . import bigquery, docgen, redshift, s3, samples, schema, stream, utils, dbt __all__ = [ "bigquery", @@ -27,6 +27,7 @@ "schema", "stream", "utils", + "dbt" ] BOOLEAN_FIELDS = {"flags_is_amp"} diff --git a/parsely_raw_data/dbt/__init__.py b/parsely_raw_data/dbt/__init__.py new file mode 100644 index 0000000..e61b783 --- /dev/null +++ b/parsely_raw_data/dbt/__init__.py @@ -0,0 +1,7 @@ +from __future__ import absolute_import + +from . import redshift as parsely_dbt_redshift + +__all__ = [ + "parsely_dbt_redshift" +] diff --git a/dbt/redshift/Parsely_DPL_DBT_etl_model.png b/parsely_raw_data/dbt/redshift/Parsely_DPL_DBT_etl_model.png similarity index 100% rename from dbt/redshift/Parsely_DPL_DBT_etl_model.png rename to parsely_raw_data/dbt/redshift/Parsely_DPL_DBT_etl_model.png diff --git a/dbt/redshift/README.md b/parsely_raw_data/dbt/redshift/README.md similarity index 100% rename from dbt/redshift/README.md rename to parsely_raw_data/dbt/redshift/README.md diff --git a/parsely_raw_data/dbt/redshift/__init__.py b/parsely_raw_data/dbt/redshift/__init__.py new file mode 100644 index 0000000..a104bea --- /dev/null +++ b/parsely_raw_data/dbt/redshift/__init__.py @@ -0,0 +1,20 @@ +from __future__ import absolute_import + +from .redshift_etl import migrate_from_s3_by_day +from .settings import ( + DBT_PROFILE_LOCATION, + DBT_PROFILE_TARGET_NAME, + ETL_END_DATE, + ETL_KEEP_RAW_DATA, + ETL_START_DATE, + PARSELY_RAW_DATA_TABLE, + REDSHIFT_DATABASE, + REDSHIFT_HOST, + REDSHIFT_PASSWORD, + REDSHIFT_PORT, + REDSHIFT_USER, + S3_AWS_ACCESS_KEY_ID, + S3_AWS_SECRET_ACCESS_KEY, + S3_NETWORK_NAME, +) +from .settings import migrate_settings \ No newline at end of file diff --git a/dbt/redshift/dbt_project.yml b/parsely_raw_data/dbt/redshift/dbt_project.yml similarity index 100% rename from dbt/redshift/dbt_project.yml rename to parsely_raw_data/dbt/redshift/dbt_project.yml diff --git a/dbt/redshift/models/base/calendar.sql b/parsely_raw_data/dbt/redshift/models/base/calendar.sql similarity index 100% rename from dbt/redshift/models/base/calendar.sql rename to parsely_raw_data/dbt/redshift/models/base/calendar.sql diff --git a/dbt/redshift/models/base/parsely_all_events.sql b/parsely_raw_data/dbt/redshift/models/base/parsely_all_events.sql similarity index 100% rename from dbt/redshift/models/base/parsely_all_events.sql rename to parsely_raw_data/dbt/redshift/models/base/parsely_all_events.sql diff --git a/dbt/redshift/models/base/parsely_base_events.sql b/parsely_raw_data/dbt/redshift/models/base/parsely_base_events.sql similarity index 100% rename from dbt/redshift/models/base/parsely_base_events.sql rename to parsely_raw_data/dbt/redshift/models/base/parsely_base_events.sql diff --git a/dbt/redshift/models/base/parsely_parent_pageview_keys.sql b/parsely_raw_data/dbt/redshift/models/base/parsely_parent_pageview_keys.sql similarity index 100% rename from dbt/redshift/models/base/parsely_parent_pageview_keys.sql rename to parsely_raw_data/dbt/redshift/models/base/parsely_parent_pageview_keys.sql diff --git a/dbt/redshift/models/base/parsely_parent_videostart_keys.sql b/parsely_raw_data/dbt/redshift/models/base/parsely_parent_videostart_keys.sql similarity index 100% rename from dbt/redshift/models/base/parsely_parent_videostart_keys.sql rename to parsely_raw_data/dbt/redshift/models/base/parsely_parent_videostart_keys.sql diff --git a/dbt/redshift/models/base/parsely_rawdata.sql b/parsely_raw_data/dbt/redshift/models/base/parsely_rawdata.sql similarity index 100% rename from dbt/redshift/models/base/parsely_rawdata.sql rename to parsely_raw_data/dbt/redshift/models/base/parsely_rawdata.sql diff --git a/dbt/redshift/models/behavior_workflow/parsely_pageview_behavior_workflow.sql b/parsely_raw_data/dbt/redshift/models/behavior_workflow/parsely_pageview_behavior_workflow.sql similarity index 100% rename from dbt/redshift/models/behavior_workflow/parsely_pageview_behavior_workflow.sql rename to parsely_raw_data/dbt/redshift/models/behavior_workflow/parsely_pageview_behavior_workflow.sql diff --git a/dbt/redshift/models/behavior_workflow/parsely_pageview_engagedtime.sql b/parsely_raw_data/dbt/redshift/models/behavior_workflow/parsely_pageview_engagedtime.sql similarity index 100% rename from dbt/redshift/models/behavior_workflow/parsely_pageview_engagedtime.sql rename to parsely_raw_data/dbt/redshift/models/behavior_workflow/parsely_pageview_engagedtime.sql diff --git a/dbt/redshift/models/behavior_workflow/parsely_pageview_hanging_engagedtime.sql b/parsely_raw_data/dbt/redshift/models/behavior_workflow/parsely_pageview_hanging_engagedtime.sql similarity index 100% rename from dbt/redshift/models/behavior_workflow/parsely_pageview_hanging_engagedtime.sql rename to parsely_raw_data/dbt/redshift/models/behavior_workflow/parsely_pageview_hanging_engagedtime.sql diff --git a/dbt/redshift/models/behavior_workflow/parsely_videoview_behavior_workflow.sql b/parsely_raw_data/dbt/redshift/models/behavior_workflow/parsely_videoview_behavior_workflow.sql similarity index 100% rename from dbt/redshift/models/behavior_workflow/parsely_videoview_behavior_workflow.sql rename to parsely_raw_data/dbt/redshift/models/behavior_workflow/parsely_videoview_behavior_workflow.sql diff --git a/dbt/redshift/models/behavior_workflow/parsely_videoview_engagedtime.sql b/parsely_raw_data/dbt/redshift/models/behavior_workflow/parsely_videoview_engagedtime.sql similarity index 100% rename from dbt/redshift/models/behavior_workflow/parsely_videoview_engagedtime.sql rename to parsely_raw_data/dbt/redshift/models/behavior_workflow/parsely_videoview_engagedtime.sql diff --git a/dbt/redshift/models/behavior_workflow/parsely_videoview_hanging_engagedtime.sql b/parsely_raw_data/dbt/redshift/models/behavior_workflow/parsely_videoview_hanging_engagedtime.sql similarity index 100% rename from dbt/redshift/models/behavior_workflow/parsely_videoview_hanging_engagedtime.sql rename to parsely_raw_data/dbt/redshift/models/behavior_workflow/parsely_videoview_hanging_engagedtime.sql diff --git a/dbt/redshift/models/campaigns/parsely_campaigns.sql b/parsely_raw_data/dbt/redshift/models/campaigns/parsely_campaigns.sql similarity index 100% rename from dbt/redshift/models/campaigns/parsely_campaigns.sql rename to parsely_raw_data/dbt/redshift/models/campaigns/parsely_campaigns.sql diff --git a/dbt/redshift/models/content/parsely_post_content.sql b/parsely_raw_data/dbt/redshift/models/content/parsely_post_content.sql similarity index 100% rename from dbt/redshift/models/content/parsely_post_content.sql rename to parsely_raw_data/dbt/redshift/models/content/parsely_post_content.sql diff --git a/dbt/redshift/models/content/parsely_video_content.sql b/parsely_raw_data/dbt/redshift/models/content/parsely_video_content.sql similarity index 100% rename from dbt/redshift/models/content/parsely_video_content.sql rename to parsely_raw_data/dbt/redshift/models/content/parsely_video_content.sql diff --git a/dbt/redshift/models/custom/parsely_custom_events.sql b/parsely_raw_data/dbt/redshift/models/custom/parsely_custom_events.sql similarity index 100% rename from dbt/redshift/models/custom/parsely_custom_events.sql rename to parsely_raw_data/dbt/redshift/models/custom/parsely_custom_events.sql diff --git a/dbt/redshift/models/excluded_events/parsely_bot_traffic.sql b/parsely_raw_data/dbt/redshift/models/excluded_events/parsely_bot_traffic.sql similarity index 100% rename from dbt/redshift/models/excluded_events/parsely_bot_traffic.sql rename to parsely_raw_data/dbt/redshift/models/excluded_events/parsely_bot_traffic.sql diff --git a/dbt/redshift/models/excluded_events/parsely_errors.sql b/parsely_raw_data/dbt/redshift/models/excluded_events/parsely_errors.sql similarity index 100% rename from dbt/redshift/models/excluded_events/parsely_errors.sql rename to parsely_raw_data/dbt/redshift/models/excluded_events/parsely_errors.sql diff --git a/dbt/redshift/models/pageviews/parsely_incoming_pageviews.sql b/parsely_raw_data/dbt/redshift/models/pageviews/parsely_incoming_pageviews.sql similarity index 100% rename from dbt/redshift/models/pageviews/parsely_incoming_pageviews.sql rename to parsely_raw_data/dbt/redshift/models/pageviews/parsely_incoming_pageviews.sql diff --git a/dbt/redshift/models/pageviews/parsely_pageviews_sessionized.sql b/parsely_raw_data/dbt/redshift/models/pageviews/parsely_pageviews_sessionized.sql similarity index 100% rename from dbt/redshift/models/pageviews/parsely_pageviews_sessionized.sql rename to parsely_raw_data/dbt/redshift/models/pageviews/parsely_pageviews_sessionized.sql diff --git a/dbt/redshift/models/sessions/parsely_entry_exit_urls.sql b/parsely_raw_data/dbt/redshift/models/sessions/parsely_entry_exit_urls.sql similarity index 100% rename from dbt/redshift/models/sessions/parsely_entry_exit_urls.sql rename to parsely_raw_data/dbt/redshift/models/sessions/parsely_entry_exit_urls.sql diff --git a/dbt/redshift/models/sessions/parsely_incoming_sessions.sql b/parsely_raw_data/dbt/redshift/models/sessions/parsely_incoming_sessions.sql similarity index 100% rename from dbt/redshift/models/sessions/parsely_incoming_sessions.sql rename to parsely_raw_data/dbt/redshift/models/sessions/parsely_incoming_sessions.sql diff --git a/dbt/redshift/models/sessions/parsely_sessions.sql b/parsely_raw_data/dbt/redshift/models/sessions/parsely_sessions.sql similarity index 100% rename from dbt/redshift/models/sessions/parsely_sessions.sql rename to parsely_raw_data/dbt/redshift/models/sessions/parsely_sessions.sql diff --git a/dbt/redshift/models/users/parsely_incoming_users.sql b/parsely_raw_data/dbt/redshift/models/users/parsely_incoming_users.sql similarity index 100% rename from dbt/redshift/models/users/parsely_incoming_users.sql rename to parsely_raw_data/dbt/redshift/models/users/parsely_incoming_users.sql diff --git a/dbt/redshift/models/users/parsely_users.sql b/parsely_raw_data/dbt/redshift/models/users/parsely_users.sql similarity index 100% rename from dbt/redshift/models/users/parsely_users.sql rename to parsely_raw_data/dbt/redshift/models/users/parsely_users.sql diff --git a/dbt/redshift/models/videoviews/parsely_incoming_videoviews.sql b/parsely_raw_data/dbt/redshift/models/videoviews/parsely_incoming_videoviews.sql similarity index 100% rename from dbt/redshift/models/videoviews/parsely_incoming_videoviews.sql rename to parsely_raw_data/dbt/redshift/models/videoviews/parsely_incoming_videoviews.sql diff --git a/dbt/redshift/models/videoviews/parsely_videoviews_sessionized.sql b/parsely_raw_data/dbt/redshift/models/videoviews/parsely_videoviews_sessionized.sql similarity index 100% rename from dbt/redshift/models/videoviews/parsely_videoviews_sessionized.sql rename to parsely_raw_data/dbt/redshift/models/videoviews/parsely_videoviews_sessionized.sql diff --git a/dbt/redshift/redshift_etl.py b/parsely_raw_data/dbt/redshift/redshift_etl.py similarity index 98% rename from dbt/redshift/redshift_etl.py rename to parsely_raw_data/dbt/redshift/redshift_etl.py index 1c0d696..1f67c0e 100644 --- a/dbt/redshift/redshift_etl.py +++ b/parsely_raw_data/dbt/redshift/redshift_etl.py @@ -7,7 +7,7 @@ from parsely_raw_data import redshift as parsely_redshift from parsely_raw_data import utils as parsely_utils -from dbt.redshift.settings.default import ( +from parsely_raw_data.dbt.redshift.settings import ( DBT_PROFILE_LOCATION, DBT_PROFILE_TARGET_NAME, ETL_END_DATE, @@ -23,7 +23,7 @@ S3_AWS_SECRET_ACCESS_KEY, S3_NETWORK_NAME, ) -from dbt.redshift.settings.merge_settings_yaml import migrate_settings +from parsely_raw_data.dbt.redshift.settings.merge_settings_yaml import migrate_settings SETTINGS_ARG_MAPPING = { 'table_name': PARSELY_RAW_DATA_TABLE, diff --git a/dbt/redshift/run_parsely_dpl.sh b/parsely_raw_data/dbt/redshift/run_parsely_dpl.sh similarity index 100% rename from dbt/redshift/run_parsely_dpl.sh rename to parsely_raw_data/dbt/redshift/run_parsely_dpl.sh diff --git a/parsely_raw_data/dbt/redshift/settings/__init__.py b/parsely_raw_data/dbt/redshift/settings/__init__.py new file mode 100644 index 0000000..4802098 --- /dev/null +++ b/parsely_raw_data/dbt/redshift/settings/__init__.py @@ -0,0 +1,3 @@ +from .default import * +from .merge_settings_yaml import migrate_settings + diff --git a/dbt/redshift/settings/default.py b/parsely_raw_data/dbt/redshift/settings/default.py similarity index 100% rename from dbt/redshift/settings/default.py rename to parsely_raw_data/dbt/redshift/settings/default.py diff --git a/dbt/redshift/settings/default.py.schema b/parsely_raw_data/dbt/redshift/settings/default.py.schema similarity index 100% rename from dbt/redshift/settings/default.py.schema rename to parsely_raw_data/dbt/redshift/settings/default.py.schema diff --git a/dbt/redshift/settings/merge_settings_yaml.py b/parsely_raw_data/dbt/redshift/settings/merge_settings_yaml.py similarity index 97% rename from dbt/redshift/settings/merge_settings_yaml.py rename to parsely_raw_data/dbt/redshift/settings/merge_settings_yaml.py index 00096c7..b9e4434 100644 --- a/dbt/redshift/settings/merge_settings_yaml.py +++ b/parsely_raw_data/dbt/redshift/settings/merge_settings_yaml.py @@ -1,5 +1,5 @@ import yaml -from dbt.redshift.settings.default import * +from .default import * SETTINGS_VAR_MAPPING = [ {'location': 'profile', 'settings': DBT_PROFILE_NAME}, diff --git a/requirements.txt b/requirements.txt index e9e6651..58f8fef 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,13 +1,16 @@ boto3>=1.4.4,<1.17 botocore>=1.5.0,<1.20 +dbt==0.17.2 google-api-core<2dev,>=1.21.0 protobuf>=3.12.0 google-api-python-client +oauth2client psycopg2cffi-compat +pytest +python-dateutil==2.8.1 +pyyaml==5.3.1 six +sqlalchemy==1.3.20 tablib -xlsxwriter tabulate -oauth2client -pytest -pyyaml<=5.1 +xlsxwriter diff --git a/setup.py b/setup.py index 4be9ca1..0b8419e 100644 --- a/setup.py +++ b/setup.py @@ -97,7 +97,7 @@ def run_setup(): 'parsely_s3 = parsely_raw_data.s3:main', 'parsely_stream = parsely_raw_data.stream:main', 'parsely_schema = parsely_raw_data.docgen:main', - 'parsely_redshift_etl = dbt.redshift.redshift_etl:main' + 'parsely_redshift_etl = parsely_raw_data.dbt.redshift.redshift_etl:main' ] }, install_requires=install_requires, From ffaff7f81a220d233f790550c0b03e3e3a5c7200 Mon Sep 17 00:00:00 2001 From: rachelannelise Date: Fri, 18 Dec 2020 13:54:14 -0500 Subject: [PATCH 04/35] pinning requirements --- requirements.txt | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/requirements.txt b/requirements.txt index 58f8fef..4850c47 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,16 +1,16 @@ boto3>=1.4.4,<1.17 botocore>=1.5.0,<1.20 dbt==0.17.2 -google-api-core<2dev,>=1.21.0 -protobuf>=3.12.0 -google-api-python-client -oauth2client +google-api-core<1.17.0,>=1.16.0 +protobuf<3.12,>=3.6.0 +google-api-python-client==1.11.0 +oauth2client==4.1.3 psycopg2cffi-compat -pytest +pytest==6.2.1 python-dateutil==2.8.1 pyyaml==5.3.1 -six +six==1.15.0 sqlalchemy==1.3.20 -tablib -tabulate -xlsxwriter +tablib==3.0.0 +tabulate==0.8.7 +xlsxwriter==1.3.7 From d769990510a6ae3395a51ca78baf8065ff0e98fb Mon Sep 17 00:00:00 2001 From: rachelannelise Date: Fri, 18 Dec 2020 14:06:12 -0500 Subject: [PATCH 05/35] pinning requirements --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 4850c47..5e17a02 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ botocore>=1.5.0,<1.20 dbt==0.17.2 google-api-core<1.17.0,>=1.16.0 protobuf<3.12,>=3.6.0 -google-api-python-client==1.11.0 +google-api-python-client==1.10.0 oauth2client==4.1.3 psycopg2cffi-compat pytest==6.2.1 From f44ffa901aa016428abbe0cb4c790af8a901ef14 Mon Sep 17 00:00:00 2001 From: rachelannelise Date: Fri, 18 Dec 2020 14:07:21 -0500 Subject: [PATCH 06/35] pinning requirements --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 5e17a02..ecb2b69 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ botocore>=1.5.0,<1.20 dbt==0.17.2 google-api-core<1.17.0,>=1.16.0 protobuf<3.12,>=3.6.0 -google-api-python-client==1.10.0 +google-api-python-client==1.8.0 oauth2client==4.1.3 psycopg2cffi-compat pytest==6.2.1 From bfe0a5aaf29d7a00198eab3d2997c22e5e1709c5 Mon Sep 17 00:00:00 2001 From: rachelannelise Date: Fri, 18 Dec 2020 14:29:31 -0500 Subject: [PATCH 07/35] adding file pathlib for yaml merge --- .../dbt/redshift/settings/merge_settings_yaml.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/parsely_raw_data/dbt/redshift/settings/merge_settings_yaml.py b/parsely_raw_data/dbt/redshift/settings/merge_settings_yaml.py index b9e4434..1fbd991 100644 --- a/parsely_raw_data/dbt/redshift/settings/merge_settings_yaml.py +++ b/parsely_raw_data/dbt/redshift/settings/merge_settings_yaml.py @@ -1,4 +1,6 @@ import yaml +from pathlib import Path + from .default import * SETTINGS_VAR_MAPPING = [ @@ -20,7 +22,10 @@ def migrate_settings(): - with open(r'dbt/redshift/dbt_project.yml') as file: + cwd = Path.cwd().parents[0] + filepath = f'{cwd}/dbt_project.yml' + + with open(filepath) as file: dbt_profile = yaml.load(file, Loader=yaml.FullLoader) for row in SETTINGS_VAR_MAPPING: @@ -31,7 +36,7 @@ def migrate_settings(): dbt_profile['vars'][row['location']] = str(row['settings']) continue - with open(r'dbt/redshift/dbt_project.yml', 'w') as file: + with open(filepath, 'w') as file: yaml.dump(dbt_profile, file, default_style='"') stored_successfully = True From 94783d28547f09910cde4412eb1b4e3ca25183e0 Mon Sep 17 00:00:00 2001 From: rachelannelise Date: Fri, 18 Dec 2020 14:40:19 -0500 Subject: [PATCH 08/35] updating relative path location --- parsely_raw_data/dbt/redshift/settings/merge_settings_yaml.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/parsely_raw_data/dbt/redshift/settings/merge_settings_yaml.py b/parsely_raw_data/dbt/redshift/settings/merge_settings_yaml.py index 1fbd991..0269e28 100644 --- a/parsely_raw_data/dbt/redshift/settings/merge_settings_yaml.py +++ b/parsely_raw_data/dbt/redshift/settings/merge_settings_yaml.py @@ -22,8 +22,7 @@ def migrate_settings(): - cwd = Path.cwd().parents[0] - filepath = f'{cwd}/dbt_project.yml' + filepath = Path(__file__).parent / "../dbt_project.yml" with open(filepath) as file: dbt_profile = yaml.load(file, Loader=yaml.FullLoader) From ac2c96465ffd929c2981de7cbf3dda3573edb71b Mon Sep 17 00:00:00 2001 From: rachelannelise Date: Fri, 18 Dec 2020 14:41:19 -0500 Subject: [PATCH 09/35] updating relative path location --- parsely_raw_data/dbt/redshift/settings/merge_settings_yaml.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parsely_raw_data/dbt/redshift/settings/merge_settings_yaml.py b/parsely_raw_data/dbt/redshift/settings/merge_settings_yaml.py index 0269e28..b9ba02a 100644 --- a/parsely_raw_data/dbt/redshift/settings/merge_settings_yaml.py +++ b/parsely_raw_data/dbt/redshift/settings/merge_settings_yaml.py @@ -22,7 +22,7 @@ def migrate_settings(): - filepath = Path(__file__).parent / "../dbt_project.yml" + filepath = Path(__file__).parent[0] / "dbt_project.yml" with open(filepath) as file: dbt_profile = yaml.load(file, Loader=yaml.FullLoader) From a87ee8b9d4a686a57aec881d93ded14aff658dd9 Mon Sep 17 00:00:00 2001 From: rachelannelise Date: Fri, 18 Dec 2020 14:45:55 -0500 Subject: [PATCH 10/35] updating relative path location --- parsely_raw_data/dbt/redshift/settings/merge_settings_yaml.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/parsely_raw_data/dbt/redshift/settings/merge_settings_yaml.py b/parsely_raw_data/dbt/redshift/settings/merge_settings_yaml.py index b9ba02a..31c4a66 100644 --- a/parsely_raw_data/dbt/redshift/settings/merge_settings_yaml.py +++ b/parsely_raw_data/dbt/redshift/settings/merge_settings_yaml.py @@ -22,7 +22,8 @@ def migrate_settings(): - filepath = Path(__file__).parent[0] / "dbt_project.yml" + base_path = Path(__file__).parent + filepath = (base_path / "../dbt_project.yml").resolve() with open(filepath) as file: dbt_profile = yaml.load(file, Loader=yaml.FullLoader) From 5125b61db4ea36a65e062d9b025cb4456b39435a Mon Sep 17 00:00:00 2001 From: Alexander Lourenco Date: Fri, 18 Dec 2020 20:02:28 -0500 Subject: [PATCH 11/35] include dbt_project file yaml --- MANIFEST.in | 1 + 1 file changed, 1 insertion(+) diff --git a/MANIFEST.in b/MANIFEST.in index 3727525..86f4fd4 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,2 +1,3 @@ include *.txt include parsely_raw_data/*.thrift +include parsely_raw_data/dbt/redshift/dbt_project.yaml From 98483111ce7a20df3fc822e63b725f598921ccc9 Mon Sep 17 00:00:00 2001 From: Alexander Lourenco Date: Fri, 18 Dec 2020 20:21:23 -0500 Subject: [PATCH 12/35] change merge_settings to use pkg_resources --- MANIFEST.in | 2 +- .../dbt/redshift/settings/merge_settings_yaml.py | 5 ++++- setup.py | 1 + 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index 86f4fd4..fdb14bc 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1,3 @@ include *.txt include parsely_raw_data/*.thrift -include parsely_raw_data/dbt/redshift/dbt_project.yaml +include parsely_raw_data/dbt/redshift/dbt_project.yml diff --git a/parsely_raw_data/dbt/redshift/settings/merge_settings_yaml.py b/parsely_raw_data/dbt/redshift/settings/merge_settings_yaml.py index 31c4a66..c2e5b4f 100644 --- a/parsely_raw_data/dbt/redshift/settings/merge_settings_yaml.py +++ b/parsely_raw_data/dbt/redshift/settings/merge_settings_yaml.py @@ -1,4 +1,5 @@ import yaml +import pkg_resources from pathlib import Path from .default import * @@ -23,7 +24,9 @@ def migrate_settings(): base_path = Path(__file__).parent - filepath = (base_path / "../dbt_project.yml").resolve() + # filepath = (base_path / "../dbt_project.yml").resolve() + # because this is a package resource, have to reference it with pkg_resources + filepath = pkg_resources.resource_filename("parsely_raw_data", "dbt/redshift/dbt_project.yml") with open(filepath) as file: dbt_profile = yaml.load(file, Loader=yaml.FullLoader) diff --git a/setup.py b/setup.py index 0b8419e..2b60088 100644 --- a/setup.py +++ b/setup.py @@ -115,6 +115,7 @@ def run_setup(): dependency_links=dependency_links, test_suite='nose.collector', include_package_data=True, + package_data={"parsely_raw_data": ["dbt/redshift/dbt_project.yml"]}, classifiers=[ "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", From abe15ed9708092fba3490b9c393d2a112e59efd2 Mon Sep 17 00:00:00 2001 From: rachelannelise Date: Sun, 20 Dec 2020 20:00:27 -0500 Subject: [PATCH 13/35] adding dbt to package data --- parsely_raw_data/dbt/redshift/redshift_etl.py | 8 ++++---- .../dbt/redshift/settings/merge_settings_yaml.py | 2 -- setup.py | 2 +- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/parsely_raw_data/dbt/redshift/redshift_etl.py b/parsely_raw_data/dbt/redshift/redshift_etl.py index 1f67c0e..99b0def 100644 --- a/parsely_raw_data/dbt/redshift/redshift_etl.py +++ b/parsely_raw_data/dbt/redshift/redshift_etl.py @@ -1,6 +1,6 @@ from __future__ import absolute_import import logging -import os +import pkg_resources import psycopg2 import subprocess from dateutil import rrule @@ -82,9 +82,9 @@ def migrate_from_s3_by_day(network=S3_NETWORK_NAME, secret_access_key=secret_access_key) # This runs dbt once all of the new data has been copied into the raw data table - dpl_wd = os.path.join(os.getcwd(), 'dbt/redshift/') - logging.info(f'Running the dbt script located at: {dpl_wd}/run_parsely_dpl.sh') - subprocess.call(dpl_wd + "run_parsely_dpl.sh " + dbt_profiles_dir + ' ' + dbt_target, shell=True, cwd=dpl_wd) + dbt_etl_script_loc = pkg_resources.resource_filename("parsely_raw_data", "dbt/redshift/run_parsely_dpl.sh") + logging.info(f'Running the dbt script located at: {dbt_etl_script_loc}') + subprocess.call(dbt_etl_script_loc + dbt_profiles_dir + ' ' + dbt_target, shell=True, cwd=dbt_etl_script_loc.par) def main(): diff --git a/parsely_raw_data/dbt/redshift/settings/merge_settings_yaml.py b/parsely_raw_data/dbt/redshift/settings/merge_settings_yaml.py index c2e5b4f..e0e3b20 100644 --- a/parsely_raw_data/dbt/redshift/settings/merge_settings_yaml.py +++ b/parsely_raw_data/dbt/redshift/settings/merge_settings_yaml.py @@ -23,8 +23,6 @@ def migrate_settings(): - base_path = Path(__file__).parent - # filepath = (base_path / "../dbt_project.yml").resolve() # because this is a package resource, have to reference it with pkg_resources filepath = pkg_resources.resource_filename("parsely_raw_data", "dbt/redshift/dbt_project.yml") diff --git a/setup.py b/setup.py index 2b60088..0ae81ce 100644 --- a/setup.py +++ b/setup.py @@ -115,7 +115,7 @@ def run_setup(): dependency_links=dependency_links, test_suite='nose.collector', include_package_data=True, - package_data={"parsely_raw_data": ["dbt/redshift/dbt_project.yml"]}, + package_data={"parsely_raw_data": ["dbt/redshift/dbt_project.yml"], "dbt": ["dbt/redshift/run_parsely_dpl.sh"]}, classifiers=[ "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", From a1f863dfb05d647bba0d4f0436cb10ef843571dc Mon Sep 17 00:00:00 2001 From: rachelannelise Date: Sun, 20 Dec 2020 20:10:30 -0500 Subject: [PATCH 14/35] adding more pkg resources --- parsely_raw_data/dbt/redshift/redshift_etl.py | 5 +++-- setup.py | 4 +++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/parsely_raw_data/dbt/redshift/redshift_etl.py b/parsely_raw_data/dbt/redshift/redshift_etl.py index 99b0def..77ea776 100644 --- a/parsely_raw_data/dbt/redshift/redshift_etl.py +++ b/parsely_raw_data/dbt/redshift/redshift_etl.py @@ -82,9 +82,10 @@ def migrate_from_s3_by_day(network=S3_NETWORK_NAME, secret_access_key=secret_access_key) # This runs dbt once all of the new data has been copied into the raw data table - dbt_etl_script_loc = pkg_resources.resource_filename("parsely_raw_data", "dbt/redshift/run_parsely_dpl.sh") + dbt_etl_script_loc = pkg_resources.resource_filename("dbt_sh", "dbt/redshift/run_parsely_dpl.sh") + dbt_etl_cwd = pkg_resources.resource_filename("dbt_cwd", "dbt/redshift/") logging.info(f'Running the dbt script located at: {dbt_etl_script_loc}') - subprocess.call(dbt_etl_script_loc + dbt_profiles_dir + ' ' + dbt_target, shell=True, cwd=dbt_etl_script_loc.par) + subprocess.call(dbt_etl_script_loc + dbt_profiles_dir + ' ' + dbt_target, shell=True, cwd=dbt_etl_cwd) def main(): diff --git a/setup.py b/setup.py index 0ae81ce..c3206ee 100644 --- a/setup.py +++ b/setup.py @@ -115,7 +115,9 @@ def run_setup(): dependency_links=dependency_links, test_suite='nose.collector', include_package_data=True, - package_data={"parsely_raw_data": ["dbt/redshift/dbt_project.yml"], "dbt": ["dbt/redshift/run_parsely_dpl.sh"]}, + package_data={"parsely_raw_data": ["dbt/redshift/dbt_project.yml"], + "dbt_sh": ["dbt/redshift/run_parsely_dpl.sh"], + "dbt_cwd": ["dbt/redshift/"]}, classifiers=[ "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", From 9191cb438f9af38601ba45926ef342de14bf401d Mon Sep 17 00:00:00 2001 From: rachelannelise Date: Sun, 20 Dec 2020 20:16:09 -0500 Subject: [PATCH 15/35] adding more pkg resources --- parsely_raw_data/dbt/redshift/redshift_etl.py | 4 ++-- setup.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/parsely_raw_data/dbt/redshift/redshift_etl.py b/parsely_raw_data/dbt/redshift/redshift_etl.py index 77ea776..defb8f7 100644 --- a/parsely_raw_data/dbt/redshift/redshift_etl.py +++ b/parsely_raw_data/dbt/redshift/redshift_etl.py @@ -82,8 +82,8 @@ def migrate_from_s3_by_day(network=S3_NETWORK_NAME, secret_access_key=secret_access_key) # This runs dbt once all of the new data has been copied into the raw data table - dbt_etl_script_loc = pkg_resources.resource_filename("dbt_sh", "dbt/redshift/run_parsely_dpl.sh") - dbt_etl_cwd = pkg_resources.resource_filename("dbt_cwd", "dbt/redshift/") + dbt_etl_script_loc = pkg_resources.resource_filename("parsely_raw_data", "dbt/redshift/run_parsely_dpl.sh") + dbt_etl_cwd = pkg_resources.resource_filename("parsely_raw_data", "dbt/redshift/") logging.info(f'Running the dbt script located at: {dbt_etl_script_loc}') subprocess.call(dbt_etl_script_loc + dbt_profiles_dir + ' ' + dbt_target, shell=True, cwd=dbt_etl_cwd) diff --git a/setup.py b/setup.py index c3206ee..748996c 100644 --- a/setup.py +++ b/setup.py @@ -115,9 +115,9 @@ def run_setup(): dependency_links=dependency_links, test_suite='nose.collector', include_package_data=True, - package_data={"parsely_raw_data": ["dbt/redshift/dbt_project.yml"], - "dbt_sh": ["dbt/redshift/run_parsely_dpl.sh"], - "dbt_cwd": ["dbt/redshift/"]}, + package_data={"parsely_raw_data": ["dbt/redshift/dbt_project.yml", + "dbt/redshift/run_parsely_dpl.sh", + "dbt/redshift/"]}, classifiers=[ "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", From b332505f4940b2d664896f6f78b160fc106888e9 Mon Sep 17 00:00:00 2001 From: rachelannelise Date: Sun, 20 Dec 2020 20:25:06 -0500 Subject: [PATCH 16/35] adding more pkg resources --- parsely_raw_data/dbt/redshift/redshift_etl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parsely_raw_data/dbt/redshift/redshift_etl.py b/parsely_raw_data/dbt/redshift/redshift_etl.py index defb8f7..328f30c 100644 --- a/parsely_raw_data/dbt/redshift/redshift_etl.py +++ b/parsely_raw_data/dbt/redshift/redshift_etl.py @@ -85,7 +85,7 @@ def migrate_from_s3_by_day(network=S3_NETWORK_NAME, dbt_etl_script_loc = pkg_resources.resource_filename("parsely_raw_data", "dbt/redshift/run_parsely_dpl.sh") dbt_etl_cwd = pkg_resources.resource_filename("parsely_raw_data", "dbt/redshift/") logging.info(f'Running the dbt script located at: {dbt_etl_script_loc}') - subprocess.call(dbt_etl_script_loc + dbt_profiles_dir + ' ' + dbt_target, shell=True, cwd=dbt_etl_cwd) + subprocess.call(dbt_etl_script_loc + ' '+ dbt_profiles_dir + ' ' + dbt_target, shell=True, cwd=dbt_etl_cwd) def main(): From f2a4c20410ce6aa1f1e3649f5c8ef36ef655445a Mon Sep 17 00:00:00 2001 From: rachelannelise Date: Sun, 20 Dec 2020 21:10:44 -0500 Subject: [PATCH 17/35] adding profile to args --- parsely_raw_data/dbt/redshift/redshift_etl.py | 4 +++- parsely_raw_data/dbt/redshift/settings/merge_settings_yaml.py | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/parsely_raw_data/dbt/redshift/redshift_etl.py b/parsely_raw_data/dbt/redshift/redshift_etl.py index 328f30c..f424304 100644 --- a/parsely_raw_data/dbt/redshift/redshift_etl.py +++ b/parsely_raw_data/dbt/redshift/redshift_etl.py @@ -96,6 +96,8 @@ def main(): help='The last day to process data from S3 to Redshift in the format YYYY-MM-DD') parser.add_argument('--dbt_profiles_dir', required=False, default=DBT_PROFILE_LOCATION, help='The location from root that contains the .dbt/profiles.yml file, example: /home/user/.dbt/') + parser.add_argument('--dbt_profile', required=False, default='parsely_dwh', + help='The name of the dbt profile located in the local /.dbt/profiles.yml file') parser.add_argument('--dbt_target', required=False, default=DBT_PROFILE_TARGET_NAME, help='The target ie. dev, prod, or test to use within the dbt profiles.yml file.') parser.add_argument('--create-table', action='store_true', default=True, @@ -103,7 +105,7 @@ def main(): args = parser.parse_args() # Reset dbt_profile to any updated settings: - settings_migration = migrate_settings() + settings_migration = migrate_settings(profile=args.dbt_profile) if not settings_migration: logging.warning("Settings not copied to dbt_profiles.yml successfully.") raise Exception("Settings not copied to dbt_profiles.yml successfully. Please edit default.py or copy the" diff --git a/parsely_raw_data/dbt/redshift/settings/merge_settings_yaml.py b/parsely_raw_data/dbt/redshift/settings/merge_settings_yaml.py index e0e3b20..41b8d7f 100644 --- a/parsely_raw_data/dbt/redshift/settings/merge_settings_yaml.py +++ b/parsely_raw_data/dbt/redshift/settings/merge_settings_yaml.py @@ -5,7 +5,6 @@ from .default import * SETTINGS_VAR_MAPPING = [ - {'location': 'profile', 'settings': DBT_PROFILE_NAME}, {'location': 'parsely:events', 'settings': PARSELY_RAW_DATA_TABLE}, {'location': 'parsely:timezone', 'settings': ETL_TIME_ZONE}, {'location': 'parsely:actions', 'settings': ETL_PARSELY_ACTIONS}, @@ -22,9 +21,10 @@ ] -def migrate_settings(): +def migrate_settings(profile=DBT_PROFILE_NAME): # because this is a package resource, have to reference it with pkg_resources filepath = pkg_resources.resource_filename("parsely_raw_data", "dbt/redshift/dbt_project.yml") + SETTINGS_VAR_MAPPING.append({'location': 'profile', 'settings': profile}) with open(filepath) as file: dbt_profile = yaml.load(file, Loader=yaml.FullLoader) From 0f02e8f9b4e6496545759026ee09e55b854957eb Mon Sep 17 00:00:00 2001 From: rachelannelise Date: Sun, 20 Dec 2020 21:19:58 -0500 Subject: [PATCH 18/35] adding all files in dbt to manifest --- MANIFEST.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MANIFEST.in b/MANIFEST.in index fdb14bc..f51d75c 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1,3 @@ include *.txt include parsely_raw_data/*.thrift -include parsely_raw_data/dbt/redshift/dbt_project.yml +include parsely_raw_data/dbt/redshift/* \ No newline at end of file From fa74fac45983d0b3ac8b10ead9de129bd3aea1ca Mon Sep 17 00:00:00 2001 From: rachelannelise Date: Mon, 21 Dec 2020 10:08:34 -0500 Subject: [PATCH 19/35] adding models to manifest --- MANIFEST.in | 3 ++- parsely_raw_data/dbt/redshift/dbt_project.yml | 2 +- parsely_raw_data/dbt/redshift/redshift_etl.py | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index f51d75c..5222e58 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1,4 @@ include *.txt include parsely_raw_data/*.thrift -include parsely_raw_data/dbt/redshift/* \ No newline at end of file +include parsely_raw_data/dbt/redshift/* +include parsely_raw_data/dbt/redshift/models/* \ No newline at end of file diff --git a/parsely_raw_data/dbt/redshift/dbt_project.yml b/parsely_raw_data/dbt/redshift/dbt_project.yml index 3e801a0..ae83020 100644 --- a/parsely_raw_data/dbt/redshift/dbt_project.yml +++ b/parsely_raw_data/dbt/redshift/dbt_project.yml @@ -18,7 +18,7 @@ - "{% if var('etl:keep_rawdata') == true %} select 1 {% else %} truncate table {{\ \ target.schema }}.parsely_rawdata {% endif %}" - "truncate table {{var('parsely:events')}}" -"profile": "parsely-dwh" +"profile": "parsely_dwh" "source-paths": - "models" "target-path": "target" diff --git a/parsely_raw_data/dbt/redshift/redshift_etl.py b/parsely_raw_data/dbt/redshift/redshift_etl.py index f424304..dbe614a 100644 --- a/parsely_raw_data/dbt/redshift/redshift_etl.py +++ b/parsely_raw_data/dbt/redshift/redshift_etl.py @@ -85,7 +85,7 @@ def migrate_from_s3_by_day(network=S3_NETWORK_NAME, dbt_etl_script_loc = pkg_resources.resource_filename("parsely_raw_data", "dbt/redshift/run_parsely_dpl.sh") dbt_etl_cwd = pkg_resources.resource_filename("parsely_raw_data", "dbt/redshift/") logging.info(f'Running the dbt script located at: {dbt_etl_script_loc}') - subprocess.call(dbt_etl_script_loc + ' '+ dbt_profiles_dir + ' ' + dbt_target, shell=True, cwd=dbt_etl_cwd) + subprocess.call(dbt_etl_script_loc + ' ' + dbt_profiles_dir + ' ' + dbt_target, shell=True, cwd=dbt_etl_cwd) def main(): From d39966bc5d98327d50d6a3199658a3cd3744a8ad Mon Sep 17 00:00:00 2001 From: rachelannelise Date: Mon, 21 Dec 2020 10:13:48 -0500 Subject: [PATCH 20/35] adding dbt models to manifest --- MANIFEST.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MANIFEST.in b/MANIFEST.in index 5222e58..28421ad 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,4 @@ include *.txt +include *.sql include parsely_raw_data/*.thrift include parsely_raw_data/dbt/redshift/* -include parsely_raw_data/dbt/redshift/models/* \ No newline at end of file From f0107f6d05f886d207378e01dff03c90ea3e6be9 Mon Sep 17 00:00:00 2001 From: rachelannelise Date: Mon, 21 Dec 2020 10:47:38 -0500 Subject: [PATCH 21/35] adding dbt models to manifest --- MANIFEST.in | 1 - setup.py | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index 28421ad..52c7d2c 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,3 @@ include *.txt -include *.sql include parsely_raw_data/*.thrift include parsely_raw_data/dbt/redshift/* diff --git a/setup.py b/setup.py index 748996c..8e1e80a 100644 --- a/setup.py +++ b/setup.py @@ -116,7 +116,8 @@ def run_setup(): test_suite='nose.collector', include_package_data=True, package_data={"parsely_raw_data": ["dbt/redshift/dbt_project.yml", - "dbt/redshift/run_parsely_dpl.sh", + "dbt/redshift/run_parsely_dpl.sh" + "dbt/redshift/models/", "dbt/redshift/"]}, classifiers=[ "Development Status :: 5 - Production/Stable", From 5567c4d2e93fb33ee6fb6db530178b46794511d6 Mon Sep 17 00:00:00 2001 From: rachelannelise Date: Mon, 21 Dec 2020 10:48:54 -0500 Subject: [PATCH 22/35] adding init to require install --- parsely_raw_data/dbt/redshift/models/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 parsely_raw_data/dbt/redshift/models/__init__.py diff --git a/parsely_raw_data/dbt/redshift/models/__init__.py b/parsely_raw_data/dbt/redshift/models/__init__.py new file mode 100644 index 0000000..e69de29 From 6db0be0fab657c54542eaea5e285ce03c0d50774 Mon Sep 17 00:00:00 2001 From: rachelannelise Date: Mon, 21 Dec 2020 10:50:49 -0500 Subject: [PATCH 23/35] adding init to require install --- parsely_raw_data/dbt/redshift/models/base/__init__.py | 0 .../dbt/redshift/models/behavior_workflow/__init__.py | 0 parsely_raw_data/dbt/redshift/models/campaigns/__init__.py | 0 parsely_raw_data/dbt/redshift/models/content/__init__.py | 0 parsely_raw_data/dbt/redshift/models/custom/__init__.py | 0 parsely_raw_data/dbt/redshift/models/excluded_events/__init__.py | 0 parsely_raw_data/dbt/redshift/models/pageviews/__init__.py | 0 parsely_raw_data/dbt/redshift/models/sessions/__init__.py | 0 parsely_raw_data/dbt/redshift/models/users/__init__.py | 0 parsely_raw_data/dbt/redshift/models/videoviews/__init__.py | 0 10 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 parsely_raw_data/dbt/redshift/models/base/__init__.py create mode 100644 parsely_raw_data/dbt/redshift/models/behavior_workflow/__init__.py create mode 100644 parsely_raw_data/dbt/redshift/models/campaigns/__init__.py create mode 100644 parsely_raw_data/dbt/redshift/models/content/__init__.py create mode 100644 parsely_raw_data/dbt/redshift/models/custom/__init__.py create mode 100644 parsely_raw_data/dbt/redshift/models/excluded_events/__init__.py create mode 100644 parsely_raw_data/dbt/redshift/models/pageviews/__init__.py create mode 100644 parsely_raw_data/dbt/redshift/models/sessions/__init__.py create mode 100644 parsely_raw_data/dbt/redshift/models/users/__init__.py create mode 100644 parsely_raw_data/dbt/redshift/models/videoviews/__init__.py diff --git a/parsely_raw_data/dbt/redshift/models/base/__init__.py b/parsely_raw_data/dbt/redshift/models/base/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/parsely_raw_data/dbt/redshift/models/behavior_workflow/__init__.py b/parsely_raw_data/dbt/redshift/models/behavior_workflow/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/parsely_raw_data/dbt/redshift/models/campaigns/__init__.py b/parsely_raw_data/dbt/redshift/models/campaigns/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/parsely_raw_data/dbt/redshift/models/content/__init__.py b/parsely_raw_data/dbt/redshift/models/content/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/parsely_raw_data/dbt/redshift/models/custom/__init__.py b/parsely_raw_data/dbt/redshift/models/custom/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/parsely_raw_data/dbt/redshift/models/excluded_events/__init__.py b/parsely_raw_data/dbt/redshift/models/excluded_events/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/parsely_raw_data/dbt/redshift/models/pageviews/__init__.py b/parsely_raw_data/dbt/redshift/models/pageviews/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/parsely_raw_data/dbt/redshift/models/sessions/__init__.py b/parsely_raw_data/dbt/redshift/models/sessions/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/parsely_raw_data/dbt/redshift/models/users/__init__.py b/parsely_raw_data/dbt/redshift/models/users/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/parsely_raw_data/dbt/redshift/models/videoviews/__init__.py b/parsely_raw_data/dbt/redshift/models/videoviews/__init__.py new file mode 100644 index 0000000..e69de29 From 756386f1bb6d10086e8635980e96113886825d3a Mon Sep 17 00:00:00 2001 From: rachelannelise Date: Mon, 21 Dec 2020 10:52:30 -0500 Subject: [PATCH 24/35] adding init to require install --- MANIFEST.in | 1 + 1 file changed, 1 insertion(+) diff --git a/MANIFEST.in b/MANIFEST.in index 52c7d2c..574cf86 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1,4 @@ include *.txt include parsely_raw_data/*.thrift include parsely_raw_data/dbt/redshift/* +include parsely_raw_data/dbt/redshift/models/*.sql From f8ccc936c1a354aa999b86aa582a4eb753fa97bc Mon Sep 17 00:00:00 2001 From: rachelannelise Date: Mon, 21 Dec 2020 11:01:19 -0500 Subject: [PATCH 25/35] removing init and adding package data --- parsely_raw_data/dbt/redshift/models/__init__.py | 0 parsely_raw_data/dbt/redshift/models/base/__init__.py | 0 .../dbt/redshift/models/behavior_workflow/__init__.py | 0 parsely_raw_data/dbt/redshift/models/campaigns/__init__.py | 0 parsely_raw_data/dbt/redshift/models/content/__init__.py | 0 parsely_raw_data/dbt/redshift/models/custom/__init__.py | 0 .../dbt/redshift/models/excluded_events/__init__.py | 0 parsely_raw_data/dbt/redshift/models/pageviews/__init__.py | 0 parsely_raw_data/dbt/redshift/models/sessions/__init__.py | 0 parsely_raw_data/dbt/redshift/models/users/__init__.py | 0 parsely_raw_data/dbt/redshift/models/videoviews/__init__.py | 0 setup.py | 2 +- 12 files changed, 1 insertion(+), 1 deletion(-) delete mode 100644 parsely_raw_data/dbt/redshift/models/__init__.py delete mode 100644 parsely_raw_data/dbt/redshift/models/base/__init__.py delete mode 100644 parsely_raw_data/dbt/redshift/models/behavior_workflow/__init__.py delete mode 100644 parsely_raw_data/dbt/redshift/models/campaigns/__init__.py delete mode 100644 parsely_raw_data/dbt/redshift/models/content/__init__.py delete mode 100644 parsely_raw_data/dbt/redshift/models/custom/__init__.py delete mode 100644 parsely_raw_data/dbt/redshift/models/excluded_events/__init__.py delete mode 100644 parsely_raw_data/dbt/redshift/models/pageviews/__init__.py delete mode 100644 parsely_raw_data/dbt/redshift/models/sessions/__init__.py delete mode 100644 parsely_raw_data/dbt/redshift/models/users/__init__.py delete mode 100644 parsely_raw_data/dbt/redshift/models/videoviews/__init__.py diff --git a/parsely_raw_data/dbt/redshift/models/__init__.py b/parsely_raw_data/dbt/redshift/models/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/parsely_raw_data/dbt/redshift/models/base/__init__.py b/parsely_raw_data/dbt/redshift/models/base/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/parsely_raw_data/dbt/redshift/models/behavior_workflow/__init__.py b/parsely_raw_data/dbt/redshift/models/behavior_workflow/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/parsely_raw_data/dbt/redshift/models/campaigns/__init__.py b/parsely_raw_data/dbt/redshift/models/campaigns/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/parsely_raw_data/dbt/redshift/models/content/__init__.py b/parsely_raw_data/dbt/redshift/models/content/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/parsely_raw_data/dbt/redshift/models/custom/__init__.py b/parsely_raw_data/dbt/redshift/models/custom/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/parsely_raw_data/dbt/redshift/models/excluded_events/__init__.py b/parsely_raw_data/dbt/redshift/models/excluded_events/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/parsely_raw_data/dbt/redshift/models/pageviews/__init__.py b/parsely_raw_data/dbt/redshift/models/pageviews/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/parsely_raw_data/dbt/redshift/models/sessions/__init__.py b/parsely_raw_data/dbt/redshift/models/sessions/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/parsely_raw_data/dbt/redshift/models/users/__init__.py b/parsely_raw_data/dbt/redshift/models/users/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/parsely_raw_data/dbt/redshift/models/videoviews/__init__.py b/parsely_raw_data/dbt/redshift/models/videoviews/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/setup.py b/setup.py index 8e1e80a..073f737 100644 --- a/setup.py +++ b/setup.py @@ -117,7 +117,7 @@ def run_setup(): include_package_data=True, package_data={"parsely_raw_data": ["dbt/redshift/dbt_project.yml", "dbt/redshift/run_parsely_dpl.sh" - "dbt/redshift/models/", + "dbt/redshift/models/*.sql", "dbt/redshift/"]}, classifiers=[ "Development Status :: 5 - Production/Stable", From aa70df4fea472af1dad3d70a7e66d754b2a3dad8 Mon Sep 17 00:00:00 2001 From: rachelannelise Date: Mon, 21 Dec 2020 11:09:04 -0500 Subject: [PATCH 26/35] adding all package data files --- setup.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index 073f737..155619a 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,5 @@ #!/usr/bin/env python +import glob import re import sys import os @@ -56,6 +57,18 @@ def read_lines(fname): setup_requires = [] +def get_package_data(): + package_data = ["dbt/redshift/dbt_project.yml", + "dbt/redshift/run_parsely_dpl.sh" + "dbt/redshift/models/*.sql", + "dbt/redshift/"] + dbt_models_path = "/dbt/redshift/models/*.sql" + file_list = glob.glob(dbt_models_path) + package_data.append(file_list) + + return package_data + + class PyTest(TestCommand): user_options = [('pytest-args=', 'a', "Arguments to pass to py.test")] @@ -115,10 +128,7 @@ def run_setup(): dependency_links=dependency_links, test_suite='nose.collector', include_package_data=True, - package_data={"parsely_raw_data": ["dbt/redshift/dbt_project.yml", - "dbt/redshift/run_parsely_dpl.sh" - "dbt/redshift/models/*.sql", - "dbt/redshift/"]}, + package_data={"parsely_raw_data": get_package_data()}, classifiers=[ "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", From d2c7318d2da966de8671d68d33a44c7b50e6bb83 Mon Sep 17 00:00:00 2001 From: rachelannelise Date: Mon, 21 Dec 2020 11:23:22 -0500 Subject: [PATCH 27/35] adding all package data files --- setup.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 155619a..a56af35 100644 --- a/setup.py +++ b/setup.py @@ -62,9 +62,11 @@ def get_package_data(): "dbt/redshift/run_parsely_dpl.sh" "dbt/redshift/models/*.sql", "dbt/redshift/"] - dbt_models_path = "/dbt/redshift/models/*.sql" - file_list = glob.glob(dbt_models_path) - package_data.append(file_list) + dbt_models_path = "/dbt/redshift/models/*" + models_path_list = glob.glob(dbt_models_path) + for directory in models_path_list: + files = glob.glob(directory + '/*') + package_data.extend(file_list) return package_data From 0562b9755ec383930a3c3b324a7473048290ed30 Mon Sep 17 00:00:00 2001 From: rachelannelise Date: Mon, 21 Dec 2020 11:27:25 -0500 Subject: [PATCH 28/35] adding all package data files --- setup.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index a56af35..e04be95 100644 --- a/setup.py +++ b/setup.py @@ -60,13 +60,14 @@ def read_lines(fname): def get_package_data(): package_data = ["dbt/redshift/dbt_project.yml", "dbt/redshift/run_parsely_dpl.sh" - "dbt/redshift/models/*.sql", + "dbt/redshift/models/", "dbt/redshift/"] dbt_models_path = "/dbt/redshift/models/*" models_path_list = glob.glob(dbt_models_path) for directory in models_path_list: + package_data.append(directory) files = glob.glob(directory + '/*') - package_data.extend(file_list) + package_data.extend(files) return package_data From 4c2622abf1a39a4358c0a56875f0bc82b4a69280 Mon Sep 17 00:00:00 2001 From: rachelannelise Date: Mon, 21 Dec 2020 12:11:36 -0500 Subject: [PATCH 29/35] adding all package data files --- MANIFEST.in | 2 +- setup.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index 574cf86..28421ad 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,4 @@ include *.txt +include *.sql include parsely_raw_data/*.thrift include parsely_raw_data/dbt/redshift/* -include parsely_raw_data/dbt/redshift/models/*.sql diff --git a/setup.py b/setup.py index e04be95..071179f 100644 --- a/setup.py +++ b/setup.py @@ -3,6 +3,7 @@ import re import sys import os +import logging from setuptools import setup, find_packages from setuptools.command.test import test as TestCommand @@ -64,11 +65,12 @@ def get_package_data(): "dbt/redshift/"] dbt_models_path = "/dbt/redshift/models/*" models_path_list = glob.glob(dbt_models_path) + package_data.extend(models_path_list) for directory in models_path_list: - package_data.append(directory) files = glob.glob(directory + '/*') package_data.extend(files) + logging.warning(f'package data {package_data}') return package_data From 5fbccefd8751bbff593dac2d54ce9738c4420bae Mon Sep 17 00:00:00 2001 From: rachelannelise Date: Mon, 21 Dec 2020 12:26:43 -0500 Subject: [PATCH 30/35] trying manifest --- MANIFEST.in | 2 +- setup.py | 7 ++----- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index 28421ad..8cfb7aa 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,4 @@ include *.txt -include *.sql include parsely_raw_data/*.thrift include parsely_raw_data/dbt/redshift/* +include *.sql diff --git a/setup.py b/setup.py index 071179f..3947484 100644 --- a/setup.py +++ b/setup.py @@ -60,12 +60,10 @@ def read_lines(fname): def get_package_data(): package_data = ["dbt/redshift/dbt_project.yml", - "dbt/redshift/run_parsely_dpl.sh" - "dbt/redshift/models/", - "dbt/redshift/"] + "dbt/redshift/run_parsely_dpl.sh", + "dbt/redshift/"] dbt_models_path = "/dbt/redshift/models/*" models_path_list = glob.glob(dbt_models_path) - package_data.extend(models_path_list) for directory in models_path_list: files = glob.glob(directory + '/*') package_data.extend(files) @@ -133,7 +131,6 @@ def run_setup(): dependency_links=dependency_links, test_suite='nose.collector', include_package_data=True, - package_data={"parsely_raw_data": get_package_data()}, classifiers=[ "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", From ae305173e3c735114fc22614e53c9c730d478ff9 Mon Sep 17 00:00:00 2001 From: rachelannelise Date: Mon, 21 Dec 2020 12:28:56 -0500 Subject: [PATCH 31/35] trying manifest --- MANIFEST.in | 1 + 1 file changed, 1 insertion(+) diff --git a/MANIFEST.in b/MANIFEST.in index 8cfb7aa..998f5f4 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,5 @@ include *.txt include parsely_raw_data/*.thrift include parsely_raw_data/dbt/redshift/* +include parsely_raw_data//dbt/redshift/models/base/* include *.sql From b645eef7b6c33a7e37595b3b05ceeece042f8efd Mon Sep 17 00:00:00 2001 From: rachelannelise Date: Mon, 21 Dec 2020 12:31:08 -0500 Subject: [PATCH 32/35] trying manifest --- MANIFEST.in | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index 998f5f4..3ba07ef 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,5 +1,4 @@ include *.txt include parsely_raw_data/*.thrift include parsely_raw_data/dbt/redshift/* -include parsely_raw_data//dbt/redshift/models/base/* -include *.sql +recursive-include parsely_raw_data *.sql From 6d4f0daf23f0ad5523b8b6ffc5ed5e45496fca85 Mon Sep 17 00:00:00 2001 From: rachelannelise Date: Mon, 21 Dec 2020 12:55:07 -0500 Subject: [PATCH 33/35] trying manifest --- parsely_raw_data/dbt/redshift/redshift_etl.py | 2 +- parsely_raw_data/dbt/redshift/settings/merge_settings_yaml.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/parsely_raw_data/dbt/redshift/redshift_etl.py b/parsely_raw_data/dbt/redshift/redshift_etl.py index dbe614a..3c680f0 100644 --- a/parsely_raw_data/dbt/redshift/redshift_etl.py +++ b/parsely_raw_data/dbt/redshift/redshift_etl.py @@ -105,7 +105,7 @@ def main(): args = parser.parse_args() # Reset dbt_profile to any updated settings: - settings_migration = migrate_settings(profile=args.dbt_profile) + settings_migration = migrate_settings(profile=args.dbt_profile, table=args.table_name) if not settings_migration: logging.warning("Settings not copied to dbt_profiles.yml successfully.") raise Exception("Settings not copied to dbt_profiles.yml successfully. Please edit default.py or copy the" diff --git a/parsely_raw_data/dbt/redshift/settings/merge_settings_yaml.py b/parsely_raw_data/dbt/redshift/settings/merge_settings_yaml.py index 41b8d7f..2315ee8 100644 --- a/parsely_raw_data/dbt/redshift/settings/merge_settings_yaml.py +++ b/parsely_raw_data/dbt/redshift/settings/merge_settings_yaml.py @@ -5,7 +5,6 @@ from .default import * SETTINGS_VAR_MAPPING = [ - {'location': 'parsely:events', 'settings': PARSELY_RAW_DATA_TABLE}, {'location': 'parsely:timezone', 'settings': ETL_TIME_ZONE}, {'location': 'parsely:actions', 'settings': ETL_PARSELY_ACTIONS}, {'location': 'etl:keep_rawdata', 'settings': ETL_KEEP_RAW_DATA}, @@ -21,10 +20,11 @@ ] -def migrate_settings(profile=DBT_PROFILE_NAME): +def migrate_settings(profile=DBT_PROFILE_NAME, table=PARSELY_RAW_DATA_TABLE): # because this is a package resource, have to reference it with pkg_resources filepath = pkg_resources.resource_filename("parsely_raw_data", "dbt/redshift/dbt_project.yml") SETTINGS_VAR_MAPPING.append({'location': 'profile', 'settings': profile}) + SETTINGS_VAR_MAPPING.append({'location': 'parsely:events', 'settings': table}) with open(filepath) as file: dbt_profile = yaml.load(file, Loader=yaml.FullLoader) From c3513b23517637e554860b01d57465096c3f58c7 Mon Sep 17 00:00:00 2001 From: rachelannelise Date: Mon, 21 Dec 2020 13:18:43 -0500 Subject: [PATCH 34/35] removing package_data --- setup.py | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/setup.py b/setup.py index 3947484..0b8419e 100644 --- a/setup.py +++ b/setup.py @@ -1,9 +1,7 @@ #!/usr/bin/env python -import glob import re import sys import os -import logging from setuptools import setup, find_packages from setuptools.command.test import test as TestCommand @@ -58,20 +56,6 @@ def read_lines(fname): setup_requires = [] -def get_package_data(): - package_data = ["dbt/redshift/dbt_project.yml", - "dbt/redshift/run_parsely_dpl.sh", - "dbt/redshift/"] - dbt_models_path = "/dbt/redshift/models/*" - models_path_list = glob.glob(dbt_models_path) - for directory in models_path_list: - files = glob.glob(directory + '/*') - package_data.extend(files) - - logging.warning(f'package data {package_data}') - return package_data - - class PyTest(TestCommand): user_options = [('pytest-args=', 'a', "Arguments to pass to py.test")] From f406e49bd2a98153d0fe2addd5a8cf2b9281e01d Mon Sep 17 00:00:00 2001 From: Annelise Date: Mon, 11 Jan 2021 16:53:04 -0700 Subject: [PATCH 35/35] changing dbt requirements --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index ecb2b69..6ca308a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ boto3>=1.4.4,<1.17 botocore>=1.5.0,<1.20 -dbt==0.17.2 +dbt>=0.15.0,<0.18 google-api-core<1.17.0,>=1.16.0 protobuf<3.12,>=3.6.0 google-api-python-client==1.8.0