Consolidate source + remove tests

.buildkite/hooks/pre-command

@@ -21,4 +21,5 @@ export CI_SNOWFLAKE_DBT_USER=$(gcloud secrets versions access latest --secret="C
 export CI_SNOWFLAKE_DBT_WAREHOUSE=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_WAREHOUSE" --project="dbt-package-testing-363917")
 export CI_DATABRICKS_DBT_HOST=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HOST" --project="dbt-package-testing-363917")
 export CI_DATABRICKS_DBT_HTTP_PATH=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HTTP_PATH" --project="dbt-package-testing-363917")
-export CI_DATABRICKS_DBT_TOKEN=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_TOKEN" --project="dbt-package-testing-363917")
+export CI_DATABRICKS_DBT_TOKEN=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_TOKEN" --project="dbt-package-testing-363917")
+export CI_DATABRICKS_DBT_CATALOG=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_CATALOG" --project="dbt-package-testing-363917")

.buildkite/pipeline.yml

@@ -69,5 +69,6 @@ steps:
             - "CI_DATABRICKS_DBT_HOST"
             - "CI_DATABRICKS_DBT_HTTP_PATH"
             - "CI_DATABRICKS_DBT_TOKEN"
+            - "CI_DATABRICKS_DBT_CATALOG"
     commands: |
       bash .buildkite/scripts/run_models.sh databricks

.github/PULL_REQUEST_TEMPLATE/maintainer_pull_request_template.md

@@ -12,13 +12,15 @@ Before marking this PR as "ready for review":
 
 ## PR Overview 
 **Package version introduced in this PR:** 
-
+- 
+
 **This PR addresses the following Issue/Feature(s):**
 <!-- Add Issue # or internal ticket reference -->
+- 
 
 **Summary of changes:**  
 <!-- 1-2 sentences describing PR changes. -->
-
+- 
 
 ### Submission Checklist  
 - [ ] Alignment meeting with the reviewer (if needed)  
@@ -27,9 +29,10 @@ Before marking this PR as "ready for review":
   - [ ] **Validation Steps:** Check for unintentional effects (e.g., add/run consistency & integrity tests)
   - [ ] **Testing Instructions:** Confirm the change addresses the issue(s)
   - [ ] **Focus Areas:** Complex logic or queries that need extra attention  
+- [ ] Merge any relevant open PRs into this PR
 
 ### Changelog  
 <!-- Recommend drafting changelog notes, then refining via ChatGPT using:  
 "Draft a changelog entry based on the following notes." -->
 - [ ] Draft changelog for PR  
-- [ ] Final changelog for release review
+- [ ] Final changelog for release review

.github/workflows/auto-release.yml

@@ -3,11 +3,12 @@ on:
   pull_request:
     types:
       - closed
-    branches:
-      - main
+      - labeled
 
 jobs:
-  call-workflow-passing-data:
-    if: github.event.pull_request.merged
+  release:
+    if: |
+      (github.event.pull_request.merged == true && github.event.pull_request.base.ref == 'main') ||
+      github.event.label.name == 'pre-release'
     uses: fivetran/dbt_package_automations/.github/workflows/auto-release.yml@main
-    secrets: inherit
+    secrets: inherit

.github/workflows/generate-docs.yml

@@ -0,0 +1,13 @@
+name: 'generate dbt docs'
+on:
+  pull_request:
+    types:
+      - labeled
+
+jobs:
+  generate-docs:
+    if: github.event.label.name == 'docs:ready'
+    uses: fivetran/dbt_package_automations/.github/workflows/generate-docs.yml@main
+    secrets: inherit
+    with:
+      schema_var_name: github_schema

.gitignore

@@ -1,10 +1,73 @@
-
-target/
+# dbt
+**/package-lock.yml
+package-lock.yml
+.dbt/
 dbt_modules/
+dbt_packages/
 logs/
-keyfile.json
+profiles.yml
+target/
+*.log
+
+# IDE files
+.idea/
+.vscode/
+*~
+*.swp
+*.swo
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# OS generated files
+**/.DS_Store
 .DS_Store
-develop/
-dbt_packages/
+.Spotlight-V100
+.Trashes
+._*
+Thumbs.db
+ehthumbs.db
+
+# Python
+*.egg
+*.egg-info/
+*.py[cod]
+*.so
+*$py.class
+.Python
+__pycache__/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.env
+.installed.cfg
+lib/
+lib64/
+MANIFEST
+parts/
+sdist/
+var/
+wheels/
+
+# Secrets and credentials
+.env.*
+.secrets
+credentials.json
+service-account.json
+
+# Temporary files
+.cache/
+*.temp
+*.tmp
+
+# Virtual environments
+.conda/
+.env
+.venv
+ENV/
 env/
-package-lock.yml
+env.bak/
+venv/
+venv.bak/

CHANGELOG.md

@@ -1,3 +1,27 @@
+# dbt_github v1.0.0
+
+[PR #67](https://github.com/fivetran/dbt_github/pull/67) includes the following updates:
+
+## Breaking Changes
+
+### Source Package Consolidation
+- Removed the dependency on the `fivetran/github_source` package.
+  - All functionality from the source package has been merged into this transformation package for improved maintainability and clarity.
+  - If you reference `fivetran/github_source` in your `packages.yml`, you must remove this dependency to avoid conflicts.
+  - Any source overrides referencing the `fivetran/github_source` package will also need to be removed or updated to reference this package.
+  - Update any github_source-scoped variables to be scoped to only under this package. See the [README](https://github.com/fivetran/dbt_github/blob/main/README.md) for how to configure the build schema of staging models.
+- As part of the consolidation, vars are no longer used to reference staging models, and only sources are represented by vars. Staging models are now referenced directly with `ref()` in downstream models.
+
+### dbt Fusion Compatibility Updates
+- Updated package to maintain compatibility with dbt-core versions both before and after v1.10.6, which introduced a breaking change to multi-argument test syntax (e.g., `unique_combination_of_columns`).
+- Temporarily removed unsupported tests to avoid errors and ensure smoother upgrades across different dbt-core versions. These tests will be reintroduced once a safe migration path is available.
+  - Removed all `dbt_utils.unique_combination_of_columns` tests.
+  - Moved `loaded_at_field: _fivetran_synced` under the `config:` block in `src_github.yml`.
+
+### Under the Hood
+- Updated conditions in `.github/workflows/auto-release.yml`.
+- Added `.github/workflows/generate-docs.yml`.
+
 # dbt_github v0.9.1
 
 [PR #64](https://github.com/fivetran/dbt_github/pull/64) includes the following updates:

README.md

@@ -1,11 +1,11 @@
-# Github dbt Package ([Docs](https://fivetran.github.io/dbt_github/))
+# GitHub dbt Package ([Docs](https://fivetran.github.io/dbt_github/))
 
 <p align="left">
     <a alt="License"
         href="https://github.com/fivetran/dbt_github/blob/main/LICENSE">
         <img src="https://img.shields.io/badge/License-Apache%202.0-blue.svg" /></a>
     <a alt="dbt-core">
-        <img src="https://img.shields.io/badge/dbt_Core™_version->=1.3.0_<2.0.0-orange.svg" /></a>
+        <img src="https://img.shields.io/badge/dbt_Core™_version->=1.3.0_,<2.0.0-orange.svg" /></a>
     <a alt="Maintained?">
         <img src="https://img.shields.io/badge/Maintained%3F-yes-green.svg" /></a>
     <a alt="PRs">
@@ -16,9 +16,7 @@
 </p>
 
 ## What does this dbt package do?
-
-- Produces modeled tables that leverage Github data from [Fivetran's connector](https://fivetran.com/docs/applications/github) in the format described by [this ERD](https://fivetran.com/docs/applications/github#schemainformation) and builds off of the output from our [github source package](https://github.com/fivetran/dbt_github_source).
-
+- Produces modeled tables that leverage GitHub data from [Fivetran's connector](https://fivetran.com/docs/applications/github) in the format described by [this ERD](https://fivetran.com/docs/applications/github#schemainformation).
 - Provides insight into GitHub issues and pull requests by enhancing these core objects with commonly used metrics.
 - Produces metrics tables, which increase understanding of your team's velocity over time. Metrics are available on a daily, weekly, monthly, and quarterly level.
 - Generates a comprehensive data dictionary of your source and modeled github data through the [dbt docs site](https://fivetran.github.io/dbt_github/).
@@ -44,7 +42,7 @@ Each Quickstart transformation job run materializes 34 models if all components
 ### Step 1: Prerequisites
 To use this dbt package, you must have the following:
 
-- At least one Fivetran Github connection syncing data into your destination.
+- At least one Fivetran GitHub connection syncing data into your destination.
 - A **BigQuery**, **Snowflake**, **Redshift**, **PostgreSQL**, or **Databricks** destination.
 
 #### Databricks Dispatch Configuration
@@ -62,24 +60,23 @@ Include the following github package version in your `packages.yml` file.
 ```yaml
 packages:
   - package: fivetran/github
-    version: [">=0.9.0", "<0.10.0"] # we recommend using ranges to capture non-breaking changes automatically
+    version: [">=1.0.0", "<1.1.0"] # we recommend using ranges to capture non-breaking changes automatically
 ```
 
-Do NOT include the `github_source` package in this file. The transformation package itself has a dependency on it and will install the source package as well.
-
+> All required sources and staging models are now bundled into this transformation package. Do not include `fivetran/github_source` in your `packages.yml` since this package has been deprecated.
 
 ### Step 3: Define database and schema variables
-By default, this package runs using your [destination](https://docs.getdbt.com/docs/running-a-dbt-project/using-the-command-line-interface/configure-your-profile) and the `github` schema. If this is not where your Github data is (for example, if your github schema is named `github_fivetran`), add the following configuration to your root `dbt_project.yml` file:
+By default, this package runs using your [destination](https://docs.getdbt.com/docs/running-a-dbt-project/using-the-command-line-interface/configure-your-profile) and the `github` schema. If this is not where your GitHub data is (for example, if your github schema is named `github_fivetran`), add the following configuration to your root `dbt_project.yml` file:
 
 ```yml
 vars:
-  github_source:
+  github:
     github_database: your_database_name
     github_schema: your_schema_name 
 ```
 
 ### Step 4: Disable models for non-existent sources
-Your Github connection might not sync every table that this package expects. If your syncs exclude certain tables, it is because you either don't use that functionality in Github or have actively excluded some tables from your syncs.
+Your GitHub connection might not sync every table that this package expects. If your syncs exclude certain tables, it is because you either don't use that functionality in GitHub or have actively excluded some tables from your syncs.
 
 If you do not have the `TEAM`, `REPO_TEAM`, `ISSUE_ASSIGNEE`, `ISSUE_LABEL`, `LABEL`, or `REQUESTED_REVIEWER_HISTORY` tables synced and are not running the package via Fivetran Quickstart, add the following variables to your `dbt_project.yml` file:
 
@@ -99,19 +96,19 @@ vars:
 <details open><summary>Expand/collapse configurations</summary>
 
 #### Change the build schema
-By default, this package builds the Github staging models within a schema titled (`<target_schema>` + `_github_source`) and your Github modeling models within a schema titled (`<target_schema>` + `_github`) in your destination. If this is not where you would like your Github data to be written to, add the following configuration to your root `dbt_project.yml` file:
+By default, this package builds the GitHub staging models within a schema titled (`<target_schema>` + `_github_source`) and your GitHub modeling models within a schema titled (`<target_schema>` + `_github`) in your destination. If this is not where you would like your GitHub data to be written to, add the following configuration to your root `dbt_project.yml` file:
 
 ```yml
 models:
-    github_source:
-      +schema: my_new_schema_name # leave blank for just the target_schema
     github:
-      +schema: my_new_schema_name # leave blank for just the target_schema
+      +schema: my_new_schema_name # Leave +schema: blank to use the default target_schema.
+      staging:
+        +schema: my_new_schema_name # Leave +schema: blank to use the default target_schema.
 ```
 #### Change the source table references
 If an individual source table has a different name than the package expects, add the table name as it appears in your destination to the respective variable:
 
-> IMPORTANT: See this project's [`dbt_project.yml`](https://github.com/fivetran/dbt_github_source/blob/main/dbt_project.yml) variable declarations to see the expected names.
+> IMPORTANT: See this project's [`dbt_project.yml`](https://github.com/fivetran/dbt_github/blob/main/dbt_project.yml) variable declarations to see the expected names.
 
 ```yml
 vars:
@@ -129,7 +126,7 @@ Fivetran offers the ability for you to orchestrate your dbt project through [Fiv
 ## Does this package have dependencies?
 This dbt package is dependent on the following dbt packages. These dependencies are installed by default within this package. For more information on the following packages, refer to the [dbt hub](https://hub.getdbt.com/) site.
 > IMPORTANT: If you have any of these dependent packages in your own `packages.yml` file, we highly recommend that you remove them from your root `packages.yml` to avoid package version conflicts.
-    
+
 ```yml
 packages:
     - package: fivetran/fivetran_utils
@@ -138,9 +135,6 @@ packages:
     - package: dbt-labs/dbt_utils
       version: [">=1.0.0", "<2.0.0"]
 
-    - package: fivetran/github_source
-      version: [">=0.9.0", "<0.10.0"]
-
     - package: dbt-labs/spark_utils
       version: [">=0.3.0", "<0.4.0"]
 ```

dbt_project.yml

@@ -1,26 +1,31 @@
 config-version: 2
 name: 'github'
-version: '0.9.1'
+version: '1.0.0'
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 models:
   github:
     +materialized: table
     +schema: github
     intermediate:
       +materialized: ephemeral
+    staging:
+      +schema: github_source
+      +materialized: table
+      tmp:
+        +materialized: view
 vars:
   github:
-    issue_assignee: "{{ ref('stg_github__issue_assignee') }}"
-    issue_closed_history: "{{ ref('stg_github__issue_closed_history') }}"
-    issue_comment: "{{ ref('stg_github__issue_comment') }}"
-    issue_label: "{{ ref('stg_github__issue_label') }}"
-    label: "{{ ref('stg_github__label') }}"
-    issue_merged: "{{ ref('stg_github__issue_merged') }}"
-    issue: "{{ ref('stg_github__issue') }}"
-    pull_request_review: "{{ ref('stg_github__pull_request_review') }}"
-    pull_request: "{{ ref('stg_github__pull_request') }}"
-    repository: "{{ ref('stg_github__repository') }}"
-    repo_team: "{{ ref('stg_github__repo_team') }}"
-    requested_reviewer_history: "{{ ref('stg_github__requested_reviewer_history') }}"
-    team: "{{ ref('stg_github__team') }}"
-    user: "{{ ref('stg_github__user') }}"
+    issue_assignee: "{{ source('github', 'issue_assignee') }}"
+    issue_closed_history: "{{ source('github', 'issue_closed_history') }}"
+    issue_comment: "{{ source('github', 'issue_comment') }}"
+    issue_label: "{{ source('github', 'issue_label') }}"
+    label: "{{ source('github', 'label') }}"
+    issue_merged: "{{ source('github', 'issue_merged') }}"
+    issue: "{{ source('github', 'issue') }}"
+    pull_request_review: "{{ source('github', 'pull_request_review') }}"
+    pull_request: "{{ source('github', 'pull_request') }}"
+    repository: "{{ source('github', 'repository') }}"
+    repo_team: "{{ source('github', 'repo_team') }}"
+    requested_reviewer_history: "{{ source('github', 'requested_reviewer_history') }}"
+    team: "{{ source('github', 'team') }}"
+    user: "{{ source('github', 'user') }}"

integration_tests/ci/sample.profiles.yml

@@ -41,10 +41,10 @@ integration_tests:
       schema: github_integration_tests_1
       threads: 8
     databricks:
-      catalog: null
+      catalog: "{{ env_var('CI_DATABRICKS_DBT_CATALOG') }}"
       host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}"
       http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}"
       schema: github_integration_tests_1
-      threads: 2
+      threads: 8
       token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}"
       type: databricks

integration_tests/dbt_project.yml

@@ -1,9 +1,9 @@
 name: 'github_integration_tests'
-version: '0.9.1'
+version: '1.0.0'
 config-version: 2
 profile: 'integration_tests'
 vars:
-  github_source:
+  github:
     github_schema: github_integration_tests_1
     github_issue_assignee_identifier: "github_issue_assignee_data"
     github_issue_closed_history_identifier: "github_issue_closed_history_data"
@@ -67,4 +67,5 @@ dispatch:
     search_order: ['spark_utils', 'dbt_utils']
 
 flags:
-  send_anonymous_usage_stats: False
+  send_anonymous_usage_stats: False
+  use_colors: True

integration_tests/requirements.txt

@@ -4,4 +4,5 @@ dbt-redshift>=1.3.0,<2.0.0
 dbt-postgres>=1.3.0,<2.0.0
 dbt-spark>=1.3.0,<2.0.0
 dbt-spark[PyHive]>=1.3.0,<2.0.0
-dbt-databricks>=1.3.0,<2.0.0
+dbt-databricks>=1.3.0,<2.0.0
+certifi==2025.1.31

macros/staging/get_issue_assignee_columns.sql

@@ -0,0 +1,11 @@
+{% macro get_issue_assignee_columns() %}
+
+{% set columns = [
+    {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()},
+    {"name": "issue_id", "datatype": dbt.type_int()},
+    {"name": "user_id", "datatype": dbt.type_int()}
+] %}
+
+{{ return(columns) }}
+
+{% endmacro %}