diff --git a/services/libs/tinybird/pipes/activityRelations_bucket_clean_enrich_copy_pipe_0.pipe b/services/libs/tinybird/pipes/activityRelations_bucket_clean_enrich_copy_pipe_0.pipe index 00b8f9241d..848fc45ef2 100644 --- a/services/libs/tinybird/pipes/activityRelations_bucket_clean_enrich_copy_pipe_0.pipe +++ b/services/libs/tinybird/pipes/activityRelations_bucket_clean_enrich_copy_pipe_0.pipe @@ -36,12 +36,8 @@ SQL > and ( ( platform IN ('git', 'gerrit', 'github', 'gitlab') - AND (channel, activityRelations_bucket_MV_ds_0.segmentId) IN ( - SELECT r.url, r.segmentId - FROM repositories r FINAL - INNER JOIN insightsProjects i FINAL ON r.insightsProjectId = i.id - WHERE isNull (r.deletedAt) AND isNull (i.deletedAt) AND r.enabled = true - ) + AND (channel, activityRelations_bucket_MV_ds_0.segmentId) + IN (SELECT channel, segmentId FROM repos_to_channels) ) OR platform NOT IN ('git', 'gerrit', 'github', 'gitlab') ) diff --git a/services/libs/tinybird/pipes/activityRelations_bucket_clean_enrich_copy_pipe_1.pipe b/services/libs/tinybird/pipes/activityRelations_bucket_clean_enrich_copy_pipe_1.pipe index 61ee900921..0efa5978c2 100644 --- a/services/libs/tinybird/pipes/activityRelations_bucket_clean_enrich_copy_pipe_1.pipe +++ b/services/libs/tinybird/pipes/activityRelations_bucket_clean_enrich_copy_pipe_1.pipe @@ -36,12 +36,8 @@ SQL > and ( ( platform IN ('git', 'gerrit', 'github', 'gitlab') - AND (channel, activityRelations_bucket_MV_ds_1.segmentId) IN ( - SELECT r.url, r.segmentId - FROM repositories r FINAL - INNER JOIN insightsProjects i FINAL ON r.insightsProjectId = i.id - WHERE isNull (r.deletedAt) AND isNull (i.deletedAt) AND r.enabled = true - ) + AND (channel, activityRelations_bucket_MV_ds_1.segmentId) + IN (SELECT channel, segmentId FROM repos_to_channels) ) OR platform NOT IN ('git', 'gerrit', 'github', 'gitlab') ) diff --git a/services/libs/tinybird/pipes/activityRelations_bucket_clean_enrich_copy_pipe_2.pipe b/services/libs/tinybird/pipes/activityRelations_bucket_clean_enrich_copy_pipe_2.pipe index ddebb000a5..6f583d89e1 100644 --- a/services/libs/tinybird/pipes/activityRelations_bucket_clean_enrich_copy_pipe_2.pipe +++ b/services/libs/tinybird/pipes/activityRelations_bucket_clean_enrich_copy_pipe_2.pipe @@ -36,12 +36,8 @@ SQL > and ( ( platform IN ('git', 'gerrit', 'github', 'gitlab') - AND (channel, activityRelations_bucket_MV_ds_2.segmentId) IN ( - SELECT r.url, r.segmentId - FROM repositories r FINAL - INNER JOIN insightsProjects i FINAL ON r.insightsProjectId = i.id - WHERE isNull (r.deletedAt) AND isNull (i.deletedAt) AND r.enabled = true - ) + AND (channel, activityRelations_bucket_MV_ds_2.segmentId) + IN (SELECT channel, segmentId FROM repos_to_channels) ) OR platform NOT IN ('git', 'gerrit', 'github', 'gitlab') ) diff --git a/services/libs/tinybird/pipes/activityRelations_bucket_clean_enrich_copy_pipe_3.pipe b/services/libs/tinybird/pipes/activityRelations_bucket_clean_enrich_copy_pipe_3.pipe index 0800087faa..3beebf43cd 100644 --- a/services/libs/tinybird/pipes/activityRelations_bucket_clean_enrich_copy_pipe_3.pipe +++ b/services/libs/tinybird/pipes/activityRelations_bucket_clean_enrich_copy_pipe_3.pipe @@ -36,12 +36,8 @@ SQL > and ( ( platform IN ('git', 'gerrit', 'github', 'gitlab') - AND (channel, activityRelations_bucket_MV_ds_3.segmentId) IN ( - SELECT r.url, r.segmentId - FROM repositories r FINAL - INNER JOIN insightsProjects i FINAL ON r.insightsProjectId = i.id - WHERE isNull (r.deletedAt) AND isNull (i.deletedAt) AND r.enabled = true - ) + AND (channel, activityRelations_bucket_MV_ds_3.segmentId) + IN (SELECT channel, segmentId FROM repos_to_channels) ) OR platform NOT IN ('git', 'gerrit', 'github', 'gitlab') ) diff --git a/services/libs/tinybird/pipes/activityRelations_bucket_clean_enrich_copy_pipe_4.pipe b/services/libs/tinybird/pipes/activityRelations_bucket_clean_enrich_copy_pipe_4.pipe index d8ad2393a6..b2af521676 100644 --- a/services/libs/tinybird/pipes/activityRelations_bucket_clean_enrich_copy_pipe_4.pipe +++ b/services/libs/tinybird/pipes/activityRelations_bucket_clean_enrich_copy_pipe_4.pipe @@ -36,12 +36,8 @@ SQL > and ( ( platform IN ('git', 'gerrit', 'github', 'gitlab') - AND (channel, activityRelations_bucket_MV_ds_4.segmentId) IN ( - SELECT r.url, r.segmentId - FROM repositories r FINAL - INNER JOIN insightsProjects i FINAL ON r.insightsProjectId = i.id - WHERE isNull (r.deletedAt) AND isNull (i.deletedAt) AND r.enabled = true - ) + AND (channel, activityRelations_bucket_MV_ds_4.segmentId) + IN (SELECT channel, segmentId FROM repos_to_channels) ) OR platform NOT IN ('git', 'gerrit', 'github', 'gitlab') ) diff --git a/services/libs/tinybird/pipes/activityRelations_bucket_clean_enrich_copy_pipe_5.pipe b/services/libs/tinybird/pipes/activityRelations_bucket_clean_enrich_copy_pipe_5.pipe index 68edc301d6..5adb1c2b18 100644 --- a/services/libs/tinybird/pipes/activityRelations_bucket_clean_enrich_copy_pipe_5.pipe +++ b/services/libs/tinybird/pipes/activityRelations_bucket_clean_enrich_copy_pipe_5.pipe @@ -36,12 +36,8 @@ SQL > and ( ( platform IN ('git', 'gerrit', 'github', 'gitlab') - AND (channel, activityRelations_bucket_MV_ds_5.segmentId) IN ( - SELECT r.url, r.segmentId - FROM repositories r FINAL - INNER JOIN insightsProjects i FINAL ON r.insightsProjectId = i.id - WHERE isNull (r.deletedAt) AND isNull (i.deletedAt) AND r.enabled = true - ) + AND (channel, activityRelations_bucket_MV_ds_5.segmentId) + IN (SELECT channel, segmentId FROM repos_to_channels) ) OR platform NOT IN ('git', 'gerrit', 'github', 'gitlab') ) diff --git a/services/libs/tinybird/pipes/activityRelations_bucket_clean_enrich_copy_pipe_6.pipe b/services/libs/tinybird/pipes/activityRelations_bucket_clean_enrich_copy_pipe_6.pipe index e2bb209a8d..32004341a6 100644 --- a/services/libs/tinybird/pipes/activityRelations_bucket_clean_enrich_copy_pipe_6.pipe +++ b/services/libs/tinybird/pipes/activityRelations_bucket_clean_enrich_copy_pipe_6.pipe @@ -36,12 +36,8 @@ SQL > and ( ( platform IN ('git', 'gerrit', 'github', 'gitlab') - AND (channel, activityRelations_bucket_MV_ds_6.segmentId) IN ( - SELECT r.url, r.segmentId - FROM repositories r FINAL - INNER JOIN insightsProjects i FINAL ON r.insightsProjectId = i.id - WHERE isNull (r.deletedAt) AND isNull (i.deletedAt) AND r.enabled = true - ) + AND (channel, activityRelations_bucket_MV_ds_6.segmentId) + IN (SELECT channel, segmentId FROM repos_to_channels) ) OR platform NOT IN ('git', 'gerrit', 'github', 'gitlab') ) diff --git a/services/libs/tinybird/pipes/activityRelations_bucket_clean_enrich_copy_pipe_7.pipe b/services/libs/tinybird/pipes/activityRelations_bucket_clean_enrich_copy_pipe_7.pipe index f0a4ef5ca7..8580ca47c6 100644 --- a/services/libs/tinybird/pipes/activityRelations_bucket_clean_enrich_copy_pipe_7.pipe +++ b/services/libs/tinybird/pipes/activityRelations_bucket_clean_enrich_copy_pipe_7.pipe @@ -36,12 +36,8 @@ SQL > and ( ( platform IN ('git', 'gerrit', 'github', 'gitlab') - AND (channel, activityRelations_bucket_MV_ds_7.segmentId) IN ( - SELECT r.url, r.segmentId - FROM repositories r FINAL - INNER JOIN insightsProjects i FINAL ON r.insightsProjectId = i.id - WHERE isNull (r.deletedAt) AND isNull (i.deletedAt) AND r.enabled = true - ) + AND (channel, activityRelations_bucket_MV_ds_7.segmentId) + IN (SELECT channel, segmentId FROM repos_to_channels) ) OR platform NOT IN ('git', 'gerrit', 'github', 'gitlab') ) diff --git a/services/libs/tinybird/pipes/activityRelations_bucket_clean_enrich_copy_pipe_8.pipe b/services/libs/tinybird/pipes/activityRelations_bucket_clean_enrich_copy_pipe_8.pipe index 87acb15cb0..1b2dc032ff 100644 --- a/services/libs/tinybird/pipes/activityRelations_bucket_clean_enrich_copy_pipe_8.pipe +++ b/services/libs/tinybird/pipes/activityRelations_bucket_clean_enrich_copy_pipe_8.pipe @@ -36,12 +36,8 @@ SQL > and ( ( platform IN ('git', 'gerrit', 'github', 'gitlab') - AND (channel, activityRelations_bucket_MV_ds_8.segmentId) IN ( - SELECT r.url, r.segmentId - FROM repositories r FINAL - INNER JOIN insightsProjects i FINAL ON r.insightsProjectId = i.id - WHERE isNull (r.deletedAt) AND isNull (i.deletedAt) AND r.enabled = true - ) + AND (channel, activityRelations_bucket_MV_ds_8.segmentId) + IN (SELECT channel, segmentId FROM repos_to_channels) ) OR platform NOT IN ('git', 'gerrit', 'github', 'gitlab') ) diff --git a/services/libs/tinybird/pipes/activityRelations_bucket_clean_enrich_copy_pipe_9.pipe b/services/libs/tinybird/pipes/activityRelations_bucket_clean_enrich_copy_pipe_9.pipe index 0cb64dc1d8..065f2909d1 100644 --- a/services/libs/tinybird/pipes/activityRelations_bucket_clean_enrich_copy_pipe_9.pipe +++ b/services/libs/tinybird/pipes/activityRelations_bucket_clean_enrich_copy_pipe_9.pipe @@ -36,12 +36,8 @@ SQL > and ( ( platform IN ('git', 'gerrit', 'github', 'gitlab') - AND (channel, activityRelations_bucket_MV_ds_9.segmentId) IN ( - SELECT r.url, r.segmentId - FROM repositories r FINAL - INNER JOIN insightsProjects i FINAL ON r.insightsProjectId = i.id - WHERE isNull (r.deletedAt) AND isNull (i.deletedAt) AND r.enabled = true - ) + AND (channel, activityRelations_bucket_MV_ds_9.segmentId) + IN (SELECT channel, segmentId FROM repos_to_channels) ) OR platform NOT IN ('git', 'gerrit', 'github', 'gitlab') ) diff --git a/services/libs/tinybird/pipes/repos_to_channels.pipe b/services/libs/tinybird/pipes/repos_to_channels.pipe index 0e321ede91..19e6de8670 100644 --- a/services/libs/tinybird/pipes/repos_to_channels.pipe +++ b/services/libs/tinybird/pipes/repos_to_channels.pipe @@ -6,7 +6,7 @@ DESCRIPTION > - Parameters: - `repos`: Optional array of repository URLs to expand. If not provided, defaults to all enabled repositories. - `excluded`: Optional boolean. When true, expands excluded repositories instead of enabled ones. Used to build exclusion filters. - - Response: `channel` - all possible channel URL formats for the given repos + - Response: `channel` - all possible channel URL formats for the given repos; `segmentId` - segment the repo belongs to (empty string when repos param is provided) TAGS "Repository URLs", "Gerrit" @@ -20,13 +20,15 @@ SQL > SELECT arrayJoin( {{ Array(repos, 'String', description="Repository URLs to expand", required=False) }} - ) AS url + ) AS url, + '' AS segmentId {% else %} - SELECT url - FROM repositories FINAL + SELECT r.url AS url, r.segmentId AS segmentId + FROM repositories r FINAL + INNER JOIN insightsProjects i FINAL ON r.insightsProjectId = i.id WHERE - isNull (deletedAt) AND enabled = true - {% if defined(excluded) and excluded %} AND excluded = true + isNull (r.deletedAt) AND r.enabled = true AND isNull (i.deletedAt) + {% if defined(excluded) and excluded %} AND r.excluded = true {% end %} {% end %} @@ -35,10 +37,16 @@ DESCRIPTION > Identify Gerrit repositories by joining with integrations table SQL > - SELECT r.url + SELECT r.url AS url, r.segmentId AS segmentId FROM repositories r FINAL JOIN integrations i FINAL ON r.sourceIntegrationId = i.id - WHERE i.platform = 'gerrit' AND isNull (r.deletedAt) AND r.url IN (SELECT url FROM repos_to_expand) + INNER JOIN insightsProjects p FINAL ON r.insightsProjectId = p.id + WHERE + i.platform = 'gerrit' + AND isNull (r.deletedAt) + AND r.enabled = true + AND isNull (p.deletedAt) + AND r.url IN (SELECT url FROM repos_to_expand) NODE expanded_urls DESCRIPTION > @@ -46,7 +54,7 @@ DESCRIPTION > SQL > -- Original URLs (all repos) - SELECT url AS channel + SELECT url AS channel, segmentId FROM repos_to_expand UNION ALL -- Gerrit channel variants: insert q/project: after the base path @@ -67,7 +75,8 @@ SQL > '/q/project:', if(path(url) = '/', '', substring(path(url), 2)) ) - END AS channel + END AS channel, + segmentId FROM gerrit_repos NODE channels_deduplicated @@ -75,4 +84,4 @@ DESCRIPTION > Final deduplicated list of all possible channel URLs SQL > - SELECT DISTINCT channel FROM expanded_urls + SELECT DISTINCT channel, segmentId FROM expanded_urls