Skip to content

fix: gerrit repositories filtering (CM-1079) #3977

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
joanagmaia wants to merge 5 commits into
base: main
Choose a base branch
from
Open

fix: gerrit repositories filtering (CM-1079) #3977

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
21bf1ba
fix: gerrit repositories filtering (CM-1079)
joanagmaia Mar 30, 2026
5907e73
fix: exclusion repos rules
joanagmaia Mar 30, 2026
acd42a7
fix: formatting
joanagmaia Mar 30, 2026
dec518e
fix: ambiguity in returned data
joanagmaia Mar 30, 2026
0c39d89
Merge branch 'main' into fix/gerrit-repositories-filtering
joanagmaia Mar 31, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 2 additions & 6 deletions services/libs/tinybird/pipes/activityRelations_bucket_clean_enrich_copy_pipe_0.pipe
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,12 +36,8 @@ SQL >
and (
(
platform IN ('git', 'gerrit', 'github', 'gitlab')
AND (channel, activityRelations_bucket_MV_ds_0.segmentId) IN (
SELECT r.url, r.segmentId
FROM repositories r FINAL
INNER JOIN insightsProjects i FINAL ON r.insightsProjectId = i.id
WHERE isNull (r.deletedAt) AND isNull (i.deletedAt) AND r.enabled = true
)
AND (channel, activityRelations_bucket_MV_ds_0.segmentId)
IN (SELECT channel, segmentId FROM repos_to_channels)
)
OR platform NOT IN ('git', 'gerrit', 'github', 'gitlab')
)
Expand Down
8 changes: 2 additions & 6 deletions services/libs/tinybird/pipes/activityRelations_bucket_clean_enrich_copy_pipe_1.pipe
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,12 +36,8 @@ SQL >
and (
(
platform IN ('git', 'gerrit', 'github', 'gitlab')
AND (channel, activityRelations_bucket_MV_ds_1.segmentId) IN (
SELECT r.url, r.segmentId
FROM repositories r FINAL
INNER JOIN insightsProjects i FINAL ON r.insightsProjectId = i.id
WHERE isNull (r.deletedAt) AND isNull (i.deletedAt) AND r.enabled = true
)
AND (channel, activityRelations_bucket_MV_ds_1.segmentId)
IN (SELECT channel, segmentId FROM repos_to_channels)
)
OR platform NOT IN ('git', 'gerrit', 'github', 'gitlab')
)
Expand Down
8 changes: 2 additions & 6 deletions services/libs/tinybird/pipes/activityRelations_bucket_clean_enrich_copy_pipe_2.pipe
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,12 +36,8 @@ SQL >
and (
(
platform IN ('git', 'gerrit', 'github', 'gitlab')
AND (channel, activityRelations_bucket_MV_ds_2.segmentId) IN (
SELECT r.url, r.segmentId
FROM repositories r FINAL
INNER JOIN insightsProjects i FINAL ON r.insightsProjectId = i.id
WHERE isNull (r.deletedAt) AND isNull (i.deletedAt) AND r.enabled = true
)
AND (channel, activityRelations_bucket_MV_ds_2.segmentId)
IN (SELECT channel, segmentId FROM repos_to_channels)
)
OR platform NOT IN ('git', 'gerrit', 'github', 'gitlab')
)
Expand Down
8 changes: 2 additions & 6 deletions services/libs/tinybird/pipes/activityRelations_bucket_clean_enrich_copy_pipe_3.pipe
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,12 +36,8 @@ SQL >
and (
(
platform IN ('git', 'gerrit', 'github', 'gitlab')
AND (channel, activityRelations_bucket_MV_ds_3.segmentId) IN (
SELECT r.url, r.segmentId
FROM repositories r FINAL
INNER JOIN insightsProjects i FINAL ON r.insightsProjectId = i.id
WHERE isNull (r.deletedAt) AND isNull (i.deletedAt) AND r.enabled = true
)
AND (channel, activityRelations_bucket_MV_ds_3.segmentId)
IN (SELECT channel, segmentId FROM repos_to_channels)
)
OR platform NOT IN ('git', 'gerrit', 'github', 'gitlab')
)
Expand Down
8 changes: 2 additions & 6 deletions services/libs/tinybird/pipes/activityRelations_bucket_clean_enrich_copy_pipe_4.pipe
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,12 +36,8 @@ SQL >
and (
(
platform IN ('git', 'gerrit', 'github', 'gitlab')
AND (channel, activityRelations_bucket_MV_ds_4.segmentId) IN (
SELECT r.url, r.segmentId
FROM repositories r FINAL
INNER JOIN insightsProjects i FINAL ON r.insightsProjectId = i.id
WHERE isNull (r.deletedAt) AND isNull (i.deletedAt) AND r.enabled = true
)
AND (channel, activityRelations_bucket_MV_ds_4.segmentId)
IN (SELECT channel, segmentId FROM repos_to_channels)
)
OR platform NOT IN ('git', 'gerrit', 'github', 'gitlab')
)
Expand Down
8 changes: 2 additions & 6 deletions services/libs/tinybird/pipes/activityRelations_bucket_clean_enrich_copy_pipe_5.pipe
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,12 +36,8 @@ SQL >
and (
(
platform IN ('git', 'gerrit', 'github', 'gitlab')
AND (channel, activityRelations_bucket_MV_ds_5.segmentId) IN (
SELECT r.url, r.segmentId
FROM repositories r FINAL
INNER JOIN insightsProjects i FINAL ON r.insightsProjectId = i.id
WHERE isNull (r.deletedAt) AND isNull (i.deletedAt) AND r.enabled = true
)
AND (channel, activityRelations_bucket_MV_ds_5.segmentId)
IN (SELECT channel, segmentId FROM repos_to_channels)
)
OR platform NOT IN ('git', 'gerrit', 'github', 'gitlab')
)
Expand Down
8 changes: 2 additions & 6 deletions services/libs/tinybird/pipes/activityRelations_bucket_clean_enrich_copy_pipe_6.pipe
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,12 +36,8 @@ SQL >
and (
(
platform IN ('git', 'gerrit', 'github', 'gitlab')
AND (channel, activityRelations_bucket_MV_ds_6.segmentId) IN (
SELECT r.url, r.segmentId
FROM repositories r FINAL
INNER JOIN insightsProjects i FINAL ON r.insightsProjectId = i.id
WHERE isNull (r.deletedAt) AND isNull (i.deletedAt) AND r.enabled = true
)
AND (channel, activityRelations_bucket_MV_ds_6.segmentId)
IN (SELECT channel, segmentId FROM repos_to_channels)
)
OR platform NOT IN ('git', 'gerrit', 'github', 'gitlab')
)
Expand Down
8 changes: 2 additions & 6 deletions services/libs/tinybird/pipes/activityRelations_bucket_clean_enrich_copy_pipe_7.pipe
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,12 +36,8 @@ SQL >
and (
(
platform IN ('git', 'gerrit', 'github', 'gitlab')
AND (channel, activityRelations_bucket_MV_ds_7.segmentId) IN (
SELECT r.url, r.segmentId
FROM repositories r FINAL
INNER JOIN insightsProjects i FINAL ON r.insightsProjectId = i.id
WHERE isNull (r.deletedAt) AND isNull (i.deletedAt) AND r.enabled = true
)
AND (channel, activityRelations_bucket_MV_ds_7.segmentId)
IN (SELECT channel, segmentId FROM repos_to_channels)
)
OR platform NOT IN ('git', 'gerrit', 'github', 'gitlab')
)
Expand Down
8 changes: 2 additions & 6 deletions services/libs/tinybird/pipes/activityRelations_bucket_clean_enrich_copy_pipe_8.pipe
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,12 +36,8 @@ SQL >
and (
(
platform IN ('git', 'gerrit', 'github', 'gitlab')
AND (channel, activityRelations_bucket_MV_ds_8.segmentId) IN (
SELECT r.url, r.segmentId
FROM repositories r FINAL
INNER JOIN insightsProjects i FINAL ON r.insightsProjectId = i.id
WHERE isNull (r.deletedAt) AND isNull (i.deletedAt) AND r.enabled = true
)
AND (channel, activityRelations_bucket_MV_ds_8.segmentId)
IN (SELECT channel, segmentId FROM repos_to_channels)
)
OR platform NOT IN ('git', 'gerrit', 'github', 'gitlab')
)
Expand Down
8 changes: 2 additions & 6 deletions services/libs/tinybird/pipes/activityRelations_bucket_clean_enrich_copy_pipe_9.pipe
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,12 +36,8 @@ SQL >
and (
(
platform IN ('git', 'gerrit', 'github', 'gitlab')
AND (channel, activityRelations_bucket_MV_ds_9.segmentId) IN (
SELECT r.url, r.segmentId
FROM repositories r FINAL
INNER JOIN insightsProjects i FINAL ON r.insightsProjectId = i.id
WHERE isNull (r.deletedAt) AND isNull (i.deletedAt) AND r.enabled = true
)
AND (channel, activityRelations_bucket_MV_ds_9.segmentId)
IN (SELECT channel, segmentId FROM repos_to_channels)
)
OR platform NOT IN ('git', 'gerrit', 'github', 'gitlab')
)
Expand Down
31 changes: 20 additions & 11 deletions services/libs/tinybird/pipes/repos_to_channels.pipe
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ DESCRIPTION >
- Parameters:
- `repos`: Optional array of repository URLs to expand. If not provided, defaults to all enabled repositories.
- `excluded`: Optional boolean. When true, expands excluded repositories instead of enabled ones. Used to build exclusion filters.
- Response: `channel` - all possible channel URL formats for the given repos
- Response: `channel` - all possible channel URL formats for the given repos; `segmentId` - segment the repo belongs to (empty string when repos param is provided)

TAGS "Repository URLs", "Gerrit"

Expand All @@ -20,13 +20,15 @@ SQL >
SELECT
arrayJoin(
{{ Array(repos, 'String', description="Repository URLs to expand", required=False) }}
) AS url
) AS url,
'' AS segmentId
{% else %}
SELECT url
FROM repositories FINAL
SELECT r.url AS url, r.segmentId AS segmentId
FROM repositories r FINAL
INNER JOIN insightsProjects i FINAL ON r.insightsProjectId = i.id
WHERE
isNull (deletedAt) AND enabled = true
{% if defined(excluded) and excluded %} AND excluded = true
isNull (r.deletedAt) AND r.enabled = true AND isNull (i.deletedAt)
{% if defined(excluded) and excluded %} AND r.excluded = true
{% end %}
{% end %}

Expand All @@ -35,18 +37,24 @@ DESCRIPTION >
Identify Gerrit repositories by joining with integrations table

SQL >
SELECT r.url
SELECT r.url AS url, r.segmentId AS segmentId
FROM repositories r FINAL
JOIN integrations i FINAL ON r.sourceIntegrationId = i.id
WHERE i.platform = 'gerrit' AND isNull (r.deletedAt) AND r.url IN (SELECT url FROM repos_to_expand)
INNER JOIN insightsProjects p FINAL ON r.insightsProjectId = p.id
WHERE
i.platform = 'gerrit'
AND isNull (r.deletedAt)
AND r.enabled = true
AND isNull (p.deletedAt)
AND r.url IN (SELECT url FROM repos_to_expand)

NODE expanded_urls
DESCRIPTION >
Output original URLs plus Gerrit channel variants

SQL >
-- Original URLs (all repos)
SELECT url AS channel
SELECT url AS channel, segmentId
FROM repos_to_expand
UNION ALL
-- Gerrit channel variants: insert q/project: after the base path
Expand All @@ -67,12 +75,13 @@ SQL >
'/q/project:',
if(path(url) = '/', '', substring(path(url), 2))
)
END AS channel
END AS channel,
segmentId
FROM gerrit_repos

NODE channels_deduplicated
DESCRIPTION >
Final deduplicated list of all possible channel URLs

SQL >
SELECT DISTINCT channel FROM expanded_urls
SELECT DISTINCT channel, segmentId FROM expanded_urls
Loading