From 6f1c00d72726b9d318e4e63f119a1e0c3ee12db8 Mon Sep 17 00:00:00 2001 From: GabhenDM Date: Fri, 1 Oct 2021 17:24:27 -0300 Subject: [PATCH 1/8] wip:automated issue opening - initial version --- collector/collector.go | 65 ++++++++++++++++++++++++++++++++++++++++++ db/db.go | 8 ++++++ rules/issue.go | 28 ++++++++++++++++++ 3 files changed, 101 insertions(+) create mode 100644 rules/issue.go diff --git a/collector/collector.go b/collector/collector.go index ed4573e..d6663d0 100644 --- a/collector/collector.go +++ b/collector/collector.go @@ -11,6 +11,8 @@ import ( "github.com/spf13/viper" "github.com/xanzy/go-gitlab" "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" _ "github.com/globocom/gitlab-lint/config" "github.com/globocom/gitlab-lint/db" @@ -78,6 +80,66 @@ func processRules(rulesList []rules.Rule) error { return nil } +func processIssues(registry *rules.Registry, git *gitlab.Client) error { + dbInstance, err := db.NewMongoSession() + if err != nil { + log.Errorf("[Collector] Error on create mongo session: %v", err) + return err + } + // iterating over matched rules + for _, r := range registry.Rules { + // searching for opened issue for project and rule + pipeline := bson.M{"$and": bson.A{bson.M{"projectId": r.ProjectID}, bson.M{"ruleId": r.RuleID}, bson.M{"state": "opened"}}} + + issue := &rules.Issue{} + err := dbInstance.Get(issue, pipeline, &options.FindOneOptions{}) + + // if any error + if err != nil && err != mongo.ErrNoDocuments { + return err + } + // if no opened issue found -> create new issue and add to DB + if err != nil && err == mongo.ErrNoDocuments { + createdIssue, _, err := git.Issues.CreateIssue(r.ProjectID, &gitlab.CreateIssueOptions{Title: &r.RuleID}) + if err != nil { + return err + } + + if _, err := dbInstance.Insert(&rules.Issue{ProjectID: r.ProjectID, RuleID: r.RuleID, IssueID: createdIssue.ID, + WebURL: r.WebURL, Title: r.RuleID, Description: "Test", State: createdIssue.State}); err != nil { + return err + } + // Opened issue found + } else { + // fetch issue info from gitlab + gitIssue, _, err := git.Issues.GetIssue(issue.ProjectID, issue.IssueID) + if err != nil { + return err + } + // if issue was closed but rule still matched + if gitIssue.State != issue.State && gitIssue.State == "closed" { + // update old issue on DB + if _, err := dbInstance.Update(&rules.Issue{}, bson.M{"_id": issue.ID}, bson.M{"$set": bson.M{"state": gitIssue.State}}, &options.UpdateOptions{}); err != nil { + return err + } + desc := "test-reopened" + // create new issue + createdIssue, _, err := git.Issues.CreateIssue(r.ProjectID, &gitlab.CreateIssueOptions{Title: &r.RuleID, Description: &desc}) + if err != nil { + return err + } + // add new issue to DB + if _, err := dbInstance.Insert(&rules.Issue{ProjectID: r.ProjectID, RuleID: r.RuleID, IssueID: createdIssue.ID, + WebURL: r.WebURL, Title: r.RuleID, Description: "Test-Reopened", State: createdIssue.State}); err != nil { + return err + } + + } + } + } + + return nil +} func insertStats(r *rules.Registry) error { dbInstance, err := db.NewMongoSession() @@ -191,4 +253,7 @@ func main() { if err := insertStats(rules.MyRegistry); err != nil { log.Errorf("[Collector] Error on insert statistics data: %v", err) } + if err := processIssues(rules.MyRegistry, git); err != nil { + log.Errorf("[Collector] Error on processing issue: %v", err) + } } diff --git a/db/db.go b/db/db.go index a6cd1a1..a2bf9ba 100644 --- a/db/db.go +++ b/db/db.go @@ -45,6 +45,7 @@ type DB interface { Count(d rules.Queryable, filter FindFilter) (int, error) DeleteMany(d rules.Queryable, q bson.M) (*mongo.DeleteResult, error) Get(d rules.Queryable, q bson.M, o *options.FindOneOptions) error + Update(d rules.Queryable, q bson.M, u bson.M, o *options.UpdateOptions) (*mongo.UpdateResult, error) GetAll(d rules.Queryable, filter FindFilter) ([]rules.Queryable, error) Insert(d rules.Queryable) (*mongo.InsertOneResult, error) InsertMany(d rules.Queryable, i []interface{}) (*mongo.InsertManyResult, error) @@ -138,6 +139,13 @@ func (m *mongoCollection) Get(d rules.Queryable, q bson.M, o *options.FindOneOpt return collection.FindOne(ctx, q).Decode(d) } +func (m *mongoCollection) Update(d rules.Queryable, q bson.M, u bson.M, o *options.UpdateOptions) (*mongo.UpdateResult, error) { + log.Debug("[DB] Update...") + collection := m.session.Database(m.dbName).Collection(d.GetCollectionName()) + ctx, _ := newDBContext() + return collection.UpdateOne(ctx, q, u, o) +} + func (m mongoCollection) GetAll(d rules.Queryable, filter FindFilter) ([]rules.Queryable, error) { log.Debug("[DB] GetAll...") collection := m.session.Database(m.dbName).Collection(d.GetCollectionName()) diff --git a/rules/issue.go b/rules/issue.go new file mode 100644 index 0000000..31efeb7 --- /dev/null +++ b/rules/issue.go @@ -0,0 +1,28 @@ +package rules + +import "go.mongodb.org/mongo-driver/bson/primitive" + +type Issue struct { + ID primitive.ObjectID `json:"_id"bson:"_id,omitempty"` + ProjectID int `json:"projectId" bson:"projectId"` + RuleID string `json:"ruleId" bson:"ruleId"` + IssueID int `json:"issueId" bson:"issueId"` + WebURL string `json:"webUrl" bson:"webUrl"` + Title string `json:"title" bson:"title"` + Description string `json:"description" bson:"description"` + State string `json:"state" bson:"state"` +} + +type Issues []Issue + +func (i Issue) Cast() Queryable { + return &i +} + +func (i Issue) GetCollectionName() string { + return "issues" +} + +func (i Issue) GetSearchableFields() []string { + return []string{"title", "projectid", "ruleid", "state"} +} From 025cde4a5d1e4641184cc0592a3fd8844dffaefa Mon Sep 17 00:00:00 2001 From: GabhenDM Date: Fri, 1 Oct 2021 23:34:13 -0300 Subject: [PATCH 2/8] wip: Added description and title access methods to rules, better issue creating logic, removed unnecessary update --- collector/collector.go | 54 ++++++++++++++++++------------------- db/db.go | 10 +------ rules/empty_repository.go | 8 ++++++ rules/fast-forward-merge.go | 8 ++++++ rules/has_open_issues.go | 8 ++++++ rules/issue.go | 1 - rules/last_activity.go | 8 ++++++ rules/ruler.go | 2 ++ rules/without_gitlab_ci.go | 8 ++++++ rules/without_readme.go | 8 ++++++ 10 files changed, 77 insertions(+), 38 deletions(-) diff --git a/collector/collector.go b/collector/collector.go index d6663d0..e9d8c60 100644 --- a/collector/collector.go +++ b/collector/collector.go @@ -4,6 +4,7 @@ package main import ( + "fmt" "sync" "time" @@ -80,6 +81,7 @@ func processRules(rulesList []rules.Rule) error { return nil } + func processIssues(registry *rules.Registry, git *gitlab.Client) error { dbInstance, err := db.NewMongoSession() if err != nil { @@ -89,26 +91,21 @@ func processIssues(registry *rules.Registry, git *gitlab.Client) error { // iterating over matched rules for _, r := range registry.Rules { // searching for opened issue for project and rule - pipeline := bson.M{"$and": bson.A{bson.M{"projectId": r.ProjectID}, bson.M{"ruleId": r.RuleID}, bson.M{"state": "opened"}}} + pipeline := bson.M{"$and": bson.A{bson.M{"projectId": r.ProjectID}, bson.M{"ruleId": r.RuleID}}} issue := &rules.Issue{} - err := dbInstance.Get(issue, pipeline, &options.FindOneOptions{}) + err := dbInstance.Get(issue, pipeline, &options.FindOneOptions{Sort: bson.M{"issueId": -1}}) - // if any error + // if any error different than noDocumentFound if err != nil && err != mongo.ErrNoDocuments { return err } + // if no opened issue found -> create new issue and add to DB if err != nil && err == mongo.ErrNoDocuments { - createdIssue, _, err := git.Issues.CreateIssue(r.ProjectID, &gitlab.CreateIssueOptions{Title: &r.RuleID}) - if err != nil { - return err - } - - if _, err := dbInstance.Insert(&rules.Issue{ProjectID: r.ProjectID, RuleID: r.RuleID, IssueID: createdIssue.ID, - WebURL: r.WebURL, Title: r.RuleID, Description: "Test", State: createdIssue.State}); err != nil { - return err - } + title := fmt.Sprintf("[Gitlab-Lint] %s", registry.RulesFn[r.RuleID].GetName()) + description := registry.RulesFn[r.RuleID].GetDescription() + createIssue(r, git, dbInstance, title, description) // Opened issue found } else { // fetch issue info from gitlab @@ -117,23 +114,12 @@ func processIssues(registry *rules.Registry, git *gitlab.Client) error { return err } // if issue was closed but rule still matched - if gitIssue.State != issue.State && gitIssue.State == "closed" { - // update old issue on DB - if _, err := dbInstance.Update(&rules.Issue{}, bson.M{"_id": issue.ID}, bson.M{"$set": bson.M{"state": gitIssue.State}}, &options.UpdateOptions{}); err != nil { - return err - } - desc := "test-reopened" + if gitIssue.State == "closed" { + // Open new issue with Reopened title + title := fmt.Sprintf("[Gitlab-Lint][Reopened] %s", registry.RulesFn[r.RuleID].GetName()) + description := registry.RulesFn[r.RuleID].GetDescription() // create new issue - createdIssue, _, err := git.Issues.CreateIssue(r.ProjectID, &gitlab.CreateIssueOptions{Title: &r.RuleID, Description: &desc}) - if err != nil { - return err - } - // add new issue to DB - if _, err := dbInstance.Insert(&rules.Issue{ProjectID: r.ProjectID, RuleID: r.RuleID, IssueID: createdIssue.ID, - WebURL: r.WebURL, Title: r.RuleID, Description: "Test-Reopened", State: createdIssue.State}); err != nil { - return err - } - + createIssue(r, git, dbInstance, title, description) } } } @@ -141,6 +127,18 @@ func processIssues(registry *rules.Registry, git *gitlab.Client) error { return nil } +func createIssue(r rules.Rule, git *gitlab.Client, dbInstance db.DB, title string, description string) error { + createdIssue, _, err := git.Issues.CreateIssue(r.ProjectID, &gitlab.CreateIssueOptions{Title: &title, Description: &description}) + if err != nil { + return err + } + if _, err := dbInstance.Insert(&rules.Issue{ProjectID: r.ProjectID, RuleID: r.RuleID, IssueID: createdIssue.IID, + WebURL: r.WebURL, Title: title, Description: r.Description}); err != nil { + return err + } + return nil +} + func insertStats(r *rules.Registry) error { dbInstance, err := db.NewMongoSession() if err != nil { diff --git a/db/db.go b/db/db.go index a2bf9ba..d6bedb4 100644 --- a/db/db.go +++ b/db/db.go @@ -45,7 +45,6 @@ type DB interface { Count(d rules.Queryable, filter FindFilter) (int, error) DeleteMany(d rules.Queryable, q bson.M) (*mongo.DeleteResult, error) Get(d rules.Queryable, q bson.M, o *options.FindOneOptions) error - Update(d rules.Queryable, q bson.M, u bson.M, o *options.UpdateOptions) (*mongo.UpdateResult, error) GetAll(d rules.Queryable, filter FindFilter) ([]rules.Queryable, error) Insert(d rules.Queryable) (*mongo.InsertOneResult, error) InsertMany(d rules.Queryable, i []interface{}) (*mongo.InsertManyResult, error) @@ -136,14 +135,7 @@ func (m *mongoCollection) Get(d rules.Queryable, q bson.M, o *options.FindOneOpt log.Debug("[DB] Get...") collection := m.session.Database(m.dbName).Collection(d.GetCollectionName()) ctx, _ := newDBContext() - return collection.FindOne(ctx, q).Decode(d) -} - -func (m *mongoCollection) Update(d rules.Queryable, q bson.M, u bson.M, o *options.UpdateOptions) (*mongo.UpdateResult, error) { - log.Debug("[DB] Update...") - collection := m.session.Database(m.dbName).Collection(d.GetCollectionName()) - ctx, _ := newDBContext() - return collection.UpdateOne(ctx, q, u, o) + return collection.FindOne(ctx, q, o).Decode(d) } func (m mongoCollection) GetAll(d rules.Queryable, filter FindFilter) ([]rules.Queryable, error) { diff --git a/rules/empty_repository.go b/rules/empty_repository.go index 0537716..d1064a9 100644 --- a/rules/empty_repository.go +++ b/rules/empty_repository.go @@ -24,6 +24,14 @@ func (e *EmptyRepository) GetLevel() string { return LevelError } +func (e *EmptyRepository) GetName() string { + return e.Name +} + +func (e *EmptyRepository) GetDescription() string { + return e.Description +} + func NewEmptyRepository() Ruler { e := &EmptyRepository{ Name: "Empty Repository", diff --git a/rules/fast-forward-merge.go b/rules/fast-forward-merge.go index d9eb816..abbd47a 100644 --- a/rules/fast-forward-merge.go +++ b/rules/fast-forward-merge.go @@ -23,6 +23,14 @@ func (w *NonFastForwardMerge) GetLevel() string { return LevelPedantic } +func (e *NonFastForwardMerge) GetName() string { + return e.Name +} + +func (e *NonFastForwardMerge) GetDescription() string { + return e.Description +} + func NewNonFastForwardMerge() Ruler { w := &NonFastForwardMerge{ Name: "Non Fast-forward Merge", diff --git a/rules/has_open_issues.go b/rules/has_open_issues.go index 8f8742b..3b4e3c2 100644 --- a/rules/has_open_issues.go +++ b/rules/has_open_issues.go @@ -26,6 +26,14 @@ func (h *HasOpenIssues) GetLevel() string { return LevelPedantic } +func (e *HasOpenIssues) GetName() string { + return e.Name +} + +func (e *HasOpenIssues) GetDescription() string { + return e.Description +} + func NewHasOpenIssues() Ruler { h := &HasOpenIssues{ Name: "Has Open Issues", diff --git a/rules/issue.go b/rules/issue.go index 31efeb7..394349f 100644 --- a/rules/issue.go +++ b/rules/issue.go @@ -10,7 +10,6 @@ type Issue struct { WebURL string `json:"webUrl" bson:"webUrl"` Title string `json:"title" bson:"title"` Description string `json:"description" bson:"description"` - State string `json:"state" bson:"state"` } type Issues []Issue diff --git a/rules/last_activity.go b/rules/last_activity.go index fefbae2..c2801d8 100644 --- a/rules/last_activity.go +++ b/rules/last_activity.go @@ -30,6 +30,14 @@ func (l *LastActivity) GetLevel() string { return LevelWarning } +func (e *LastActivity) GetName() string { + return e.Name +} + +func (e *LastActivity) GetDescription() string { + return e.Description +} + func NewLastActivity() Ruler { l := &LastActivity{ Name: "Last Activity > 1 year", diff --git a/rules/ruler.go b/rules/ruler.go index 8cdd5f1..51afd4e 100644 --- a/rules/ruler.go +++ b/rules/ruler.go @@ -7,6 +7,8 @@ import "github.com/xanzy/go-gitlab" type Ruler interface { Run(client *gitlab.Client, p *gitlab.Project) bool + GetName() string + GetDescription() string GetSlug() string GetLevel() string } diff --git a/rules/without_gitlab_ci.go b/rules/without_gitlab_ci.go index db0ffd2..a87446c 100644 --- a/rules/without_gitlab_ci.go +++ b/rules/without_gitlab_ci.go @@ -31,6 +31,14 @@ func (w *WithoutGitlabCI) GetLevel() string { return LevelInfo } +func (e *WithoutGitlabCI) GetName() string { + return e.Name +} + +func (e *WithoutGitlabCI) GetDescription() string { + return e.Description +} + func NewWithoutGitlabCI() Ruler { w := &WithoutGitlabCI{ Name: "Without Gitlab CI", diff --git a/rules/without_readme.go b/rules/without_readme.go index 90096b9..426bbe9 100644 --- a/rules/without_readme.go +++ b/rules/without_readme.go @@ -23,6 +23,14 @@ func (w *WithoutReadme) GetLevel() string { return LevelError } +func (e *WithoutReadme) GetName() string { + return e.Name +} + +func (e *WithoutReadme) GetDescription() string { + return e.Description +} + func NewWithoutReadme() Ruler { w := &WithoutReadme{ Name: "Without Readme", From ef4c67fb04a422c403d7899ee9721c0c5590c5df Mon Sep 17 00:00:00 2001 From: Pablo Aguilar Date: Tue, 5 Oct 2021 08:29:57 -0300 Subject: [PATCH 3/8] Creates `GoVendorFolder` rule (#20) --- rules/go_vendor_folder.go | 119 ++++++++++++++++++++++++++++++++++++++ rules/my_registry.go | 1 + 2 files changed, 120 insertions(+) create mode 100644 rules/go_vendor_folder.go diff --git a/rules/go_vendor_folder.go b/rules/go_vendor_folder.go new file mode 100644 index 0000000..2a6e519 --- /dev/null +++ b/rules/go_vendor_folder.go @@ -0,0 +1,119 @@ +// Copyright (c) 2021, Pablo Aguilar +// Licensed under the BSD 3-Clause License + +package rules + +import ( + log "github.com/sirupsen/logrus" + "github.com/xanzy/go-gitlab" +) + +// GoVendorFolder is a rule to verify if a repository has or not the go vendor folder. +// It look at the repository root searching for the `go.mod` file first since a project without +// that file means that it doesn't use go modules. If there's a `go.mod` it'll search for a +// file called `modules.txt` inside a `vendor` folder. That's the pattern the projects that use +// go modules and vendor its dependencies follows. +// If "go.mod" and "vendor/modules.txt" exist this rule will return `true`. +type GoVendorFolder struct { + Description string `json:"description"` + ID string `json:"ruleId"` + Level string `json:"level"` + Name string `json:"name"` +} + +// NewGoVendorFolder returns an instance of GoVendorFolder with its attributes filled +func NewGoVendorFolder() Ruler { + v := &GoVendorFolder{ + Name: "Go Vendor Folder", + Description: "This rule identifies if a repo has the vendor folder for a project that uses go modules", + } + v.ID = v.GetSlug() + v.Level = v.GetLevel() + return v +} + +func (f *GoVendorFolder) Run(c *gitlab.Client, p *gitlab.Project) bool { + if p.EmptyRepo { + return false + } + + hasGoMod, err := f.searchForGoModFile(p.ID, c) + if err != nil { + log.Errorf(`[%s] error searching for "go.mod" file: %s`, f.GetSlug(), err) + return false + } + + if !hasGoMod { + return false + } + + hasGoVendor, err := f.searchGoVendorModulesFile(p.ID, c) + if err != nil { + log.Errorf(`[%s] error searching for "vendor/modules.txt" file: %s`, f.GetSlug(), err) + return false + } + + return hasGoVendor +} + +func (f *GoVendorFolder) searchForGoModFile(projectID int, c *gitlab.Client) (bool, error) { + return f.searchForFile( + projectID, + "go.mod", + "", + c, + ) +} + +func (f *GoVendorFolder) searchGoVendorModulesFile(projectID int, c *gitlab.Client) (bool, error) { + return f.searchForFile( + projectID, + "modules.txt", + "vendor", + c, + ) +} + +func (f *GoVendorFolder) searchForFile( + projectID int, + fileName string, + path string, + c *gitlab.Client, +) (bool, error) { + listOpts := gitlab.ListTreeOptions{ + ListOptions: gitlab.ListOptions{ + Page: 1, + }, + Recursive: gitlab.Bool(false), + Path: gitlab.String(path), + } + + for { + nodes, resp, err := c.Repositories.ListTree(projectID, &listOpts) + if err != nil { + return false, err + } + + for _, node := range nodes { + if node.Type == "blob" && node.Name == fileName { + return true, nil + } + } + + if resp.CurrentPage >= resp.TotalPages { + break + } + + listOpts.Page = resp.NextPage + } + + return false, nil +} + +func (f *GoVendorFolder) GetSlug() string { + return "go-vendor-folder" +} + +func (f *GoVendorFolder) GetLevel() string { + return LevelWarning +} diff --git a/rules/my_registry.go b/rules/my_registry.go index a239002..08393da 100644 --- a/rules/my_registry.go +++ b/rules/my_registry.go @@ -11,6 +11,7 @@ var MyRegistry = &Registry{ func init() { MyRegistry.AddRule(NewEmptyRepository()) + MyRegistry.AddRule(NewGoVendorFolder()) MyRegistry.AddRule(NewHasOpenIssues()) MyRegistry.AddRule(NewLastActivity()) MyRegistry.AddRule(NewNonFastForwardMerge()) From 095d506eb6f84072fd7a3c52e4ea0231e63eff62 Mon Sep 17 00:00:00 2001 From: GabhenDM Date: Thu, 7 Oct 2021 19:11:11 -0300 Subject: [PATCH 4/8] Fixing no error validation bugs --- collector/collector.go | 8 ++++++-- rules/issue.go | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/collector/collector.go b/collector/collector.go index e9d8c60..a5cf5ed 100644 --- a/collector/collector.go +++ b/collector/collector.go @@ -105,7 +105,9 @@ func processIssues(registry *rules.Registry, git *gitlab.Client) error { if err != nil && err == mongo.ErrNoDocuments { title := fmt.Sprintf("[Gitlab-Lint] %s", registry.RulesFn[r.RuleID].GetName()) description := registry.RulesFn[r.RuleID].GetDescription() - createIssue(r, git, dbInstance, title, description) + if err := createIssue(r, git, dbInstance, title, description); err != nil { + return err + } // Opened issue found } else { // fetch issue info from gitlab @@ -119,7 +121,9 @@ func processIssues(registry *rules.Registry, git *gitlab.Client) error { title := fmt.Sprintf("[Gitlab-Lint][Reopened] %s", registry.RulesFn[r.RuleID].GetName()) description := registry.RulesFn[r.RuleID].GetDescription() // create new issue - createIssue(r, git, dbInstance, title, description) + if err := createIssue(r, git, dbInstance, title, description); err != nil { + return err + } } } } diff --git a/rules/issue.go b/rules/issue.go index 394349f..ce8c107 100644 --- a/rules/issue.go +++ b/rules/issue.go @@ -3,7 +3,7 @@ package rules import "go.mongodb.org/mongo-driver/bson/primitive" type Issue struct { - ID primitive.ObjectID `json:"_id"bson:"_id,omitempty"` + ID primitive.ObjectID `json:"_id" bson:"_id,omitempty"` ProjectID int `json:"projectId" bson:"projectId"` RuleID string `json:"ruleId" bson:"ruleId"` IssueID int `json:"issueId" bson:"issueId"` From 2e9e3b6ed4fbe51da58c089d91e1230d58ea566f Mon Sep 17 00:00:00 2001 From: GabhenDM Date: Fri, 1 Oct 2021 17:24:27 -0300 Subject: [PATCH 5/8] wip:automated issue opening - initial version --- collector/collector.go | 65 ++++++++++++++++++++++++++++++++++++++++++ db/db.go | 8 ++++++ rules/issue.go | 28 ++++++++++++++++++ 3 files changed, 101 insertions(+) create mode 100644 rules/issue.go diff --git a/collector/collector.go b/collector/collector.go index ed4573e..d6663d0 100644 --- a/collector/collector.go +++ b/collector/collector.go @@ -11,6 +11,8 @@ import ( "github.com/spf13/viper" "github.com/xanzy/go-gitlab" "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" _ "github.com/globocom/gitlab-lint/config" "github.com/globocom/gitlab-lint/db" @@ -78,6 +80,66 @@ func processRules(rulesList []rules.Rule) error { return nil } +func processIssues(registry *rules.Registry, git *gitlab.Client) error { + dbInstance, err := db.NewMongoSession() + if err != nil { + log.Errorf("[Collector] Error on create mongo session: %v", err) + return err + } + // iterating over matched rules + for _, r := range registry.Rules { + // searching for opened issue for project and rule + pipeline := bson.M{"$and": bson.A{bson.M{"projectId": r.ProjectID}, bson.M{"ruleId": r.RuleID}, bson.M{"state": "opened"}}} + + issue := &rules.Issue{} + err := dbInstance.Get(issue, pipeline, &options.FindOneOptions{}) + + // if any error + if err != nil && err != mongo.ErrNoDocuments { + return err + } + // if no opened issue found -> create new issue and add to DB + if err != nil && err == mongo.ErrNoDocuments { + createdIssue, _, err := git.Issues.CreateIssue(r.ProjectID, &gitlab.CreateIssueOptions{Title: &r.RuleID}) + if err != nil { + return err + } + + if _, err := dbInstance.Insert(&rules.Issue{ProjectID: r.ProjectID, RuleID: r.RuleID, IssueID: createdIssue.ID, + WebURL: r.WebURL, Title: r.RuleID, Description: "Test", State: createdIssue.State}); err != nil { + return err + } + // Opened issue found + } else { + // fetch issue info from gitlab + gitIssue, _, err := git.Issues.GetIssue(issue.ProjectID, issue.IssueID) + if err != nil { + return err + } + // if issue was closed but rule still matched + if gitIssue.State != issue.State && gitIssue.State == "closed" { + // update old issue on DB + if _, err := dbInstance.Update(&rules.Issue{}, bson.M{"_id": issue.ID}, bson.M{"$set": bson.M{"state": gitIssue.State}}, &options.UpdateOptions{}); err != nil { + return err + } + desc := "test-reopened" + // create new issue + createdIssue, _, err := git.Issues.CreateIssue(r.ProjectID, &gitlab.CreateIssueOptions{Title: &r.RuleID, Description: &desc}) + if err != nil { + return err + } + // add new issue to DB + if _, err := dbInstance.Insert(&rules.Issue{ProjectID: r.ProjectID, RuleID: r.RuleID, IssueID: createdIssue.ID, + WebURL: r.WebURL, Title: r.RuleID, Description: "Test-Reopened", State: createdIssue.State}); err != nil { + return err + } + + } + } + } + + return nil +} func insertStats(r *rules.Registry) error { dbInstance, err := db.NewMongoSession() @@ -191,4 +253,7 @@ func main() { if err := insertStats(rules.MyRegistry); err != nil { log.Errorf("[Collector] Error on insert statistics data: %v", err) } + if err := processIssues(rules.MyRegistry, git); err != nil { + log.Errorf("[Collector] Error on processing issue: %v", err) + } } diff --git a/db/db.go b/db/db.go index a6cd1a1..a2bf9ba 100644 --- a/db/db.go +++ b/db/db.go @@ -45,6 +45,7 @@ type DB interface { Count(d rules.Queryable, filter FindFilter) (int, error) DeleteMany(d rules.Queryable, q bson.M) (*mongo.DeleteResult, error) Get(d rules.Queryable, q bson.M, o *options.FindOneOptions) error + Update(d rules.Queryable, q bson.M, u bson.M, o *options.UpdateOptions) (*mongo.UpdateResult, error) GetAll(d rules.Queryable, filter FindFilter) ([]rules.Queryable, error) Insert(d rules.Queryable) (*mongo.InsertOneResult, error) InsertMany(d rules.Queryable, i []interface{}) (*mongo.InsertManyResult, error) @@ -138,6 +139,13 @@ func (m *mongoCollection) Get(d rules.Queryable, q bson.M, o *options.FindOneOpt return collection.FindOne(ctx, q).Decode(d) } +func (m *mongoCollection) Update(d rules.Queryable, q bson.M, u bson.M, o *options.UpdateOptions) (*mongo.UpdateResult, error) { + log.Debug("[DB] Update...") + collection := m.session.Database(m.dbName).Collection(d.GetCollectionName()) + ctx, _ := newDBContext() + return collection.UpdateOne(ctx, q, u, o) +} + func (m mongoCollection) GetAll(d rules.Queryable, filter FindFilter) ([]rules.Queryable, error) { log.Debug("[DB] GetAll...") collection := m.session.Database(m.dbName).Collection(d.GetCollectionName()) diff --git a/rules/issue.go b/rules/issue.go new file mode 100644 index 0000000..31efeb7 --- /dev/null +++ b/rules/issue.go @@ -0,0 +1,28 @@ +package rules + +import "go.mongodb.org/mongo-driver/bson/primitive" + +type Issue struct { + ID primitive.ObjectID `json:"_id"bson:"_id,omitempty"` + ProjectID int `json:"projectId" bson:"projectId"` + RuleID string `json:"ruleId" bson:"ruleId"` + IssueID int `json:"issueId" bson:"issueId"` + WebURL string `json:"webUrl" bson:"webUrl"` + Title string `json:"title" bson:"title"` + Description string `json:"description" bson:"description"` + State string `json:"state" bson:"state"` +} + +type Issues []Issue + +func (i Issue) Cast() Queryable { + return &i +} + +func (i Issue) GetCollectionName() string { + return "issues" +} + +func (i Issue) GetSearchableFields() []string { + return []string{"title", "projectid", "ruleid", "state"} +} From 144afc3e28adfab54107b4fcfbbb79881c967c04 Mon Sep 17 00:00:00 2001 From: GabhenDM Date: Fri, 1 Oct 2021 23:34:13 -0300 Subject: [PATCH 6/8] wip: Added description and title access methods to rules, better issue creating logic, removed unnecessary update --- collector/collector.go | 54 ++++++++++++++++++------------------- db/db.go | 10 +------ rules/empty_repository.go | 8 ++++++ rules/fast-forward-merge.go | 8 ++++++ rules/has_open_issues.go | 8 ++++++ rules/issue.go | 1 - rules/last_activity.go | 8 ++++++ rules/ruler.go | 2 ++ rules/without_gitlab_ci.go | 8 ++++++ rules/without_readme.go | 8 ++++++ 10 files changed, 77 insertions(+), 38 deletions(-) diff --git a/collector/collector.go b/collector/collector.go index d6663d0..e9d8c60 100644 --- a/collector/collector.go +++ b/collector/collector.go @@ -4,6 +4,7 @@ package main import ( + "fmt" "sync" "time" @@ -80,6 +81,7 @@ func processRules(rulesList []rules.Rule) error { return nil } + func processIssues(registry *rules.Registry, git *gitlab.Client) error { dbInstance, err := db.NewMongoSession() if err != nil { @@ -89,26 +91,21 @@ func processIssues(registry *rules.Registry, git *gitlab.Client) error { // iterating over matched rules for _, r := range registry.Rules { // searching for opened issue for project and rule - pipeline := bson.M{"$and": bson.A{bson.M{"projectId": r.ProjectID}, bson.M{"ruleId": r.RuleID}, bson.M{"state": "opened"}}} + pipeline := bson.M{"$and": bson.A{bson.M{"projectId": r.ProjectID}, bson.M{"ruleId": r.RuleID}}} issue := &rules.Issue{} - err := dbInstance.Get(issue, pipeline, &options.FindOneOptions{}) + err := dbInstance.Get(issue, pipeline, &options.FindOneOptions{Sort: bson.M{"issueId": -1}}) - // if any error + // if any error different than noDocumentFound if err != nil && err != mongo.ErrNoDocuments { return err } + // if no opened issue found -> create new issue and add to DB if err != nil && err == mongo.ErrNoDocuments { - createdIssue, _, err := git.Issues.CreateIssue(r.ProjectID, &gitlab.CreateIssueOptions{Title: &r.RuleID}) - if err != nil { - return err - } - - if _, err := dbInstance.Insert(&rules.Issue{ProjectID: r.ProjectID, RuleID: r.RuleID, IssueID: createdIssue.ID, - WebURL: r.WebURL, Title: r.RuleID, Description: "Test", State: createdIssue.State}); err != nil { - return err - } + title := fmt.Sprintf("[Gitlab-Lint] %s", registry.RulesFn[r.RuleID].GetName()) + description := registry.RulesFn[r.RuleID].GetDescription() + createIssue(r, git, dbInstance, title, description) // Opened issue found } else { // fetch issue info from gitlab @@ -117,23 +114,12 @@ func processIssues(registry *rules.Registry, git *gitlab.Client) error { return err } // if issue was closed but rule still matched - if gitIssue.State != issue.State && gitIssue.State == "closed" { - // update old issue on DB - if _, err := dbInstance.Update(&rules.Issue{}, bson.M{"_id": issue.ID}, bson.M{"$set": bson.M{"state": gitIssue.State}}, &options.UpdateOptions{}); err != nil { - return err - } - desc := "test-reopened" + if gitIssue.State == "closed" { + // Open new issue with Reopened title + title := fmt.Sprintf("[Gitlab-Lint][Reopened] %s", registry.RulesFn[r.RuleID].GetName()) + description := registry.RulesFn[r.RuleID].GetDescription() // create new issue - createdIssue, _, err := git.Issues.CreateIssue(r.ProjectID, &gitlab.CreateIssueOptions{Title: &r.RuleID, Description: &desc}) - if err != nil { - return err - } - // add new issue to DB - if _, err := dbInstance.Insert(&rules.Issue{ProjectID: r.ProjectID, RuleID: r.RuleID, IssueID: createdIssue.ID, - WebURL: r.WebURL, Title: r.RuleID, Description: "Test-Reopened", State: createdIssue.State}); err != nil { - return err - } - + createIssue(r, git, dbInstance, title, description) } } } @@ -141,6 +127,18 @@ func processIssues(registry *rules.Registry, git *gitlab.Client) error { return nil } +func createIssue(r rules.Rule, git *gitlab.Client, dbInstance db.DB, title string, description string) error { + createdIssue, _, err := git.Issues.CreateIssue(r.ProjectID, &gitlab.CreateIssueOptions{Title: &title, Description: &description}) + if err != nil { + return err + } + if _, err := dbInstance.Insert(&rules.Issue{ProjectID: r.ProjectID, RuleID: r.RuleID, IssueID: createdIssue.IID, + WebURL: r.WebURL, Title: title, Description: r.Description}); err != nil { + return err + } + return nil +} + func insertStats(r *rules.Registry) error { dbInstance, err := db.NewMongoSession() if err != nil { diff --git a/db/db.go b/db/db.go index a2bf9ba..d6bedb4 100644 --- a/db/db.go +++ b/db/db.go @@ -45,7 +45,6 @@ type DB interface { Count(d rules.Queryable, filter FindFilter) (int, error) DeleteMany(d rules.Queryable, q bson.M) (*mongo.DeleteResult, error) Get(d rules.Queryable, q bson.M, o *options.FindOneOptions) error - Update(d rules.Queryable, q bson.M, u bson.M, o *options.UpdateOptions) (*mongo.UpdateResult, error) GetAll(d rules.Queryable, filter FindFilter) ([]rules.Queryable, error) Insert(d rules.Queryable) (*mongo.InsertOneResult, error) InsertMany(d rules.Queryable, i []interface{}) (*mongo.InsertManyResult, error) @@ -136,14 +135,7 @@ func (m *mongoCollection) Get(d rules.Queryable, q bson.M, o *options.FindOneOpt log.Debug("[DB] Get...") collection := m.session.Database(m.dbName).Collection(d.GetCollectionName()) ctx, _ := newDBContext() - return collection.FindOne(ctx, q).Decode(d) -} - -func (m *mongoCollection) Update(d rules.Queryable, q bson.M, u bson.M, o *options.UpdateOptions) (*mongo.UpdateResult, error) { - log.Debug("[DB] Update...") - collection := m.session.Database(m.dbName).Collection(d.GetCollectionName()) - ctx, _ := newDBContext() - return collection.UpdateOne(ctx, q, u, o) + return collection.FindOne(ctx, q, o).Decode(d) } func (m mongoCollection) GetAll(d rules.Queryable, filter FindFilter) ([]rules.Queryable, error) { diff --git a/rules/empty_repository.go b/rules/empty_repository.go index 0537716..d1064a9 100644 --- a/rules/empty_repository.go +++ b/rules/empty_repository.go @@ -24,6 +24,14 @@ func (e *EmptyRepository) GetLevel() string { return LevelError } +func (e *EmptyRepository) GetName() string { + return e.Name +} + +func (e *EmptyRepository) GetDescription() string { + return e.Description +} + func NewEmptyRepository() Ruler { e := &EmptyRepository{ Name: "Empty Repository", diff --git a/rules/fast-forward-merge.go b/rules/fast-forward-merge.go index d9eb816..abbd47a 100644 --- a/rules/fast-forward-merge.go +++ b/rules/fast-forward-merge.go @@ -23,6 +23,14 @@ func (w *NonFastForwardMerge) GetLevel() string { return LevelPedantic } +func (e *NonFastForwardMerge) GetName() string { + return e.Name +} + +func (e *NonFastForwardMerge) GetDescription() string { + return e.Description +} + func NewNonFastForwardMerge() Ruler { w := &NonFastForwardMerge{ Name: "Non Fast-forward Merge", diff --git a/rules/has_open_issues.go b/rules/has_open_issues.go index 8f8742b..3b4e3c2 100644 --- a/rules/has_open_issues.go +++ b/rules/has_open_issues.go @@ -26,6 +26,14 @@ func (h *HasOpenIssues) GetLevel() string { return LevelPedantic } +func (e *HasOpenIssues) GetName() string { + return e.Name +} + +func (e *HasOpenIssues) GetDescription() string { + return e.Description +} + func NewHasOpenIssues() Ruler { h := &HasOpenIssues{ Name: "Has Open Issues", diff --git a/rules/issue.go b/rules/issue.go index 31efeb7..394349f 100644 --- a/rules/issue.go +++ b/rules/issue.go @@ -10,7 +10,6 @@ type Issue struct { WebURL string `json:"webUrl" bson:"webUrl"` Title string `json:"title" bson:"title"` Description string `json:"description" bson:"description"` - State string `json:"state" bson:"state"` } type Issues []Issue diff --git a/rules/last_activity.go b/rules/last_activity.go index fefbae2..c2801d8 100644 --- a/rules/last_activity.go +++ b/rules/last_activity.go @@ -30,6 +30,14 @@ func (l *LastActivity) GetLevel() string { return LevelWarning } +func (e *LastActivity) GetName() string { + return e.Name +} + +func (e *LastActivity) GetDescription() string { + return e.Description +} + func NewLastActivity() Ruler { l := &LastActivity{ Name: "Last Activity > 1 year", diff --git a/rules/ruler.go b/rules/ruler.go index 8cdd5f1..51afd4e 100644 --- a/rules/ruler.go +++ b/rules/ruler.go @@ -7,6 +7,8 @@ import "github.com/xanzy/go-gitlab" type Ruler interface { Run(client *gitlab.Client, p *gitlab.Project) bool + GetName() string + GetDescription() string GetSlug() string GetLevel() string } diff --git a/rules/without_gitlab_ci.go b/rules/without_gitlab_ci.go index db0ffd2..a87446c 100644 --- a/rules/without_gitlab_ci.go +++ b/rules/without_gitlab_ci.go @@ -31,6 +31,14 @@ func (w *WithoutGitlabCI) GetLevel() string { return LevelInfo } +func (e *WithoutGitlabCI) GetName() string { + return e.Name +} + +func (e *WithoutGitlabCI) GetDescription() string { + return e.Description +} + func NewWithoutGitlabCI() Ruler { w := &WithoutGitlabCI{ Name: "Without Gitlab CI", diff --git a/rules/without_readme.go b/rules/without_readme.go index 90096b9..426bbe9 100644 --- a/rules/without_readme.go +++ b/rules/without_readme.go @@ -23,6 +23,14 @@ func (w *WithoutReadme) GetLevel() string { return LevelError } +func (e *WithoutReadme) GetName() string { + return e.Name +} + +func (e *WithoutReadme) GetDescription() string { + return e.Description +} + func NewWithoutReadme() Ruler { w := &WithoutReadme{ Name: "Without Readme", From 8ad0294267ff9b472e6f6a757396f48994bd4276 Mon Sep 17 00:00:00 2001 From: GabhenDM Date: Thu, 7 Oct 2021 19:11:11 -0300 Subject: [PATCH 7/8] Fixing no error validation bugs --- collector/collector.go | 8 ++++++-- rules/issue.go | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/collector/collector.go b/collector/collector.go index e9d8c60..a5cf5ed 100644 --- a/collector/collector.go +++ b/collector/collector.go @@ -105,7 +105,9 @@ func processIssues(registry *rules.Registry, git *gitlab.Client) error { if err != nil && err == mongo.ErrNoDocuments { title := fmt.Sprintf("[Gitlab-Lint] %s", registry.RulesFn[r.RuleID].GetName()) description := registry.RulesFn[r.RuleID].GetDescription() - createIssue(r, git, dbInstance, title, description) + if err := createIssue(r, git, dbInstance, title, description); err != nil { + return err + } // Opened issue found } else { // fetch issue info from gitlab @@ -119,7 +121,9 @@ func processIssues(registry *rules.Registry, git *gitlab.Client) error { title := fmt.Sprintf("[Gitlab-Lint][Reopened] %s", registry.RulesFn[r.RuleID].GetName()) description := registry.RulesFn[r.RuleID].GetDescription() // create new issue - createIssue(r, git, dbInstance, title, description) + if err := createIssue(r, git, dbInstance, title, description); err != nil { + return err + } } } } diff --git a/rules/issue.go b/rules/issue.go index 394349f..ce8c107 100644 --- a/rules/issue.go +++ b/rules/issue.go @@ -3,7 +3,7 @@ package rules import "go.mongodb.org/mongo-driver/bson/primitive" type Issue struct { - ID primitive.ObjectID `json:"_id"bson:"_id,omitempty"` + ID primitive.ObjectID `json:"_id" bson:"_id,omitempty"` ProjectID int `json:"projectId" bson:"projectId"` RuleID string `json:"ruleId" bson:"ruleId"` IssueID int `json:"issueId" bson:"issueId"` From afe25127867d1a617054bfd2509786f028e59727 Mon Sep 17 00:00:00 2001 From: GabhenDM Date: Fri, 8 Oct 2021 12:06:20 -0300 Subject: [PATCH 8/8] Fixing go vendor rule --- rules/go_vendor_folder.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/rules/go_vendor_folder.go b/rules/go_vendor_folder.go index 2a6e519..1289bb3 100644 --- a/rules/go_vendor_folder.go +++ b/rules/go_vendor_folder.go @@ -117,3 +117,11 @@ func (f *GoVendorFolder) GetSlug() string { func (f *GoVendorFolder) GetLevel() string { return LevelWarning } + +func (e *GoVendorFolder) GetName() string { + return e.Name +} + +func (e *GoVendorFolder) GetDescription() string { + return e.Description +}