From c189daa59f4339ca9e57120ba6daafe59da7d038 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Canet?= Date: Tue, 14 Feb 2023 15:56:17 +0100 Subject: [PATCH] gmail: Use Label Get() in order to get better progress estimate. The ResultSizeEstimate field is only an estimate. This leads the backup progress computation to use a wrong total account of mails hence displaying bogus results for non small mailboxes. This commit creates a method to be used to compute something nearing the mail count (the label count) in order to compute a better backup progress estimation. This works by retrieving the Labels() then iterating on the resulting list and filtering only by the label names that make sense. The two cases this patch implements are: 1 ) No label is to be used for filtering: Then we take into account labels in the following set: { "DRAFT" , "INBOX", "SENT", "TRASH" }. The rationale for this is that these labels fit 'real' mails contrary to the other labels. Note that account of mail labelled "TRASH" must be substracted from the account computed above. 2 ) A label was specified in order to make a selective backup: Then simply account the number of mails having this label. Finally the messages sent to feed the progress display goroutine takes into account the relevant label count of each mails if we are into the 1) case. Again relevant labels are in { "DRAFT" , "INBOX", "SENT", "TRASH" }. --- lib/gmail/gmail.go | 105 +++++++++++++++++++++++++++++++++++++++---- lib/gmail/service.go | 11 +++++ lib/progress.go | 4 +- main.go | 5 ++- 4 files changed, 114 insertions(+), 11 deletions(-) diff --git a/lib/gmail/gmail.go b/lib/gmail/gmail.go index 765699e..c8e756d 100644 --- a/lib/gmail/gmail.go +++ b/lib/gmail/gmail.go @@ -364,6 +364,87 @@ func shardForMsgId(id string) int { return int(shard) } +// This function return true if the given label parameter +// is not an extra (decorating) label. +func isToBeAccounted(label string) bool { + // The following build a set of the real mails labels. + // Only these mails types are to take into account when + // retrieving a full mailbox without filtering by a given + // label. + type void struct{} + var member void + toAccountLabels := make(map[string]void) + toAccountLabels["DRAFT"] = member + toAccountLabels["INBOX"] = member + toAccountLabels["SENT"] = member + toAccountLabels["TRASH"] = member + _, toBeAccounted := toAccountLabels[label] + return toBeAccounted +} + +// This function takes a list of labels and count +// the number of real labels discarding extra labels. +func toBeAccountedLabelsCount(g *Gmail, labels []string) int64 { + // If a label was specified on the command line + // then we are interested only on this given label + // and we return 1. + if len(g.label) != 0 { + return 1 + } + // No label was specified so return the relevant labels + // count. + count := int64(0) + for _, label := range labels { + if isToBeAccounted(label) { + count += 1 + } + } + return count +} + +func getLabelsCount(g *Gmail) (int64, error) { + // Get a list of partially filled labels. + // In order to get the message count of the resulting + // labels another call to UsersLabelsService.Get() must be + // done later per label. + labelsResponse, err := g.svc.GetLabels() + if err != nil { + return 0, err + } + + totalCount := int64(0) + for _, label := range labelsResponse.Labels { + // Get the complete data for a specific label + fullLabel, err := g.svc.GetLabel(label.Id) + if err != nil { + return 0, err + } + + toAccount := int64(fullLabel.MessagesTotal) + // 1) If the current label is TRASH and no label was + // specified then it's message count must be + // substracted from the totalCount if we are + // in 2) below. + if label.Name == "TRASH" { + toAccount *= -1 + } + + // No label was specified when calling the NewGmail() + // constructor: add to the total account of labels + // matching this loop's label excepted if the are in 1) + // (like above) then we must withdraw the account. + if len(g.labelId) == 0 && isToBeAccounted(label.Name) { + totalCount += toAccount + } else if len(g.labelId) != 0 && fullLabel.Id == g.labelId { + // If we are looking for a specific label then + // simply return the message count of this label. + return fullLabel.MessagesTotal, nil + } + } + + return totalCount, nil +} + func (g *Gmail) incremental(historyId uint64) error { log.Println("Performing incremental sync.") page := "" @@ -406,7 +487,11 @@ func (g *Gmail) incremental(historyId uint64) error { close(ops) }() - t := uint(0) // Total count, for progress reporting. + t, err := getLabelsCount(g) // Total count, for progress reporting + if err != nil { + return err + } + go func() { for true { r, err := g.svc.GetHistory(historyId, g.labelId, page) @@ -419,7 +504,6 @@ func (g *Gmail) incremental(historyId uint64) error { return } page = r.NextPageToken - t += uint(len(r.History)) for _, m := range r.History { if m.Id > historyId { historyId = m.Id @@ -474,13 +558,13 @@ func (g *Gmail) incremental(historyId uint64) error { close(h) } }() - i := uint(0) + i := int64(0) for o := range ops { // Update progress bar. if g.progress != nil { g.progress <- lib.Progress{Current: i, Total: t} } - i++ + i += toBeAccountedLabelsCount(g, o.Labels) if o.Error != nil { return o.Error } @@ -533,7 +617,12 @@ func (g *Gmail) full() error { close(ops) }() seen := make(map[string]struct{}) // Used to compute deletes. - t := uint(0) // Total count, for progress reporting. + + t, err := getLabelsCount(g) // Total count, for progress reporting + if err != nil { + return err + } + go func() { defer close(newMsgs) page := "" @@ -544,7 +633,7 @@ func (g *Gmail) full() error { return } page = r.NextPageToken - t += uint(r.ResultSizeEstimate) + for _, m := range r.Messages { newMsgs <- m.Id seen[m.Id] = struct{}{} @@ -555,13 +644,13 @@ func (g *Gmail) full() error { } }() historyId := uint64(0) - i := uint(0) // For updating progress bar. + i := int64(0) // For updating progress bar. for o := range ops { // Update progress bar. if g.progress != nil { g.progress <- lib.Progress{Current: i, Total: t} } - i++ + i += toBeAccountedLabelsCount(g, o.Labels) if o.Error != nil { return o.Error } diff --git a/lib/gmail/service.go b/lib/gmail/service.go index 0c1863c..fe52092 100644 --- a/lib/gmail/service.go +++ b/lib/gmail/service.go @@ -17,6 +17,7 @@ type gmailService interface { GetRawMessage(id string) (string, error) GetMetadata(id string) (*gmail.Message, error) GetLabels() (*gmail.ListLabelsResponse, error) + GetLabel(id string) (*gmail.Label, error) GetHistory(historyIndex uint64, label, page string) (*gmail.ListHistoryResponse, error) GetMessages(q, page string) (*gmail.ListMessagesResponse, error) } @@ -79,6 +80,16 @@ func (s *restGmailService) GetLabels() (*gmail.ListLabelsResponse, error) { return r, err } +func (s *restGmailService) GetLabel(id string) (*gmail.Label, error) { + var r *gmail.Label + var err error + err = s.limiter.DoWithBackoff(func() (error, bool) { + r, err = s.svc.Labels.Get("me", id).Do() + return isRateLimited(err) + }) + return r, err +} + func (s *restGmailService) GetHistory(historyIndex uint64, labelId, page string) (*gmail.ListHistoryResponse, error) { hist := s.svc.History.List("me").StartHistoryId(historyIndex) if labelId != "" { diff --git a/lib/progress.go b/lib/progress.go index 1d0fb01..fc15502 100644 --- a/lib/progress.go +++ b/lib/progress.go @@ -2,6 +2,6 @@ package lib // Progress represents a simple "done xxx out of yyy"-style progress report. type Progress struct { - Current uint - Total uint + Current int64 + Total int64 } diff --git a/main.go b/main.go index 26b099b..9da86a8 100644 --- a/main.go +++ b/main.go @@ -78,11 +78,14 @@ func main() { } progress := make(chan lib.Progress) go func() { + // Given how the label mail counting work we are only able to render the progress + // relative to the total number of mail label processed. + // So we specify below that the progress is done against labels and not mails. l := time.Time{} for p := range progress { if time.Since(l).Seconds() > progressUpdateFreqSecs { l = time.Now() - fmt.Printf("\r%d / %d %.2f%% ", p.Current, p.Total, float32(p.Current)/float32(p.Total)*100) + fmt.Printf("\r%d / %d labels : %.2f%% ", p.Current, p.Total, float32(p.Current)/float32(p.Total)*100) } } fmt.Println()