Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,7 @@ See [docs/agentic-features-overview.md](docs/agentic-features-overview.md) for a
| `/gen` | Alias of `/generate-report` |
| `/list` | List this week's work items |
| `/check` | List missing members with nudge buttons |
| `/nudge` | Send a test nudge DM (self by default; managers can target one member) |
| `/retrospect` | Analyze corrections and suggest improvements |
| `/stats` | View classification accuracy dashboard |
| `/help` | Show help and usage |
Expand Down Expand Up @@ -376,6 +377,14 @@ Anyone can view this week's items:

**On-demand**: `/check` lists team members who haven't reported this week, with a "Nudge" button next to each member and a "Nudge All" button at the bottom. Clicking opens a confirmation before sending the DM.

**Testing**:

```text
/nudge # Send a test nudge to your own DM
/nudge Member Name # Manager only: send a test nudge to one member
/nudge U123ABC456 # Manager only: target a Slack user ID directly
```

On Monday before `monday_cutoff_time` (default `12:00`) in configured `timezone`, report commands use the previous calendar week.

Accepts any day name: `Monday`, `Tuesday`, ..., `Sunday`.
Expand Down
2 changes: 1 addition & 1 deletion internal/app/app.go
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ func Main() {
slack.OptionAppLevelToken(cfg.SlackAppToken),
)

nudge.StartNudgeScheduler(cfg, api)
nudge.StartNudgeScheduler(cfg, db, api)
fetch.StartAutoFetchScheduler(cfg, db, api)

log.Println("Starting Engineering Report Bot...")
Expand Down
101 changes: 99 additions & 2 deletions internal/integrations/llm/llm.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@ import (
"log"
"net/http"
"os"
"regexp"
"strconv"
"strings"
"sync"

Expand Down Expand Up @@ -65,6 +67,8 @@ const defaultOpenAIModel = "gpt-4o-mini"
const maxTemplateGuidanceChars = 8000
const defaultLLMMaxConcurrentBatches = 4

var confidenceFieldRe = regexp.MustCompile(`("confidence"\s*:\s*)([^,}\]]+)`)

func llmBatchConcurrencyLimit(totalBatches int) int {
if totalBatches <= 0 {
return 1
Expand Down Expand Up @@ -295,7 +299,7 @@ Also:
- choose normalized_status from: done, in testing, in progress, other
- extract ticket IDs if present (e.g. [1247202] or bare ticket numbers)
- if this item is the same underlying work as an existing item, set duplicate_of to that existing key (Kxx); otherwise empty string
- set confidence between 0 and 1.
- set confidence between 0 and 1, using digits only (example: 0.91). Never spell out numbers.
%s%s

Respond with JSON only (no markdown):
Expand Down Expand Up @@ -362,7 +366,14 @@ func parseSectionClassifiedResponse(responseText string) (map[int64]LLMSectionDe

var classified []sectionClassifiedItem
if err := json.Unmarshal([]byte(responseText), &classified); err != nil {
return nil, fmt.Errorf("parsing LLM section response: %w (response: %s)", err, responseText)
repaired := repairSectionClassifiedResponse(responseText)
if repaired == responseText {
return nil, fmt.Errorf("parsing LLM section response: %w (response: %s)", err, responseText)
}
if repairErr := json.Unmarshal([]byte(repaired), &classified); repairErr != nil {
return nil, fmt.Errorf("parsing LLM section response after repair: %w (original error: %v, response: %s)", repairErr, err, repaired)
}
log.Printf("llm section response repaired before parse")
}

decisions := make(map[int64]LLMSectionDecision)
Expand Down Expand Up @@ -424,6 +435,92 @@ func parseTicketIDsField(raw json.RawMessage) string {
return ""
}

func repairSectionClassifiedResponse(responseText string) string {
return confidenceFieldRe.ReplaceAllStringFunc(responseText, func(match string) string {
parts := confidenceFieldRe.FindStringSubmatch(match)
if len(parts) != 3 {
return match
}
return parts[1] + normalizeConfidenceLiteral(parts[2])
})
}

func normalizeConfidenceLiteral(raw string) string {
text := strings.TrimSpace(strings.Trim(raw, `"`))
if text == "" {
return "0"
}
if parsed, ok := parseLooseConfidence(text); ok {
return strconv.FormatFloat(parsed, 'f', -1, 64)
}
return "0"
}

func parseLooseConfidence(raw string) (float64, bool) {
text := strings.TrimSpace(raw)
if text == "" {
return 0, false
}
if parsed, err := strconv.ParseFloat(text, 64); err == nil {
return clampConfidence(parsed), true
}

compact := strings.ReplaceAll(strings.ToLower(text), " ", "")
if parsed, err := strconv.ParseFloat(compact, 64); err == nil {
return clampConfidence(parsed), true
}

if strings.HasPrefix(compact, "0.") {
if digit, ok := digitWord(compact[2:]); ok {
return float64(digit) / 10, true
}
}
if compact == "zero" {
return 0, true
}
if compact == "one" {
return 1, true
}
return 0, false
}

func digitWord(s string) (int, bool) {
switch s {
case "zero":
return 0, true
case "one":
return 1, true
case "two":
return 2, true
case "three":
return 3, true
case "four":
return 4, true
case "five":
return 5, true
case "six":
return 6, true
case "seven":
return 7, true
case "eight":
return 8, true
case "nine":
return 9, true
default:
return 0, false
}
}

func clampConfidence(v float64) float64 {
if v < 0 {
return 0
}
if v > 1 {
return 1
}
return v
}

func loadGlossaryIfConfigured(cfg Config) (*LLMGlossary, error) {
if strings.TrimSpace(cfg.LLMGlossaryPath) == "" {
return nil, nil
Expand Down
41 changes: 41 additions & 0 deletions internal/integrations/llm/llm_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -129,6 +129,47 @@ func TestParseSectionClassifiedResponse_AcceptsArrayTicketIDs(t *testing.T) {
}
}

func TestParseSectionClassifiedResponse_RepairsMalformedConfidence(t *testing.T) {
response := `[
{"id": 1, "section_id": "S0_0", "normalized_status": "done", "ticket_ids": "", "duplicate_of": "", "confidence": 0. Nine},
{"id": 2, "section_id": "S0_1", "normalized_status": "in progress", "ticket_ids": "", "duplicate_of": "", "confidence": 0. seven},
{"id": 3, "section_id": "S0_2", "normalized_status": "done", "ticket_ids": "", "duplicate_of": "", "confidence": nope}
]`

got, err := parseSectionClassifiedResponse(response)
if err != nil {
t.Fatalf("parseSectionClassifiedResponse should repair malformed confidence: %v", err)
}
if got[1].Confidence != 0.9 {
t.Fatalf("expected repaired confidence 0.9, got %v", got[1].Confidence)
}
if got[2].Confidence != 0.7 {
t.Fatalf("expected repaired confidence 0.7, got %v", got[2].Confidence)
}
if got[3].Confidence != 0 {
t.Fatalf("expected fallback confidence 0, got %v", got[3].Confidence)
}
}

func TestNormalizeConfidenceLiteral(t *testing.T) {
tests := []struct {
raw string
want string
}{
{raw: "0.91", want: "0.91"},
{raw: `"0.82"`, want: "0.82"},
{raw: "0. Nine", want: "0.9"},
{raw: "0. seven", want: "0.7"},
{raw: "garbage", want: "0"},
}

for _, tt := range tests {
if got := normalizeConfidenceLiteral(tt.raw); got != tt.want {
t.Fatalf("normalizeConfidenceLiteral(%q) = %q, want %q", tt.raw, got, tt.want)
}
}
}

func TestParseTicketIDsField_MixedArray(t *testing.T) {
raw := json.RawMessage(`[ "123", 456, "", " 789 " ]`)
got := parseTicketIDsField(raw)
Expand Down
20 changes: 18 additions & 2 deletions internal/integrations/slack/deps.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ type ClassificationStats = domain.ClassificationStats
type sectionOption = llm.SectionOption
type BuildResult = report.BuildResult
type LLMSectionDecision = llm.LLMSectionDecision
type RenderedNudge = nudge.RenderedNudge

type loadStatus int

Expand All @@ -32,6 +33,13 @@ const (
templateFirstEver
)

const (
actionNudgeDone = nudge.ActionDone
actionNudgeMore = nudge.ActionMore
actionNudgePagePrev = nudge.ActionPagePrev
actionNudgePageNext = nudge.ActionPageNext
)

func ReportWeekRange(cfg Config, now time.Time) (time.Time, time.Time) {
return domain.ReportWeekRange(cfg, now)
}
Expand Down Expand Up @@ -136,6 +144,10 @@ func UpdateWorkItemTextAndStatus(db *sql.DB, id int64, description, status strin
return sqlite.UpdateWorkItemTextAndStatus(db, id, description, status)
}

func UpdateWorkItemStatus(db *sql.DB, id int64, status string) error {
return sqlite.UpdateWorkItemStatus(db, id, status)
}

func UpdateWorkItemCategory(db *sql.DB, id int64, category string) error {
return sqlite.UpdateWorkItemCategory(db, id, category)
}
Expand Down Expand Up @@ -180,8 +192,12 @@ func extractGlossaryPhrase(description string) string {
return llm.ExtractGlossaryPhrase(description)
}

func sendNudges(api *slack.Client, cfg Config, memberIDs []string, reportChannelID string) {
nudge.SendNudges(api, cfg, memberIDs, reportChannelID)
func sendNudges(api *slack.Client, db *sql.DB, cfg Config, memberIDs []string, reportChannelID string) {
nudge.SendNudges(api, db, cfg, memberIDs, reportChannelID)
}

func RenderNudgeForUser(api *slack.Client, db *sql.DB, cfg Config, userID, reportChannelID string, now time.Time, page int, updated bool) (RenderedNudge, error) {
return nudge.RenderNudgeForUser(api, db, cfg, userID, reportChannelID, now, page, updated)
}

func normalizeTextToken(s string) string {
Expand Down
3 changes: 3 additions & 0 deletions internal/integrations/slack/functional_slack_github_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,7 @@ func withMockGitHubAPI(t *testing.T) {

func newMockSlackAPI(t *testing.T) (*slack.Client, *int) {
t.Helper()
resetUserCacheForTest(t)

postEphemeralCalls := 0
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
Expand Down Expand Up @@ -171,6 +172,7 @@ func newMockSlackAPI(t *testing.T) (*slack.Client, *int) {

func newMockSlackAPIWithUsers(t *testing.T) *slack.Client {
t.Helper()
resetUserCacheForTest(t)

server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
path := strings.TrimPrefix(r.URL.Path, "/api/")
Expand Down Expand Up @@ -214,6 +216,7 @@ func newMockSlackAPIWithUsers(t *testing.T) *slack.Client {

func newMockSlackAPIWithManagerNotify(t *testing.T) (*slack.Client, *int, *string) {
t.Helper()
resetUserCacheForTest(t)

managerMsgCalls := 0
lastManagerMsg := ""
Expand Down
Loading