Errors overhaul

.golangci.yaml

@@ -2,7 +2,6 @@
 version: "2"
 formatters:
   enable:
-    - gci
     - gofumpt
     - goimports
     - golines
@@ -15,6 +14,7 @@ formatters:
         - pattern: "interface{}"
           replacement: "any"
     gofumpt:
+      module-path: github.com/ozontech/testo
       extra-rules: true
 
 linters:
@@ -57,8 +57,10 @@ linters:
           - thelper
 
   settings:
+    cyclop:
+      max-complexity: 15
     funlen:
-      lines: 70
+      lines: 80
     gosec:
       excludes:
         - G304

CHANGELOG.md

@@ -5,6 +5,17 @@ All notable changes to this project are documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## Unreleased
+
+## Added
+
+- Added strict mode which turns warnings into hard errors. Can be enabled with `-testo.strict` command line argument and `TESTO_STRICT` environment variable (lower priority).
+
+### Changed
+
+- Testo error messages are now more descriptive.
+- Malformed method names (e.g. lowercase letter after `Test` prefix) raise fatal errors, similar to native `go test` runner.
+
 ## [1.4.0] - 2026-06-07
 
 ### Added

collector.go

@@ -1,9 +1,7 @@
 package testo
 
 import (
-	"fmt"
 	"maps"
-	"os"
 	"reflect"
 	"slices"
 	"strings"
@@ -81,7 +79,16 @@ func suiteCasesOf[Suite suite[T], T CommonT](tb testing.TB) map[string]suiteCase
 		const prefix = "Cases"
 
 		if !isTest(method.Name, prefix) {
-			continue
+			if !strings.HasPrefix(method.Name, prefix) {
+				continue
+			}
+
+			tb.Fatalf(
+				"testo: (%s).%s has malformed name: first letter after '%s' must not be lowercase",
+				reflect.TypeFor[Suite](),
+				method.Name,
+				prefix,
+			)
 		}
 
 		name := strings.TrimPrefix(method.Name, prefix)
@@ -95,7 +102,7 @@ func suiteCasesOf[Suite suite[T], T CommonT](tb testing.TB) map[string]suiteCase
 
 		if !isValidIn || !isValidOut {
 			tb.Fatalf(
-				"testo: wrong signature for %[1]s.%[2]s, must be: func (%[1]s) %[2]s() []...",
+				"testo: wrong signature for %[1]s.%[2]s, must be: func (%[1]s) %[2]s() []Type",
 				reflect.TypeFor[Suite](), method.Name, tb,
 			)
 		}
@@ -150,6 +157,10 @@ type suiteTests[Suite suite[T], T CommonT] struct {
 	Parametrized []suiteTestParametrized[Suite, T]
 }
 
+func (st suiteTests[Suite, T]) isEmpty() bool {
+	return len(st.Regular) == 0 && len(st.Parametrized) == 0
+}
+
 type annotatedSuiteTest[Suite suite[T], T CommonT] struct {
 	suiteTest[Suite, T]
 
@@ -164,9 +175,12 @@ type annotatedSuiteTest[Suite suite[T], T CommonT] struct {
 // Suite instance is required here to get
 // parameter cases (CasesXXX funcs), not to invoke the actual tests.
 func (st suiteTests[Suite, T]) Collect(
+	tb testing.TB,
 	s Suite,
 	name func(string) string,
 ) []annotatedSuiteTest[Suite, T] {
+	tb.Helper()
+
 	tests := make([]annotatedSuiteTest[Suite, T], 0, len(st.Regular))
 
 	for _, r := range st.Regular {
@@ -177,7 +191,7 @@ func (st suiteTests[Suite, T]) Collect(
 	}
 
 	for _, p := range st.Parametrized {
-		cases := p.Tests(s)
+		cases := p.Tests(tb, s)
 
 		tests = append(tests, cases...)
 	}
@@ -217,30 +231,44 @@ func (tc *testsCollector[Suite, T]) testName(base string) string {
 }
 
 //nolint:cyclop,funlen,gocognit // splitting it would make it even more complex
-func (tc *testsCollector[Suite, T]) Collect(
-	tb testing.TB,
-) suiteTests[Suite, T] {
+func (tc *testsCollector[Suite, T]) Collect(tb testing.TB) suiteTests[Suite, T] {
 	tb.Helper()
 
-	cases := suiteCasesOf[Suite](tb)
-
 	suiteTyp := reflect.TypeFor[Suite]()
 
+	if suiteTyp == reflect.TypeFor[singleton[T]]() {
+		return suiteTests[Suite, T]{}
+	}
+
+	cases := suiteCasesOf[Suite](tb)
+
 	var tests suiteTests[Suite, T]
 
 	for i := range suiteTyp.NumMethod() {
 		method := suiteTyp.Method(i)
 
-		if !isTest(method.Name, "Test") {
-			continue
+		const prefix = "Test"
+
+		if !isTest(method.Name, prefix) {
+			if !strings.HasPrefix(method.Name, prefix) {
+				continue
+			}
+
+			// identical to native go test behavior
+			tb.Fatalf(
+				"testo: (%s).%s has malformed name: first letter after '%s' must not be lowercase",
+				suiteTyp,
+				method.Name,
+				prefix,
+			)
 		}
 
 		raiseWrongSignatureError := func() {
 			tb.Helper()
 
 			//nolint:lll // it's a long message
 			tb.Fatalf(
-				"testo: wrong signature for (%[1]s).%[2]s, must be: func (%[1]s).%[2]s(%[3]s) or func (%[1]s).%[2]s(%[3]s, struct{...})",
+				"testo: wrong signature for (%[1]s).%[2]s, must be either:\nfunc (%[1]s).%[2]s(%[3]s)\nfunc (%[1]s).%[2]s(%[3]s, struct{...})",
 				suiteTyp,
 				method.Name,
 				reflect.TypeFor[T](),
@@ -328,19 +356,25 @@ func (tc *testsCollector[Suite, T]) Collect(
 		}
 	}
 
+	if tests.isEmpty() {
+		warnf(tb, "suite %s has no tests", suiteTyp)
+	}
+
 	return tests
 }
 
 type suiteTestParametrized[Suite suite[T], T CommonT] struct {
-	Tests func(Suite) []annotatedSuiteTest[Suite, T]
+	Tests func(testing.TB, Suite) []annotatedSuiteTest[Suite, T]
 }
 
 func (tc *testsCollector[Suite, T]) newParametrizedTest(
 	method reflect.Method,
 	cases map[string]suiteCase[Suite, T],
 ) suiteTestParametrized[Suite, T] {
 	return suiteTestParametrized[Suite, T]{
-		Tests: func(s Suite) []annotatedSuiteTest[Suite, T] {
+		Tests: func(tb testing.TB, s Suite) []annotatedSuiteTest[Suite, T] {
+			tb.Helper()
+
 			casesValues := make(map[string][]reflect.Value, len(cases))
 
 			for caseName, c := range cases {
@@ -349,9 +383,9 @@ func (tc *testsCollector[Suite, T]) newParametrizedTest(
 				if len(values) == 0 {
 					structName := method.Type.In(0).String()
 
-					fmt.Fprintf(
-						os.Stderr,
-						"testo: warning: (%[1]s).Cases%[2]s provides zero values, (%[1]s).%[3]s will not run\n",
+					warnf(
+						tb,
+						"(%[1]s).Cases%[2]s provides zero values, (%[1]s).%[3]s won't run",
 						structName,
 						caseName,
 						method.Name,

docs/how-to.md

@@ -40,6 +40,26 @@ TestFoo(name=Joe, age=60)
 TestFoo(name=Joe, age=6)
 ```
 
+If for at least one required parameter function `CasesXxx`
+returns zero values Testo will log a warning with similar message:
+
+```txt
+main_test.go:15: testo: (*main.Suite).CasesName provides zero values, (*main.Suite).TestFoo won't run
+```
+
+To turn this log into fatal error and do not proceed with further execution
+pass flag `-testo.strict` to the `go test` command invocation:
+
+```bash
+go test ./... -testo.strict
+```
+
+> [!TIP]
+> You can also set `TESTO_STRICT` environment variable to `true`
+> for the same effect.
+>
+> Flags have higher priority than environment variables.
+
 ## How to write parallel tests
 
 You can use your regular `t.Parallel` method to mark a test as parallel.

docs/technical-overview.md

@@ -7,9 +7,9 @@ A technical overview of Testo.
 When you call `testo.RunSuite` the following happens:
 
 ```txt
-Suite tests are collected and verified.
-
 A root test named the same as a suite is run {
+    Suite tests are collected and verified.
+
     Plugins are collected and initialized with ".Plugin(parent: nil, options)" method call, if implemented. Innermost plugins are initialized first.
 
     "BeforeAll" plugin hooks are called.

examples/06_errors/main_test.go

@@ -22,21 +22,39 @@ func (InvalidCases) CasesFoo() []string { return []string{"one", "two"} }
 // testo: wrong param signature for (*main.InvalidCases).Test: (*main.InvalidCases).CasesFoo provides string values, not assignable to param "Foo" of type int
 func (InvalidCases) Test(t T, p struct{ Foo int }) {}
 
+type MalformedCases struct{ testo.Suite[T] }
+
+// testo: (*main.MalformedCases).Casesfoo has malformed name: first letter after 'Cases' must not be lowercase
+func (MalformedCases) Casesfoo() []string { return []string{"a", "b"} }
+
+func (MalformedCases) Test(t T, p struct{ Foo string }) {}
+
+type MalformedTest struct{ testo.Suite[T] }
+
+// testo: (*main.MalformedTest).Testimony has malformed name: first letter after 'Test' must not be lowercase
+func (MalformedTest) Testimony(t T) {}
+
 type EmptyCases struct{ testo.Suite[T] }
 
 func (EmptyCases) CasesFoo() []int { return nil }
 
-// testo: warning: (*main.EmptyCases).CasesFoo provides zero values, (*main.EmptyCases).Test will not run
+// testo: warning: (*main.EmptyCases).CasesFoo provides zero values, (*main.EmptyCases).Test won't run
 func (EmptyCases) Test(t T, p struct{ Foo int }) {}
 
 type WrongT struct{ testo.Suite[T] }
 
 // testo: wrong signature for (*main.WrongT).Test, must be: func (*main.WrongT).Test(main.T) or func (*main.WrongT).Test(main.T, struct{...})
 func (WrongT) Test(t *testing.T) {}
 
+// testo: warning: suite *main.MissingTests has no tests
+type MissingTests struct{ testo.Suite[T] }
+
 func Test(t *testing.T) {
+	t.Run("malformed cases", func(t *testing.T) { testo.RunSuite(t, new(MalformedCases)) })
+	t.Run("malformed test", func(t *testing.T) { testo.RunSuite(t, new(MalformedTest)) })
 	t.Run("missing cases", func(t *testing.T) { testo.RunSuite(t, new(MissingCases)) })
 	t.Run("invalid cases", func(t *testing.T) { testo.RunSuite(t, new(InvalidCases)) })
 	t.Run("empty cases", func(t *testing.T) { testo.RunSuite(t, new(EmptyCases)) })
 	t.Run("wrong t", func(t *testing.T) { testo.RunSuite(t, new(WrongT)) })
+	t.Run("missing tests", func(t *testing.T) { testo.RunSuite(t, new(MissingTests)) })
 }

examples/06_errors/output.golden

@@ -1,21 +1,46 @@
 === RUN   Test
+=== RUN   Test/malformed_cases
+=== RUN   Test/malformed_cases/MalformedCases
+    main_test.go:53: testo: (*main.MalformedCases).Casesfoo has malformed name: first letter after 'Cases' must not be lowercase
+=== RUN   Test/malformed_test
+=== RUN   Test/malformed_test/MalformedTest
+    main_test.go:54: testo: (*main.MalformedTest).Testimony has malformed name: first letter after 'Test' must not be lowercase
 === RUN   Test/missing_cases
-    main_test.go:38: testo: wrong param signature for (*main.MissingCases).Test: missing (*main.MissingCases).CasesFoo() []int for param "Foo"
+=== RUN   Test/missing_cases/MissingCases
+    main_test.go:55: testo: wrong param signature for (*main.MissingCases).Test: missing (*main.MissingCases).CasesFoo() []int for param "Foo"
 === RUN   Test/invalid_cases
-    main_test.go:39: testo: wrong param signature for (*main.InvalidCases).Test: (*main.InvalidCases).CasesFoo provides string values, not assignable to param "Foo" of type int
+=== RUN   Test/invalid_cases/InvalidCases
+    main_test.go:56: testo: wrong param signature for (*main.InvalidCases).Test: (*main.InvalidCases).CasesFoo provides string values, not assignable to param "Foo" of type int
 === RUN   Test/empty_cases
 === RUN   Test/empty_cases/EmptyCases
-testo: warning: (*main.EmptyCases).CasesFoo provides zero values, (*main.EmptyCases).Test will not run
+    main_test.go:57: testo: warning: (*main.EmptyCases).CasesFoo provides zero values, (*main.EmptyCases).Test won't run
 === RUN   Test/empty_cases/EmptyCases/testo!
 === RUN   Test/wrong_t
-    main_test.go:41: testo: wrong signature for (*main.WrongT).Test, must be: func (*main.WrongT).Test(main.T) or func (*main.WrongT).Test(main.T, struct{...})
+=== RUN   Test/wrong_t/WrongT
+    main_test.go:58: testo: wrong signature for (*main.WrongT).Test, must be either:
+        func (*main.WrongT).Test(main.T)
+        func (*main.WrongT).Test(main.T, struct{...})
+=== RUN   Test/missing_tests
+=== RUN   Test/missing_tests/MissingTests
+    main_test.go:59: testo: warning: suite *main.MissingTests has no tests
+=== RUN   Test/missing_tests/MissingTests/testo!
 --- FAIL: Test (0.00s)
+    --- FAIL: Test/malformed_cases (0.00s)
+        --- FAIL: Test/malformed_cases/MalformedCases (0.00s)
+    --- FAIL: Test/malformed_test (0.00s)
+        --- FAIL: Test/malformed_test/MalformedTest (0.00s)
     --- FAIL: Test/missing_cases (0.00s)
+        --- FAIL: Test/missing_cases/MissingCases (0.00s)
     --- FAIL: Test/invalid_cases (0.00s)
+        --- FAIL: Test/invalid_cases/InvalidCases (0.00s)
     --- PASS: Test/empty_cases (0.00s)
         --- PASS: Test/empty_cases/EmptyCases (0.00s)
             --- PASS: Test/empty_cases/EmptyCases/testo! (0.00s)
     --- FAIL: Test/wrong_t (0.00s)
+        --- FAIL: Test/wrong_t/WrongT (0.00s)
+    --- PASS: Test/missing_tests (0.00s)
+        --- PASS: Test/missing_tests/MissingTests (0.00s)
+            --- PASS: Test/missing_tests/MissingTests/testo! (0.00s)
 FAIL
-FAIL	github.com/ozontech/testo/examples/06_errors	0.316s
+FAIL	github.com/ozontech/testo/examples/06_errors	0.334s
 FAIL

examples_test.go

@@ -71,7 +71,11 @@ func requireEqualLines(t *testing.T, a, b string, ignoreOrder bool) {
 	}
 
 	if !slices.Equal(aLines, bLines) {
-		t.Fatalf("lines not equal:\n\n%s\n\n%s", aLines, bLines)
+		t.Fatalf(
+			"lines not equal:\n\n%s\n\n%s",
+			strings.Join(aLines, "\n"),
+			strings.Join(bLines, "\n"),
+		)
 	}
 }
 

flag.go

@@ -4,10 +4,19 @@ import (
 	"flag"
 	"regexp"
 
+	"github.com/ozontech/testo/internal/env"
+	"github.com/ozontech/testo/internal/parse"
+
 	// so that cache flags are always available.
 	_ "github.com/ozontech/testo/testocache"
 )
 
+var flagStrict = flag.Bool(
+	"testo.strict",
+	parse.Bool(env.TestoStrict.Get()),
+	"turn warnings into errors",
+)
+
 var flagMethod = flagRegexp{Regexp: regexp.MustCompile("")}
 
 func init() {

internal/env/env.go

@@ -0,0 +1,16 @@
+// Package env holds all environment variables used by Testo.
+package env
+
+import "os"
+
+const (
+	TestoStrict       Env = "TESTO_STRICT"
+	TestoCacheDir     Env = "TESTO_CACHE_DIR"
+	TestoCacheDisable Env = "TESTO_CACHE_DISABLE"
+)
+
+type Env string
+
+func (e Env) Get() string {
+	return os.Getenv(string(e))
+}