diff --git a/.golangci.yaml b/.golangci.yaml index 3d215f4..cb16585 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -2,7 +2,6 @@ version: "2" formatters: enable: - - gci - gofumpt - goimports - golines @@ -15,6 +14,7 @@ formatters: - pattern: "interface{}" replacement: "any" gofumpt: + module-path: github.com/ozontech/testo extra-rules: true linters: @@ -57,8 +57,10 @@ linters: - thelper settings: + cyclop: + max-complexity: 15 funlen: - lines: 70 + lines: 80 gosec: excludes: - G304 diff --git a/CHANGELOG.md b/CHANGELOG.md index 458e8b4..94cb10d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,17 @@ All notable changes to this project are documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## Unreleased + +## Added + +- Added strict mode which turns warnings into hard errors. Can be enabled with `-testo.strict` command line argument and `TESTO_STRICT` environment variable (lower priority). + +### Changed + +- Testo error messages are now more descriptive. +- Malformed method names (e.g. lowercase letter after `Test` prefix) raise fatal errors, similar to native `go test` runner. + ## [1.4.0] - 2026-06-07 ### Added diff --git a/collector.go b/collector.go index cf8d752..b2e24fc 100644 --- a/collector.go +++ b/collector.go @@ -1,9 +1,7 @@ package testo import ( - "fmt" "maps" - "os" "reflect" "slices" "strings" @@ -81,7 +79,16 @@ func suiteCasesOf[Suite suite[T], T CommonT](tb testing.TB) map[string]suiteCase const prefix = "Cases" if !isTest(method.Name, prefix) { - continue + if !strings.HasPrefix(method.Name, prefix) { + continue + } + + tb.Fatalf( + "testo: (%s).%s has malformed name: first letter after '%s' must not be lowercase", + reflect.TypeFor[Suite](), + method.Name, + prefix, + ) } name := strings.TrimPrefix(method.Name, prefix) @@ -95,7 +102,7 @@ func suiteCasesOf[Suite suite[T], T CommonT](tb testing.TB) map[string]suiteCase if !isValidIn || !isValidOut { tb.Fatalf( - "testo: wrong signature for %[1]s.%[2]s, must be: func (%[1]s) %[2]s() []...", + "testo: wrong signature for %[1]s.%[2]s, must be: func (%[1]s) %[2]s() []Type", reflect.TypeFor[Suite](), method.Name, tb, ) } @@ -150,6 +157,10 @@ type suiteTests[Suite suite[T], T CommonT] struct { Parametrized []suiteTestParametrized[Suite, T] } +func (st suiteTests[Suite, T]) isEmpty() bool { + return len(st.Regular) == 0 && len(st.Parametrized) == 0 +} + type annotatedSuiteTest[Suite suite[T], T CommonT] struct { suiteTest[Suite, T] @@ -164,9 +175,12 @@ type annotatedSuiteTest[Suite suite[T], T CommonT] struct { // Suite instance is required here to get // parameter cases (CasesXXX funcs), not to invoke the actual tests. func (st suiteTests[Suite, T]) Collect( + tb testing.TB, s Suite, name func(string) string, ) []annotatedSuiteTest[Suite, T] { + tb.Helper() + tests := make([]annotatedSuiteTest[Suite, T], 0, len(st.Regular)) for _, r := range st.Regular { @@ -177,7 +191,7 @@ func (st suiteTests[Suite, T]) Collect( } for _, p := range st.Parametrized { - cases := p.Tests(s) + cases := p.Tests(tb, s) tests = append(tests, cases...) } @@ -217,22 +231,36 @@ func (tc *testsCollector[Suite, T]) testName(base string) string { } //nolint:cyclop,funlen,gocognit // splitting it would make it even more complex -func (tc *testsCollector[Suite, T]) Collect( - tb testing.TB, -) suiteTests[Suite, T] { +func (tc *testsCollector[Suite, T]) Collect(tb testing.TB) suiteTests[Suite, T] { tb.Helper() - cases := suiteCasesOf[Suite](tb) - suiteTyp := reflect.TypeFor[Suite]() + if suiteTyp == reflect.TypeFor[singleton[T]]() { + return suiteTests[Suite, T]{} + } + + cases := suiteCasesOf[Suite](tb) + var tests suiteTests[Suite, T] for i := range suiteTyp.NumMethod() { method := suiteTyp.Method(i) - if !isTest(method.Name, "Test") { - continue + const prefix = "Test" + + if !isTest(method.Name, prefix) { + if !strings.HasPrefix(method.Name, prefix) { + continue + } + + // identical to native go test behavior + tb.Fatalf( + "testo: (%s).%s has malformed name: first letter after '%s' must not be lowercase", + suiteTyp, + method.Name, + prefix, + ) } raiseWrongSignatureError := func() { @@ -240,7 +268,7 @@ func (tc *testsCollector[Suite, T]) Collect( //nolint:lll // it's a long message tb.Fatalf( - "testo: wrong signature for (%[1]s).%[2]s, must be: func (%[1]s).%[2]s(%[3]s) or func (%[1]s).%[2]s(%[3]s, struct{...})", + "testo: wrong signature for (%[1]s).%[2]s, must be either:\nfunc (%[1]s).%[2]s(%[3]s)\nfunc (%[1]s).%[2]s(%[3]s, struct{...})", suiteTyp, method.Name, reflect.TypeFor[T](), @@ -328,11 +356,15 @@ func (tc *testsCollector[Suite, T]) Collect( } } + if tests.isEmpty() { + warnf(tb, "suite %s has no tests", suiteTyp) + } + return tests } type suiteTestParametrized[Suite suite[T], T CommonT] struct { - Tests func(Suite) []annotatedSuiteTest[Suite, T] + Tests func(testing.TB, Suite) []annotatedSuiteTest[Suite, T] } func (tc *testsCollector[Suite, T]) newParametrizedTest( @@ -340,7 +372,9 @@ func (tc *testsCollector[Suite, T]) newParametrizedTest( cases map[string]suiteCase[Suite, T], ) suiteTestParametrized[Suite, T] { return suiteTestParametrized[Suite, T]{ - Tests: func(s Suite) []annotatedSuiteTest[Suite, T] { + Tests: func(tb testing.TB, s Suite) []annotatedSuiteTest[Suite, T] { + tb.Helper() + casesValues := make(map[string][]reflect.Value, len(cases)) for caseName, c := range cases { @@ -349,9 +383,9 @@ func (tc *testsCollector[Suite, T]) newParametrizedTest( if len(values) == 0 { structName := method.Type.In(0).String() - fmt.Fprintf( - os.Stderr, - "testo: warning: (%[1]s).Cases%[2]s provides zero values, (%[1]s).%[3]s will not run\n", + warnf( + tb, + "(%[1]s).Cases%[2]s provides zero values, (%[1]s).%[3]s won't run", structName, caseName, method.Name, diff --git a/docs/how-to.md b/docs/how-to.md index facc865..81a52c9 100644 --- a/docs/how-to.md +++ b/docs/how-to.md @@ -40,6 +40,26 @@ TestFoo(name=Joe, age=60) TestFoo(name=Joe, age=6) ``` +If for at least one required parameter function `CasesXxx` +returns zero values Testo will log a warning with similar message: + +```txt +main_test.go:15: testo: (*main.Suite).CasesName provides zero values, (*main.Suite).TestFoo won't run +``` + +To turn this log into fatal error and do not proceed with further execution +pass flag `-testo.strict` to the `go test` command invocation: + +```bash +go test ./... -testo.strict +``` + +> [!TIP] +> You can also set `TESTO_STRICT` environment variable to `true` +> for the same effect. +> +> Flags have higher priority than environment variables. + ## How to write parallel tests You can use your regular `t.Parallel` method to mark a test as parallel. diff --git a/docs/technical-overview.md b/docs/technical-overview.md index 4bac8ba..904164c 100644 --- a/docs/technical-overview.md +++ b/docs/technical-overview.md @@ -7,9 +7,9 @@ A technical overview of Testo. When you call `testo.RunSuite` the following happens: ```txt -Suite tests are collected and verified. - A root test named the same as a suite is run { + Suite tests are collected and verified. + Plugins are collected and initialized with ".Plugin(parent: nil, options)" method call, if implemented. Innermost plugins are initialized first. "BeforeAll" plugin hooks are called. diff --git a/examples/06_errors/main_test.go b/examples/06_errors/main_test.go index c79c311..fddd4aa 100644 --- a/examples/06_errors/main_test.go +++ b/examples/06_errors/main_test.go @@ -22,11 +22,23 @@ func (InvalidCases) CasesFoo() []string { return []string{"one", "two"} } // testo: wrong param signature for (*main.InvalidCases).Test: (*main.InvalidCases).CasesFoo provides string values, not assignable to param "Foo" of type int func (InvalidCases) Test(t T, p struct{ Foo int }) {} +type MalformedCases struct{ testo.Suite[T] } + +// testo: (*main.MalformedCases).Casesfoo has malformed name: first letter after 'Cases' must not be lowercase +func (MalformedCases) Casesfoo() []string { return []string{"a", "b"} } + +func (MalformedCases) Test(t T, p struct{ Foo string }) {} + +type MalformedTest struct{ testo.Suite[T] } + +// testo: (*main.MalformedTest).Testimony has malformed name: first letter after 'Test' must not be lowercase +func (MalformedTest) Testimony(t T) {} + type EmptyCases struct{ testo.Suite[T] } func (EmptyCases) CasesFoo() []int { return nil } -// testo: warning: (*main.EmptyCases).CasesFoo provides zero values, (*main.EmptyCases).Test will not run +// testo: warning: (*main.EmptyCases).CasesFoo provides zero values, (*main.EmptyCases).Test won't run func (EmptyCases) Test(t T, p struct{ Foo int }) {} type WrongT struct{ testo.Suite[T] } @@ -34,9 +46,15 @@ type WrongT struct{ testo.Suite[T] } // testo: wrong signature for (*main.WrongT).Test, must be: func (*main.WrongT).Test(main.T) or func (*main.WrongT).Test(main.T, struct{...}) func (WrongT) Test(t *testing.T) {} +// testo: warning: suite *main.MissingTests has no tests +type MissingTests struct{ testo.Suite[T] } + func Test(t *testing.T) { + t.Run("malformed cases", func(t *testing.T) { testo.RunSuite(t, new(MalformedCases)) }) + t.Run("malformed test", func(t *testing.T) { testo.RunSuite(t, new(MalformedTest)) }) t.Run("missing cases", func(t *testing.T) { testo.RunSuite(t, new(MissingCases)) }) t.Run("invalid cases", func(t *testing.T) { testo.RunSuite(t, new(InvalidCases)) }) t.Run("empty cases", func(t *testing.T) { testo.RunSuite(t, new(EmptyCases)) }) t.Run("wrong t", func(t *testing.T) { testo.RunSuite(t, new(WrongT)) }) + t.Run("missing tests", func(t *testing.T) { testo.RunSuite(t, new(MissingTests)) }) } diff --git a/examples/06_errors/output.golden b/examples/06_errors/output.golden index db0f609..e3afc51 100644 --- a/examples/06_errors/output.golden +++ b/examples/06_errors/output.golden @@ -1,21 +1,46 @@ === RUN Test +=== RUN Test/malformed_cases +=== RUN Test/malformed_cases/MalformedCases + main_test.go:53: testo: (*main.MalformedCases).Casesfoo has malformed name: first letter after 'Cases' must not be lowercase +=== RUN Test/malformed_test +=== RUN Test/malformed_test/MalformedTest + main_test.go:54: testo: (*main.MalformedTest).Testimony has malformed name: first letter after 'Test' must not be lowercase === RUN Test/missing_cases - main_test.go:38: testo: wrong param signature for (*main.MissingCases).Test: missing (*main.MissingCases).CasesFoo() []int for param "Foo" +=== RUN Test/missing_cases/MissingCases + main_test.go:55: testo: wrong param signature for (*main.MissingCases).Test: missing (*main.MissingCases).CasesFoo() []int for param "Foo" === RUN Test/invalid_cases - main_test.go:39: testo: wrong param signature for (*main.InvalidCases).Test: (*main.InvalidCases).CasesFoo provides string values, not assignable to param "Foo" of type int +=== RUN Test/invalid_cases/InvalidCases + main_test.go:56: testo: wrong param signature for (*main.InvalidCases).Test: (*main.InvalidCases).CasesFoo provides string values, not assignable to param "Foo" of type int === RUN Test/empty_cases === RUN Test/empty_cases/EmptyCases -testo: warning: (*main.EmptyCases).CasesFoo provides zero values, (*main.EmptyCases).Test will not run + main_test.go:57: testo: warning: (*main.EmptyCases).CasesFoo provides zero values, (*main.EmptyCases).Test won't run === RUN Test/empty_cases/EmptyCases/testo! === RUN Test/wrong_t - main_test.go:41: testo: wrong signature for (*main.WrongT).Test, must be: func (*main.WrongT).Test(main.T) or func (*main.WrongT).Test(main.T, struct{...}) +=== RUN Test/wrong_t/WrongT + main_test.go:58: testo: wrong signature for (*main.WrongT).Test, must be either: + func (*main.WrongT).Test(main.T) + func (*main.WrongT).Test(main.T, struct{...}) +=== RUN Test/missing_tests +=== RUN Test/missing_tests/MissingTests + main_test.go:59: testo: warning: suite *main.MissingTests has no tests +=== RUN Test/missing_tests/MissingTests/testo! --- FAIL: Test (0.00s) + --- FAIL: Test/malformed_cases (0.00s) + --- FAIL: Test/malformed_cases/MalformedCases (0.00s) + --- FAIL: Test/malformed_test (0.00s) + --- FAIL: Test/malformed_test/MalformedTest (0.00s) --- FAIL: Test/missing_cases (0.00s) + --- FAIL: Test/missing_cases/MissingCases (0.00s) --- FAIL: Test/invalid_cases (0.00s) + --- FAIL: Test/invalid_cases/InvalidCases (0.00s) --- PASS: Test/empty_cases (0.00s) --- PASS: Test/empty_cases/EmptyCases (0.00s) --- PASS: Test/empty_cases/EmptyCases/testo! (0.00s) --- FAIL: Test/wrong_t (0.00s) + --- FAIL: Test/wrong_t/WrongT (0.00s) + --- PASS: Test/missing_tests (0.00s) + --- PASS: Test/missing_tests/MissingTests (0.00s) + --- PASS: Test/missing_tests/MissingTests/testo! (0.00s) FAIL -FAIL github.com/ozontech/testo/examples/06_errors 0.316s +FAIL github.com/ozontech/testo/examples/06_errors 0.334s FAIL diff --git a/examples_test.go b/examples_test.go index 8942d00..a31d540 100644 --- a/examples_test.go +++ b/examples_test.go @@ -71,7 +71,11 @@ func requireEqualLines(t *testing.T, a, b string, ignoreOrder bool) { } if !slices.Equal(aLines, bLines) { - t.Fatalf("lines not equal:\n\n%s\n\n%s", aLines, bLines) + t.Fatalf( + "lines not equal:\n\n%s\n\n%s", + strings.Join(aLines, "\n"), + strings.Join(bLines, "\n"), + ) } } diff --git a/flag.go b/flag.go index f8aa6b1..ba1f25a 100644 --- a/flag.go +++ b/flag.go @@ -4,10 +4,19 @@ import ( "flag" "regexp" + "github.com/ozontech/testo/internal/env" + "github.com/ozontech/testo/internal/parse" + // so that cache flags are always available. _ "github.com/ozontech/testo/testocache" ) +var flagStrict = flag.Bool( + "testo.strict", + parse.Bool(env.TestoStrict.Get()), + "turn warnings into errors", +) + var flagMethod = flagRegexp{Regexp: regexp.MustCompile("")} func init() { diff --git a/internal/env/env.go b/internal/env/env.go new file mode 100644 index 0000000..1a44349 --- /dev/null +++ b/internal/env/env.go @@ -0,0 +1,16 @@ +// Package env holds all environment variables used by Testo. +package env + +import "os" + +const ( + TestoStrict Env = "TESTO_STRICT" + TestoCacheDir Env = "TESTO_CACHE_DIR" + TestoCacheDisable Env = "TESTO_CACHE_DISABLE" +) + +type Env string + +func (e Env) Get() string { + return os.Getenv(string(e)) +} diff --git a/internal/env/env_test.go b/internal/env/env_test.go new file mode 100644 index 0000000..14ba06b --- /dev/null +++ b/internal/env/env_test.go @@ -0,0 +1,17 @@ +package env + +import ( + "os" + "testing" +) + +func TestEnv(t *testing.T) { + const Var Env = "TEST_EXAMPLE" + const value = "lorem ipsum" + + t.Setenv(string(Var), value) + + if Var.Get() != os.Getenv(string(Var)) { + t.Fatal("Env.Get must be equal to os.Getenv") + } +} diff --git a/internal/parse/parse.go b/internal/parse/parse.go new file mode 100644 index 0000000..ff1ee50 --- /dev/null +++ b/internal/parse/parse.go @@ -0,0 +1,15 @@ +// Package parse provides parsing utilities. +package parse + +import ( + "strconv" + "strings" +) + +// Bool parses string as bool treating it as false +// in case of a failure. +func Bool(s string) bool { + b, _ := strconv.ParseBool(strings.TrimSpace(s)) + + return b +} diff --git a/internal/parse/parse_test.go b/internal/parse/parse_test.go new file mode 100644 index 0000000..ed70841 --- /dev/null +++ b/internal/parse/parse_test.go @@ -0,0 +1,46 @@ +package parse + +import "testing" + +func TestBool(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + s string + want bool + }{ + { + name: "simple value", + s: "true", + want: true, + }, + { + name: "padded value", + s: " 1 ", + want: true, + }, + { + name: "empty value", + s: "", + want: false, + }, + { + name: "padded false value", + s: " f ", + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + got := Bool(tt.s) + + if tt.want != got { + t.Errorf("Bool() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/runner.go b/runner.go index eff605f..dd04c50 100644 --- a/runner.go +++ b/runner.go @@ -237,9 +237,7 @@ func newRunner[Suite suite[T], T CommonT](t common) runner[Suite, T] { } } -func (r *runner[Suite, T]) collectTests( - t TestingT, -) suiteTests[Suite, T] { +func (r *runner[Suite, T]) collectTests(t TestingT) suiteTests[Suite, T] { t.Helper() collector := testsCollector[Suite, T]{ @@ -260,8 +258,6 @@ func (r *runner[Suite, T]) runSuite( options = append(getOptions(), options...) - tests := r.collectTests(testingT) - suiteInfo := testoreflect.SuiteInfo{ Parent: parentSuite, Name: r.suiteName, @@ -273,6 +269,8 @@ func (r *runner[Suite, T]) runSuite( return testingT.Run(r.suiteName, func(testingT *testing.T) { testingT.Helper() + tests := r.collectTests(testingT) + t := construct[T]( testingT, nil, @@ -337,7 +335,7 @@ func (r *runner[Suite, T]) runSuiteTests(t T, s Suite, tests suiteTests[Suite, T allTests := r.applyPlan( t, suiteInfo, - tests.Collect(s, func(name string) string { + tests.Collect(t, s, func(name string) string { return r.testNamer.Name(r.caller, name) }), ) diff --git a/t.go b/t.go index 3a2f62d..dabaf09 100644 --- a/t.go +++ b/t.go @@ -584,3 +584,21 @@ func (a *atomicInt[T]) Store(value T) { func (a *atomicInt[T]) CompareAndSwap(oldvalue, newvalue T) bool { return (*atomic.Int64)(a).CompareAndSwap(int64(oldvalue), int64(newvalue)) } + +func warnf(tb testing.TB, f string, args ...any) { + tb.Helper() + + warn(tb, fmt.Sprintf(f, args...)) +} + +func warn(tb testing.TB, args ...any) { + tb.Helper() + + const prefix = "testo: " + + if *flagStrict { + tb.Fatal(prefix + fmt.Sprint(args...)) + } + + tb.Log(prefix + "warning: " + fmt.Sprint(args...)) +} diff --git a/testocache/cache.go b/testocache/cache.go index 0c5ddf4..95cf154 100644 --- a/testocache/cache.go +++ b/testocache/cache.go @@ -25,17 +25,20 @@ import ( "slices" "strconv" "sync" + + "github.com/ozontech/testo/internal/env" + "github.com/ozontech/testo/internal/parse" ) var ( flagDir = flag.String( "cache.dir", - cmp.Or(os.Getenv("TESTO_CACHE_DIR"), ".testo_cache"), + cmp.Or(env.TestoCacheDir.Get(), ".testo_cache"), "directory where the testo cache is stored", ) flagDisable = flag.Bool( "cache.disable", - parseBool(os.Getenv("TESTO_CACHE_DISABLE")), + parse.Bool(env.TestoCacheDisable.Get()), "disable caching in testo", ) ) @@ -282,12 +285,6 @@ func cacheDir() (string, error) { return dir, nil } -func parseBool(s string) bool { - b, _ := strconv.ParseBool(s) - - return b -} - func validate(key string) error { if slices.Contains([]byte(key), 0) { return ErrInvalidKey