stopwords/stopwords_test.go at main · OpenSystemsLab/stopwords · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
package stopwords

import (
	"testing"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

func TestIsStopWord(t *testing.T) {
	// Clear any previously loaded languages
	Clear()

	tt := []struct {
		lang string
		word string
		want bool
	}{
		{"fr", "au", true},
		{"fr", "aux", true},
		{"fr", "avec", true},
		{"fr", "ce", true},
		{"fr", "ces", true},
		{"fr", "Voiture", false},
		{"bad", "bad", false}, // This should return false since language won't be loaded
		{"en", "the", true},
		{"en", "car", false},
	}

	// Register required languages for testing
	err := RegisterLanguages("fr", "en")
	require.NoError(t, err)

	for _, tc := range tt {
		t.Run(tc.word, func(t *testing.T) {
			got := IsStopWord(tc.lang, tc.word)
			assert.Equal(t, tc.want, got)
		})
	}
}

func TestRegisterLanguage(t *testing.T) {
	// Clear any previously loaded languages
	Clear()

	// Test that language is not loaded initially
	assert.False(t, IsLanguageLoaded("en"))

	// Register English
	err := RegisterLanguage("en")
	require.NoError(t, err)

	// Test that language is now loaded
	assert.True(t, IsLanguageLoaded("en"))

	// Test that stopwords work
	assert.True(t, IsStopWord("en", "the"))
	assert.False(t, IsStopWord("en", "car"))

	// Test registering invalid language
	err = RegisterLanguage("invalid")
	require.Error(t, err)
}

func TestRegisterLanguages(t *testing.T) {
	// Clear any previously loaded languages
	Clear()

	// Register multiple languages
	err := RegisterLanguages("en", "fr", "es")
	require.NoError(t, err)

	// Test all are loaded
	assert.True(t, IsLanguageLoaded("en"))
	assert.True(t, IsLanguageLoaded("fr"))
	assert.True(t, IsLanguageLoaded("es"))

	// Test loaded languages list
	loaded := LoadedLanguages()
	assert.Len(t, loaded, 3)
	assert.Contains(t, loaded, "en")
	assert.Contains(t, loaded, "fr")
	assert.Contains(t, loaded, "es")
}

func TestUnregisterLanguage(t *testing.T) {
	// Clear any previously loaded languages
	Clear()

	// Register a language
	err := RegisterLanguage("en")
	require.NoError(t, err)
	assert.True(t, IsLanguageLoaded("en"))

	// Unregister it
	UnregisterLanguage("en")
	assert.False(t, IsLanguageLoaded("en"))

	// Test that stopwords no longer work
	assert.False(t, IsStopWord("en", "the"))
}

func TestGetSupportedLanguages(t *testing.T) {
	supported := GetSupportedLanguages()

	// Should have many languages
	assert.Greater(t, len(supported), 50)

	// Should contain common languages
	assert.Contains(t, supported, "en")
	assert.Contains(t, supported, "fr")
	assert.Contains(t, supported, "es")
	assert.Contains(t, supported, "de")
}

func TestRegistryIsolation(t *testing.T) {
	// Test that different registries are isolated
	reg1 := NewRegistry()
	reg2 := NewRegistry()

	// Load language in first registry
	err := reg1.RegisterLanguage("en")
	require.NoError(t, err)

	// Should only be loaded in first registry
	assert.True(t, reg1.IsLanguageLoaded("en"))
	assert.False(t, reg2.IsLanguageLoaded("en"))

	// Should only work in first registry
	assert.True(t, reg1.IsStopWord("en", "the"))
	assert.False(t, reg2.IsStopWord("en", "the"))
}