Skip to content

Commit f7cbeda

Browse files
FllipEisFllipEis
authored
Merge pull request #38 from simplecloudapp/feat/single-envoy-port
feat: single port in envoy that can handle all droplets
2 parents 19d8c5d + 3c5927d commit f7cbeda

File tree

4 files changed

+174
-95
lines changed

4 files changed

+174
-95
lines changed

‎controller-runtime/src/main/kotlin/app/simplecloud/controller/runtime/ControllerRuntime.kt‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,14 @@ class ControllerRuntime(
4343
private val serverRepository = ServerRepository(database, numericalIdRepository)
4444
private val hostRepository = ServerHostRepository()
4545
private val serverHostAttacher = ServerHostAttacher(hostRepository, serverRepository)
46-
private val dropletRepository = DropletRepository(authCallCredentials, serverHostAttacher, controllerStartCommand.envoyStartPort, hostRepository)
46+
private val dropletRepository = DropletRepository(
47+
authCallCredentials,
48+
serverHostAttacher,
49+
controllerStartCommand.grpcHost,
50+
controllerStartCommand.envoyPort,
51+
controllerStartCommand.envoyStartPort,
52+
hostRepository
53+
)
4754
private val pubSubService = PubSubService()
4855
private val controlPlaneServer = ControlPlaneServer(controllerStartCommand, dropletRepository)
4956
private val authServer = OAuthServer(controllerStartCommand, database)

‎controller-runtime/src/main/kotlin/app/simplecloud/controller/runtime/droplet/DropletRepository.kt‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,12 +16,14 @@ import kotlinx.coroutines.launch
1616
class DropletRepository(
1717
private val authCallCredentials: AuthCallCredentials,
1818
private val serverHostAttacher: ServerHostAttacher,
19+
envoyHost: String,
20+
envoyPort: Int,
1921
private val envoyStartPort: Int,
2022
private val serverHostRepository: ServerHostRepository,
2123
) : Repository<Droplet, String> {
2224

2325
private val currentDroplets = mutableListOf<Droplet>()
24-
private val dropletCache = DropletCache()
26+
private val dropletCache = DropletCache(envoyHost, envoyPort)
2527

2628
override suspend fun getAll(): List<Droplet> {
2729
return currentDroplets

‎controller-runtime/src/main/kotlin/app/simplecloud/controller/runtime/envoy/DropletCache.kt‎

Lines changed: 160 additions & 93 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
package app.simplecloud.controller.runtime.envoy
22

3-
import app.simplecloud.controller.runtime.droplet.DropletRepository
43
import app.simplecloud.droplet.api.droplet.Droplet
54
import com.google.protobuf.Any
65
import com.google.protobuf.Duration
@@ -9,7 +8,6 @@ import io.envoyproxy.controlplane.cache.ConfigWatcher
98
import io.envoyproxy.controlplane.cache.v3.SimpleCache
109
import io.envoyproxy.controlplane.cache.v3.Snapshot
1110
import io.envoyproxy.envoy.config.cluster.v3.Cluster
12-
import io.envoyproxy.envoy.config.cluster.v3.Cluster.EdsClusterConfig
1311
import io.envoyproxy.envoy.config.core.v3.*
1412
import io.envoyproxy.envoy.config.endpoint.v3.ClusterLoadAssignment
1513
import io.envoyproxy.envoy.config.endpoint.v3.Endpoint
@@ -19,7 +17,6 @@ import io.envoyproxy.envoy.config.listener.v3.Filter
1917
import io.envoyproxy.envoy.config.listener.v3.FilterChain
2018
import io.envoyproxy.envoy.config.listener.v3.Listener
2119
import io.envoyproxy.envoy.config.route.v3.*
22-
import io.envoyproxy.envoy.extensions.filters.http.connect_grpc_bridge.v3.FilterConfig
2320
import io.envoyproxy.envoy.extensions.filters.http.grpc_web.v3.GrpcWeb
2421
import io.envoyproxy.envoy.extensions.filters.http.router.v3.Router
2522
import io.envoyproxy.envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
@@ -28,147 +25,217 @@ import io.envoyproxy.envoy.extensions.upstreams.http.v3.HttpProtocolOptions
2825
import org.apache.logging.log4j.LogManager
2926
import java.util.*
3027

31-
/**
32-
* This class handles the remapping of the [DropletRepository] to a [SimpleCache] of [Snapshot]s, which are used by the envoy ADS service.
33-
*/
34-
class DropletCache {
28+
class DropletCache(
29+
private val grpcHost: String,
30+
private val envoyPort: Int,
31+
) {
3532
private val cache = SimpleCache(SimpleCloudNodeGroup())
3633
private val logger = LogManager.getLogger(DropletCache::class.java)
3734

38-
//Create a new Snapshot by the droplet repository's data
3935
fun update(droplets: List<Droplet>) {
4036
logger.info("Detected new droplets in DropletRepository, adding to ADS...")
37+
4138
val clusters = mutableListOf<Cluster>()
42-
val listeners = mutableListOf<Listener>()
4339
val clas = mutableListOf<ClusterLoadAssignment>()
44-
droplets.forEach {
45-
clusters.add(createCluster(it))
46-
listeners.add(createListener(it))
47-
clas.add(createCLA(it))
40+
val listeners = mutableListOf<Listener>()
41+
42+
droplets.forEach { droplet ->
43+
clusters.add(createCluster(droplet))
44+
clas.add(createCLA(droplet))
45+
listeners.add(createOriginalListener(droplet))
4846
}
47+
48+
listeners.add(createMainListener(droplets))
49+
4950
cache.setSnapshot(
5051
SimpleCloudNodeGroup.GROUP,
5152
Snapshot.create(
5253
clusters,
5354
clas,
5455
listeners,
55-
listOf(), // We don't need routes
56-
listOf(), // We don't need secrets
57-
UUID.randomUUID()
58-
.toString() //This can be anything, used internally for versioning. THIS HAS TO BE DIFFERENT FOR EVERY SNAPSHOT
56+
listOf(),
57+
listOf(),
58+
UUID.randomUUID().toString()
5959
)
6060
)
6161
}
6262

63-
//Creates endpoints users can connect with later
64-
private fun createListener(it: Droplet): Listener {
65-
return Listener.newBuilder().setName("${it.type}-${it.id}").setAddress(
66-
Address.newBuilder().setSocketAddress(
67-
SocketAddress.newBuilder().setProtocol(SocketAddress.Protocol.TCP).setAddress("0.0.0.0")
68-
.setPortValue(it.envoyPort)
63+
private fun createMainListener(droplets: List<Droplet>): Listener {
64+
return Listener.newBuilder()
65+
.setName("main_listener")
66+
.setAddress(
67+
Address.newBuilder().setSocketAddress(
68+
SocketAddress.newBuilder()
69+
.setProtocol(SocketAddress.Protocol.TCP)
70+
.setAddress(grpcHost)
71+
.setPortValue(envoyPort)
72+
)
6973
)
70-
).setDefaultFilterChain(createListenerFilterChain("${it.type}-${it.id}")).build()
71-
74+
.addFilterChains(createMainFilterChain(droplets))
75+
.build()
7276
}
7377

74-
//Creates load assignments for new droplets (I don't yet know if they need to be called every time?)
75-
private fun createCLA(it: Droplet): ClusterLoadAssignment {
76-
return ClusterLoadAssignment.newBuilder().setClusterName("${it.type}-${it.id}")
77-
.addEndpoints(
78-
LocalityLbEndpoints.newBuilder().addLbEndpoints(
79-
LbEndpoint.newBuilder().setEndpoint(
80-
Endpoint.newBuilder()
81-
.setAddress(
82-
Address.newBuilder().setSocketAddress(
83-
SocketAddress.newBuilder().setPortValue(it.port).setAddress(it.host)
84-
.setProtocol(SocketAddress.Protocol.TCP)
78+
private fun createMainFilterChain(droplets: List<Droplet>): FilterChain {
79+
val routes = droplets.map { droplet ->
80+
Route.newBuilder()
81+
.setMatch(
82+
RouteMatch.newBuilder()
83+
.setPrefix("/${droplet.id}/")
84+
)
85+
.setRoute(
86+
RouteAction.newBuilder()
87+
.setCluster("${droplet.type}-${droplet.id}")
88+
.setPrefixRewrite("/")
89+
)
90+
.build()
91+
}
92+
93+
return FilterChain.newBuilder()
94+
.addFilters(
95+
Filter.newBuilder()
96+
.setName("envoy.filters.network.http_connection_manager")
97+
.setTypedConfig(
98+
Any.pack(
99+
HttpConnectionManager.newBuilder()
100+
.setStatPrefix("ingress_http")
101+
.setCodecType(HttpConnectionManager.CodecType.AUTO)
102+
.setRouteConfig(
103+
RouteConfiguration.newBuilder()
104+
.setName("local_route")
105+
.addVirtualHosts(
106+
VirtualHost.newBuilder()
107+
.setName("local_service")
108+
.addDomains("*")
109+
.addAllRoutes(routes)
110+
)
85111
)
86-
)
112+
.addHttpFilters(
113+
HttpFilter.newBuilder()
114+
.setName("envoy.filters.http.grpc_web")
115+
.setTypedConfig(Any.pack(GrpcWeb.getDefaultInstance()))
116+
)
117+
.addHttpFilters(
118+
HttpFilter.newBuilder()
119+
.setName("envoy.filters.http.router")
120+
.setTypedConfig(Any.pack(Router.getDefaultInstance()))
121+
)
122+
.build()
123+
)
87124
)
88-
)
89125
)
90126
.build()
91127
}
92128

93-
//Creates clusters listening to droplets
94-
private fun createCluster(it: Droplet): Cluster {
95-
return Cluster.newBuilder().setName("${it.type}-${it.id}")
96-
.setConnectTimeout(Duration.newBuilder().setSeconds(5))
97-
.setType(Cluster.DiscoveryType.EDS)
98-
.setEdsClusterConfig(
99-
EdsClusterConfig.newBuilder()
100-
.setEdsConfig(ConfigSource.newBuilder().setAds(AggregatedConfigSource.getDefaultInstance()))
101-
)
102-
.setLbPolicy(Cluster.LbPolicy.ROUND_ROBIN)
103-
.setLoadAssignment(
104-
ClusterLoadAssignment.newBuilder().setClusterName("${it.type}-${it.id}")
105-
.addEndpoints(
106-
LocalityLbEndpoints.newBuilder()
107-
.addLbEndpoints(
108-
LbEndpoint.newBuilder().setEndpoint(
109-
Endpoint.newBuilder()
110-
.setAddress(
111-
Address.newBuilder().setSocketAddress(
112-
SocketAddress.newBuilder().setPortValue(it.port).setAddress(it.host)
113-
)
114-
)
115-
)
116-
)
117-
)
118-
).putTypedExtensionProtocolOptions(
119-
"envoy.extensions.upstreams.http.v3.HttpProtocolOptions", Any.pack(
120-
HttpProtocolOptions.newBuilder().setExplicitHttpConfig(
121-
HttpProtocolOptions.ExplicitHttpConfig.newBuilder().setHttp2ProtocolOptions(
122-
Http2ProtocolOptions.newBuilder().setMaxConcurrentStreams(
123-
UInt32Value.of(100)
124-
)
125-
)
126-
).build()
129+
private fun createOriginalListener(droplet: Droplet): Listener {
130+
return Listener.newBuilder()
131+
.setName("original_listener_${droplet.type}_${droplet.id}")
132+
.setAddress(
133+
Address.newBuilder().setSocketAddress(
134+
SocketAddress.newBuilder()
135+
.setProtocol(SocketAddress.Protocol.TCP)
136+
.setAddress("0.0.0.0")
137+
.setPortValue(droplet.envoyPort)
127138
)
128139
)
140+
.addFilterChains(createOriginalFilterChain(droplet))
129141
.build()
130-
131142
}
132143

133-
//Creates a filter chain that remaps http to grpc
134-
private fun createListenerFilterChain(cluster: String): FilterChain.Builder {
144+
private fun createOriginalFilterChain(droplet: Droplet): FilterChain {
135145
return FilterChain.newBuilder()
136146
.addFilters(
137-
Filter.newBuilder().setName("envoy.filters.network.http_connection_manager")
147+
Filter.newBuilder()
148+
.setName("envoy.filters.network.http_connection_manager")
138149
.setTypedConfig(
139150
Any.pack(
140-
HttpConnectionManager.newBuilder().setStatPrefix("ingress_http")
151+
HttpConnectionManager.newBuilder()
152+
.setStatPrefix("ingress_http")
141153
.setCodecType(HttpConnectionManager.CodecType.AUTO)
142154
.setRouteConfig(
143-
RouteConfiguration.newBuilder().setName("local_route")
155+
RouteConfiguration.newBuilder()
156+
.setName("local_route")
144157
.addVirtualHosts(
145-
VirtualHost.newBuilder().setName("local_service").addDomains("*")
158+
VirtualHost.newBuilder()
159+
.setName("local_service")
160+
.addDomains("*")
146161
.addRoutes(
147-
Route.newBuilder().setRoute(
148-
RouteAction.newBuilder().setCluster(cluster)
149-
.setTimeout(Duration.newBuilder().setSeconds(0).setNanos(0))
150-
).setMatch(RouteMatch.newBuilder().setPrefix("/"))
162+
Route.newBuilder()
163+
.setMatch(
164+
RouteMatch.newBuilder()
165+
.setPrefix("/")
166+
)
167+
.setRoute(
168+
RouteAction.newBuilder()
169+
.setCluster("${droplet.type}-${droplet.id}")
170+
)
151171
)
152172
)
153-
).addHttpFilters(
154-
HttpFilter.newBuilder().setName("envoy.filters.http.connect_grpc_bridge")
155-
.setTypedConfig(Any.pack(FilterConfig.getDefaultInstance()))
156-
).addHttpFilters(
157-
HttpFilter.newBuilder().setName("envoy.filters.http.grpc_web")
173+
)
174+
.addHttpFilters(
175+
HttpFilter.newBuilder()
176+
.setName("envoy.filters.http.grpc_web")
158177
.setTypedConfig(Any.pack(GrpcWeb.getDefaultInstance()))
159-
).addHttpFilters(
160-
HttpFilter.newBuilder().setName("envoy.filters.http.router")
178+
)
179+
.addHttpFilters(
180+
HttpFilter.newBuilder()
181+
.setName("envoy.filters.http.router")
161182
.setTypedConfig(Any.pack(Router.getDefaultInstance()))
162183
)
163-
164184
.build()
165185
)
166186
)
167187
)
188+
.build()
189+
}
190+
191+
private fun createCluster(droplet: Droplet): Cluster {
192+
return Cluster.newBuilder()
193+
.setName("${droplet.type}-${droplet.id}")
194+
.setConnectTimeout(Duration.newBuilder().setSeconds(5))
195+
.setType(Cluster.DiscoveryType.STRICT_DNS)
196+
.setLbPolicy(Cluster.LbPolicy.ROUND_ROBIN)
197+
.putTypedExtensionProtocolOptions(
198+
"envoy.extensions.upstreams.http.v3.HttpProtocolOptions",
199+
Any.pack(
200+
HttpProtocolOptions.newBuilder()
201+
.setExplicitHttpConfig(
202+
HttpProtocolOptions.ExplicitHttpConfig.newBuilder()
203+
.setHttp2ProtocolOptions(
204+
Http2ProtocolOptions.newBuilder()
205+
.setMaxConcurrentStreams(UInt32Value.of(100))
206+
)
207+
).build()
208+
)
209+
)
210+
.setLoadAssignment(createCLA(droplet))
211+
.build()
212+
}
213+
214+
private fun createCLA(droplet: Droplet): ClusterLoadAssignment {
215+
return ClusterLoadAssignment.newBuilder()
216+
.setClusterName("${droplet.type}-${droplet.id}")
217+
.addEndpoints(
218+
LocalityLbEndpoints.newBuilder()
219+
.addLbEndpoints(
220+
LbEndpoint.newBuilder()
221+
.setEndpoint(
222+
Endpoint.newBuilder()
223+
.setAddress(
224+
Address.newBuilder()
225+
.setSocketAddress(
226+
SocketAddress.newBuilder()
227+
.setPortValue(droplet.port)
228+
.setAddress(droplet.host)
229+
.setProtocol(SocketAddress.Protocol.TCP)
230+
)
231+
)
232+
)
233+
)
234+
)
235+
.build()
168236
}
169237

170238
fun getCache(): ConfigWatcher {
171239
return cache
172240
}
173-
174-
}
241+
}

‎controller-runtime/src/main/kotlin/app/simplecloud/controller/runtime/launcher/ControllerStartCommand.kt‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,9 @@ class ControllerStartCommand(
5353
val envoyStartPort: Int by option(help = "Envoy start port (default: 8080)", envvar = "ENVOY_START_PORT").int()
5454
.default(8080)
5555

56+
val envoyPort: Int by option(help = "Envoy port (default: 8090)", envvar = "ENVOY_PORT").int()
57+
.default(1337)
58+
5659
private val authSecretPath: Path by option(
5760
help = "Path to auth secret file (default: .auth.secret)",
5861
envvar = "AUTH_SECRET_PATH"

0 commit comments

Comments
 (0)