Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
37 commits
Select commit Hold shift + click to select a range
2e4f813
Added VR Health check feature
Nov 11, 2019
2467818
Fix pylint errors from pycodestyle for travis failures
Nov 12, 2019
ce4b1d1
Removed dependency from /opt/bin to local health_checks directory
Nov 12, 2019
348317e
Added router version, CPU and Memory checks
Nov 13, 2019
3f586ba
Minor message updatE
Nov 13, 2019
986bb33
Pylint errors
Nov 13, 2019
0d6a10c
make hardware required for tests as scripts can only run in actual ro…
Nov 13, 2019
2910abb
Added database layer on management server for keeping router health c…
Dec 9, 2019
9cb367f
More fixes
Dec 10, 2019
74b3c37
Add mechanism to not keep files in processed cache on VR
Dec 11, 2019
2efff2e
More fixes to avoid caching repetitive health data
Dec 11, 2019
73b7b40
Added skipping unavailabe data for checks, and changed UI to add tables
Dec 13, 2019
cf081ac
Added more fixes + Code review comments
Dec 16, 2019
32f244a
Fix failing test after schema changes
Dec 16, 2019
06c53e2
More CR comments and minor nits
Dec 17, 2019
26bc9a8
Cleanup code and more review comments
Dec 17, 2019
5db6fc7
Cleanup some more unitended changes
Dec 17, 2019
8699af2
Fix pylint errors
Dec 18, 2019
23b5c77
Added missing license to file
Dec 18, 2019
2f1c0a5
Fix typo mistake in JS Script
Dec 18, 2019
b846b91
Add NPE checks for defensive failures
Dec 18, 2019
8b52c5d
More CR comments
Dec 19, 2019
6a0558a
Increase ping attempts, fix some bugs and cleanup in health check scr…
Dec 23, 2019
fbe436c
Fix travis pycodestyle failure
Dec 23, 2019
46b51e3
Bug fixes
Jan 10, 2020
8ffadc3
Add more fixes and expand HAProxy checks
Jan 13, 2020
5878429
Added events to mark test failures and router recreation
Jan 14, 2020
5550e8e
Refactor code
Jan 14, 2020
ffd51a0
Refactor for readability
Jan 14, 2020
8288552
Added support for VPC routers to monitor services + bug fixes/enhance…
Jan 17, 2020
d23ecd7
Pycodestyle errors
Jan 17, 2020
4bd1020
Code review comments
Jan 17, 2020
ea9536b
Split failing checks files into two
Jan 18, 2020
9b3d11c
Some fixes and improvements - add commas, delete skipped tests, add l…
Jan 20, 2020
4f44ed9
Some minor code review comments
Jan 20, 2020
bd2ee15
Handle case for monitoring services in VPC when additional guest netw…
Jan 21, 2020
3210c35
Only use ssh service as default for VPC Routers
Jan 21, 2020
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 9 additions & 6 deletions api/src/main/java/com/cloud/event/EventTypes.java
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,13 @@
import java.util.HashMap;
import java.util.Map;

import org.apache.cloudstack.acl.Role;
import org.apache.cloudstack.acl.RolePermission;
import org.apache.cloudstack.annotation.Annotation;
import org.apache.cloudstack.config.Configuration;
import org.apache.cloudstack.ha.HAConfig;
import org.apache.cloudstack.usage.Usage;

import com.cloud.dc.DataCenter;
import com.cloud.dc.Pod;
import com.cloud.dc.StorageNetworkIpRange;
Expand Down Expand Up @@ -69,12 +76,6 @@
import com.cloud.vm.Nic;
import com.cloud.vm.NicSecondaryIp;
import com.cloud.vm.VirtualMachine;
import org.apache.cloudstack.acl.Role;
import org.apache.cloudstack.acl.RolePermission;
import org.apache.cloudstack.annotation.Annotation;
import org.apache.cloudstack.config.Configuration;
import org.apache.cloudstack.ha.HAConfig;
import org.apache.cloudstack.usage.Usage;

public class EventTypes {

Expand Down Expand Up @@ -106,6 +107,7 @@ public class EventTypes {
public static final String EVENT_ROUTER_HA = "ROUTER.HA";
public static final String EVENT_ROUTER_UPGRADE = "ROUTER.UPGRADE";
public static final String EVENT_ROUTER_DIAGNOSTICS = "ROUTER.DIAGNOSTICS";
public static final String EVENT_ROUTER_HEALTH_CHECKS = "ROUTER.HEALTH.CHECKS";

// Console proxy
public static final String EVENT_PROXY_CREATE = "PROXY.CREATE";
Expand Down Expand Up @@ -603,6 +605,7 @@ public class EventTypes {
entityEventDetails.put(EVENT_ROUTER_HA, VirtualRouter.class);
entityEventDetails.put(EVENT_ROUTER_UPGRADE, VirtualRouter.class);
entityEventDetails.put(EVENT_ROUTER_DIAGNOSTICS, VirtualRouter.class);
entityEventDetails.put(EVENT_ROUTER_HEALTH_CHECKS, VirtualRouter.class);

entityEventDetails.put(EVENT_PROXY_CREATE, VirtualMachine.class);
entityEventDetails.put(EVENT_PROXY_DESTROY, VirtualMachine.class);
Expand Down
4 changes: 3 additions & 1 deletion api/src/main/java/com/cloud/network/NetworkService.java
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,9 @@ IpAddress allocatePortableIP(Account ipOwner, int regionId, Long zoneId, Long ne

boolean deleteNetwork(long networkId, boolean forced);

boolean restartNetwork(RestartNetworkCmd cmd, boolean cleanup, boolean makeRedundant) throws ConcurrentOperationException, ResourceUnavailableException, InsufficientCapacityException;
boolean restartNetwork(Long networkId, boolean cleanup, boolean makeRedundant, User user) throws ConcurrentOperationException, ResourceUnavailableException, InsufficientCapacityException;

boolean restartNetwork(RestartNetworkCmd cmd) throws ConcurrentOperationException, ResourceUnavailableException, InsufficientCapacityException;

int getActiveNicsInNetwork(long networkId);

Expand Down
34 changes: 34 additions & 0 deletions api/src/main/java/com/cloud/network/RouterHealthCheckResult.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

package com.cloud.network;

import java.util.Date;

public interface RouterHealthCheckResult {
long getRouterId();

String getCheckName();

String getCheckType();

boolean getCheckResult();

Date getLastUpdateTime();

String getParsedCheckDetails();
}
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,7 @@ public interface VirtualNetworkApplianceService {
/**
* Starts domain router
*
* @param cmd
* the command specifying router's id
* @param cmd the command specifying router's id
* @return DomainRouter object
*/
VirtualRouter startRouter(long routerId, boolean reprogramNetwork) throws ConcurrentOperationException, ResourceUnavailableException, InsufficientCapacityException;
Expand All @@ -51,10 +50,8 @@ public interface VirtualNetworkApplianceService {
/**
* Stops domain router
*
* @param id
* of the router
* @param forced
* just do it. caller knows best.
* @param id of the router
* @param forced just do it. caller knows best.
* @return router if successful, null otherwise
* @throws ResourceUnavailableException
* @throws ConcurrentOperationException
Expand All @@ -68,4 +65,13 @@ public interface VirtualNetworkApplianceService {
VirtualRouter findRouter(long routerId);

List<Long> upgradeRouterTemplate(UpgradeRouterTemplateCmd cmd);

/**
* Updates router with latest health checkdata, runs health checks and persists health checks on virtual router if feasible.
* Throws relevant exception if feature is disabled or failures occur.
*
* @param routerId id of the router
* @return
*/
boolean performRouterHealthChecks(long routerId);
}
6 changes: 5 additions & 1 deletion api/src/main/java/com/cloud/network/vpc/VpcService.java
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@

import org.apache.cloudstack.api.command.user.vpc.ListPrivateGatewaysCmd;
import org.apache.cloudstack.api.command.user.vpc.ListStaticRoutesCmd;
import org.apache.cloudstack.api.command.user.vpc.RestartVPCCmd;

import com.cloud.exception.ConcurrentOperationException;
import com.cloud.exception.InsufficientAddressCapacityException;
Expand All @@ -29,6 +30,7 @@
import com.cloud.exception.ResourceAllocationException;
import com.cloud.exception.ResourceUnavailableException;
import com.cloud.network.IpAddress;
import com.cloud.user.User;
import com.cloud.utils.Pair;

public interface VpcService {
Expand Down Expand Up @@ -132,7 +134,9 @@ public Pair<List<? extends Vpc>, Integer> listVpcs(Long id, String vpcName, Stri
* @return
* @throws InsufficientCapacityException
*/
boolean restartVpc(long id, boolean cleanUp, boolean makeredundant) throws ConcurrentOperationException, ResourceUnavailableException, InsufficientCapacityException;
boolean restartVpc(RestartVPCCmd cmd) throws ConcurrentOperationException, ResourceUnavailableException, InsufficientCapacityException;

boolean restartVpc(Long networkId, boolean cleanup, boolean makeRedundant, User user) throws ConcurrentOperationException, ResourceUnavailableException, InsufficientCapacityException;

/**
* Returns a Private gateway found in the VPC by id
Expand Down
8 changes: 8 additions & 0 deletions api/src/main/java/org/apache/cloudstack/api/ApiConstants.java
Original file line number Diff line number Diff line change
Expand Up @@ -724,6 +724,7 @@ public class ApiConstants {
public static final String VIRTUAL_SIZE = "virtualsize";
public static final String NETSCALER_CONTROLCENTER_ID = "netscalercontrolcenterid";
public static final String NETSCALER_SERVICEPACKAGE_ID = "netscalerservicepackageid";
public static final String FETCH_ROUTER_HEALTH_CHECK_RESULTS = "fetchhealthcheckresults";

public static final String ZONE_ID_LIST = "zoneids";
public static final String DESTINATION_ZONE_ID_LIST = "destzoneids";
Expand All @@ -745,6 +746,13 @@ public class ApiConstants {
public static final String FILES = "files";
public static final String VOLUME_IDS = "volumeids";

public static final String ROUTER_ID = "routerid";
public static final String ROUTER_HEALTH_CHECKS = "healthchecks";
public static final String ROUTER_CHECK_NAME = "checkname";
public static final String ROUTER_CHECK_TYPE = "checktype";
public static final String LAST_UPDATED = "lastupdated";
public static final String PERFORM_FRESH_CHECKS = "performfreshchecks";

public enum HostDetails {
all, capacity, events, stats, min;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import java.util.Map;
import java.util.Set;

import org.apache.cloudstack.api.response.RouterHealthCheckResultResponse;
import org.apache.cloudstack.management.ManagementServerHost;
import org.apache.cloudstack.affinity.AffinityGroup;
import org.apache.cloudstack.affinity.AffinityGroupResponse;
Expand Down Expand Up @@ -146,6 +147,7 @@
import com.cloud.network.PhysicalNetworkServiceProvider;
import com.cloud.network.PhysicalNetworkTrafficType;
import com.cloud.network.RemoteAccessVpn;
import com.cloud.network.RouterHealthCheckResult;
import com.cloud.network.Site2SiteCustomerGateway;
import com.cloud.network.Site2SiteVpnConnection;
import com.cloud.network.Site2SiteVpnGateway;
Expand Down Expand Up @@ -466,4 +468,6 @@ List<TemplateResponse> createTemplateResponses(ResponseView view, VirtualMachine
SSHKeyPairResponse createSSHKeyPairResponse(SSHKeyPair sshkeyPair, boolean privatekey);

ManagementServerResponse createManagementResponse(ManagementServerHost mgmt);

List<RouterHealthCheckResultResponse> createHealthCheckResponse(VirtualMachine router, List<RouterHealthCheckResult> healthCheckResults);
}
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
// under the License.
package org.apache.cloudstack.api.command.admin.internallb;

import org.apache.commons.lang.BooleanUtils;
import org.apache.log4j.Logger;

import org.apache.cloudstack.api.APICommand;
Expand Down Expand Up @@ -73,6 +74,11 @@ public class ListInternalLBVMsCmd extends BaseListProjectAndAccountResourcesCmd
@Parameter(name = ApiConstants.FOR_VPC, type = CommandType.BOOLEAN, description = "if true is passed for this parameter, list only VPC Internal LB VMs")
private Boolean forVpc;


@Parameter(name = ApiConstants.FETCH_ROUTER_HEALTH_CHECK_RESULTS, type = CommandType.BOOLEAN, since = "4.14",
description = "if true is passed for this parameter, also fetch last executed health check results for the VM. Default is false")
private Boolean fetchHealthCheckResults;

/////////////////////////////////////////////////////
/////////////////// Accessors ///////////////////////
/////////////////////////////////////////////////////
Expand Down Expand Up @@ -117,6 +123,10 @@ public String getRole() {
return Role.INTERNAL_LB_VM.toString();
}

public boolean shouldFetchHealthCheckResults() {
return BooleanUtils.isTrue(fetchHealthCheckResults);
}

/////////////////////////////////////////////////////
/////////////// API Implementation///////////////////
/////////////////////////////////////////////////////
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,117 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

package org.apache.cloudstack.api.command.admin.router;

import java.util.List;

import org.apache.cloudstack.api.APICommand;
import org.apache.cloudstack.api.ApiConstants;
import org.apache.cloudstack.api.ApiErrorCode;
import org.apache.cloudstack.api.BaseCmd;
import org.apache.cloudstack.api.Parameter;
import org.apache.cloudstack.api.ServerApiException;
import org.apache.cloudstack.api.response.DomainRouterResponse;
import org.apache.cloudstack.api.response.RouterHealthCheckResultResponse;
import org.apache.cloudstack.api.response.RouterHealthCheckResultsListResponse;
import org.apache.cloudstack.context.CallContext;
import org.apache.commons.lang.BooleanUtils;
import org.apache.log4j.Logger;

import com.cloud.exception.InvalidParameterValueException;
import com.cloud.exception.ResourceUnavailableException;
import com.cloud.network.router.VirtualRouter;
import com.cloud.user.Account;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.vm.VirtualMachine;

@APICommand(name = GetRouterHealthCheckResultsCmd.APINAME,
responseObject = RouterHealthCheckResultsListResponse.class,
description = "Starts a router.",
entityType = {VirtualMachine.class},
requestHasSensitiveInfo = false,
responseHasSensitiveInfo = false,
since = "4.14.0")
public class GetRouterHealthCheckResultsCmd extends BaseCmd {
public static final Logger s_logger = Logger.getLogger(GetRouterHealthCheckResultsCmd.class.getName());
public static final String APINAME = "getRouterHealthCheckResults";

/////////////////////////////////////////////////////
//////////////// API parameters /////////////////////
/////////////////////////////////////////////////////

@Parameter(name = ApiConstants.ROUTER_ID, type = CommandType.UUID, entityType = DomainRouterResponse.class,
required = true, description = "the ID of the router")
private Long routerId;

@Parameter(name = ApiConstants.PERFORM_FRESH_CHECKS, type = CommandType.BOOLEAN, description = "if true is passed for this parameter, " +
"health checks are performed on the fly. Else last performed checks data is fetched")
private Boolean performFreshChecks;

/////////////////////////////////////////////////////
/////////////////// Accessors ///////////////////////
/////////////////////////////////////////////////////

public Long getRouterId() {
return routerId;
}

public boolean shouldPerformFreshChecks() {
return BooleanUtils.isTrue(performFreshChecks);
}

/////////////////////////////////////////////////////
/////////////// API Implementation///////////////////
/////////////////////////////////////////////////////

@Override
public String getCommandName() {
return APINAME.toLowerCase() + BaseCmd.RESPONSE_SUFFIX;
}

@Override
public long getEntityOwnerId() {
VirtualRouter router = _entityMgr.findById(VirtualRouter.class, getRouterId());
if (router != null) {
return router.getAccountId();
}

return Account.ACCOUNT_ID_SYSTEM;
}

@Override
public void execute() throws ResourceUnavailableException, InvalidParameterValueException, ServerApiException {
CallContext.current().setEventDetails("Router Id: " + this._uuidMgr.getUuid(VirtualMachine.class, getRouterId()));
VirtualRouter router = _routerService.findRouter(getRouterId());
if (router == null || router.getRole() != VirtualRouter.Role.VIRTUAL_ROUTER) {
throw new InvalidParameterValueException("Can't find router by routerId");
}

try {
List<RouterHealthCheckResultResponse> healthChecks = _queryService.listRouterHealthChecks(this);
RouterHealthCheckResultsListResponse routerResponse = new RouterHealthCheckResultsListResponse();
routerResponse.setRouterId(router.getUuid());
routerResponse.setHealthChecks(healthChecks);
routerResponse.setObjectName("routerhealthchecks");
routerResponse.setResponseName(getCommandName());
setResponseObject(routerResponse);
} catch (CloudRuntimeException ex){
ex.printStackTrace();
throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to execute command due to exception: " + ex.getLocalizedMessage());
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
// under the License.
package org.apache.cloudstack.api.command.admin.router;

import org.apache.commons.lang.BooleanUtils;
import org.apache.log4j.Logger;

import org.apache.cloudstack.api.APICommand;
Expand Down Expand Up @@ -80,6 +81,10 @@ public class ListRoutersCmd extends BaseListProjectAndAccountResourcesCmd {
@Parameter(name = ApiConstants.VERSION, type = CommandType.STRING, description = "list virtual router elements by version")
private String version;

@Parameter(name = ApiConstants.FETCH_ROUTER_HEALTH_CHECK_RESULTS, type = CommandType.BOOLEAN, since = "4.14",
description = "if true is passed for this parameter, also fetch last executed health check results for the router. Default is false")
private Boolean fetchHealthCheckResults;

/////////////////////////////////////////////////////
/////////////////// Accessors ///////////////////////
/////////////////////////////////////////////////////
Expand Down Expand Up @@ -132,6 +137,11 @@ public String getRole() {
return Role.VIRTUAL_ROUTER.toString();
}

public boolean shouldFetchHealthCheckResults() {
return BooleanUtils.isTrue(fetchHealthCheckResults);
}


/////////////////////////////////////////////////////
/////////////// API Implementation///////////////////
/////////////////////////////////////////////////////
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ public static String getResultObjectName() {

@Override
public void execute() throws ResourceUnavailableException, ResourceAllocationException, ConcurrentOperationException, InsufficientCapacityException {
boolean result = _networkService.restartNetwork(this, getCleanup(), getMakeRedundant());
boolean result = _networkService.restartNetwork(this);
if (result) {
SuccessResponse response = new SuccessResponse(getCommandName());
setResponseObject(response);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@ public long getEntityOwnerId() {
@Override
public void execute() {
try {
final boolean result = _vpcService.restartVpc(getId(), getCleanup(), getMakeredundant());
final boolean result = _vpcService.restartVpc(this);
if (result) {
final SuccessResponse response = new SuccessResponse(getCommandName());
setResponseObject(response);
Expand Down
Loading