From 2e4f813f0913b491409f6bedf3c243899c6c1542 Mon Sep 17 00:00:00 2001 From: Anurag Awasthi Date: Mon, 11 Nov 2019 23:50:44 +0530 Subject: [PATCH 01/37] Added VR Health check feature --- .../main/java/com/cloud/event/EventTypes.java | 15 +- .../VirtualNetworkApplianceService.java | 20 +- .../apache/cloudstack/api/ApiConstants.java | 1 + .../cloudstack/api/ResponseGenerator.java | 3 + .../internallb/ListInternalLBVMsCmd.java | 9 + .../GetRouterHealthCheckResultsCmd.java | 114 ++++++ .../command/admin/router/ListRoutersCmd.java | 9 + ...omainRouterHealthCheckResultsResponse.java | 78 +++++ .../api/response/DomainRouterResponse.java | 15 +- .../GetRouterMonitorResultsAnswer.java | 46 +++ .../GetRouterMonitorResultsCommand.java | 38 ++ .../LoadRouterHealthChecksConfigCommand.java | 48 +++ .../api/routing/NetworkElementCommand.java | 1 - .../api/routing/SetMonitorServiceCommand.java | 33 +- .../resource/virtualnetwork/VRScripts.java | 3 + .../VirtualRoutingResource.java | 68 +++- .../facade/SetMonitorServiceConfigItem.java | 32 +- .../virtualnetwork/model/MonitorService.java | 56 ++- .../resource/HypervDirectConnectResource.java | 5 + .../java/com/cloud/api/ApiResponseHelper.java | 16 +- .../com/cloud/api/query/QueryManagerImpl.java | 26 +- .../VirtualNetworkApplianceManager.java | 34 ++ .../VirtualNetworkApplianceManagerImpl.java | 325 +++++++++++++++++- .../cloud/server/ManagementServerImpl.java | 2 + ...MockVpcVirtualNetworkApplianceManager.java | 5 + systemvm/debian/opt/cloud/bin/cs/CsMonitor.py | 52 ++- .../debian/opt/cloud/bin/cs_monitorservice.py | 11 + .../opt/cloud/bin/getRouterMonitorResults.sh | 48 +++ .../opt/cloud/bin/healthchecksutility.py | 44 +++ .../debian/root/health_scripts/dhcp_check.py | 49 +++ .../root/health_scripts/disk_space_check.py | 46 +++ .../debian/root/health_scripts/dns_check.py | 48 +++ .../root/health_scripts/gateways_check.py | 56 +++ .../root/health_scripts/haproxy_check.py | 112 ++++++ .../root/health_scripts/iptables_check.py | 71 ++++ systemvm/debian/root/monitorServices.py | 142 ++++++-- test/integration/component/test_routers.py | 64 +++- .../scripts/configure_systemvm_services.sh | 1 + ui/l10n/en.js | 4 + ui/scripts/system.js | 192 +++++++++-- ui/scripts/ui/widgets/listView.js | 7 +- 41 files changed, 1822 insertions(+), 127 deletions(-) create mode 100644 api/src/main/java/org/apache/cloudstack/api/command/admin/router/GetRouterHealthCheckResultsCmd.java create mode 100644 api/src/main/java/org/apache/cloudstack/api/response/DomainRouterHealthCheckResultsResponse.java create mode 100644 core/src/main/java/com/cloud/agent/api/routing/GetRouterMonitorResultsAnswer.java create mode 100644 core/src/main/java/com/cloud/agent/api/routing/GetRouterMonitorResultsCommand.java create mode 100644 core/src/main/java/com/cloud/agent/api/routing/LoadRouterHealthChecksConfigCommand.java create mode 100755 systemvm/debian/opt/cloud/bin/getRouterMonitorResults.sh create mode 100644 systemvm/debian/opt/cloud/bin/healthchecksutility.py create mode 100755 systemvm/debian/root/health_scripts/dhcp_check.py create mode 100644 systemvm/debian/root/health_scripts/disk_space_check.py create mode 100644 systemvm/debian/root/health_scripts/dns_check.py create mode 100644 systemvm/debian/root/health_scripts/gateways_check.py create mode 100644 systemvm/debian/root/health_scripts/haproxy_check.py create mode 100644 systemvm/debian/root/health_scripts/iptables_check.py diff --git a/api/src/main/java/com/cloud/event/EventTypes.java b/api/src/main/java/com/cloud/event/EventTypes.java index a30518aaf176..d9a70b98e89e 100644 --- a/api/src/main/java/com/cloud/event/EventTypes.java +++ b/api/src/main/java/com/cloud/event/EventTypes.java @@ -19,6 +19,13 @@ import java.util.HashMap; import java.util.Map; +import org.apache.cloudstack.acl.Role; +import org.apache.cloudstack.acl.RolePermission; +import org.apache.cloudstack.annotation.Annotation; +import org.apache.cloudstack.config.Configuration; +import org.apache.cloudstack.ha.HAConfig; +import org.apache.cloudstack.usage.Usage; + import com.cloud.dc.DataCenter; import com.cloud.dc.Pod; import com.cloud.dc.StorageNetworkIpRange; @@ -69,12 +76,6 @@ import com.cloud.vm.Nic; import com.cloud.vm.NicSecondaryIp; import com.cloud.vm.VirtualMachine; -import org.apache.cloudstack.acl.Role; -import org.apache.cloudstack.acl.RolePermission; -import org.apache.cloudstack.annotation.Annotation; -import org.apache.cloudstack.config.Configuration; -import org.apache.cloudstack.ha.HAConfig; -import org.apache.cloudstack.usage.Usage; public class EventTypes { @@ -106,6 +107,7 @@ public class EventTypes { public static final String EVENT_ROUTER_HA = "ROUTER.HA"; public static final String EVENT_ROUTER_UPGRADE = "ROUTER.UPGRADE"; public static final String EVENT_ROUTER_DIAGNOSTICS = "ROUTER.DIAGNOSTICS"; + public static final String EVENT_ROUTER_HEALTH_CHECKS = "ROUTER.HEALTH.CHECKS"; // Console proxy public static final String EVENT_PROXY_CREATE = "PROXY.CREATE"; @@ -603,6 +605,7 @@ public class EventTypes { entityEventDetails.put(EVENT_ROUTER_HA, VirtualRouter.class); entityEventDetails.put(EVENT_ROUTER_UPGRADE, VirtualRouter.class); entityEventDetails.put(EVENT_ROUTER_DIAGNOSTICS, VirtualRouter.class); + entityEventDetails.put(EVENT_ROUTER_HEALTH_CHECKS, VirtualRouter.class); entityEventDetails.put(EVENT_PROXY_CREATE, VirtualMachine.class); entityEventDetails.put(EVENT_PROXY_DESTROY, VirtualMachine.class); diff --git a/api/src/main/java/com/cloud/network/VirtualNetworkApplianceService.java b/api/src/main/java/com/cloud/network/VirtualNetworkApplianceService.java index 815ae4d6ae31..8db3404b229a 100644 --- a/api/src/main/java/com/cloud/network/VirtualNetworkApplianceService.java +++ b/api/src/main/java/com/cloud/network/VirtualNetworkApplianceService.java @@ -17,6 +17,7 @@ package com.cloud.network; import java.util.List; +import java.util.Map; import org.apache.cloudstack.api.command.admin.router.UpgradeRouterCmd; import org.apache.cloudstack.api.command.admin.router.UpgradeRouterTemplateCmd; @@ -31,8 +32,7 @@ public interface VirtualNetworkApplianceService { /** * Starts domain router * - * @param cmd - * the command specifying router's id + * @param cmd the command specifying router's id * @return DomainRouter object */ VirtualRouter startRouter(long routerId, boolean reprogramNetwork) throws ConcurrentOperationException, ResourceUnavailableException, InsufficientCapacityException; @@ -51,10 +51,8 @@ public interface VirtualNetworkApplianceService { /** * Stops domain router * - * @param id - * of the router - * @param forced - * just do it. caller knows best. + * @param id of the router + * @param forced just do it. caller knows best. * @return router if successful, null otherwise * @throws ResourceUnavailableException * @throws ConcurrentOperationException @@ -68,4 +66,14 @@ public interface VirtualNetworkApplianceService { VirtualRouter findRouter(long routerId); List upgradeRouterTemplate(UpgradeRouterTemplateCmd cmd); + + /** + * Returns the health check results for the router. It can run the health checks on demand if runChecks is true. Otherwise, + * it fetches the previously executed health checks. + * + * @param routerId id of the router + * @param runChecks + * @return + */ + Map getRouterHealthCheckResults(long routerId, boolean runChecks); } diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index 44c53f690fb8..4a8a0350f1cb 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -724,6 +724,7 @@ public class ApiConstants { public static final String VIRTUAL_SIZE = "virtualsize"; public static final String NETSCALER_CONTROLCENTER_ID = "netscalercontrolcenterid"; public static final String NETSCALER_SERVICEPACKAGE_ID = "netscalerservicepackageid"; + public static final String INCLUDE_ROUTER_HEALTH_CHECK_RESULTS = "includehealthcheckresults"; public static final String ZONE_ID_LIST = "zoneids"; public static final String DESTINATION_ZONE_ID_LIST = "destzoneids"; diff --git a/api/src/main/java/org/apache/cloudstack/api/ResponseGenerator.java b/api/src/main/java/org/apache/cloudstack/api/ResponseGenerator.java index 740ee468702f..1d96fbd48225 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ResponseGenerator.java +++ b/api/src/main/java/org/apache/cloudstack/api/ResponseGenerator.java @@ -22,6 +22,7 @@ import java.util.Map; import java.util.Set; +import org.apache.cloudstack.api.response.DomainRouterHealthCheckResultsResponse; import org.apache.cloudstack.management.ManagementServerHost; import org.apache.cloudstack.affinity.AffinityGroup; import org.apache.cloudstack.affinity.AffinityGroupResponse; @@ -466,4 +467,6 @@ List createTemplateResponses(ResponseView view, VirtualMachine SSHKeyPairResponse createSSHKeyPairResponse(SSHKeyPair sshkeyPair, boolean privatekey); ManagementServerResponse createManagementResponse(ManagementServerHost mgmt); + + DomainRouterHealthCheckResultsResponse createHealthCheckResponse(VirtualMachine router, Map healthCheckResults); } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/internallb/ListInternalLBVMsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/internallb/ListInternalLBVMsCmd.java index ba2054c3c24b..0a32d96bd281 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/internallb/ListInternalLBVMsCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/internallb/ListInternalLBVMsCmd.java @@ -73,6 +73,11 @@ public class ListInternalLBVMsCmd extends BaseListProjectAndAccountResourcesCmd @Parameter(name = ApiConstants.FOR_VPC, type = CommandType.BOOLEAN, description = "if true is passed for this parameter, list only VPC Internal LB VMs") private Boolean forVpc; + + @Parameter(name = ApiConstants.INCLUDE_ROUTER_HEALTH_CHECK_RESULTS, type = CommandType.BOOLEAN, since = "4.14", + description = "if true is passed for this parameter, also fetch last executed health check results for the VM. Default is false") + private Boolean includeHealthCheckResults; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -117,6 +122,10 @@ public String getRole() { return Role.INTERNAL_LB_VM.toString(); } + public boolean shouldIncludeHealthCheckResults() { + return includeHealthCheckResults == null ? false : includeHealthCheckResults.booleanValue(); + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/router/GetRouterHealthCheckResultsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/router/GetRouterHealthCheckResultsCmd.java new file mode 100644 index 000000000000..b1e2cae50a41 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/router/GetRouterHealthCheckResultsCmd.java @@ -0,0 +1,114 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.api.command.admin.router; + +import java.util.Map; + +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.DomainRouterHealthCheckResultsResponse; +import org.apache.cloudstack.api.response.DomainRouterResponse; +import org.apache.cloudstack.context.CallContext; +import org.apache.log4j.Logger; + +import com.cloud.exception.ConcurrentOperationException; +import com.cloud.exception.InsufficientCapacityException; +import com.cloud.exception.InvalidParameterValueException; +import com.cloud.exception.ResourceUnavailableException; +import com.cloud.network.router.VirtualRouter; +import com.cloud.user.Account; +import com.cloud.vm.VirtualMachine; + +@APICommand(name = GetRouterHealthCheckResultsCmd.APINAME, + responseObject = DomainRouterHealthCheckResultsResponse.class, + description = "Starts a router.", + entityType = {VirtualMachine.class}, + requestHasSensitiveInfo = false, + responseHasSensitiveInfo = false, + since = "4.13.1") +public class GetRouterHealthCheckResultsCmd extends BaseCmd { + public static final Logger s_logger = Logger.getLogger(GetRouterHealthCheckResultsCmd.class.getName()); + public static final String APINAME = "getRouterHealthCheckResults"; + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + + @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = DomainRouterResponse.class, + required = true, description = "the ID of the router") + private Long id; + + @Parameter(name = "performfreshchecks", type = CommandType.BOOLEAN, description = "if true is passed for this parameter, " + + "health checks are performed on the fly. Else last performed checks data is fetched") + private Boolean performFreshChecks; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + public Long getId() { + return id; + } + + public boolean shouldPerformFreshChecks() { + return performFreshChecks == null ? false : performFreshChecks.booleanValue(); + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + + @Override + public String getCommandName() { + return APINAME.toLowerCase() + BaseCmd.RESPONSE_SUFFIX; + } + + @Override + public long getEntityOwnerId() { + VirtualRouter router = _entityMgr.findById(VirtualRouter.class, getId()); + if (router != null) { + return router.getAccountId(); + } + + return Account.ACCOUNT_ID_SYSTEM; + } + + @Override + public void execute() throws ConcurrentOperationException, ResourceUnavailableException, InsufficientCapacityException { + CallContext.current().setEventDetails("Router Id: " + this._uuidMgr.getUuid(VirtualMachine.class, getId())); + VirtualRouter router = _routerService.findRouter(getId()); + Map result = null; + if (router == null || router.getRole() != VirtualRouter.Role.VIRTUAL_ROUTER) { + throw new InvalidParameterValueException("Can't find router by id"); + } else { + result = _routerService.getRouterHealthCheckResults(getId(), shouldPerformFreshChecks()); + } + + if (result != null) { + DomainRouterHealthCheckResultsResponse routerResponse = _responseGenerator.createHealthCheckResponse(router, result); + routerResponse.setResponseName(getCommandName()); + setResponseObject(routerResponse); + } else { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to start router"); + } + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/router/ListRoutersCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/router/ListRoutersCmd.java index 121fc5bc14d2..bb7b5500de8b 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/router/ListRoutersCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/router/ListRoutersCmd.java @@ -80,6 +80,10 @@ public class ListRoutersCmd extends BaseListProjectAndAccountResourcesCmd { @Parameter(name = ApiConstants.VERSION, type = CommandType.STRING, description = "list virtual router elements by version") private String version; + @Parameter(name = ApiConstants.INCLUDE_ROUTER_HEALTH_CHECK_RESULTS, type = CommandType.BOOLEAN, since = "4.14", + description = "if true is passed for this parameter, also fetch last executed health check results for the router. Default is false") + private Boolean includeHealthCheckResults; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -132,6 +136,11 @@ public String getRole() { return Role.VIRTUAL_ROUTER.toString(); } + public boolean shouldIncludeHealthCheckResults() { + return includeHealthCheckResults == null ? false : includeHealthCheckResults.booleanValue(); + } + + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/response/DomainRouterHealthCheckResultsResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/DomainRouterHealthCheckResultsResponse.java new file mode 100644 index 000000000000..4376177979e5 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/response/DomainRouterHealthCheckResultsResponse.java @@ -0,0 +1,78 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.api.response; + +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseResponse; + +import com.cloud.serializer.Param; +import com.google.gson.annotations.SerializedName; + +public class DomainRouterHealthCheckResultsResponse extends BaseResponse { + @SerializedName("routerId") + @Param(description = "the id of the router") + private String routerId; + + @SerializedName(ApiConstants.NAME) + @Param(description = "the name of the router") + private String name; + + @SerializedName(ApiConstants.RESULT) + @Param(description = "result of management server's attempt to fetch data.") + private String result; + + @SerializedName(ApiConstants.DETAILS) + @Param(description = "detailed data fetched from management server") + private String details; + + public DomainRouterHealthCheckResultsResponse(String objectName) { + super(objectName); + } + + public String getRouterId() { + return routerId; + } + + public String getName() { + return name; + } + + public String getResult() { + return result; + } + + public String getDetails() { + return details; + } + + public void setRouterId(String id) { + this.routerId = id; + } + + public void setName(String name) { + this.name = name; + } + + public void setResult(String result) { + this.result = result; + } + + public void setDetails(String details) { + this.details = details; + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/response/DomainRouterResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/DomainRouterResponse.java index 131e3e1de7ed..3c7413d0afdb 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/DomainRouterResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/DomainRouterResponse.java @@ -20,8 +20,6 @@ import java.util.LinkedHashSet; import java.util.Set; -import com.google.gson.annotations.SerializedName; - import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.BaseResponse; import org.apache.cloudstack.api.EntityReference; @@ -29,6 +27,7 @@ import com.cloud.serializer.Param; import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachine.State; +import com.google.gson.annotations.SerializedName; @EntityReference(value = VirtualMachine.class) @SuppressWarnings("unused") @@ -217,6 +216,10 @@ public class DomainRouterResponse extends BaseResponse implements ControlledView @Param(description = "true if the router template requires upgrader") private boolean requiresUpgrade; + @SerializedName("healthcheckresults") + @Param(description = "Last executed health check result for the router", responseObject = DomainRouterHealthCheckResultsResponse.class, since = "4.14") + DomainRouterHealthCheckResultsResponse healthCheckResults; + public DomainRouterResponse() { nics = new LinkedHashSet(); } @@ -278,6 +281,10 @@ public String getHypervisor() { return hypervisor; } + public DomainRouterHealthCheckResultsResponse getHealthCheckResults() { + return healthCheckResults; + } + public void setHypervisor(String hypervisor) { this.hypervisor = hypervisor; } @@ -446,4 +453,8 @@ public boolean requiresUpgrade() { public void setRequiresUpgrade(boolean requiresUpgrade) { this.requiresUpgrade = requiresUpgrade; } + + public void setHealthCheckResults(DomainRouterHealthCheckResultsResponse healthCheckResults) { + this.healthCheckResults = healthCheckResults; + } } diff --git a/core/src/main/java/com/cloud/agent/api/routing/GetRouterMonitorResultsAnswer.java b/core/src/main/java/com/cloud/agent/api/routing/GetRouterMonitorResultsAnswer.java new file mode 100644 index 000000000000..4db59dfac335 --- /dev/null +++ b/core/src/main/java/com/cloud/agent/api/routing/GetRouterMonitorResultsAnswer.java @@ -0,0 +1,46 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.agent.api.routing; + +import java.util.List; + +import com.cloud.agent.api.Answer; +import com.cloud.agent.api.Command; + +public class GetRouterMonitorResultsAnswer extends Answer { + private List failingChecks; + private String monitoringResults; + + protected GetRouterMonitorResultsAnswer() { + super(); + } + + public GetRouterMonitorResultsAnswer(Command cmd, boolean success, List failingChecks, String monitoringResults) { + super(cmd, success, monitoringResults); + this.failingChecks = failingChecks; + this.monitoringResults = monitoringResults; + } + + public List getFailingChecks() { + return failingChecks; + } + + public String getMonitoringResults() { + return monitoringResults; + } +} diff --git a/core/src/main/java/com/cloud/agent/api/routing/GetRouterMonitorResultsCommand.java b/core/src/main/java/com/cloud/agent/api/routing/GetRouterMonitorResultsCommand.java new file mode 100644 index 000000000000..779a0f45a57f --- /dev/null +++ b/core/src/main/java/com/cloud/agent/api/routing/GetRouterMonitorResultsCommand.java @@ -0,0 +1,38 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.agent.api.routing; + +public class GetRouterMonitorResultsCommand extends NetworkElementCommand { + private boolean performFreshChecks; + + protected GetRouterMonitorResultsCommand() { + } + + public GetRouterMonitorResultsCommand(boolean performFreshChecks) { + this.performFreshChecks = performFreshChecks; + } + + @Override + public boolean isQuery() { + return true; + } + + public boolean shouldPerformFreshChecks() { + return performFreshChecks; + } +} \ No newline at end of file diff --git a/core/src/main/java/com/cloud/agent/api/routing/LoadRouterHealthChecksConfigCommand.java b/core/src/main/java/com/cloud/agent/api/routing/LoadRouterHealthChecksConfigCommand.java new file mode 100644 index 000000000000..6b2c119406d1 --- /dev/null +++ b/core/src/main/java/com/cloud/agent/api/routing/LoadRouterHealthChecksConfigCommand.java @@ -0,0 +1,48 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.agent.api.routing; + +import java.util.HashMap; +import java.util.Map; + +/** + * Loads new and updates old configuration details on VR for health checks. + */ +public class LoadRouterHealthChecksConfigCommand extends NetworkElementCommand { +// public static String CHECKS_RUN_INTERVAL = "checks_run_interval"; +// public static String CHECKS_EXCLUDED = "checks_excluded"; + + private Map details; + + protected LoadRouterHealthChecksConfigCommand() { + details = new HashMap<>(); + } + + public void addDetail(String key, String value) { + this.details.put(key, value); + } + + public Map getDetails() { + return details; +// StringBuilder detailsBuilder = new StringBuilder(); +// for (Map.Entry detail : details.entrySet()) { +// detailsBuilder.append(detail.getKey() + "::" + detail.getValue() + ";"); +// } +// return detailsBuilder.toString(); + } +} diff --git a/core/src/main/java/com/cloud/agent/api/routing/NetworkElementCommand.java b/core/src/main/java/com/cloud/agent/api/routing/NetworkElementCommand.java index ae482ac71ec7..de3843e2b837 100644 --- a/core/src/main/java/com/cloud/agent/api/routing/NetworkElementCommand.java +++ b/core/src/main/java/com/cloud/agent/api/routing/NetworkElementCommand.java @@ -38,7 +38,6 @@ public abstract class NetworkElementCommand extends Command { public static final String GUEST_BRIDGE = "guest.bridge"; public static final String VPC_PRIVATE_GATEWAY = "vpc.gateway.private"; public static final String FIREWALL_EGRESS_DEFAULT = "firewall.egress.default"; - public static final String ROUTER_MONITORING_ENABLE = "router.monitor.enable"; public static final String NETWORK_PUB_LAST_IP = "network.public.last.ip"; private String routerAccessIp; diff --git a/core/src/main/java/com/cloud/agent/api/routing/SetMonitorServiceCommand.java b/core/src/main/java/com/cloud/agent/api/routing/SetMonitorServiceCommand.java index a5377039dd63..0e1c9e066eb8 100644 --- a/core/src/main/java/com/cloud/agent/api/routing/SetMonitorServiceCommand.java +++ b/core/src/main/java/com/cloud/agent/api/routing/SetMonitorServiceCommand.java @@ -20,6 +20,7 @@ package com.cloud.agent.api.routing; import java.util.List; +import java.util.Map; import com.cloud.agent.api.to.MonitorServiceTO; @@ -29,9 +30,17 @@ * how to access the components inside the command. */ public class SetMonitorServiceCommand extends NetworkElementCommand { - MonitorServiceTO[] services; + public static final String ROUTER_MONITORING_ENABLED = "router.monitor.enabled"; + public static final String ROUTER_HEALTH_CHECKS_ENABLED = "router.health.checks.enabled"; + public static final String ROUTER_HEALTH_CHECKS_BASIC_INTERVAL = "router.health.checks.basic.interval"; + public static final String ROUTER_HEALTH_CHECKS_ADVANCED_INTERVAL = "router.health.checks.advanced.interval"; + public static final String ROUTER_HEALTH_CHECKS_EXCLUDED = "router.health.checks.excluded"; - protected SetMonitorServiceCommand() { + private MonitorServiceTO[] services; + private Map additionalData; + private boolean reconfigureAfterUpdate; + + public SetMonitorServiceCommand() { } public SetMonitorServiceCommand(List services) { @@ -43,7 +52,9 @@ public MonitorServiceTO[] getRules() { } public String getConfiguration() { - + if (services == null) { + return null; + } StringBuilder sb = new StringBuilder(); for (MonitorServiceTO service : services) { sb.append("[").append(service.getService()).append("]").append(":"); @@ -55,4 +66,20 @@ public String getConfiguration() { return sb.toString(); } + + public Map getAdditionalData() { + return additionalData; + } + + public void setAdditionalData(Map additionalData) { + this.additionalData = additionalData; + } + + public boolean shouldReconfigureAfterUpdate() { + return reconfigureAfterUpdate; + } + + public void setReconfigureAfterUpdate(boolean reconfigureAfterUpdate) { + this.reconfigureAfterUpdate = reconfigureAfterUpdate; + } } diff --git a/core/src/main/java/com/cloud/agent/resource/virtualnetwork/VRScripts.java b/core/src/main/java/com/cloud/agent/resource/virtualnetwork/VRScripts.java index b9d6487de561..cf3b2d4931ff 100644 --- a/core/src/main/java/com/cloud/agent/resource/virtualnetwork/VRScripts.java +++ b/core/src/main/java/com/cloud/agent/resource/virtualnetwork/VRScripts.java @@ -47,6 +47,8 @@ public class VRScripts { // New scripts for use with chef public static final String UPDATE_CONFIG = "update_config.py"; + public static final String CONFIGURE = "configure.py"; // Typically needs to be called if only update_config.py is called + // Script still in use - mostly by HyperV public static final String S2SVPN_CHECK = "checkbatchs2svpn.sh"; @@ -66,6 +68,7 @@ public class VRScripts { public static final String VPC_STATIC_ROUTE = "vpc_staticroute.sh"; public static final String VPN_L2TP = "vpn_l2tp.sh"; public static final String UPDATE_HOST_PASSWD = "update_host_passwd.sh"; + public static final String ROUTER_MONITOR_RESULTS = "getRouterMonitorResults.sh"; public static final String VR_CFG = "vr_cfg.sh"; diff --git a/core/src/main/java/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResource.java b/core/src/main/java/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResource.java index 191a62263f36..42f59499266c 100644 --- a/core/src/main/java/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResource.java +++ b/core/src/main/java/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResource.java @@ -22,13 +22,6 @@ import java.io.IOException; import java.net.InetSocketAddress; import java.nio.channels.SocketChannel; - -import org.apache.cloudstack.diagnostics.DeleteFileInVrCommand; -import org.apache.cloudstack.diagnostics.DiagnosticsAnswer; -import org.apache.cloudstack.diagnostics.DiagnosticsCommand; -import org.apache.cloudstack.diagnostics.PrepareFilesAnswer; -import org.apache.cloudstack.diagnostics.PrepareFilesCommand; -import org.joda.time.Duration; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -45,8 +38,14 @@ import org.apache.cloudstack.ca.SetupCertificateCommand; import org.apache.cloudstack.ca.SetupKeyStoreCommand; import org.apache.cloudstack.ca.SetupKeystoreAnswer; +import org.apache.cloudstack.diagnostics.DeleteFileInVrCommand; +import org.apache.cloudstack.diagnostics.DiagnosticsAnswer; +import org.apache.cloudstack.diagnostics.DiagnosticsCommand; +import org.apache.cloudstack.diagnostics.PrepareFilesAnswer; +import org.apache.cloudstack.diagnostics.PrepareFilesCommand; import org.apache.cloudstack.utils.security.KeyStoreUtils; import org.apache.log4j.Logger; +import org.joda.time.Duration; import com.cloud.agent.api.Answer; import com.cloud.agent.api.CheckRouterAnswer; @@ -59,6 +58,8 @@ import com.cloud.agent.api.routing.AggregationControlCommand; import com.cloud.agent.api.routing.AggregationControlCommand.Action; import com.cloud.agent.api.routing.GetRouterAlertsCommand; +import com.cloud.agent.api.routing.GetRouterMonitorResultsAnswer; +import com.cloud.agent.api.routing.GetRouterMonitorResultsCommand; import com.cloud.agent.api.routing.GroupAnswer; import com.cloud.agent.api.routing.NetworkElementCommand; import com.cloud.agent.resource.virtualnetwork.facade.AbstractConfigItemFacade; @@ -204,6 +205,8 @@ private Answer executeQueryCommand(NetworkElementCommand cmd) { return execute((PrepareFilesCommand) cmd); } else if (cmd instanceof DeleteFileInVrCommand) { return execute((DeleteFileInVrCommand)cmd); + } else if (cmd instanceof GetRouterMonitorResultsCommand) { + return execute((GetRouterMonitorResultsCommand)cmd); } else { s_logger.error("Unknown query command in VirtualRoutingResource!"); return Answer.createUnsupportedCommandAnswer(cmd); @@ -225,10 +228,7 @@ private ExecutionResult applyConfigToVR(String routerAccessIp, ConfigItem c, Dur throw new CloudRuntimeException("Unable to apply unknown configitem of type " + c.getClass().getSimpleName()); } - private Answer applyConfig(NetworkElementCommand cmd, List cfg) { - - if (cfg.isEmpty()) { return new Answer(cmd, true, "Nothing to do"); } @@ -256,7 +256,6 @@ private Answer applyConfig(NetworkElementCommand cmd, List cfg) { s_logger.warn("Expected " + cmd.getAnswersCount() + " answers while executing " + cmd.getClass().getSimpleName() + " but received " + results.size()); } - if (results.size() == 1) { return new Answer(cmd, finalResult, results.get(0).getDetails()); } else { @@ -275,6 +274,53 @@ private CheckS2SVpnConnectionsAnswer execute(CheckS2SVpnConnectionsCommand cmd) return new CheckS2SVpnConnectionsAnswer(cmd, result.isSuccess(), result.getDetails()); } + private GetRouterMonitorResultsAnswer execute(GetRouterMonitorResultsCommand cmd) { + + String routerIp = cmd.getAccessDetail(NetworkElementCommand.ROUTER_IP); + String args = cmd.shouldPerformFreshChecks() ? "true" : "false"; + s_logger.debug("Fetching health check result for " + routerIp + " and executing fresh checks: " + args); + ExecutionResult result = _vrDeployer.executeInVR(routerIp, VRScripts.ROUTER_MONITOR_RESULTS, args); + if (result.isSuccess()) { + List failingChecks = new ArrayList<>(); + StringBuilder monitorResults = new StringBuilder(); + if (!result.getDetails().isEmpty()) { + String[] lines = result.getDetails().trim().split("\n"); + boolean readingFailedChecks = false, readingMonitorResults = false; + for (String line : lines) { + line = line.trim(); + if (line.contains("FAILING CHECKS")) { + readingFailedChecks = true; + readingMonitorResults = false; + continue; + } else if (line.contains("MONITOR RESULTS")) { + readingFailedChecks = false; + readingMonitorResults = true; + continue; + } + if (readingFailedChecks && !readingMonitorResults) { + for (String w : line.split(",")) { + if (!w.trim().isEmpty()) { + failingChecks.add(w.trim()); + } + } + } else if (!readingFailedChecks && readingMonitorResults) { + monitorResults.append(line); + } else { + s_logger.info("Unexpected state of lines reached while parsing response. Skipping line."); + } + } + } else { + s_logger.warn("Received no results back in monitor results."); + } + if (monitorResults.length() == 0) { + monitorResults.append("No results available."); + } + return new GetRouterMonitorResultsAnswer(cmd, true, failingChecks, monitorResults.toString()); + } else { + return new GetRouterMonitorResultsAnswer(cmd, false, null, result.getDetails()); + } + } + private GetRouterAlertsAnswer execute(GetRouterAlertsCommand cmd) { String routerIp = cmd.getAccessDetail(NetworkElementCommand.ROUTER_IP); diff --git a/core/src/main/java/com/cloud/agent/resource/virtualnetwork/facade/SetMonitorServiceConfigItem.java b/core/src/main/java/com/cloud/agent/resource/virtualnetwork/facade/SetMonitorServiceConfigItem.java index 2cf03e445fc1..03b61ea4296b 100644 --- a/core/src/main/java/com/cloud/agent/resource/virtualnetwork/facade/SetMonitorServiceConfigItem.java +++ b/core/src/main/java/com/cloud/agent/resource/virtualnetwork/facade/SetMonitorServiceConfigItem.java @@ -21,21 +21,49 @@ import java.util.List; +import org.apache.log4j.Logger; + import com.cloud.agent.api.routing.NetworkElementCommand; import com.cloud.agent.api.routing.SetMonitorServiceCommand; import com.cloud.agent.resource.virtualnetwork.ConfigItem; +import com.cloud.agent.resource.virtualnetwork.ScriptConfigItem; import com.cloud.agent.resource.virtualnetwork.VRScripts; import com.cloud.agent.resource.virtualnetwork.model.ConfigBase; import com.cloud.agent.resource.virtualnetwork.model.MonitorService; public class SetMonitorServiceConfigItem extends AbstractConfigItemFacade { + private static final Logger s_logger = Logger.getLogger(SetMonitorServiceConfigItem.class); @Override public List generateConfig(final NetworkElementCommand cmd) { final SetMonitorServiceCommand command = (SetMonitorServiceCommand) cmd; - final MonitorService monitorService = new MonitorService(command.getConfiguration(), cmd.getAccessDetail(NetworkElementCommand.ROUTER_MONITORING_ENABLE)); - return generateConfigItems(monitorService); + final MonitorService monitorService = new MonitorService( + command.getConfiguration(), + cmd.getAccessDetail(SetMonitorServiceCommand.ROUTER_MONITORING_ENABLED), + cmd.getAccessDetail(SetMonitorServiceCommand.ROUTER_HEALTH_CHECKS_ENABLED)); + + try { + monitorService.setHealthChecksBasicRunInterval(Integer.parseInt(cmd.getAccessDetail(SetMonitorServiceCommand.ROUTER_HEALTH_CHECKS_BASIC_INTERVAL))); + } catch (NumberFormatException exception) { + s_logger.error("Unexpected health check basic interval set" + cmd.getAccessDetail(SetMonitorServiceCommand.ROUTER_HEALTH_CHECKS_BASIC_INTERVAL) + + ". Exception: " + exception + "Will use default value"); + } + + try { + monitorService.setHealthChecksAdvanceRunInterval(Integer.parseInt(cmd.getAccessDetail(SetMonitorServiceCommand.ROUTER_HEALTH_CHECKS_ADVANCED_INTERVAL))); + } catch (NumberFormatException exception) { + s_logger.error("Unexpected health check advance interval set" + cmd.getAccessDetail(SetMonitorServiceCommand.ROUTER_HEALTH_CHECKS_ADVANCED_INTERVAL) + + ". Exception: " + exception + "Will use default value"); + } + + monitorService.setExcludedHealthChecks(cmd.getAccessDetail(SetMonitorServiceCommand.ROUTER_HEALTH_CHECKS_EXCLUDED)); + monitorService.setAdditionalData(command.getAdditionalData()); + List configItems = generateConfigItems(monitorService); + if (configItems != null && command.shouldReconfigureAfterUpdate()) { + configItems.add(new ScriptConfigItem(VRScripts.CONFIGURE, "monitor_service.json")); + } + return configItems; } @Override diff --git a/core/src/main/java/com/cloud/agent/resource/virtualnetwork/model/MonitorService.java b/core/src/main/java/com/cloud/agent/resource/virtualnetwork/model/MonitorService.java index fdf9e473f35c..cab44fe6f1cb 100644 --- a/core/src/main/java/com/cloud/agent/resource/virtualnetwork/model/MonitorService.java +++ b/core/src/main/java/com/cloud/agent/resource/virtualnetwork/model/MonitorService.java @@ -19,34 +19,80 @@ package com.cloud.agent.resource.virtualnetwork.model; +import java.util.Map; + public class MonitorService extends ConfigBase { public String config, disableMonitoring; + public Boolean healthChecksEnabled; + public Integer healthChecksBasicRunInterval; + public Integer healthChecksAdvanceRunInterval; + public String excludedHealthChecks; + public Map additionalData; public MonitorService() { super(ConfigBase.MONITORSERVICE); } - public MonitorService(String config, String disableMonitoring) { + public MonitorService(String config, String disableMonitoring, String healthChecksEnabled) { super(ConfigBase.MONITORSERVICE); this.config = config; this.disableMonitoring = disableMonitoring; + this.healthChecksEnabled = Boolean.parseBoolean(healthChecksEnabled); } public String getConfig() { return config; } - public void setConfig(String config) { - this.config = config; - } - public String getDisableMonitoring() { return disableMonitoring; } + public Boolean getHealthChecksEnabled() { + return healthChecksEnabled; + } + + public Integer getHealthChecksBasicRunInterval() { + return healthChecksBasicRunInterval; + } + + public Integer getHealthChecksAdvanceRunInterval() { + return healthChecksAdvanceRunInterval; + } + + public String getExcludedHealthChecks() { + return excludedHealthChecks; + } + + public Map getAdditionalData() { + return additionalData; + } + + public void setConfig(String config) { + this.config = config; + } + public void setDisableMonitoring(String disableMonitoring) { this.disableMonitoring = disableMonitoring; } + public void setHealthChecksEnabled(Boolean healthChecksEnabled) { + this.healthChecksEnabled = healthChecksEnabled; + } + public void setHealthChecksBasicRunInterval(Integer healthChecksBasicRunInterval) { + this.healthChecksBasicRunInterval = healthChecksBasicRunInterval; + } + + public void setHealthChecksAdvanceRunInterval(Integer healthChecksAdvanceRunInterval) { + this.healthChecksAdvanceRunInterval = healthChecksAdvanceRunInterval; + } + + public void setExcludedHealthChecks(String excludedHealthChecks) { + this.excludedHealthChecks = excludedHealthChecks; + } + + public void setAdditionalData(Map additionalData) { + this.additionalData = additionalData; + } } diff --git a/plugins/hypervisors/hyperv/src/main/java/com/cloud/hypervisor/hyperv/resource/HypervDirectConnectResource.java b/plugins/hypervisors/hyperv/src/main/java/com/cloud/hypervisor/hyperv/resource/HypervDirectConnectResource.java index 979be732f2bb..038661b58adf 100644 --- a/plugins/hypervisors/hyperv/src/main/java/com/cloud/hypervisor/hyperv/resource/HypervDirectConnectResource.java +++ b/plugins/hypervisors/hyperv/src/main/java/com/cloud/hypervisor/hyperv/resource/HypervDirectConnectResource.java @@ -2085,6 +2085,11 @@ protected Answer execute(final SetMonitorServiceCommand cmd) { final String controlIp = getRouterSshControlIp(cmd); final String config = cmd.getConfiguration(); + if (org.apache.commons.lang.StringUtils.isBlank(config)) { + s_logger.error("SetMonitorServiceCommand should have config for this case"); + return new Answer(cmd, false, "SetMonitorServiceCommand failed due to missing config"); + } + final String args = String.format(" %s %s", "-c", config); final String command = String.format("%s%s %s", "/opt/cloud/bin/", VRScripts.MONITOR_SERVICE, args); diff --git a/server/src/main/java/com/cloud/api/ApiResponseHelper.java b/server/src/main/java/com/cloud/api/ApiResponseHelper.java index b8e60325ea22..6bf369d5ea3c 100644 --- a/server/src/main/java/com/cloud/api/ApiResponseHelper.java +++ b/server/src/main/java/com/cloud/api/ApiResponseHelper.java @@ -31,8 +31,6 @@ import javax.inject.Inject; -import com.cloud.vm.snapshot.VMSnapshotVO; -import com.cloud.vm.snapshot.dao.VMSnapshotDao; import org.apache.cloudstack.acl.ControlledEntity; import org.apache.cloudstack.acl.ControlledEntity.ACLType; import org.apache.cloudstack.affinity.AffinityGroup; @@ -63,6 +61,7 @@ import org.apache.cloudstack.api.response.CreateSSHKeyPairResponse; import org.apache.cloudstack.api.response.DiskOfferingResponse; import org.apache.cloudstack.api.response.DomainResponse; +import org.apache.cloudstack.api.response.DomainRouterHealthCheckResultsResponse; import org.apache.cloudstack.api.response.DomainRouterResponse; import org.apache.cloudstack.api.response.EventResponse; import org.apache.cloudstack.api.response.ExtractResponse; @@ -336,6 +335,8 @@ import com.cloud.vm.dao.NicExtraDhcpOptionDao; import com.cloud.vm.dao.NicSecondaryIpVO; import com.cloud.vm.snapshot.VMSnapshot; +import com.cloud.vm.snapshot.VMSnapshotVO; +import com.cloud.vm.snapshot.dao.VMSnapshotDao; public class ApiResponseHelper implements ResponseGenerator { @@ -1349,6 +1350,7 @@ public DomainRouterResponse createDomainRouterResponse(VirtualRouter router) { return listVrs.get(0); } + @Override public SystemVmResponse createSystemVmResponse(VirtualMachine vm) { SystemVmResponse vmResponse = new SystemVmResponse(); @@ -4205,4 +4207,14 @@ public ManagementServerResponse createManagementResponse(ManagementServerHost mg response.setState(mgmt.getState()); return response; } + + @Override + public DomainRouterHealthCheckResultsResponse createHealthCheckResponse(VirtualMachine router, Map healthCheckResults) { + DomainRouterHealthCheckResultsResponse healthCheckResponse = new DomainRouterHealthCheckResultsResponse("healthData"); + healthCheckResponse.setRouterId(router.getUuid()); + healthCheckResponse.setName(router.getInstanceName()); + healthCheckResponse.setResult(healthCheckResults.get("success")); + healthCheckResponse.setDetails(healthCheckResults.get("message")); + return healthCheckResponse; + } } diff --git a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java index 1ba083f88aca..7032c8df99b0 100644 --- a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java +++ b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java @@ -31,9 +31,6 @@ import javax.inject.Inject; -import com.cloud.agent.api.storage.OVFProperty; -import com.cloud.storage.TemplateOVFPropertyVO; -import com.cloud.storage.dao.TemplateOVFPropertiesDao; import org.apache.cloudstack.acl.ControlledEntity.ACLType; import org.apache.cloudstack.affinity.AffinityGroupDomainMapVO; import org.apache.cloudstack.affinity.AffinityGroupResponse; @@ -42,6 +39,7 @@ import org.apache.cloudstack.affinity.dao.AffinityGroupVMMapDao; import org.apache.cloudstack.api.BaseListProjectAndAccountResourcesCmd; import org.apache.cloudstack.api.ResourceDetail; +import org.apache.cloudstack.api.ResponseGenerator; import org.apache.cloudstack.api.ResponseObject.ResponseView; import org.apache.cloudstack.api.command.admin.account.ListAccountsCmdByAdmin; import org.apache.cloudstack.api.command.admin.domain.ListDomainsCmd; @@ -123,6 +121,7 @@ import org.apache.log4j.Logger; import org.springframework.stereotype.Component; +import com.cloud.agent.api.storage.OVFProperty; import com.cloud.api.query.dao.AccountJoinDao; import com.cloud.api.query.dao.AffinityGroupJoinDao; import com.cloud.api.query.dao.AsyncJobJoinDao; @@ -182,6 +181,7 @@ import com.cloud.ha.HighAvailabilityManager; import com.cloud.hypervisor.Hypervisor; import com.cloud.hypervisor.Hypervisor.HypervisorType; +import com.cloud.network.VpcVirtualNetworkApplianceService; import com.cloud.network.security.SecurityGroupVMMapVO; import com.cloud.network.security.dao.SecurityGroupVMMapDao; import com.cloud.org.Grouping; @@ -206,9 +206,11 @@ import com.cloud.storage.Storage.ImageFormat; import com.cloud.storage.Storage.TemplateType; import com.cloud.storage.StoragePoolTagVO; +import com.cloud.storage.TemplateOVFPropertyVO; import com.cloud.storage.VMTemplateVO; import com.cloud.storage.Volume; import com.cloud.storage.dao.StoragePoolTagsDao; +import com.cloud.storage.dao.TemplateOVFPropertiesDao; import com.cloud.storage.dao.VMTemplateDao; import com.cloud.tags.ResourceTagVO; import com.cloud.tags.dao.ResourceTagDao; @@ -395,6 +397,12 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q @Inject TemplateOVFPropertiesDao templateOVFPropertiesDao; + @Inject + public VpcVirtualNetworkApplianceService routerService; + + @Inject + private ResponseGenerator responseGenerator; + /* * (non-Javadoc) * @@ -1160,6 +1168,12 @@ public ListResponse searchForRouters(ListRoutersCmd cmd) { ListResponse response = new ListResponse(); List routerResponses = ViewResponseHelper.createDomainRouterResponse(result.first().toArray(new DomainRouterJoinVO[result.first().size()])); + if (cmd.shouldIncludeHealthCheckResults()) { + for (DomainRouterResponse res : routerResponses) { + DomainRouterVO resRouter = _routerDao.findByUuid(res.getId()); + res.setHealthCheckResults(responseGenerator.createHealthCheckResponse(resRouter, routerService.getRouterHealthCheckResults(resRouter.getId(), false))); + } + } response.setResponses(routerResponses, result.second()); return response; } @@ -1171,6 +1185,12 @@ public ListResponse searchForInternalLbVms(ListInternalLBV ListResponse response = new ListResponse(); List routerResponses = ViewResponseHelper.createDomainRouterResponse(result.first().toArray(new DomainRouterJoinVO[result.first().size()])); + if (cmd.shouldIncludeHealthCheckResults()) { + for (DomainRouterResponse res : routerResponses) { + DomainRouterVO resRouter = _routerDao.findByUuid(res.getId()); + res.setHealthCheckResults(responseGenerator.createHealthCheckResponse(resRouter, routerService.getRouterHealthCheckResults(resRouter.getId(), false))); + } + } response.setResponses(routerResponses, result.second()); return response; } diff --git a/server/src/main/java/com/cloud/network/router/VirtualNetworkApplianceManager.java b/server/src/main/java/com/cloud/network/router/VirtualNetworkApplianceManager.java index a291b3590b25..3527ebcb793d 100644 --- a/server/src/main/java/com/cloud/network/router/VirtualNetworkApplianceManager.java +++ b/server/src/main/java/com/cloud/network/router/VirtualNetworkApplianceManager.java @@ -45,6 +45,15 @@ public interface VirtualNetworkApplianceManager extends Manager, VirtualNetworkA static final String SetServiceMonitorCK = "network.router.EnableServiceMonitoring"; static final String RouterAlertsCheckIntervalCK = "router.alerts.check.interval"; + static final String RouterHealthChecksEnabledCK = "router.health.checks.enabled"; + static final String RouterHealthChecksBasicIntervalCK = "router.health.checks.basic.interval"; + static final String RouterHealthChecksAdvancedIntervalCK = "router.health.checks.advanced.interval"; + static final String RouterHealthChecksDataRefreshIntervalCK = "router.health.checks.data.refresh.interval"; + static final String RouterHealthChecksResultFetchIntervalCK = "router.health.checks.results.fetch.interval"; + static final String RouterHealthChecksFailuresToRestartVrCK = "router.health.checks.failures.to.restart.vr"; + static final String RouterHealthChecksToExcludeCK = "router.health.checks.to.exclude"; + static final String RouterHealthChecksFreeDiskSpaceThresholdCK = "router.health.checks.free.disk.space.threshold"; + static final ConfigKey RouterTemplateXen = new ConfigKey(String.class, RouterTemplateXenCK, "Advanced", "SystemVM Template (XenServer)", "Name of the default router template on Xenserver.", true, ConfigKey.Scope.Zone, null); static final ConfigKey RouterTemplateKvm = new ConfigKey(String.class, RouterTemplateKvmCK, "Advanced", "SystemVM Template (KVM)", @@ -68,6 +77,31 @@ public interface VirtualNetworkApplianceManager extends Manager, VirtualNetworkA static final ConfigKey UseExternalDnsServers = new ConfigKey(Boolean.class, "use.external.dns", "Advanced", "false", "Bypass internal dns, use external dns1 and dns2", true, ConfigKey.Scope.Zone, null); + // Health checks + static final ConfigKey RouterHealthChecksEnabled = new ConfigKey(Boolean.class, RouterHealthChecksEnabledCK, "Advanced", "true", + "If true, router health checks are performed periodically as per other configurations", + true, ConfigKey.Scope.Zone, null); + static final ConfigKey RouterHealthChecksBasicInterval = new ConfigKey(Integer.class, RouterHealthChecksBasicIntervalCK, "Advanced", "3", + "Intervals (in minutes) at which basic router health checks are performed in minutes. Checks are disabled if set to zero.", + true, ConfigKey.Scope.Global, null); + static final ConfigKey RouterHealthChecksAdvancedInterval = new ConfigKey(Integer.class, RouterHealthChecksAdvancedIntervalCK, "Advanced", "15", + "Intervals (in minutes) at which advanced router health checks are performed in minutes. Checks are disabled if set to zero.", + true, ConfigKey.Scope.Global, null); + static final ConfigKey RouterHealthChecksDataRefreshInterval = new ConfigKey(Integer.class, RouterHealthChecksDataRefreshIntervalCK, "Advanced", "10", + "Intervals (in minutes) at which router health checks data - such as scheduling interval, excluded checks, etc is updated on all VRs.", + true, ConfigKey.Scope.Global, null); + static final ConfigKey RouterHealthChecksResultFetchInterval = new ConfigKey(Integer.class, RouterHealthChecksResultFetchIntervalCK, "Advanced", "10", + "Intervals (in minutes) at which router health checks results are fetched in minutes. On each check management server evaluates need to restart as per " + RouterHealthChecksFailuresToRestartVrCK, + true, ConfigKey.Scope.Global, null); + static final ConfigKey RouterHealthChecksFailuresToRestartVr = new ConfigKey(String.class, RouterHealthChecksFailuresToRestartVrCK, "Advanced", "", + "Health checks failures that should cause router to restart. If empty the restart never happens. Put 'any' to restart on any failure", + true, ConfigKey.Scope.Zone, null); + static final ConfigKey RouterHealthChecksToExclude = new ConfigKey(String.class, RouterHealthChecksToExcludeCK, "Advanced", "", + "Health checks that should be excluded when executing scheduled checks", true, ConfigKey.Scope.Cluster, null); + static final ConfigKey RouterHealthChecksFreeDiskSpaceThreshold = new ConfigKey(Integer.class, RouterHealthChecksFreeDiskSpaceThresholdCK, + "Advanced", "100", "Free disk space in MB threshold on VR below which the VR needs to be restarted as is considered a failure", + true, ConfigKey.Scope.Zone, null); + public static final int DEFAULT_ROUTER_VM_RAMSIZE = 256; // 256M public static final int DEFAULT_ROUTER_CPU_MHZ = 500; // 500 MHz public static final boolean USE_POD_VLAN = false; diff --git a/server/src/main/java/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java b/server/src/main/java/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java index c8ea47b31db1..5c72f1b9fe3b 100644 --- a/server/src/main/java/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java +++ b/server/src/main/java/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java @@ -25,6 +25,7 @@ import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Calendar; +import java.util.Collections; import java.util.Date; import java.util.HashMap; import java.util.HashSet; @@ -42,6 +43,9 @@ import javax.inject.Inject; import javax.naming.ConfigurationException; +import org.apache.cloudstack.lb.ApplicationLoadBalancerRuleVO; +import org.apache.cloudstack.lb.dao.ApplicationLoadBalancerRuleDao; +import org.apache.commons.lang.StringUtils; import org.apache.log4j.Logger; import org.cloud.network.router.deployment.RouterDeploymentDefinitionBuilder; import org.springframework.beans.factory.annotation.Autowired; @@ -87,6 +91,9 @@ import com.cloud.agent.api.routing.AggregationControlCommand; import com.cloud.agent.api.routing.AggregationControlCommand.Action; import com.cloud.agent.api.routing.GetRouterAlertsCommand; +import com.cloud.agent.api.routing.GetRouterMonitorResultsAnswer; +import com.cloud.agent.api.routing.GetRouterMonitorResultsCommand; +import com.cloud.agent.api.routing.GroupAnswer; import com.cloud.agent.api.routing.IpAliasTO; import com.cloud.agent.api.routing.NetworkElementCommand; import com.cloud.agent.api.routing.SetMonitorServiceCommand; @@ -95,6 +102,10 @@ import com.cloud.alert.AlertManager; import com.cloud.api.ApiAsyncJobDispatcher; import com.cloud.api.ApiGsonHelper; +import com.cloud.api.query.dao.DomainRouterJoinDao; +import com.cloud.api.query.dao.UserVmJoinDao; +import com.cloud.api.query.vo.DomainRouterJoinVO; +import com.cloud.api.query.vo.UserVmJoinVO; import com.cloud.cluster.ManagementServerHostVO; import com.cloud.cluster.dao.ManagementServerHostDao; import com.cloud.configuration.Config; @@ -146,6 +157,7 @@ import com.cloud.network.dao.IPAddressVO; import com.cloud.network.dao.LoadBalancerDao; import com.cloud.network.dao.LoadBalancerVMMapDao; +import com.cloud.network.dao.LoadBalancerVMMapVO; import com.cloud.network.dao.LoadBalancerVO; import com.cloud.network.dao.MonitoringServiceDao; import com.cloud.network.dao.MonitoringServiceVO; @@ -175,6 +187,7 @@ import com.cloud.network.rules.FirewallRuleVO; import com.cloud.network.rules.LoadBalancerContainer.Scheme; import com.cloud.network.rules.PortForwardingRule; +import com.cloud.network.rules.PortForwardingRuleVO; import com.cloud.network.rules.RulesManager; import com.cloud.network.rules.StaticNat; import com.cloud.network.rules.StaticNatImpl; @@ -214,6 +227,7 @@ import com.cloud.utils.db.Filter; import com.cloud.utils.db.GlobalLock; import com.cloud.utils.db.QueryBuilder; +import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; import com.cloud.utils.db.Transaction; import com.cloud.utils.db.TransactionCallbackNoReturn; @@ -311,6 +325,11 @@ public class VirtualNetworkApplianceManagerImpl extends ManagerBase implements V @Inject protected NetworkTopologyContext _networkTopologyContext; + @Inject private UserVmJoinDao userVmJoinDao; + @Inject private DomainRouterJoinDao domainRouterJoinDao; + @Inject private PortForwardingRulesDao portForwardingDao; + @Inject private ApplicationLoadBalancerRuleDao applicationLoadBalancerRuleDao; + @Autowired @Qualifier("networkHelper") protected NetworkHelper _nwHelper; @@ -658,7 +677,21 @@ public boolean start() { if (routerAlertsCheckInterval > 0) { _checkExecutor.scheduleAtFixedRate(new CheckRouterAlertsTask(), routerAlertsCheckInterval, routerAlertsCheckInterval, TimeUnit.SECONDS); } else { - s_logger.debug("router.alerts.check.interval - " + routerAlertsCheckInterval + " so not scheduling the router alerts checking thread"); + s_logger.debug(RouterAlertsCheckIntervalCK + "=" + routerAlertsCheckInterval + " so not scheduling the router alerts checking thread"); + } + + final int routerHealthCheckDataRefreshInterval = RouterHealthChecksDataRefreshInterval.value(); + if (routerHealthCheckDataRefreshInterval > 0) { + _checkExecutor.scheduleAtFixedRate(new UpdateRouterHealthChecksConfigDataTask(), routerHealthCheckDataRefreshInterval, routerHealthCheckDataRefreshInterval, TimeUnit.MINUTES); + } else { + s_logger.debug(RouterHealthChecksDataRefreshIntervalCK + "=" + routerAlertsCheckInterval + " so not scheduling the router health check data thread"); + } + + final int routerHealthChecksFetchInterval = RouterHealthChecksResultFetchInterval.value(); + if (routerHealthChecksFetchInterval > 0) { + _checkExecutor.scheduleAtFixedRate(new AnalyseRouterMonitorResultsTask(), routerHealthChecksFetchInterval, routerHealthChecksFetchInterval, TimeUnit.MINUTES); + } else { + s_logger.debug(RouterHealthChecksResultFetchIntervalCK + "=" + routerAlertsCheckInterval + " so not scheduling the router checks fetching thread"); } return true; @@ -1186,6 +1219,259 @@ protected void pushToUpdateQueue(final List networks) throws Interrup } } + protected class AnalyseRouterMonitorResultsTask extends ManagedContextRunnable { + public AnalyseRouterMonitorResultsTask() { + } + + @Override + protected void runInContext() { + try { + final List routers = _routerDao.listByStateAndManagementServer(VirtualMachine.State.Running, mgmtSrvrId); + s_logger.debug("Found " + routers.size() + " running routers. "); + + for (final DomainRouterVO router : routers) { + GetRouterMonitorResultsAnswer answer = getMonitorResults(router, false); + String checkFailsToRestartVr = RouterHealthChecksFailuresToRestartVr.valueIn(router.getDataCenterId()); + if (answer != null && answer.getFailingChecks().size() > 0 && StringUtils.isNotBlank(checkFailsToRestartVr)) { + for (String failedCheck : answer.getFailingChecks()) { + if (checkFailsToRestartVr.contains(failedCheck)) { + rebootRouter(router.getId(), true); + } + } + } + } + } catch (final Exception ex) { + s_logger.error("Fail to complete the AnalyseRouterMonitorResultsTask! ", ex); + } + } + } + + // Returns null if health checks are not enabled + private GetRouterMonitorResultsAnswer getMonitorResults(DomainRouterVO router, boolean performFreshChecks) { + if (!RouterHealthChecksEnabled.valueIn(router.getDataCenterId())) { + return null; + } + + String controlIP = getRouterControlIP(router); + if (StringUtils.isNotBlank(controlIP) && !controlIP.equals("0.0.0.0")) { + final GetRouterMonitorResultsCommand command = new GetRouterMonitorResultsCommand(performFreshChecks); + command.setAccessDetail(NetworkElementCommand.ROUTER_IP, controlIP); + command.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); + try { + final Answer answer = _agentMgr.easySend(router.getHostId(), command); + + if (answer == null) { + s_logger.warn("Unable to fetch monitoring results data from router " + router.getHostName()); + return null; + } + if (answer instanceof GetRouterMonitorResultsAnswer) { + return (GetRouterMonitorResultsAnswer) answer; + } else { + s_logger.warn("Unable to fetch health checks results to router " + router.getHostName() + " Received answer " + answer.getDetails()); + return new GetRouterMonitorResultsAnswer(command, false, null, answer.getDetails()); + } + } catch (final Exception e) { + s_logger.warn("Error while collecting alerts from router: " + router.getInstanceName(), e); + return null; + } + } + + return null; + } + + @Override + public Map getRouterHealthCheckResults(long routerId, boolean runChecks) { + DomainRouterVO router = _routerDao.findById(routerId); + Map result = new HashMap<>(); + + if (router == null) { + result.put("success", "False"); + result.put("message", "Router not found"); + return result; + } + + if (!RouterHealthChecksEnabled.valueIn(router.getDataCenterId())) { + result.put("success", "False"); + result.put("message", "Router id not valid. Either router not found or it contains VPC."); + return result; + } + + GetRouterMonitorResultsAnswer answer = getMonitorResults(router, runChecks); + if (answer == null) { + result.put("success", "False"); + result.put("message", "Router is unreachable."); + return result; + } + + result.put("success", String.valueOf(answer.getResult())); + result.put("message", answer.getDetails()); + + return result; + } + + protected class UpdateRouterHealthChecksConfigDataTask extends ManagedContextRunnable { + public UpdateRouterHealthChecksConfigDataTask() { + } + + @Override + protected void runInContext() { + try { + final List routers = _routerDao.listByStateAndManagementServer(VirtualMachine.State.Running, mgmtSrvrId); + s_logger.debug("Found " + routers.size() + " running routers. "); + + for (final DomainRouterVO router : routers) { + if (!RouterHealthChecksEnabled.valueIn(router.getDataCenterId())) { + continue; + } + + String controlIP = getRouterControlIP(router); + if (StringUtils.isNotBlank(controlIP) && !controlIP.equals("0.0.0.0")) { + + final SetMonitorServiceCommand command = new SetMonitorServiceCommand(); + command.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIP(router)); + command.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); + command.setAccessDetail(SetMonitorServiceCommand.ROUTER_HEALTH_CHECKS_ENABLED, RouterHealthChecksEnabled.valueIn(router.getDataCenterId()).toString()); + command.setAccessDetail(SetMonitorServiceCommand.ROUTER_HEALTH_CHECKS_BASIC_INTERVAL, RouterHealthChecksBasicInterval.value().toString()); + command.setAccessDetail(SetMonitorServiceCommand.ROUTER_HEALTH_CHECKS_ADVANCED_INTERVAL, RouterHealthChecksAdvancedInterval.value().toString()); + command.setAccessDetail(SetMonitorServiceCommand.ROUTER_HEALTH_CHECKS_EXCLUDED, RouterHealthChecksToExclude.valueIn(router.getDataCenterId())); + command.setAdditionalData(getAdditionalDataForRouterHealthChecks(router)); + command.setReconfigureAfterUpdate(true); + + try { + final Answer origAnswer = _agentMgr.easySend(router.getHostId(), command); + GroupAnswer answer = null; + + if (origAnswer == null) { + s_logger.warn("Unable to update health checks data to router " + router.getHostName()); + continue; + } + if (origAnswer instanceof GroupAnswer) { + answer = (GroupAnswer) origAnswer; + } else { + s_logger.warn("Unable to update health checks data to router " + router.getHostName() + " Received answer " + origAnswer.getDetails()); + continue; + } + if (!answer.getResult()) { + s_logger.warn("Unable to update health checks data to router " + router.getHostName() + ", details : " + answer.getDetails()); + continue; + } + } catch (final Exception e) { + s_logger.warn("Error while collecting alerts from router: " + router.getInstanceName(), e); + continue; + } + } + } + } catch (final Exception ex) { + s_logger.error("Fail to complete the UpdateRouterHealthChecksConfigDataTask! ", ex); + } + } + } + + private Map getAdditionalDataForRouterHealthChecks(final DomainRouterVO router) { + s_logger.info("Updating data for router health checks for all routers"); + Map data = new HashMap<>(); + List routerJoinVOs = domainRouterJoinDao.searchByIds(router.getId()); + StringBuilder vmsData = new StringBuilder(); + StringBuilder portData = new StringBuilder(); + StringBuilder loadBalancingData = new StringBuilder(); + StringBuilder gateways = new StringBuilder(); + gateways.append("gatewaysIps="); + for (DomainRouterJoinVO routerJoinVO : routerJoinVOs) { + if (StringUtils.isNotBlank(routerJoinVO.getGateway())) { + gateways.append(routerJoinVO.getGateway() + " "); + } + SearchBuilder sbvm = userVmJoinDao.createSearchBuilder(); + sbvm.and("networkId", sbvm.entity().getNetworkId(), SearchCriteria.Op.EQ); + SearchCriteria scvm = sbvm.create(); + scvm.setParameters("networkId", routerJoinVO.getNetworkId()); + List vms = userVmJoinDao.search(scvm, null); + for (UserVmJoinVO vm : vms) { + if (vm.getState() != VirtualMachine.State.Running) { + continue; + } + + vmsData.append("vmName=").append(vm.getName()) + .append(",macAddress=").append(vm.getMacAddress()) + .append(",ip=").append(vm.getIpAddress()).append(";"); + SearchBuilder sbpf = portForwardingDao.createSearchBuilder(); + sbpf.and("networkId", sbpf.entity().getNetworkId(), SearchCriteria.Op.EQ); + sbpf.and("instanceId", sbpf.entity().getVirtualMachineId(), SearchCriteria.Op.EQ); + SearchCriteria scpf = sbpf.create(); + scpf.setParameters("networkId", routerJoinVO.getNetworkId()); + scpf.setParameters("instanceId", vm.getId()); + List portForwardingRules = portForwardingDao.search(scpf, null); + for (PortForwardingRuleVO portForwardingRule : portForwardingRules) { + portData.append("sourceIp=").append(_ipAddressDao.findById(portForwardingRule.getSourceIpAddressId()).getAddress().toString()) + .append(",sourcePortStart=").append(portForwardingRule.getSourcePortStart()) + .append(",sourcePortEnd=").append(portForwardingRule.getSourcePortEnd()) + .append(",destIp=").append(portForwardingRule.getDestinationIpAddress()) + .append(",destPortStart=").append(portForwardingRule.getDestinationPortStart()) + .append(",destPortEnd=").append(portForwardingRule.getDestinationPortEnd()).append(";"); + } + } + + List loadBalancerVOs = this.getLBRules(routerJoinVO); + for (FirewallRuleVO firewallRuleVO : loadBalancerVOs) { + List vmMapVOs = _loadBalancerVMMapDao.listByLoadBalancerId(firewallRuleVO.getId(), false); + if (vmMapVOs.size() > 0) { + + final NetworkOffering offering = _networkOfferingDao.findById(_networkDao.findById(routerJoinVO.getNetworkId()).getNetworkOfferingId()); + if (offering.getConcurrentConnections() == null) { + loadBalancingData.append("maxconn=").append(_configDao.getValue(Config.NetworkLBHaproxyMaxConn.key())); + } else { + loadBalancingData.append("maxconn=").append(offering.getConcurrentConnections().toString()); + } + + loadBalancingData.append(",sourcePortStart=").append(firewallRuleVO.getSourcePortStart()) + .append(",sourcePortEnd=").append(firewallRuleVO.getSourcePortEnd()); + if (firewallRuleVO instanceof LoadBalancerVO) { + LoadBalancerVO loadBalancerVO = (LoadBalancerVO) firewallRuleVO; + loadBalancingData.append(",sourceIp=").append(_ipAddressDao.findById(loadBalancerVO.getSourceIpAddressId()).getAddress().toString()) + .append(",destPortStart=").append(loadBalancerVO.getDefaultPortStart()) + .append(",destPortEnd=").append(loadBalancerVO.getDefaultPortEnd()) + .append(",algorithm=").append(loadBalancerVO.getAlgorithm()).append(",vmIps="); + } else if (firewallRuleVO instanceof ApplicationLoadBalancerRuleVO) { + ApplicationLoadBalancerRuleVO appLoadBalancerVO = (ApplicationLoadBalancerRuleVO) firewallRuleVO; + loadBalancingData.append(",sourceIp=").append(appLoadBalancerVO.getSourceIp()) + .append(",destPortStart=").append(appLoadBalancerVO.getDefaultPortStart()) + .append(",destPortEnd=").append(appLoadBalancerVO.getDefaultPortEnd()) + .append(",algorithm=").append(appLoadBalancerVO.getAlgorithm()).append(",vmIps="); + + } + for (LoadBalancerVMMapVO vmMapVO : vmMapVOs) { + loadBalancingData.append(vmMapVO.getInstanceIp()).append(" "); + } + loadBalancingData.setCharAt(loadBalancingData.length() - 1, ';'); + } + } + } + data.put("virtualMachines", vmsData.toString()); + data.put("gateways", gateways.toString()); + data.put("portForwarding", portData.toString()); + data.put("haproxyData", loadBalancingData.toString()); + data.put("systemThresholds", "minSpaceNeeded=" + RouterHealthChecksFreeDiskSpaceThreshold.valueIn(router.getDataCenterId()).toString()); + return data; + } + + private List getLBRules(final DomainRouterJoinVO router) { + if (router.getRole() == Role.VIRTUAL_ROUTER) { + SearchBuilder sblb = _loadBalancerDao.createSearchBuilder(); + sblb.and("networkId", sblb.entity().getNetworkId(), SearchCriteria.Op.EQ); + sblb.and("sourceIpAddressId", sblb.entity().getSourceIpAddressId(), SearchCriteria.Op.NNULL); + SearchCriteria sclb = sblb.create(); + sclb.setParameters("networkId", router.getNetworkId()); + return _loadBalancerDao.search(sclb, null); + } else if (router.getRole() == Role.INTERNAL_LB_VM) { + SearchBuilder sbalb = applicationLoadBalancerRuleDao.createSearchBuilder(); + sbalb.and("networkId", sbalb.entity().getNetworkId(), SearchCriteria.Op.EQ); + sbalb.and("sourceIpAddress", sbalb.entity().getSourceIp(), SearchCriteria.Op.NNULL); + SearchCriteria sclb = sbalb.create(); + sclb.setParameters("networkId", router.getNetworkId()); + return applicationLoadBalancerRuleDao.search(sclb, null); + } + return Collections.emptyList(); + } + protected class CheckRouterAlertsTask extends ManagedContextRunnable { public CheckRouterAlertsTask() { } @@ -1205,7 +1491,6 @@ protected void getRouterAlerts() { final List routers = _routerDao.listByStateAndManagementServer(VirtualMachine.State.Running, mgmtSrvrId); s_logger.debug("Found " + routers.size() + " running routers. "); - for (final DomainRouterVO router : routers) { final String serviceMonitoringFlag = SetServiceMonitor.valueIn(router.getDataCenterId()); // Skip the routers in VPC network or skip the routers where @@ -1253,7 +1538,7 @@ protected void getRouterAlerts() { final String alerts[] = answer.getAlerts(); if (alerts != null) { final String lastAlertTimeStamp = answer.getTimeStamp(); - final SimpleDateFormat sdfrmt = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss"); + final SimpleDateFormat sdfrmt = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); sdfrmt.setLenient(false); try { sdfrmt.parse(lastAlertTimeStamp); @@ -1674,9 +1959,9 @@ public boolean finalizeCommandsOnStart(final Commands cmds, final VirtualMachine final String serviceMonitringSet = _configDao.getValue(Config.EnableServiceMonitoring.key()); if (serviceMonitringSet != null && serviceMonitringSet.equalsIgnoreCase("true")) { - finalizeMonitorServiceOnStrat(cmds, profile, router, provider, guestNetworkId, true); + finalizeMonitorServiceOnStart(cmds, profile, router, provider, guestNetworkId, true); } else { - finalizeMonitorServiceOnStrat(cmds, profile, router, provider, guestNetworkId, false); + finalizeMonitorServiceOnStart(cmds, profile, router, provider, guestNetworkId, false); } } @@ -1692,8 +1977,8 @@ public boolean finalizeCommandsOnStart(final Commands cmds, final VirtualMachine return true; } - private void finalizeMonitorServiceOnStrat(final Commands cmds, final VirtualMachineProfile profile, final DomainRouterVO router, final Provider provider, - final long networkId, final Boolean add) { + private void finalizeMonitorServiceOnStart(final Commands cmds, final VirtualMachineProfile profile, final DomainRouterVO router, final Provider provider, + final long networkId, final Boolean add) { final NetworkVO network = _networkDao.findById(networkId); @@ -1734,14 +2019,23 @@ private void finalizeMonitorServiceOnStrat(final Commands cmds, final VirtualMac if (controlNic == null) { throw new CloudRuntimeException("VirtualMachine " + profile.getInstanceName() + " doesn't have a control interface"); } + final SetMonitorServiceCommand command = new SetMonitorServiceCommand(servicesTO); command.setAccessDetail(NetworkElementCommand.ROUTER_IP, controlNic.getIPv4Address()); command.setAccessDetail(NetworkElementCommand.ROUTER_GUEST_IP, _routerControlHelper.getRouterIpInNetwork(networkId, router.getId())); command.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); if (!add) { - command.setAccessDetail(NetworkElementCommand.ROUTER_MONITORING_ENABLE, add.toString()); + command.setAccessDetail(SetMonitorServiceCommand.ROUTER_MONITORING_ENABLED, add.toString()); } + + command.setAccessDetail(SetMonitorServiceCommand.ROUTER_HEALTH_CHECKS_ENABLED, RouterHealthChecksEnabled.valueIn(router.getDataCenterId()).toString()); + command.setAccessDetail(SetMonitorServiceCommand.ROUTER_HEALTH_CHECKS_BASIC_INTERVAL, RouterHealthChecksBasicInterval.value().toString()); + command.setAccessDetail(SetMonitorServiceCommand.ROUTER_HEALTH_CHECKS_ADVANCED_INTERVAL, RouterHealthChecksAdvancedInterval.value().toString()); + command.setAccessDetail(SetMonitorServiceCommand.ROUTER_HEALTH_CHECKS_EXCLUDED, RouterHealthChecksToExclude.valueIn(router.getDataCenterId())); + command.setAdditionalData(getAdditionalDataForRouterHealthChecks(router)); + command.setReconfigureAfterUpdate(false); // As part of aggregate command we don't need to reconfigure + cmds.addCommand("monitor", command); } @@ -2599,7 +2893,20 @@ public String getConfigComponentName() { @Override public ConfigKey[] getConfigKeys() { - return new ConfigKey[] { UseExternalDnsServers, routerVersionCheckEnabled, SetServiceMonitor, RouterAlertsCheckInterval }; + return new ConfigKey[] { + UseExternalDnsServers, + routerVersionCheckEnabled, + SetServiceMonitor, + RouterAlertsCheckInterval, + RouterHealthChecksEnabled, + RouterHealthChecksBasicInterval, + RouterHealthChecksAdvancedInterval, + RouterHealthChecksDataRefreshInterval, + RouterHealthChecksResultFetchInterval, + RouterHealthChecksFailuresToRestartVr, + RouterHealthChecksToExclude, + RouterHealthChecksFreeDiskSpaceThreshold + }; } @Override diff --git a/server/src/main/java/com/cloud/server/ManagementServerImpl.java b/server/src/main/java/com/cloud/server/ManagementServerImpl.java index 147c527b2739..01fe2eb88336 100644 --- a/server/src/main/java/com/cloud/server/ManagementServerImpl.java +++ b/server/src/main/java/com/cloud/server/ManagementServerImpl.java @@ -171,6 +171,7 @@ import org.apache.cloudstack.api.command.admin.router.ConfigureVirtualRouterElementCmd; import org.apache.cloudstack.api.command.admin.router.CreateVirtualRouterElementCmd; import org.apache.cloudstack.api.command.admin.router.DestroyRouterCmd; +import org.apache.cloudstack.api.command.admin.router.GetRouterHealthCheckResultsCmd; import org.apache.cloudstack.api.command.admin.router.ListOvsElementsCmd; import org.apache.cloudstack.api.command.admin.router.ListRoutersCmd; import org.apache.cloudstack.api.command.admin.router.ListVirtualRouterElementsCmd; @@ -3115,6 +3116,7 @@ public List> getCommands() { cmdList.add(ListMgmtsCmd.class); cmdList.add(GetUploadParamsForIsoCmd.class); cmdList.add(ListTemplateOVFProperties.class); + cmdList.add(GetRouterHealthCheckResultsCmd.class); // Out-of-band management APIs for admins cmdList.add(EnableOutOfBandManagementForHostCmd.class); diff --git a/server/src/test/java/com/cloud/vpc/MockVpcVirtualNetworkApplianceManager.java b/server/src/test/java/com/cloud/vpc/MockVpcVirtualNetworkApplianceManager.java index a85d039cd13f..62f2f2327a18 100644 --- a/server/src/test/java/com/cloud/vpc/MockVpcVirtualNetworkApplianceManager.java +++ b/server/src/test/java/com/cloud/vpc/MockVpcVirtualNetworkApplianceManager.java @@ -248,6 +248,11 @@ public List upgradeRouterTemplate(final UpgradeRouterTemplateCmd cmd) { return null; //To change body of implemented methods use File | Settings | File Templates. } + @Override + public Map getRouterHealthCheckResults(long routerId, boolean runChecks) { + return null; + } + @Override public boolean prepareAggregatedExecution(final Network network, final List routers) throws AgentUnavailableException { return true; //To change body of implemented methods use File | Settings | File Templates. diff --git a/systemvm/debian/opt/cloud/bin/cs/CsMonitor.py b/systemvm/debian/opt/cloud/bin/cs/CsMonitor.py index 6b194238b1ac..55ff11c6a099 100755 --- a/systemvm/debian/opt/cloud/bin/cs/CsMonitor.py +++ b/systemvm/debian/opt/cloud/bin/cs/CsMonitor.py @@ -17,27 +17,55 @@ import logging from cs.CsDatabag import CsDataBag from CsFile import CsFile +import json MON_CONFIG = "/etc/monitor.conf" +HC_CONFIG = "/root/health_checks_data.json" class CsMonitor(CsDataBag): """ Manage dhcp entries """ def process(self): - if "config" not in self.dbag: - return - procs = [x.strip() for x in self.dbag['config'].split(',')] - file = CsFile(MON_CONFIG) - for proc in procs: - bits = [x for x in proc.split(':')] - if len(bits) < 5: - continue - for i in range(0, 4): - file.add(bits[i], -1) - file.commit() + if "config" in self.dbag: + procs = [x.strip() for x in self.dbag['config'].split(',')] + file = CsFile(MON_CONFIG) + for proc in procs: + bits = [x for x in proc.split(':')] + if len(bits) < 5: + continue + for i in range(0, 4): + file.add(bits[i], -1) + file.commit() + + hc_data = {} + + cron_rep_basic = self.dbag["health_checks_basic_run_interval"] if "health_checks_basic_run_interval" in self.dbag else 3 + cron_rep_advance = self.dbag["health_checks_advance_run_interval"] if "health_checks_advance_run_interval" in self.dbag else 0 + hc_data["health_checks_basic_run_interval"] = cron_rep_basic + hc_data["health_checks_advance_run_interval"] = cron_rep_advance + + hc_data["health_checks_enabled"] = self.dbag["health_checks_enabled"] if "health_checks_enabled" in self.dbag else False + + if "excluded_health_checks" in self.dbag: + excluded_checks = self.dbag["excluded_health_checks"] + hc_data["excluded_health_checks"] = [ch.strip() for ch in excluded_checks.split(",")] if len(excluded_checks) > 0 else [] + else: + hc_data["excluded_health_checks"] = [] + if "additional_data" in self.dbag: + hc_data["additional_data"] = self.dbag["additional_data"] + else: + hc_data["additional_data"] = {} + cron = CsFile("/etc/cron.d/process") + cron.deleteLine("root /usr/bin/python /root/monitorServices.py") cron.add("SHELL=/bin/bash", 0) cron.add("PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin", 1) - cron.add("*/3 * * * * root /usr/bin/python /root/monitorServices.py", -1) + if cron_rep_basic > 0: + cron.add("*/" + str(cron_rep_basic) + " * * * * root /usr/bin/python /root/monitorServices.py basic", -1) + if cron_rep_advance > 0: + cron.add("*/" + str(cron_rep_advance) + " * * * * root /usr/bin/python /root/monitorServices.py advance", -1) cron.commit() + + with open(HC_CONFIG, 'w') as f: + json.dump(hc_data, f, ensure_ascii=False, indent=4) diff --git a/systemvm/debian/opt/cloud/bin/cs_monitorservice.py b/systemvm/debian/opt/cloud/bin/cs_monitorservice.py index 75a7c95d6fac..cf2dcc3bbeff 100755 --- a/systemvm/debian/opt/cloud/bin/cs_monitorservice.py +++ b/systemvm/debian/opt/cloud/bin/cs_monitorservice.py @@ -22,4 +22,15 @@ def merge(dbag, data): if "config" in data: dbag['config'] = data["config"] + if "health_checks_enabled" in data: + dbag["health_checks_enabled"] = data["health_checks_enabled"] + if "health_checks_basic_run_interval" in data: + dbag["health_checks_basic_run_interval"] = data["health_checks_basic_run_interval"] + if "health_checks_advance_run_interval" in data: + dbag["health_checks_advance_run_interval"] = data["health_checks_advance_run_interval"] + if "excluded_health_checks" in data: + dbag["excluded_health_checks"] = data["excluded_health_checks"] + if "additional_data" in data: + dbag["additional_data"] = data["additional_data"] + return dbag diff --git a/systemvm/debian/opt/cloud/bin/getRouterMonitorResults.sh b/systemvm/debian/opt/cloud/bin/getRouterMonitorResults.sh new file mode 100755 index 000000000000..fc9ef2fd13e6 --- /dev/null +++ b/systemvm/debian/opt/cloud/bin/getRouterMonitorResults.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# getRouterMonitorResults.sh --- Send the monitor results to Management Server + +if [ "$1" == "true" ] +then + python /root/monitorServices.py > /dev/null +fi + +if [ -f /root/failing_health_checks ] +then + printf "FAILING CHECKS:\n" + echo `cat /root/failing_health_checks` +fi + +printf "MONITOR RESULTS:\n" +echo "{\"basic\":" +if [ -f /root/basic_monitor_results.json ] +then + echo `cat /root/basic_monitor_results.json` +else + echo "\"Not available yet\"" +fi +echo ", \"advance\":" +if [ -f /root/advance_monitor_results.json ] +then + echo `cat /root/advance_monitor_results.json` +else + echo "\"Not available yet\"" +fi + +echo "}" diff --git a/systemvm/debian/opt/cloud/bin/healthchecksutility.py b/systemvm/debian/opt/cloud/bin/healthchecksutility.py new file mode 100644 index 000000000000..3a105d2a93d4 --- /dev/null +++ b/systemvm/debian/opt/cloud/bin/healthchecksutility.py @@ -0,0 +1,44 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import json + +def getHealthChecksData(additionalDataKey = None): + with open('health_checks_data.json', 'r') as hc_data_file: + hc_data = json.load(hc_data_file) + if additionalDataKey == None: + return hc_data + if additionalDataKey in hc_data["additional_data"]: + data = hc_data["additional_data"][additionalDataKey].strip().split(";") + addData = [] + for line in data: + line = line.strip() + if len(line) == 0: + continue + entries = line.split(',') + d = {} + for entry in entries: + entry = entry.strip() + if len(entry) == 0: + continue + keyVal = entry.split("=") + if len(keyVal) == 2: + d[keyVal[0].strip()] = keyVal[1].strip() + if len(d) > 0: + addData.append(d) + return addData + return None \ No newline at end of file diff --git a/systemvm/debian/root/health_scripts/dhcp_check.py b/systemvm/debian/root/health_scripts/dhcp_check.py new file mode 100755 index 000000000000..a25baf9d828e --- /dev/null +++ b/systemvm/debian/root/health_scripts/dhcp_check.py @@ -0,0 +1,49 @@ +#!/usr/bin/python +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from os import sys, path + +sys.path.append('/opt/cloud/bin') +from healthchecksutility import getHealthChecksData + +def main(): + vMs = getHealthChecksData("virtualMachines") + if vMs != None and len(vMs) > 0: + with open('/etc/dhcphosts.txt', 'r') as hostsFile: + allHosts = hostsFile.readlines() + hostsFile.close() + for vM in vMs: + entry = vM["macAddress"] + "," + vM["ip"] + "," + vM["vmName"] + foundEntry = False + for host in allHosts: + if host.strip().find(entry) == 0: + foundEntry = True + break + + if foundEntry == False: + print "Missing entry in dhcphosts.txt - " + entry + exit(1) + + print "All " + str(len(vMs)) + " VMs are present in dhcphosts.txt" + else: + print "No VMs running data available" + exit(0) + +if __name__ == "__main__": + if len(sys.argv) == 2 and sys.argv[1] == "advance": + main() \ No newline at end of file diff --git a/systemvm/debian/root/health_scripts/disk_space_check.py b/systemvm/debian/root/health_scripts/disk_space_check.py new file mode 100644 index 000000000000..3d6fd028aad8 --- /dev/null +++ b/systemvm/debian/root/health_scripts/disk_space_check.py @@ -0,0 +1,46 @@ +#!/usr/bin/python +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from os import sys, path, statvfs + +sys.path.append('/opt/cloud/bin') +from healthchecksutility import getHealthChecksData + +def main(): + entries = getHealthChecksData("systemThresholds") + data = {} + if len(entries) == 1: + data = entries[0] + + if "minSpaceNeeded" in data: + minSpaceNeeded = int(data["minSpaceNeeded"]) * 1024 + s = statvfs('/') + freeSpace = (s.f_bavail * s.f_frsize) / 1024 + if (freeSpace < minSpaceNeeded): + print "Insufficient free space is " + str(freeSpace/1024) + " MB" + exit(1) + else: + print "Sufficient free space is " + str(freeSpace/1024) + " MB" + exit(0) + else: + print "Missing config in health_checks_data systemThresholds > minSpaceNeeded" + exit(1) + +if __name__ == "__main__": + if len(sys.argv) == 2 and sys.argv[1] == "basic": + main() \ No newline at end of file diff --git a/systemvm/debian/root/health_scripts/dns_check.py b/systemvm/debian/root/health_scripts/dns_check.py new file mode 100644 index 000000000000..626512b23e07 --- /dev/null +++ b/systemvm/debian/root/health_scripts/dns_check.py @@ -0,0 +1,48 @@ +#!/usr/bin/python +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from os import sys, path + +sys.path.append('/opt/cloud/bin') +from healthchecksutility import getHealthChecksData + +def main(): + vMs = getHealthChecksData("virtualMachines") + if vMs != None and len(vMs) > 0: + with open('/etc/hosts', 'r') as hostsFile: + allHosts = hostsFile.readlines() + hostsFile.close() + for vM in vMs: + foundEntry = False + for host in allHosts: + if host.find(vM["ip"]) != -1 and host.find(vM["vmName"]) != -1: + foundEntry = True + break + + if foundEntry == False: + print "Missing entry in /etc/hosts - " + vM["ip"] + " " + vM["vmName"] + exit(1) + + print "All " + str(len(vMs)) + " VMs are present in /etc/hosts" + else: + print "No VMs running data available" + exit(0) + +if __name__ == "__main__": + if len(sys.argv) == 2 and sys.argv[1] == "advance": + main() \ No newline at end of file diff --git a/systemvm/debian/root/health_scripts/gateways_check.py b/systemvm/debian/root/health_scripts/gateways_check.py new file mode 100644 index 000000000000..2faef127a1b4 --- /dev/null +++ b/systemvm/debian/root/health_scripts/gateways_check.py @@ -0,0 +1,56 @@ +#!/usr/bin/python +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from os import sys, path +from subprocess import * + +sys.path.append('/opt/cloud/bin') +from healthchecksutility import getHealthChecksData + +def main(): + gws = getHealthChecksData("gateways") + if gws != None and len(gws) > 0: + unreachableGateWays = [] + gwsList = gws[0]["gatewaysIps"].strip().split(' ') + for gw in gwsList: + if len(gw) == 0: + continue + reachableGw = False + for i in range(3): + pingCmd = "ping " + gw + " -c " + str(i + 1) + pout = Popen(pingCmd, shell=True, stdout=PIPE) + if pout.wait() == 0: + reachableGw = True + break + + if not reachableGw: + unreachableGateWays.append(gw) + + if len(unreachableGateWays) == 0: + print "All " + str(len(gws)) + " gateways are reachable via ping" + exit(0) + else: + print "Unreachable gateways found " + unreachableGateWays + exit(1) + else: + print "No gateways data available" + exit(0) + +if __name__ == "__main__": + if len(sys.argv) == 2 and sys.argv[1] == "basic": + main() \ No newline at end of file diff --git a/systemvm/debian/root/health_scripts/haproxy_check.py b/systemvm/debian/root/health_scripts/haproxy_check.py new file mode 100644 index 000000000000..d582d79bffbc --- /dev/null +++ b/systemvm/debian/root/health_scripts/haproxy_check.py @@ -0,0 +1,112 @@ +#!/usr/bin/python +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from os import sys, path + +sys.path.append('/opt/cloud/bin') +from healthchecksutility import getHealthChecksData + +def checkMaxconn(haproxyData, haCfgSections): + if "maxconn" in haproxyData and "maxconn" in haCfgSections["global"]: + if haproxyData["maxconn"] != haCfgSections["global"]["maxconn"]: + print "global maxconn mismatch occured" + return False + + return True + +def formatPort(portStart, portEnd, delim = "-"): + return portStart if portStart == portEnd else portStart + delim + portEnd + +def checkLoadBalance(haproxyData, haCfgSections): + correct = True + for lbSec in haproxyData: + srcServer = lbSec["sourceIp"].replace('.', '_') + "-" + formatPort(lbSec["sourcePortStart"], lbSec["sourcePortEnd"]) + secName = "listen " + srcServer + + if secName not in haCfgSections: + print "Missing section for load balancing " + secName + "\n" + correct = False + else: + cfgSection = haCfgSections[secName] + if "server" in cfgSection: + if lbSec["algorithm"] != cfgSection["balance"][0]: + print "Incorrect balance method for source " + secName + "Expected : " + lbSec["algorithm"] + " but found " + cfgSection["balance"][0] + "\n" + correct = False + expectedServerIps = lbSec["vmIps"].split(" ") + for expectedServerIp in expectedServerIps: + pattern = expectedServerIp + ":" + formatPort(lbSec["destPortStart"], lbSec["destPortEnd"]) + foundPattern = False + for server in cfgSection["server"]: + if server.find(srcServer) != -1 and server.find(pattern) != -1: + foundPattern = True + break + + if not foundPattern: + correct = False + print "Missing load balancing for " + pattern + ". " + + + return correct + + +def main(): + haproxyData = getHealthChecksData("haproxyData") + if haproxyData == None or len(haproxyData) == 0: + print "No data provided to check" + exit(0) + + with open("/etc/haproxy/haproxy.cfg", 'r') as haCfgFile: + haCfgLines = haCfgFile.readlines() + haCfgFile.close() + + if len(haCfgLines) == 0: + print "Unable to read config file /etc/haproxy/haproxy.cfg" + exit(1) + + haCfgSections = {} + currSection = None + currSectionDict = {} + for line in haCfgLines: + line = line.strip() + if len(line) == 0: + if currSection is not None and len(currSectionDict) > 0: + haCfgSections[currSection] = currSectionDict + + currSection = None + currSectionDict = {} + continue + + if currSection is None: + currSection = line + else: + lineSec = line.split(' ', 1) + if lineSec[0] not in currSectionDict: + currSectionDict[lineSec[0]] = [] + + currSectionDict[lineSec[0]].append(lineSec[1] if len(lineSec) > 1 else '') + + if checkMaxconn(haproxyData, haCfgSections) and checkLoadBalance(haproxyData, haCfgSections): + print "All checks pass" + exit(0) + else: + exit(1) + + +if __name__ == "__main__": + if len(sys.argv) == 2 and sys.argv[1] == "advance": + main() diff --git a/systemvm/debian/root/health_scripts/iptables_check.py b/systemvm/debian/root/health_scripts/iptables_check.py new file mode 100644 index 000000000000..dad2ec356fce --- /dev/null +++ b/systemvm/debian/root/health_scripts/iptables_check.py @@ -0,0 +1,71 @@ +#!/usr/bin/python +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from os import sys, path +from subprocess import * + +sys.path.append('/opt/cloud/bin') +from healthchecksutility import getHealthChecksData + +def main(): + portForwards = getHealthChecksData("portForwarding") + if portForwards != None and len(portForwards) > 0: + entriesExpected = [] + for portForward in portForwards: + srcIp = portForward["sourceIp"] + srcPortStart = portForward["sourcePortStart"] + srcPortEnd = portForward["sourcePortEnd"] + destIp = portForward["destIp"] + destPortStart = portForward["destPortStart"] + destPortEnd = portForward["destPortEnd"] + entriesExpected.append(["-d " + srcIp, "--dport " + (srcPortStart if (srcPortStart == srcPortEnd) else (srcPortStart + ":" + srcPortEnd)), "--to-destination " + destIp + ":" + (destPortStart if (destPortStart == destPortEnd) else (destPortStart + "-" + destPortEnd))]) + + pout = Popen("iptables-save | grep " + destIp, shell=True, stdout=PIPE) + if pout.wait() != 0: + print "Unable to execute iptables-save command for fetching rules" + exit(1) + + ipTablesMatchingEntries = pout.communicate()[0].strip().split('\n') + for pfEntryListExpected in entriesExpected: + foundPfEntryList = False + for ipTableEntry in ipTablesMatchingEntries: + # Check if all expected parts of pfEntryList is present in this ipTableEntry + foundAll = True + for expectedEntry in pfEntryListExpected: + if ipTableEntry.find(expectedEntry) == -1: + foundAll = False + break + + if foundAll: + foundPfEntryList = True + break + + if foundPfEntryList == False: + print "Missing entry for port forwarding rules in Iptables - " + print pfEntryListExpected + exit(1) + + print "Found all entries (count " + str(len(portForwards)) + ") in iptables" + else: + print "No portforwarding rules provided to check" + + exit(0) + +if __name__ == "__main__": + if len(sys.argv) == 2 and sys.argv[1] == "advance": + main() diff --git a/systemvm/debian/root/monitorServices.py b/systemvm/debian/root/monitorServices.py index 75d10043816d..78099ef311db 100755 --- a/systemvm/debian/root/monitorServices.py +++ b/systemvm/debian/root/monitorServices.py @@ -16,16 +16,17 @@ # specific language governing permissions and limitations # under the License. - - - - from ConfigParser import SafeConfigParser from subprocess import * -from os import path +from datetime import datetime import time import os import logging +import json +from os import sys, path + +sys.path.append('/opt/cloud/bin') +from healthchecksutility import getHealthChecksData class StatusCodes: SUCCESS = 0 @@ -48,9 +49,11 @@ class Config: RETRY_FOR_RESTART = 5 MONITOR_LOG = '/var/log/monitor.log' UNMONIT_PS_FILE = '/etc/unmonit_psList.txt' + HEALTH_CHECKS_SCRIPTS_DIR = 'health_scripts' + MONITOR_RESULT_FILE_SUFFIX = 'monitor_results.json' + FAILING_CHECKS_FILE = 'failing_health_checks' - -def getConfig( config_file_path = "/etc/monitor.conf" ): +def getServicesConfig( config_file_path = "/etc/monitor.conf" ): """ Reads the process configuration from the config file. Config file contains the processes to be monitored. @@ -66,7 +69,7 @@ def getConfig( config_file_path = "/etc/monitor.conf" ): for name, value in parser.items(section): process_dict[section][name] = value -# printd (" %s = %r" % (name, value)) + printd (" %s = %r" % (name, value)) return process_dict @@ -77,12 +80,12 @@ def printd (msg): #for debug #print msg - return 0 - f= open(Config.MONITOR_LOG,'r+') + f= open(Config.MONITOR_LOG, 'w' if not path.isfile(Config.MONITOR_LOG) else 'r+') f.seek(0, 2) f.write(str(msg)+"\n") f.close() + print str(msg) def raisealert(severity, msg, process_name=None): """ Writes the alert message""" @@ -97,6 +100,7 @@ def raisealert(severity, msg, process_name=None): logging.info(log) msg = 'logger -t monit '+ log pout = Popen(msg, shell=True, stdout=PIPE) + print "[Alert] " + msg def isPidMatchPidFile(pidfile, pids): @@ -126,7 +130,7 @@ def isPidMatchPidFile(pidfile, pids): fd.close() return StatusCodes.FAILED - printd("file content "+str(inp)) + printd("file content of pidfile " + pidfile + " = " + str(inp).strip()) printd(pids) tocheck_pid = inp.strip() for item in pids: @@ -152,7 +156,7 @@ def checkProcessRunningStatus(process_name, pidFile): #check there is only one pid or not if exitStatus == 0: - pids = temp_out.split(' ') + pids = temp_out.strip().split(' ') printd("pid(s) of process %s are %s " %(process_name, pids)) #there is more than one process so match the pid file @@ -181,8 +185,6 @@ def restartService(service_name): return False - - def checkProcessStatus( process ): """ Check the process running status, if not running tries to restart @@ -203,7 +205,7 @@ def checkProcessStatus( process ): if status == True: printd("The process is running ....") - return StatusCodes.RUNNING + return StatusCodes.RUNNING else: printd("Process %s is not running trying to recover" %process_name) #Retry the process state for few seconds @@ -245,16 +247,20 @@ def checkProcessStatus( process ): printd("Restart failed after number of retries") return StatusCodes.STOPPED - return StatusCodes.RUNNING + return StatusCodes.RUNNING def monitProcess( processes_info ): """ Monitors the processes which got from the config file """ + service_status = {} + failing_services = [] if len( processes_info ) == 0: - printd("Invalid Input") - return StatusCodes.INVALID_INP + printd("No config items provided - means a redundant VR or a VPC Router") + return service_status, failing_services + + print "[Process Info] " + json.dumps(processes_info) dict_unmonit={} umonit_update={} @@ -270,22 +276,28 @@ def monitProcess( processes_info ): csec = repr(time.time()).split('.')[0] for process,properties in processes_info.items(): + serviceName = process + ".service" #skip the process it its time stamp less than Config.MONIT_AFTER_MINS - printd ("checking the service %s \n" %process) - + printd ("---------------------------\nchecking the service %s\n---------------------------- " %process) if not is_emtpy(dict_unmonit): if dict_unmonit.has_key(process): ts = dict_unmonit[process] if checkPsTimeStampForMonitor (csec, ts, properties) == False: + service_status[serviceName] = {"success": "False", "message": "down since" + str(ts)} + failing_services.append(serviceName) unMonitPs = True continue - if checkProcessStatus( properties) != StatusCodes.RUNNING: + if checkProcessStatus(properties) != StatusCodes.RUNNING: printd( "\n Service %s is not Running"%process) #add this process into unmonit list printd ("updating the service for unmonit %s\n" %process) umonit_update[process]=csec + service_status[serviceName] = {"success": "False", "message": "down since" + str(csec)} + failing_services.append(serviceName) + else: + service_status[serviceName] = {"success": "True", "message": "service is running"} #if dict is not empty write to file else delete it if not is_emtpy(umonit_update): @@ -294,6 +306,7 @@ def monitProcess( processes_info ): if is_emtpy(umonit_update) and unMonitPs == False: #delete file it is there removeFile(Config.UNMONIT_PS_FILE) + return service_status, failing_services def checkPsTimeStampForMonitor(csec,ts, process): @@ -364,17 +377,92 @@ def is_emtpy(struct): else: return True -def main(): +def execute(script, checkType = "basic"): + cmd = "./" + script + " " + checkType + printd ("Executing health check script command: " + cmd) + + pout = Popen(cmd, shell=True, stdout=PIPE) + exitStatus = pout.wait() + output = pout.communicate()[0].strip() + + if exitStatus == 0: + if len(output) > 0: + printd("Successful execution of " + script) + return {"success": "True", "message": output} + return {} #Skip script if no output is received + else: + printd("Script execution failed " + script) + return {"success": "False", "message": output} + +def main(checkType = "basic"): + startTime = int(time.time()) ''' - Step1 : Get Config + Step1 : Get Services Config ''' printd("monitoring started") - temp_dict = getConfig() + configDict = getServicesConfig() ''' - Step2: Monitor and Raise Alert + Step2: Monitor services and Raise Alerts ''' - monitProcess( temp_dict ) + monitResult = {} + failingChecks = [] + if checkType == "basic": + monitResult, failingChecks = monitProcess(configDict) + + ''' + Step3: Run health check scripts as needed + ''' + hc_data = getHealthChecksData() + + if "health_checks_enabled" in hc_data and hc_data['health_checks_enabled']: + hc_exclude = hc_data["excluded_health_checks"] if "excluded_health_checks" in hc_data else [] + for f in os.listdir(Config.HEALTH_CHECKS_SCRIPTS_DIR): + if f in hc_exclude: + continue + fpath = path.join(Config.HEALTH_CHECKS_SCRIPTS_DIR, f) + if path.isfile(fpath) and os.access(fpath, os.X_OK): + ret = execute(fpath, checkType) + if len(ret) == 0: + continue + if "success" in ret and not ret["success"]: + failingChecks.append(f) + monitResult[f] = ret + + ''' + Step4: Write results to the json file for admins/management server to read + ''' + + endTime = int(time.time()) + monitResult["lastRun"] = { + "start": str(datetime.fromtimestamp(startTime)), + "end": str(datetime.fromtimestamp(endTime)), + "duration": str(endTime - startTime) + } + + with open(checkType + "_" + Config.MONITOR_RESULT_FILE_SUFFIX, 'w') as f: + json.dump(monitResult, f, ensure_ascii=False) + + failChecksFile = Config.FAILING_CHECKS_FILE + if len(failingChecks) > 0: + fcs = "" + for fc in failingChecks: + fcs = fcs + fc + "," + fcs = fcs[0, -1] + with open(failChecksFile, 'w') as f: + f.write(fcs) + elif path.isfile(failChecksFile): + os.remove(failChecksFile) if __name__ == "__main__": - main() + checkType = "basic" + if len(sys.argv) == 2: + if sys.argv[1] == "advance": + main("advance") + elif sys.argv[1] == "basic": + main("basic") + else: + printd("Error: Unknown type of test: " + sys.argv) + else: + main("basic") + main("advance") diff --git a/test/integration/component/test_routers.py b/test/integration/component/test_routers.py index 45e2853db898..fbf3f94d2296 100644 --- a/test/integration/component/test_routers.py +++ b/test/integration/component/test_routers.py @@ -21,7 +21,8 @@ from marvin.cloudstackTestCase import cloudstackTestCase from marvin.cloudstackAPI import (stopVirtualMachine, stopRouter, - startRouter) + startRouter, + getRouterHealthCheckResults) from marvin.lib.utils import (cleanup_resources, get_process_status) from marvin.lib.base import (ServiceOffering, @@ -594,6 +595,67 @@ def test_03_RouterStartOnVmDeploy(self): return + @attr(tags=["advanced"], required_hardware="false") + def test_04_RouterHealthChecksResults(self): + """Test advanced zone router list contains health check records + """ + + routers = list_routers( + self.apiclient, + account=self.account.name, + domainid=self.account.domainid, + includehealthcheckresults=True + ) + + self.assertEqual(isinstance(routers, list), True, + "Check for list routers response return valid data" + ) + self.assertNotEqual( + len(routers), 0, + "Check list router response" + ) + + router = routers[0] + self.info("Router ID: %s & Router state: %s" % ( + router.id, router.state + )) + self.assertEqual(router.id, router.healthcheckresults.routerId, + "Router response should contain it's health check result so id should match" + ) + self.assertEqual(router.name, router.healthcheckresults.name, + "Router response should contain it's health check result so router name should match" + ) + + for checkType in ["basic", "advance"]: + self.assertTrue( + checkType in router.healthcheckresults.details, + "Router should contain health check results info for type " + checkType + ) + + cmd = getRouterHealthCheckResults.getRouterHealthCheckResultsCmd() + cmd.id = router.id + cmd.performfreshchecks = True # Perform fresh checks as a newly created router may not have results + healthData = self.api_client.getRouterHealthCheckResults(cmd) + print healthData + self.info("Router ID: %s & Router state: %s" % ( + router.id, router.state + )) + self.assertEqual(router.id, healthData.routerId, + "Router response should contain it's health check result so id should match" + ) + self.assertEqual(router.name, healthData.name, + "Router response should contain it's health check result so router name should match" + ) + self.assertTrue(healthData.success == "true", + "Router health check result should be successful" + ) + + for detailPattern in ["basic", "advance", "lastRun", "dns_check.py", "dhcp_check.py", "haproxy_check.py", "disk_space_check.py", "iptables_check.py", "gateways_check.py"]: + self.assertTrue( + detailPattern in healthData.details, + "Router health check details string should contain " + detailPattern + ) + class TestRouterStopCreatePF(cloudstackTestCase): diff --git a/tools/appliance/systemvmtemplate/scripts/configure_systemvm_services.sh b/tools/appliance/systemvmtemplate/scripts/configure_systemvm_services.sh index 56406b711f52..f82f360550c2 100644 --- a/tools/appliance/systemvmtemplate/scripts/configure_systemvm_services.sh +++ b/tools/appliance/systemvmtemplate/scripts/configure_systemvm_services.sh @@ -48,6 +48,7 @@ function install_cloud_scripts() { /root/{clearUsageRules.sh,reconfigLB.sh,monitorServices.py} \ /etc/profile.d/cloud.sh /etc/cron.daily/* /etc/cron.hourly/* + chmod -R +x /root/health/ chmod -x /etc/systemd/system/* systemctl daemon-reload diff --git a/ui/l10n/en.js b/ui/l10n/en.js index 87deba8142b2..4ac1fb473f52 100644 --- a/ui/l10n/en.js +++ b/ui/l10n/en.js @@ -291,6 +291,8 @@ var dictionary = { "label.action.stop.instance.processing":"Stopping Instance....", "label.action.stop.router":"Stop Router", "label.action.stop.router.processing":"Stopping Router....", +"label.action.router.health.checks":"Get health checks result", +"label.perform.fresh.checks":"Perform fresh checks", "label.action.stop.systemvm":"Stop System VM", "label.action.stop.systemvm.processing":"Stopping System VM....", "label.action.take.snapshot":"Take Snapshot", @@ -579,6 +581,7 @@ var dictionary = { "label.continue":"Continue", "label.continue.basic.install":"Continue with basic installation", "label.copying.iso":"Copying ISO", +"label.copy.text": "Copy Text", "label.corrections.saved":"Corrections saved", "label.counter":"Counter", "label.cpu":"CPU", @@ -1972,6 +1975,7 @@ var dictionary = { "message.action.start.systemvm":"Please confirm that you want to start this system VM.", "message.action.stop.instance":"Please confirm that you want to stop this instance.", "message.action.stop.router":"All services provided by this virtual router will be interrupted. Please confirm that you want to stop this router.", +"message.action.router.health.checks":"Health checks result will be fetched from router.", "message.action.stop.systemvm":"Please confirm that you want to stop this system VM.", "message.action.take.snapshot":"Please confirm that you want to take a snapshot of this volume.", "message.action.unmanage.cluster":"Please confirm that you want to unmanage the cluster.", diff --git a/ui/scripts/system.js b/ui/scripts/system.js index 2ae2f466043c..a56f34b4a394 100755 --- a/ui/scripts/system.js +++ b/ui/scripts/system.js @@ -9914,7 +9914,8 @@ indicator: { 'Running': 'on', 'Stopped': 'off', - 'Error': 'off' + 'Error': 'off', + 'Alert': 'warning' } }, requiresupgrade: { @@ -9936,6 +9937,7 @@ } var data2 = { + includehealthcheckresults: true // forvpc: false }; @@ -9979,47 +9981,93 @@ json.listroutersresponse.router:[]; $(items).map(function (index, item) { + if (item.state && item.state === 'Running' && item.healthcheckresults && item.healthcheckresults.details) { + try { + if (!item.healthcheckresults.success || item.healthcheckresults.success.toLowerCase() !== 'true') { + item.state = 'Alert' + } else { + var hcData = JSON.parse(item.healthcheckresults.details) + var checkTypes = ['basic', 'advance'] + $.each(checkTypes, function (cId, checkType) { + if (hcData[checkType] && $.type(hcData[checkType]) === "object") { + var checks = hcData[checkType] + $.each(Object.keys(checks), function (idx, key) { + if (key !== 'lastRun' && (!checks[key].success || checks[key].success.toLowerCase() !== 'true')) { + item.state = 'Alert' + } + }) + } + }) + } + } catch (ex) { + console.error('Unable to parse health check results') + console.log(item) + } + } routers.push(item); }); - /* - * In project view, the first listRotuers API(without projectid=-1) will return the same objects as the second listRouters API(with projectid=-1), - * because in project view, all API calls are appended with projectid=[projectID]. - * Therefore, we only call the second listRouters API(with projectid=-1) in non-project view. - */ - if (cloudStack.context && cloudStack.context.projects == null) { //non-project view - /* - * account parameter(account+domainid) and project parameter(projectid) are not allowed to be passed together to listXXXXXXX API. - * So, remove account parameter(account+domainid) from data2 - */ - if ("account" in data2) { - delete data2.account; - } - if ("domainid" in data2) { - delete data2.domainid; - } - - $.ajax({ - url: createURL("listRouters&listAll=true&page=" + args.page + "&pagesize=" + pageSize + array1.join("") + "&projectid=-1"), - data: data2, - async: false, - success: function (json) { - var items = json.listroutersresponse.router ? - json.listroutersresponse.router:[]; + /* + * In project view, the first listRotuers API(without projectid=-1) will return the same objects as the second listRouters API(with projectid=-1), + * because in project view, all API calls are appended with projectid=[projectID]. + * Therefore, we only call the second listRouters API(with projectid=-1) in non-project view. + */ + if (cloudStack.context && cloudStack.context.projects == null) { //non-project view + /* + * account parameter(account+domainid) and project parameter(projectid) are not allowed to be passed together to listXXXXXXX API. + * So, remove account parameter(account+domainid) from data2 + */ + if ("account" in data2) { + delete data2.account; + } + if ("domainid" in data2) { + delete data2.domainid; + } - $(items).map(function (index, item) { - routers.push(item); - }); + $.ajax({ + url: createURL("listRouters&listAll=true&page=" + args.page + "&pagesize=" + pageSize + array1.join("") + "&projectid=-1"), + data: data2, + async: false, + success: function (json) { + var items = json.listroutersresponse.router ? + json.listroutersresponse.router:[]; + + $(items).map(function (index, item) { + if (item.state && item.state === 'Running' && item.healthcheckresults && item.healthcheckresults.details) { + try { + if (!item.healthcheckresults.success || item.healthcheckresults.success.toLowerCase() !== 'true') { + item.state = 'Alert' + } else { + var hcData = JSON.parse(item.healthcheckresults.details) + var checkTypes = ['basic', 'advance'] + $.each(checkTypes, function (cId, checkType) { + if (hcData[checkType] && $.type(hcData[checkType]) === "object") { + var checks = hcData[checkType] + $.each(Object.keys(checks), function (idx, key) { + if (key !== 'lastRun' && (!checks[key].success || checks[key].success.toLowerCase() !== 'true')) { + item.state = 'Alert' + } + }) + } + }) + } + } catch (ex) { + console.error('Unable to parse health check results') + console.log(item) + } + } + routers.push(item); + }); + } + }); } - }); - } - args.response.success({ - actionFilter: routerActionfilter, - data: $(routers).map(mapRouterType) - }); - } + args.response.success({ + actionFilter: routerActionfilter, + data: $(routers).map(mapRouterType) }); + } + }); }, detailView: { name: 'label.virtual.appliance.details', @@ -10542,6 +10590,79 @@ height: 640 } } + }, + + healthChecks: { + label: 'label.action.router.health.checks', + createForm: { + title: 'label.action.router.health.checks', + desc: 'message.action.router.health.checks', + fields: { + performfreshchecks: { + label: 'label.perform.fresh.checks', + isBoolean: true + } + } + }, + action: function (args) { + var data = { + 'id': args.context.routers[0].id, + 'performfreshchecks': (args.data.performfreshchecks === 'on') + }; + $.ajax({ + url: createURL('getRouterHealthCheckResults'), + dataType: 'json', + data: data, + async: false, + success: function (json) { + var hcdata = json.getrouterhealthcheckresultsresponse + $('div.overlay').remove() + $('.loading-overlay').remove() + $('div.loading-overlay').remove() + var textArea = $('