From 6f4c5f91d526807e93bdab6f1fb64a135b7e239a Mon Sep 17 00:00:00 2001 From: "Ivan A. Kudryavtsev" Date: Tue, 6 Nov 2018 21:41:25 -0500 Subject: [PATCH 1/4] CLOUDSTACK-3009: Fixed resource calculation CPU, RAM for accounts. --- .dockerignore | 22 ++ .gitignore | 1 + .../dao/ResourceCountDaoImpl.java | 2 +- .../component/test_browse_templates.py | 10 +- .../smoke/test_resource_accounting.py | 252 ++++++++++++++++++ 5 files changed, 281 insertions(+), 6 deletions(-) create mode 100644 .dockerignore create mode 100644 test/integration/smoke/test_resource_accounting.py diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000000..6ca3ad4aa232 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,22 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +cloudstack/tools/docker/Dockerfile +.dockerignore +.idea +.git +venv diff --git a/.gitignore b/.gitignore index 1a73724c1172..2f5c5167e2d8 100644 --- a/.gitignore +++ b/.gitignore @@ -99,3 +99,4 @@ plugins/hypervisors/kvm/.pydevproject scripts/.pydevproject *.qcow2 *.raw +venv diff --git a/engine/schema/src/com/cloud/configuration/dao/ResourceCountDaoImpl.java b/engine/schema/src/com/cloud/configuration/dao/ResourceCountDaoImpl.java index 9cc8913f8fa1..1de2b65aed12 100644 --- a/engine/schema/src/com/cloud/configuration/dao/ResourceCountDaoImpl.java +++ b/engine/schema/src/com/cloud/configuration/dao/ResourceCountDaoImpl.java @@ -258,7 +258,7 @@ public long removeEntriesByOwner(long ownerId, ResourceOwnerType ownerType) { + " ELSE CONVERT(vmd.value, UNSIGNED INTEGER) " + " END)) as total " + " from vm_instance vm " - + " join service_offering_view so on so.id = vm.service_offering_id " + + " join service_offering so on so.id = vm.service_offering_id " + " left join user_vm_details vmd on vmd.vm_id = vm.id and vmd.name = '%s' " + " where vm.type = 'User' and state not in ('Destroyed', 'Error', 'Expunging') and display_vm = true and account_id = ? "; diff --git a/test/integration/component/test_browse_templates.py b/test/integration/component/test_browse_templates.py index 80e9a135b80a..4340092b9a06 100644 --- a/test/integration/component/test_browse_templates.py +++ b/test/integration/component/test_browse_templates.py @@ -230,7 +230,7 @@ def gettemplatelimts(self): return(totaltemplates) - def getstoragelimts(self,rtype): + def getstoragelimits(self, rtype): cmd=updateResourceCount.updateResourceCountCmd() cmd.account=self.account.name @@ -1652,7 +1652,7 @@ def test_07_Browser_Upload_template_secondary_storage_resource_limits(self): self.debug("========================= Test 22 Upload template and verify secondary storage limits========================") - initialsecondarystoragelimit=self.getstoragelimts(11) + initialsecondarystoragelimit=self.getstoragelimits(11) browseup_template1=self.browse_upload_template() tmpldetails=Template.list( @@ -1662,7 +1662,7 @@ def test_07_Browser_Upload_template_secondary_storage_resource_limits(self): zoneid=self.zone.id) - afteruploadsecondarystoragelimit=self.getstoragelimts(11) + afteruploadsecondarystoragelimit=self.getstoragelimits(11) if afteruploadsecondarystoragelimit!=(initialsecondarystoragelimit+tmpldetails[0].size): self.fail("Secondary Storage Resouce Count is not updated") @@ -1709,10 +1709,10 @@ def test_09_Browser_Upload_Volume_secondary_storage_resource_limits_after_deleti templatefilter="all", zoneid=self.zone.id) - initialuploadprimarystoragelimit=self.getstoragelimts(11) + initialuploadprimarystoragelimit=self.getstoragelimits(11) self.delete_template(browseup_template1) - afteruploadprimarystoragelimit=self.getstoragelimts(11) + afteruploadprimarystoragelimit=self.getstoragelimits(11) if afteruploadprimarystoragelimit!=(initialuploadprimarystoragelimit-tmpldetails[0].size): self.fail("Secondary Storage Resource Count is not updated after deletion") diff --git a/test/integration/smoke/test_resource_accounting.py b/test/integration/smoke/test_resource_accounting.py new file mode 100644 index 000000000000..53dbc19e111f --- /dev/null +++ b/test/integration/smoke/test_resource_accounting.py @@ -0,0 +1,252 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" P1 tests for Account +""" + +from nose.plugins.attrib import attr + +# Import Local Modules +from marvin.cloudstackTestCase import cloudstackTestCase +from marvin.lib.base import (Domain, + Account, + ServiceOffering, + VirtualMachine, updateResourceCount) +from marvin.lib.common import (get_zone, + get_test_template) +from marvin.lib.utils import (cleanup_resources) + + +class Services: + """Test Account Services + """ + + def __init__(self): + self.services = { + "domain": { + "name": "Domain", + }, + "account": { + "email": "test@test.com", + "firstname": "Test", + "lastname": "User", + "username": "test", + # Random characters are appended for unique + # username + "password": "fr3sca", + }, + "user": { + "email": "user@test.com", + "firstname": "User", + "lastname": "User", + "username": "User", + # Random characters are appended for unique + # username + "password": "fr3sca", + }, + "service_offering_it_1": { + "name": "InstanceType-1", + "displaytext": "Tiny Instance", + "cpunumber": 1, + "cpuspeed": 100, + "memory": 128, + }, + "service_offering_it_2": { + "name": "InstanceType-2", + "displaytext": "Tiny Instance", + "cpunumber": 4, + "cpuspeed": 100, + "memory": 512, + }, + "virtual_machine_1": { + "displayname": "Test VM1", + "username": "root", + "password": "password", + "ssh_port": 22, + "hypervisor": 'XenServer', + "privateport": 22, + "publicport": 22, + "protocol": 'TCP', + }, + "virtual_machine_2": { + "displayname": "Test VM2", + "username": "root", + "password": "password", + "ssh_port": 22, + "hypervisor": 'XenServer', + "privateport": 22, + "publicport": 22, + "protocol": 'TCP', + }, + "template": { + "displaytext": "Public Template", + "name": "Public template", + "ostype": 'CentOS 5.6 (64-bit)', + "url": "", + "hypervisor": '', + "format": '', + "isfeatured": True, + "ispublic": True, + "isextractable": True, + "templatefilter": "self" + }, + "natrule": { + "publicport": 22, + "privateport": 22, + "protocol": 'TCP', + }, + "ostype": 'CentOS 5.6 (64-bit)', + "sleep": 60, + "timeout": 10, + } + + +class TestRAMCPUResourceAccounting(cloudstackTestCase): + + @classmethod + def setUpClass(cls): + cls.testClient = super( + TestRAMCPUResourceAccounting, + cls).getClsTestClient() + cls.api_client = cls.testClient.getApiClient() + + cls.services = Services().services + cls.zone = get_zone(cls.api_client, cls.testClient.getZoneForTests()) + cls.services['mode'] = cls.zone.networktype + cls.hypervisor = cls.testClient.getHypervisorInfo() + # Create an account, domain etc + cls.domain = Domain.create( + cls.api_client, + cls.services["domain"], + ) + cls.account = Account.create( + cls.api_client, + cls.services["account"], + admin=True, + domainid=cls.domain.id + ) + + cls.template = get_test_template( + cls.api_client, + cls.zone.id, + cls.hypervisor) + + cls.services["virtual_machine_1"]["zoneid"] = cls.zone.id + cls.services["virtual_machine_1"]["template"] = cls.template.id + + cls.services["virtual_machine_2"]["zoneid"] = cls.zone.id + cls.services["virtual_machine_2"]["template"] = cls.template.id + + cls._cleanup = [ + cls.account, + cls.domain + ] + return + + @classmethod + def tearDownClass(cls): + try: + # Cleanup resources used + cleanup_resources(cls.api_client, cls._cleanup) + except Exception as e: + raise Exception("Warning: Exception during cleanup : %s" % e) + return + + def setUp(self): + self.apiclient = self.testClient.getApiClient() + self.dbclient = self.testClient.getDbConnection() + self.cleanup = [] + return + + def tearDown(self): + try: + # Clean up, terminate the created accounts + cleanup_resources(self.apiclient, self.cleanup) + except Exception as e: + raise Exception("Warning: Exception during cleanup : %s" % e) + return + + def get_resource_amount(self, resource_type): + cmd = updateResourceCount.updateResourceCountCmd() + cmd.account = self.account.name + cmd.domainid = self.domain.id + cmd.resourcetype = resource_type + response = self.apiclient.updateResourceCount(cmd) + amount = response[0].resourcecount + return amount + + @attr(tags=["advanced", "basic"], required_hardware="false") + def test_01_so_removal_resource_update(self): + + self.service_offering_it_1 = ServiceOffering.create( + self.api_client, + self.services["service_offering_it_1"], + domainid=self.domain.id + ) + + self.cleanup.append(self.service_offering_it_1) + + self.service_offering_it_2 = ServiceOffering.create( + self.api_client, + self.services["service_offering_it_2"], + domainid=self.domain.id + ) + + self.cleanup.append(self.service_offering_it_2) + + vm_1 = VirtualMachine.create( + self.apiclient, + self.services["virtual_machine_1"], + accountid=self.account.name, + domainid=self.account.domainid, + serviceofferingid=self.service_offering_it_1.id + ) + + self.debug("Deployed VM in account: %s, ID: %s" % (self.account.name, vm_1.id)) + self.cleanup.append(vm_1) + + vm_2 = VirtualMachine.create( + self.apiclient, + self.services["virtual_machine_2"], + accountid=self.account.name, + domainid=self.account.domainid, + serviceofferingid=self.service_offering_it_2.id + ) + + self.debug("Deployed VM in account: %s, ID: %s" % (self.account.name, vm_2.id)) + self.cleanup.append(vm_2) + + CPU_RESOURCE_ID = 8 + RAM_RESOURCE_ID = 9 + + cores = int(self.get_resource_amount(CPU_RESOURCE_ID)) + ram = int(self.get_resource_amount(RAM_RESOURCE_ID)) + + self.assertEqual(cores, self.services['service_offering_it_1']['cpunumber'] + self.services['service_offering_it_2']['cpunumber']) + self.assertEqual(ram, self.services['service_offering_it_1']['memory'] + self.services['service_offering_it_2']['memory']) + + self.service_offering_it_2.delete(self.apiclient) + + self.cleanup = self.cleanup[0:-1] + + cores = int(self.get_resource_amount(CPU_RESOURCE_ID)) + ram = int(self.get_resource_amount(RAM_RESOURCE_ID)) + + self.assertEqual(cores, self.services['service_offering_it_1']['cpunumber'] + self.services['service_offering_it_2']['cpunumber']) + self.assertEqual(ram, self.services['service_offering_it_1']['memory'] + self.services['service_offering_it_2']['memory']) + + pass + From 937cd75f433d15ade1025631903f4b49f6f405cd Mon Sep 17 00:00:00 2001 From: Rohit Yadav Date: Mon, 12 Nov 2018 14:27:33 +0530 Subject: [PATCH 2/4] clean python code --- test/integration/smoke/test_resource_accounting.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/test/integration/smoke/test_resource_accounting.py b/test/integration/smoke/test_resource_accounting.py index 53dbc19e111f..a0ee43173954 100644 --- a/test/integration/smoke/test_resource_accounting.py +++ b/test/integration/smoke/test_resource_accounting.py @@ -14,8 +14,6 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -""" P1 tests for Account -""" from nose.plugins.attrib import attr @@ -154,7 +152,6 @@ def setUpClass(cls): cls.account, cls.domain ] - return @classmethod def tearDownClass(cls): @@ -163,13 +160,11 @@ def tearDownClass(cls): cleanup_resources(cls.api_client, cls._cleanup) except Exception as e: raise Exception("Warning: Exception during cleanup : %s" % e) - return def setUp(self): self.apiclient = self.testClient.getApiClient() self.dbclient = self.testClient.getDbConnection() self.cleanup = [] - return def tearDown(self): try: @@ -177,7 +172,6 @@ def tearDown(self): cleanup_resources(self.apiclient, self.cleanup) except Exception as e: raise Exception("Warning: Exception during cleanup : %s" % e) - return def get_resource_amount(self, resource_type): cmd = updateResourceCount.updateResourceCountCmd() @@ -247,6 +241,3 @@ def test_01_so_removal_resource_update(self): self.assertEqual(cores, self.services['service_offering_it_1']['cpunumber'] + self.services['service_offering_it_2']['cpunumber']) self.assertEqual(ram, self.services['service_offering_it_1']['memory'] + self.services['service_offering_it_2']['memory']) - - pass - From c6d643d6bd06253106a84e9ca659f55e9e3d2364 Mon Sep 17 00:00:00 2001 From: Rohit Yadav Date: Mon, 12 Nov 2018 14:28:48 +0530 Subject: [PATCH 3/4] add new smoketest to travis --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index d5fd173d71ed..43f2bb29035d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -78,6 +78,7 @@ env: smoke/test_pvlan smoke/test_regions smoke/test_reset_vm_on_reboot + smoke/test_resource_accounting smoke/test_resource_detail smoke/test_router_dhcphosts smoke/test_router_dns From 1cec04bfaccc1e70919450e9f6bfaf7af3ec4912 Mon Sep 17 00:00:00 2001 From: "Ivan A. Kudryavtsev" Date: Mon, 12 Nov 2018 09:37:05 -0500 Subject: [PATCH 4/4] Fixed SO to request less resources to fit Trillian env. --- test/integration/smoke/test_resource_accounting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/smoke/test_resource_accounting.py b/test/integration/smoke/test_resource_accounting.py index 53dbc19e111f..e4b01d967da3 100644 --- a/test/integration/smoke/test_resource_accounting.py +++ b/test/integration/smoke/test_resource_accounting.py @@ -67,7 +67,7 @@ def __init__(self): "service_offering_it_2": { "name": "InstanceType-2", "displaytext": "Tiny Instance", - "cpunumber": 4, + "cpunumber": 2, "cpuspeed": 100, "memory": 512, },