[root@trl-283-v-M7-boris-stoyanov-mgmt1 ~]# tailf /var/log/cloudstack/management/management-server.log 2022-03-10 07:00:58,787 DEBUG [c.c.h.d.HostDaoImpl] (ClusteredAgentManager Timer:ctx-f6cb111d) (logid:46625178) Acquiring hosts for clusters already owned by this management server 2022-03-10 07:00:58,787 DEBUG [c.c.h.d.HostDaoImpl] (ClusteredAgentManager Timer:ctx-f6cb111d) (logid:46625178) Completed acquiring hosts for clusters already owned by this management server 2022-03-10 07:00:58,788 DEBUG [c.c.h.d.HostDaoImpl] (ClusteredAgentManager Timer:ctx-f6cb111d) (logid:46625178) Acquiring hosts for clusters not owned by any management server 2022-03-10 07:00:58,788 DEBUG [c.c.h.d.HostDaoImpl] (ClusteredAgentManager Timer:ctx-f6cb111d) (logid:46625178) Completed acquiring hosts for clusters not owned by any management server 2022-03-10 07:00:58,841 DEBUG [o.a.c.h.HAManagerImpl] (BackgroundTaskPollManager-6:ctx-2669928d) (logid:8df7fb0a) HA health check task is running... 2022-03-10 07:01:01,702 DEBUG [c.c.a.m.AgentManagerImpl] (AgentManager-Handler-6:null) (logid:) SeqA 5-6800: Processing Seq 5-6800: { Cmd , MgmtId: -1, via: 5, Ver: v1, Flags: 11, [{"com.cloud.agent.api.ConsoleProxyLoadReportCommand":{"_proxyVmId":"2","_loadInfo":"{ "connections": [] }","wait":"0","bypassHostMaintenance":"false"}}] } 2022-03-10 07:01:01,706 DEBUG [c.c.a.m.AgentManagerImpl] (AgentManager-Handler-6:null) (logid:) SeqA 5-6800: Sending Seq 5-6800: { Ans: , MgmtId: 32988737831924, via: 5, Ver: v1, Flags: 100010, [{"com.cloud.agent.api.AgentControlAnswer":{"result":"true","wait":"0","bypassHostMaintenance":"false"}}] } 2022-03-10 07:01:02,842 DEBUG [o.a.c.h.HAManagerImpl] (BackgroundTaskPollManager-3:ctx-fe188c12) (logid:a86c3d52) HA health check task is running... 2022-03-10 07:01:05,087 DEBUG [c.c.a.ApiServlet] (qtp1747352992-19:ctx-52a3ea07) (logid:45bafbd7) ===START=== 10.0.3.151 -- POST command=deployVirtualMachine&response=json 2022-03-10 07:01:05,093 DEBUG [c.c.a.ApiServer] (qtp1747352992-19:ctx-52a3ea07 ctx-d724b83e) (logid:45bafbd7) CIDRs from which account 'Acct[11c652b3-9fc4-11ec-81d4-1e00ca0003f4-admin] -- Account {"id": 2, "name": "admin", "uuid": "11c652b3-9fc4-11ec-81d4-1e00ca0003f4"}' is allowed to perform API calls: 0.0.0.0/0,::/0 2022-03-10 07:01:05,106 DEBUG [c.c.u.AccountManagerImpl] (qtp1747352992-19:ctx-52a3ea07 ctx-d724b83e) (logid:45bafbd7) Access granted to Acct[11c652b3-9fc4-11ec-81d4-1e00ca0003f4-admin] -- Account {"id": 2, "name": "admin", "uuid": "11c652b3-9fc4-11ec-81d4-1e00ca0003f4"} to org.apache.cloudstack.quota.vo.ServiceOfferingVO$$EnhancerByCGLIB$$1c247de4@24022cc by AffinityGroupAccessChecker 2022-03-10 07:01:05,107 DEBUG [c.c.u.AccountManagerImpl] (qtp1747352992-19:ctx-52a3ea07 ctx-d724b83e) (logid:45bafbd7) Access granted to Acct[11c652b3-9fc4-11ec-81d4-1e00ca0003f4-admin] -- Account {"id": 2, "name": "admin", "uuid": "11c652b3-9fc4-11ec-81d4-1e00ca0003f4"} to null by AffinityGroupAccessChecker 2022-03-10 07:01:05,115 DEBUG [c.c.n.NetworkModelImpl] (qtp1747352992-19:ctx-52a3ea07 ctx-d724b83e) (logid:45bafbd7) Service SecurityGroup is not supported in the network id=209 2022-03-10 07:01:05,124 INFO [c.c.v.UserVmManagerImpl] (qtp1747352992-19:ctx-52a3ea07 ctx-d724b83e) (logid:45bafbd7) VM cannot be configured to be dynamically scalable if any of the service offering's dynamic scaling property, template's dynamic scaling property or global setting is false 2022-03-10 07:01:05,133 DEBUG [c.c.v.UserVmManagerImpl] (qtp1747352992-19:ctx-52a3ea07 ctx-d724b83e) (logid:45bafbd7) Allocating in the DB for vm 2022-03-10 07:01:05,140 INFO [c.c.v.VirtualMachineManagerImpl] (qtp1747352992-19:ctx-52a3ea07 ctx-d724b83e) (logid:45bafbd7) allocating virtual machine from template:f0a45ec8-9fc3-11ec-81d4-1e00ca0003f4 with hostname:i-2-12-VM and 1 networks 2022-03-10 07:01:05,143 DEBUG [c.c.v.VirtualMachineManagerImpl] (qtp1747352992-19:ctx-52a3ea07 ctx-d724b83e) (logid:45bafbd7) Allocating entries for VM: VM instance {id: "12", name: "i-2-12-VM", uuid: "fe006474-e8c7-4f78-8982-e72c058c6a3e", type="User"} 2022-03-10 07:01:05,144 DEBUG [c.c.v.VirtualMachineManagerImpl] (qtp1747352992-19:ctx-52a3ea07 ctx-d724b83e) (logid:45bafbd7) Allocating nics for VM instance {id: "12", name: "i-2-12-VM", uuid: "fe006474-e8c7-4f78-8982-e72c058c6a3e", type="User"} 2022-03-10 07:01:05,144 DEBUG [o.a.c.e.o.NetworkOrchestrator] (qtp1747352992-19:ctx-52a3ea07 ctx-d724b83e) (logid:45bafbd7) Allocating nic for vm VM instance {id: "12", name: "i-2-12-VM", uuid: "fe006474-e8c7-4f78-8982-e72c058c6a3e", type="User"} in network Ntwk[209|Guest|8] with requested profile NicProfile[0-0-null-null-null] 2022-03-10 07:01:05,164 DEBUG [c.c.n.NetworkModelImpl] (qtp1747352992-19:ctx-52a3ea07 ctx-d724b83e) (logid:45bafbd7) Service SecurityGroup is not supported in the network id=209 2022-03-10 07:01:05,165 DEBUG [c.c.v.VirtualMachineManagerImpl] (qtp1747352992-19:ctx-52a3ea07 ctx-d724b83e) (logid:45bafbd7) Allocating disks for VM instance {id: "12", name: "i-2-12-VM", uuid: "fe006474-e8c7-4f78-8982-e72c058c6a3e", type="User"} 2022-03-10 07:01:05,165 INFO [o.a.c.e.o.VolumeOrchestrator] (qtp1747352992-19:ctx-52a3ea07 ctx-d724b83e) (logid:45bafbd7) adding disk object ROOT-12 to i-2-12-VM 2022-03-10 07:01:05,175 DEBUG [c.c.r.ResourceLimitManagerImpl] (qtp1747352992-19:ctx-52a3ea07 ctx-d724b83e) (logid:45bafbd7) Updating resource Type = volume count for Account = 2 Operation = increasing Amount = 1 2022-03-10 07:01:05,177 DEBUG [c.c.r.ResourceLimitManagerImpl] (qtp1747352992-19:ctx-52a3ea07 ctx-d724b83e) (logid:45bafbd7) Updating resource Type = primary_storage count for Account = 2 Operation = increasing Amount = (2.00 GB) 2147483648 2022-03-10 07:01:05,180 DEBUG [c.c.v.VirtualMachineManagerImpl] (qtp1747352992-19:ctx-52a3ea07 ctx-d724b83e) (logid:45bafbd7) Allocation completed for VM: VM instance {id: "12", name: "i-2-12-VM", uuid: "fe006474-e8c7-4f78-8982-e72c058c6a3e", type="User"} 2022-03-10 07:01:05,180 DEBUG [c.c.v.UserVmManagerImpl] (qtp1747352992-19:ctx-52a3ea07 ctx-d724b83e) (logid:45bafbd7) Successfully allocated DB entry for VM instance {id: "12", name: "i-2-12-VM", uuid: "fe006474-e8c7-4f78-8982-e72c058c6a3e", type="User"} 2022-03-10 07:01:05,182 DEBUG [c.c.r.ResourceLimitManagerImpl] (qtp1747352992-19:ctx-52a3ea07 ctx-d724b83e) (logid:45bafbd7) Updating resource Type = user_vm count for Account = 2 Operation = increasing Amount = 1 2022-03-10 07:01:05,184 DEBUG [c.c.r.ResourceLimitManagerImpl] (qtp1747352992-19:ctx-52a3ea07 ctx-d724b83e) (logid:45bafbd7) Updating resource Type = cpu count for Account = 2 Operation = increasing Amount = 1 2022-03-10 07:01:05,187 DEBUG [c.c.r.ResourceLimitManagerImpl] (qtp1747352992-19:ctx-52a3ea07 ctx-d724b83e) (logid:45bafbd7) Updating resource Type = memory count for Account = 2 Operation = increasing Amount = 512 2022-03-10 07:01:05,200 DEBUG [c.c.n.NetworkModelImpl] (qtp1747352992-19:ctx-52a3ea07 ctx-d724b83e) (logid:45bafbd7) Service SecurityGroup is not supported in the network id=209 2022-03-10 07:01:05,202 DEBUG [c.c.n.NetworkModelImpl] (qtp1747352992-19:ctx-52a3ea07 ctx-d724b83e) (logid:45bafbd7) Service SecurityGroup is not supported in the network id=209 2022-03-10 07:01:05,215 DEBUG [o.a.c.f.j.i.AsyncJobManagerImpl] (qtp1747352992-19:ctx-52a3ea07 ctx-d724b83e) (logid:45bafbd7) submit async job-78, details: AsyncJobVO: {id:78, userId: 2, accountId: 2, instanceType: VirtualMachine, instanceId: 12, cmd: org.apache.cloudstack.api.command.admin.vm.DeployVMCmdByAdmin, cmdInfo: {"iptonetworklist[0].networkid":"bde8a9ed-8633-47b5-91f1-413f6708ead2","boottype":"BIOS","httpmethod":"POST","templateid":"f0a45ec8-9fc3-11ec-81d4-1e00ca0003f4","ctxAccountId":"2","uuid":"fe006474-e8c7-4f78-8982-e72c058c6a3e","cmdEventType":"VM.CREATE","startvm":"true","bootmode":"LEGACY","serviceofferingid":"92b40982-957a-4f87-b280-9788a43df57c","response":"json","ctxUserId":"2","displayname":"vmware","name":"vmware","zoneid":"fa30df44-1240-499d-a0a2-0e8307a2c565","ctxStartEventId":"175","id":"12","ctxDetails":"{\"interface com.cloud.template.VirtualMachineTemplate\":\"f0a45ec8-9fc3-11ec-81d4-1e00ca0003f4\",\"interface com.cloud.offering.ServiceOffering\":\"92b40982-957a-4f87-b280-9788a43df57c\",\"interface com.cloud.vm.VirtualMachine\":\"fe006474-e8c7-4f78-8982-e72c058c6a3e\",\"interface com.cloud.dc.DataCenter\":\"fa30df44-1240-499d-a0a2-0e8307a2c565\"}","affinitygroupids":""}, cmdVersion: 0, status: IN_PROGRESS, processStatus: 0, resultCode: 0, result: null, initMsid: 32988737831924, completeMsid: null, lastUpdated: null, lastPolled: null, created: null, removed: null} 2022-03-10 07:01:05,217 DEBUG [c.c.a.ApiServlet] (qtp1747352992-19:ctx-52a3ea07 ctx-d724b83e) (logid:45bafbd7) ===END=== 10.0.3.151 -- POST command=deployVirtualMachine&response=json 2022-03-10 07:01:05,219 INFO [o.a.c.f.j.i.AsyncJobMonitor] (API-Job-Executor-2:ctx-949eff6a job-78) (logid:83de71a1) Add job-78 into job monitoring 2022-03-10 07:01:05,223 DEBUG [o.a.c.f.j.i.AsyncJobManagerImpl] (API-Job-Executor-2:ctx-949eff6a job-78) (logid:a882c7d6) Executing AsyncJobVO: {id:78, userId: 2, accountId: 2, instanceType: VirtualMachine, instanceId: 12, cmd: org.apache.cloudstack.api.command.admin.vm.DeployVMCmdByAdmin, cmdInfo: {"iptonetworklist[0].networkid":"bde8a9ed-8633-47b5-91f1-413f6708ead2","boottype":"BIOS","httpmethod":"POST","templateid":"f0a45ec8-9fc3-11ec-81d4-1e00ca0003f4","ctxAccountId":"2","uuid":"fe006474-e8c7-4f78-8982-e72c058c6a3e","cmdEventType":"VM.CREATE","startvm":"true","bootmode":"LEGACY","serviceofferingid":"92b40982-957a-4f87-b280-9788a43df57c","response":"json","ctxUserId":"2","displayname":"vmware","name":"vmware","zoneid":"fa30df44-1240-499d-a0a2-0e8307a2c565","ctxStartEventId":"175","id":"12","ctxDetails":"{\"interface com.cloud.template.VirtualMachineTemplate\":\"f0a45ec8-9fc3-11ec-81d4-1e00ca0003f4\",\"interface com.cloud.offering.ServiceOffering\":\"92b40982-957a-4f87-b280-9788a43df57c\",\"interface com.cloud.vm.VirtualMachine\":\"fe006474-e8c7-4f78-8982-e72c058c6a3e\",\"interface com.cloud.dc.DataCenter\":\"fa30df44-1240-499d-a0a2-0e8307a2c565\"}","affinitygroupids":""}, cmdVersion: 0, status: IN_PROGRESS, processStatus: 0, resultCode: 0, result: null, initMsid: 32988737831924, completeMsid: null, lastUpdated: null, lastPolled: null, created: null, removed: null} 2022-03-10 07:01:05,228 DEBUG [c.c.a.ApiServlet] (qtp1747352992-18:ctx-76c0ce04) (logid:471b0d87) ===START=== 10.0.3.151 -- GET jobId=a882c7d6-b61b-4684-8285-079287e3c320&command=queryAsyncJobResult&response=json 2022-03-10 07:01:05,233 DEBUG [c.c.a.ApiServer] (qtp1747352992-18:ctx-76c0ce04 ctx-e63337a2) (logid:471b0d87) CIDRs from which account 'Acct[11c652b3-9fc4-11ec-81d4-1e00ca0003f4-admin] -- Account {"id": 2, "name": "admin", "uuid": "11c652b3-9fc4-11ec-81d4-1e00ca0003f4"}' is allowed to perform API calls: 0.0.0.0/0,::/0 2022-03-10 07:01:05,245 DEBUG [c.c.a.ApiServlet] (qtp1747352992-18:ctx-76c0ce04 ctx-e63337a2) (logid:471b0d87) ===END=== 10.0.3.151 -- GET jobId=a882c7d6-b61b-4684-8285-079287e3c320&command=queryAsyncJobResult&response=json 2022-03-10 07:01:05,260 DEBUG [c.c.n.NetworkModelImpl] (API-Job-Executor-2:ctx-949eff6a job-78 ctx-7b924ad9) (logid:a882c7d6) Service SecurityGroup is not supported in the network id=209 2022-03-10 07:01:05,262 DEBUG [c.c.n.NetworkModelImpl] (API-Job-Executor-2:ctx-949eff6a job-78 ctx-7b924ad9) (logid:a882c7d6) Service SecurityGroup is not supported in the network id=209 2022-03-10 07:01:05,269 DEBUG [c.c.d.DeploymentPlanningManagerImpl] (API-Job-Executor-2:ctx-949eff6a job-78 ctx-7b924ad9) (logid:a882c7d6) Adding pods to avoid lists for non-explicit VM deployment: [] 2022-03-10 07:01:05,269 DEBUG [c.c.d.DeploymentPlanningManagerImpl] (API-Job-Executor-2:ctx-949eff6a job-78 ctx-7b924ad9) (logid:a882c7d6) Adding clusters to avoid lists for non-explicit VM deployment: [] 2022-03-10 07:01:05,269 DEBUG [c.c.d.DeploymentPlanningManagerImpl] (API-Job-Executor-2:ctx-949eff6a job-78 ctx-7b924ad9) (logid:a882c7d6) Adding hosts to avoid lists for non-explicit VM deployment: [] 2022-03-10 07:01:05,269 DEBUG [c.c.d.DeploymentPlanningManagerImpl] (API-Job-Executor-2:ctx-949eff6a job-78 ctx-7b924ad9) (logid:a882c7d6) DeploymentPlanner allocation algorithm: null 2022-03-10 07:01:05,270 DEBUG [c.c.d.DeploymentPlanningManagerImpl] (API-Job-Executor-2:ctx-949eff6a job-78 ctx-7b924ad9) (logid:a882c7d6) Trying to allocate a host and storage pools from dc:1, pod:null,cluster:null, requested cpu: 500, requested ram: (512.00 MB) 536870912 2022-03-10 07:01:05,270 DEBUG [c.c.d.DeploymentPlanningManagerImpl] (API-Job-Executor-2:ctx-949eff6a job-78 ctx-7b924ad9) (logid:a882c7d6) Is ROOT volume READY (pool already allocated)?: No 2022-03-10 07:01:05,273 DEBUG [c.c.d.DeploymentPlanningManagerImpl] (API-Job-Executor-2:ctx-949eff6a job-78 ctx-7b924ad9) (logid:a882c7d6) Deploy avoids pods: [], clusters: [], hosts: [] 2022-03-10 07:01:05,274 DEBUG [c.c.d.FirstFitPlanner] (API-Job-Executor-2:ctx-949eff6a job-78 ctx-7b924ad9) (logid:a882c7d6) Searching all possible resources under this Zone: 1 2022-03-10 07:01:05,275 DEBUG [c.c.d.FirstFitPlanner] (API-Job-Executor-2:ctx-949eff6a job-78 ctx-7b924ad9) (logid:a882c7d6) Listing clusters in order of aggregate capacity, that have (at least one host with) enough CPU and RAM capacity under this Zone: 1 2022-03-10 07:01:05,278 DEBUG [c.c.d.FirstFitPlanner] (API-Job-Executor-2:ctx-949eff6a job-78 ctx-7b924ad9) (logid:a882c7d6) Removing from the clusterId list these clusters from avoid set: [] 2022-03-10 07:01:05,296 DEBUG [c.c.d.DeploymentPlanningManagerImpl] (API-Job-Executor-2:ctx-949eff6a job-78 ctx-7b924ad9) (logid:a882c7d6) Checking resources in Cluster: 1 under Pod: 1 2022-03-10 07:01:05,298 INFO [c.c.a.m.a.i.FirstFitAllocator] (API-Job-Executor-2:ctx-949eff6a job-78 ctx-7b924ad9 FirstFitRoutingAllocator) (logid:a882c7d6) Guest VM is requested with Custom[UEFI] Boot Type false 2022-03-10 07:01:05,298 DEBUG [c.c.a.m.a.i.FirstFitAllocator] (API-Job-Executor-2:ctx-949eff6a job-78 ctx-7b924ad9 FirstFitRoutingAllocator) (logid:a882c7d6) Looking for hosts in dc: 1 pod:1 cluster:1 2022-03-10 07:01:05,301 DEBUG [c.c.a.m.a.i.FirstFitAllocator] (API-Job-Executor-2:ctx-949eff6a job-78 ctx-7b924ad9 FirstFitRoutingAllocator) (logid:a882c7d6) FirstFitAllocator has 2 hosts to check for allocation: [Host {"id": "2", "name": "10.0.34.192", "uuid": "3980320d-7451-49cd-b253-7086d4ad4e51", "type"="Routing"}, Host {"id": "1", "name": "10.0.33.183", "uuid": "14c10bb4-0403-4eab-8d02-b37bdafaad25", "type"="Routing"}] 2022-03-10 07:01:05,304 DEBUG [c.c.a.m.a.i.FirstFitAllocator] (API-Job-Executor-2:ctx-949eff6a job-78 ctx-7b924ad9 FirstFitRoutingAllocator) (logid:a882c7d6) Found 2 hosts for allocation after prioritization: [Host {"id": "2", "name": "10.0.34.192", "uuid": "3980320d-7451-49cd-b253-7086d4ad4e51", "type"="Routing"}, Host {"id": "1", "name": "10.0.33.183", "uuid": "14c10bb4-0403-4eab-8d02-b37bdafaad25", "type"="Routing"}] 2022-03-10 07:01:05,304 DEBUG [c.c.a.m.a.i.FirstFitAllocator] (API-Job-Executor-2:ctx-949eff6a job-78 ctx-7b924ad9 FirstFitRoutingAllocator) (logid:a882c7d6) Looking for speed=500Mhz, Ram=512 MB 2022-03-10 07:01:05,308 DEBUG [c.c.c.CapacityManagerImpl] (API-Job-Executor-2:ctx-949eff6a job-78 ctx-7b924ad9 FirstFitRoutingAllocator) (logid:a882c7d6) Host: 2 has cpu capability (cpu:3, speed:2100) to support requested CPU: 1 and requested speed: 500 2022-03-10 07:01:05,308 DEBUG [c.c.c.CapacityManagerImpl] (API-Job-Executor-2:ctx-949eff6a job-78 ctx-7b924ad9 FirstFitRoutingAllocator) (logid:a882c7d6) Checking if host: 2 has enough capacity for requested CPU: 500 and requested RAM: (512.00 MB) 536870912 , cpuOverprovisioningFactor: 2.0 2022-03-10 07:01:05,310 DEBUG [c.c.c.CapacityManagerImpl] (API-Job-Executor-2:ctx-949eff6a job-78 ctx-7b924ad9 FirstFitRoutingAllocator) (logid:a882c7d6) Hosts's actual total CPU: 6300 and CPU after applying overprovisioning: 12600 2022-03-10 07:01:05,310 DEBUG [c.c.c.CapacityManagerImpl] (API-Job-Executor-2:ctx-949eff6a job-78 ctx-7b924ad9 FirstFitRoutingAllocator) (logid:a882c7d6) Free CPU: 12600 , Requested CPU: 500 2022-03-10 07:01:05,310 DEBUG [c.c.c.CapacityManagerImpl] (API-Job-Executor-2:ctx-949eff6a job-78 ctx-7b924ad9 FirstFitRoutingAllocator) (logid:a882c7d6) Free RAM: (8.00 GB) 8589328384 , Requested RAM: (512.00 MB) 536870912 2022-03-10 07:01:05,310 DEBUG [c.c.c.CapacityManagerImpl] (API-Job-Executor-2:ctx-949eff6a job-78 ctx-7b924ad9 FirstFitRoutingAllocator) (logid:a882c7d6) Host has enough CPU and RAM available 2022-03-10 07:01:05,310 DEBUG [c.c.c.CapacityManagerImpl] (API-Job-Executor-2:ctx-949eff6a job-78 ctx-7b924ad9 FirstFitRoutingAllocator) (logid:a882c7d6) STATS: Can alloc CPU from host: 2, used: 0, reserved: 0, actual total: 6300, total with overprovisioning: 12600; requested cpu:500,alloc_from_last_host?:false ,considerReservedCapacity?: true 2022-03-10 07:01:05,310 DEBUG [c.c.c.CapacityManagerImpl] (API-Job-Executor-2:ctx-949eff6a job-78 ctx-7b924ad9 FirstFitRoutingAllocator) (logid:a882c7d6) STATS: Can alloc MEM from host: 2, used: (0 bytes) 0, reserved: (0 bytes) 0, total: (8.00 GB) 8589328384; requested mem: (512.00 MB) 536870912, alloc_from_last_host?: false , considerReservedCapacity?: true 2022-03-10 07:01:05,310 DEBUG [c.c.a.m.a.i.FirstFitAllocator] (API-Job-Executor-2:ctx-949eff6a job-78 ctx-7b924ad9 FirstFitRoutingAllocator) (logid:a882c7d6) Found a suitable host, adding to list: 2 2022-03-10 07:01:05,314 DEBUG [c.c.c.CapacityManagerImpl] (API-Job-Executor-2:ctx-949eff6a job-78 ctx-7b924ad9 FirstFitRoutingAllocator) (logid:a882c7d6) Host: 1 has cpu capability (cpu:3, speed:2100) to support requested CPU: 1 and requested speed: 500 2022-03-10 07:01:05,314 DEBUG [c.c.c.CapacityManagerImpl] (API-Job-Executor-2:ctx-949eff6a job-78 ctx-7b924ad9 FirstFitRoutingAllocator) (logid:a882c7d6) Checking if host: 1 has enough capacity for requested CPU: 500 and requested RAM: (512.00 MB) 536870912 , cpuOverprovisioningFactor: 2.0 2022-03-10 07:01:05,315 DEBUG [c.c.c.CapacityManagerImpl] (API-Job-Executor-2:ctx-949eff6a job-78 ctx-7b924ad9 FirstFitRoutingAllocator) (logid:a882c7d6) Hosts's actual total CPU: 6300 and CPU after applying overprovisioning: 12600 2022-03-10 07:01:05,315 DEBUG [c.c.c.CapacityManagerImpl] (API-Job-Executor-2:ctx-949eff6a job-78 ctx-7b924ad9 FirstFitRoutingAllocator) (logid:a882c7d6) Free CPU: 12100 , Requested CPU: 500 2022-03-10 07:01:05,315 DEBUG [c.c.c.CapacityManagerImpl] (API-Job-Executor-2:ctx-949eff6a job-78 ctx-7b924ad9 FirstFitRoutingAllocator) (logid:a882c7d6) Free RAM: (7.00 GB) 7515586560 , Requested RAM: (512.00 MB) 536870912 2022-03-10 07:01:05,315 DEBUG [c.c.c.CapacityManagerImpl] (API-Job-Executor-2:ctx-949eff6a job-78 ctx-7b924ad9 FirstFitRoutingAllocator) (logid:a882c7d6) Host has enough CPU and RAM available 2022-03-10 07:01:05,315 DEBUG [c.c.c.CapacityManagerImpl] (API-Job-Executor-2:ctx-949eff6a job-78 ctx-7b924ad9 FirstFitRoutingAllocator) (logid:a882c7d6) STATS: Can alloc CPU from host: 1, used: 500, reserved: 0, actual total: 6300, total with overprovisioning: 12600; requested cpu:500,alloc_from_last_host?:false ,considerReservedCapacity?: true 2022-03-10 07:01:05,315 DEBUG [c.c.c.CapacityManagerImpl] (API-Job-Executor-2:ctx-949eff6a job-78 ctx-7b924ad9 FirstFitRoutingAllocator) (logid:a882c7d6) STATS: Can alloc MEM from host: 1, used: (1.00 GB) 1073741824, reserved: (0 bytes) 0, total: (8.00 GB) 8589328384; requested mem: (512.00 MB) 536870912, alloc_from_last_host?: false , considerReservedCapacity?: true 2022-03-10 07:01:05,315 DEBUG [c.c.a.m.a.i.FirstFitAllocator] (API-Job-Executor-2:ctx-949eff6a job-78 ctx-7b924ad9 FirstFitRoutingAllocator) (logid:a882c7d6) Found a suitable host, adding to list: 1 2022-03-10 07:01:05,315 DEBUG [c.c.a.m.a.i.FirstFitAllocator] (API-Job-Executor-2:ctx-949eff6a job-78 ctx-7b924ad9 FirstFitRoutingAllocator) (logid:a882c7d6) Host Allocator returning 2 suitable hosts 2022-03-10 07:01:05,317 DEBUG [c.c.d.DeploymentPlanningManagerImpl] (API-Job-Executor-2:ctx-949eff6a job-78 ctx-7b924ad9) (logid:a882c7d6) Checking suitable pools for volume (Id, Type): (12,ROOT) 2022-03-10 07:01:05,317 DEBUG [c.c.d.DeploymentPlanningManagerImpl] (API-Job-Executor-2:ctx-949eff6a job-78 ctx-7b924ad9) (logid:a882c7d6) We need to allocate new storagepool for this volume 2022-03-10 07:01:05,317 DEBUG [c.c.d.DeploymentPlanningManagerImpl] (API-Job-Executor-2:ctx-949eff6a job-78 ctx-7b924ad9) (logid:a882c7d6) Calling StoragePoolAllocators to find suitable pools 2022-03-10 07:01:05,318 DEBUG [o.a.c.s.a.LocalStoragePoolAllocator] (API-Job-Executor-2:ctx-949eff6a job-78 ctx-7b924ad9) (logid:a882c7d6) LocalStoragePoolAllocator trying to find storage pool to fit the vm 2022-03-10 07:01:05,318 DEBUG [o.a.c.s.a.ClusterScopeStoragePoolAllocator] (API-Job-Executor-2:ctx-949eff6a job-78 ctx-7b924ad9) (logid:a882c7d6) ClusterScopeStoragePoolAllocator looking for storage pool 2022-03-10 07:01:05,318 DEBUG [o.a.c.s.a.ClusterScopeStoragePoolAllocator] (API-Job-Executor-2:ctx-949eff6a job-78 ctx-7b924ad9) (logid:a882c7d6) Looking for pools in dc: 1 pod:1 cluster:1. Disabled pools will be ignored. 2022-03-10 07:01:05,319 DEBUG [o.a.c.s.a.ClusterScopeStoragePoolAllocator] (API-Job-Executor-2:ctx-949eff6a job-78 ctx-7b924ad9) (logid:a882c7d6) Found pools matching tags: [Pool[1|NetworkFilesystem]] 2022-03-10 07:01:05,320 DEBUG [o.a.c.s.a.AbstractStoragePoolAllocator] (API-Job-Executor-2:ctx-949eff6a job-78 ctx-7b924ad9) (logid:a882c7d6) Checking if storage pool is suitable, name: trl-283-v-M7-boris-stoyanov-esxi-pri1 ,poolId: 1 2022-03-10 07:01:05,324 INFO [c.c.s.StorageManagerImpl] (API-Job-Executor-2:ctx-949eff6a job-78 ctx-7b924ad9) (logid:a882c7d6) Storage pool trl-283-v-M7-boris-stoyanov-esxi-pri1 (1) does not supply IOPS capacity, assuming enough capacity 2022-03-10 07:01:05,325 DEBUG [c.c.s.StorageManagerImpl] (API-Job-Executor-2:ctx-949eff6a job-78 ctx-7b924ad9) (logid:a882c7d6) Checking pool 1 for storage, totalSize: 2197949513728, usedBytes: 1326611099648, usedPct: 0.6035675939607457, disable threshold: 0.85 2022-03-10 07:01:05,325 DEBUG [c.c.s.StorageManagerImpl] (API-Job-Executor-2:ctx-949eff6a job-78 ctx-7b924ad9) (logid:a882c7d6) Destination pool id: 1 2022-03-10 07:01:05,336 DEBUG [c.c.s.StorageManagerImpl] (API-Job-Executor-2:ctx-949eff6a job-78 ctx-7b924ad9) (logid:a882c7d6) Pool ID for the volume with ID 12 is null 2022-03-10 07:01:05,339 DEBUG [c.c.s.StorageManagerImpl] (API-Job-Executor-2:ctx-949eff6a job-78 ctx-7b924ad9) (logid:a882c7d6) Found storage pool trl-283-v-M7-boris-stoyanov-esxi-pri1 of type NetworkFilesystem with over-provisioning factor 2 2022-03-10 07:01:05,339 DEBUG [c.c.s.StorageManagerImpl] (API-Job-Executor-2:ctx-949eff6a job-78 ctx-7b924ad9) (logid:a882c7d6) Total over-provisioned capacity calculated is 2 * (1.9990 TB) 2197949513728 2022-03-10 07:01:05,339 DEBUG [c.c.s.StorageManagerImpl] (API-Job-Executor-2:ctx-949eff6a job-78 ctx-7b924ad9) (logid:a882c7d6) Total capacity of the pool trl-283-v-M7-boris-stoyanov-esxi-pri1 with ID 1 is (3.9980 TB) 4395899027456 2022-03-10 07:01:05,340 DEBUG [c.c.s.StorageManagerImpl] (API-Job-Executor-2:ctx-949eff6a job-78 ctx-7b924ad9) (logid:a882c7d6) Checking pool: 1 for storage allocation , maxSize : (3.9980 TB) 4395899027456, totalAllocatedSize : (5.91 GB) 6341787648, askingSize : (2.00 GB) 2147483648, allocated disable threshold: 0.85 2022-03-10 07:01:05,340 DEBUG [o.a.c.s.a.ClusterScopeStoragePoolAllocator] (API-Job-Executor-2:ctx-949eff6a job-78 ctx-7b924ad9) (logid:a882c7d6) ClusterScopeStoragePoolAllocator returning 1 suitable storage pools 2022-03-10 07:01:05,342 DEBUG [c.c.d.DeploymentPlanningManagerImpl] (API-Job-Executor-2:ctx-949eff6a job-78 ctx-7b924ad9) (logid:a882c7d6) Trying to find a potenial host and associated storage pools from the suitable host/pool lists for this VM 2022-03-10 07:01:05,343 DEBUG [c.c.d.DeploymentPlanningManagerImpl] (API-Job-Executor-2:ctx-949eff6a job-78 ctx-7b924ad9) (logid:a882c7d6) Checking if host: 2 can access any suitable storage pool for volume: ROOT 2022-03-10 07:01:05,344 DEBUG [c.c.d.DeploymentPlanningManagerImpl] (API-Job-Executor-2:ctx-949eff6a job-78 ctx-7b924ad9) (logid:a882c7d6) Host: 2 can access pool: 1 2022-03-10 07:01:05,347 DEBUG [c.c.d.DeploymentPlanningManagerImpl] (API-Job-Executor-2:ctx-949eff6a job-78 ctx-7b924ad9) (logid:a882c7d6) Found a potential host id: 2 name: 10.0.34.192 and associated storage pools for this VM 2022-03-10 07:01:05,348 DEBUG [c.c.d.DeploymentPlanningManagerImpl] (API-Job-Executor-2:ctx-949eff6a job-78 ctx-7b924ad9) (logid:a882c7d6) Returning Deployment Destination: Dest[Zone(Id)-Pod(Id)-Cluster(Id)-Host(Id)-Storage(Volume(Id|Type-->Pool(Id))] : Dest[Zone(1)-Pod(1)-Cluster(1)-Host(2)-Storage(Volume(12|ROOT-->Pool(1))] 2022-03-10 07:01:05,362 DEBUG [o.a.c.f.j.i.AsyncJobManagerImpl] (API-Job-Executor-2:ctx-949eff6a job-78 ctx-7b924ad9) (logid:a882c7d6) Sync job-79 execution on object VmWorkJobQueue.12 2022-03-10 07:01:05,410 DEBUG [c.c.a.ApiServlet] (qtp1747352992-13:ctx-d6cc70d2) (logid:3bdfc99a) ===START=== 10.0.3.151 -- GET listall=true&page=1&pagesize=20&showIcon=true&command=listVirtualMachinesMetrics&response=json 2022-03-10 07:01:05,415 DEBUG [c.c.a.ApiServer] (qtp1747352992-13:ctx-d6cc70d2 ctx-e27449eb) (logid:3bdfc99a) CIDRs from which account 'Acct[11c652b3-9fc4-11ec-81d4-1e00ca0003f4-admin] -- Account {"id": 2, "name": "admin", "uuid": "11c652b3-9fc4-11ec-81d4-1e00ca0003f4"}' is allowed to perform API calls: 0.0.0.0/0,::/0 2022-03-10 07:01:05,450 DEBUG [c.c.a.ApiServlet] (qtp1747352992-13:ctx-d6cc70d2 ctx-e27449eb) (logid:3bdfc99a) ===END=== 10.0.3.151 -- GET listall=true&page=1&pagesize=20&showIcon=true&command=listVirtualMachinesMetrics&response=json 2022-03-10 07:01:05,587 DEBUG [c.c.a.ApiServlet] (qtp1747352992-21:ctx-0e6236b0) (logid:ea547ecf) ===START=== 10.0.3.151 -- GET id=f0da01b8-9fc3-11ec-81d4-1e00ca0003f4&command=listOsTypes&response=json 2022-03-10 07:01:05,592 DEBUG [c.c.a.ApiServer] (qtp1747352992-21:ctx-0e6236b0 ctx-30e05037) (logid:ea547ecf) CIDRs from which account 'Acct[11c652b3-9fc4-11ec-81d4-1e00ca0003f4-admin] -- Account {"id": 2, "name": "admin", "uuid": "11c652b3-9fc4-11ec-81d4-1e00ca0003f4"}' is allowed to perform API calls: 0.0.0.0/0,::/0 2022-03-10 07:01:05,597 DEBUG [c.c.a.ApiServlet] (qtp1747352992-21:ctx-0e6236b0 ctx-30e05037) (logid:ea547ecf) ===END=== 10.0.3.151 -- GET id=f0da01b8-9fc3-11ec-81d4-1e00ca0003f4&command=listOsTypes&response=json 2022-03-10 07:01:05,598 DEBUG [c.c.a.ApiServlet] (qtp1747352992-21:ctx-2377f3a4) (logid:93ec556f) ===START=== 10.0.3.151 -- GET id=f0aa8182-9fc3-11ec-81d4-1e00ca0003f4&command=listOsTypes&response=json 2022-03-10 07:01:05,602 DEBUG [c.c.a.ApiServer] (qtp1747352992-21:ctx-2377f3a4 ctx-72ccabdf) (logid:93ec556f) CIDRs from which account 'Acct[11c652b3-9fc4-11ec-81d4-1e00ca0003f4-admin] -- Account {"id": 2, "name": "admin", "uuid": "11c652b3-9fc4-11ec-81d4-1e00ca0003f4"}' is allowed to perform API calls: 0.0.0.0/0,::/0 2022-03-10 07:01:05,606 DEBUG [c.c.a.ApiServlet] (qtp1747352992-21:ctx-2377f3a4 ctx-72ccabdf) (logid:93ec556f) ===END=== 10.0.3.151 -- GET id=f0aa8182-9fc3-11ec-81d4-1e00ca0003f4&command=listOsTypes&response=json 2022-03-10 07:01:05,610 DEBUG [o.a.c.f.j.i.AsyncJobManagerImpl] (AsyncJobMgr-Heartbeat-1:ctx-36d318a2) (logid:a1ee22c4) Execute sync-queue item: SyncQueueItemVO {id:27, queueId: 27, contentType: AsyncJob, contentId: 79, lastProcessMsid: 32988737831924, lastprocessNumber: 1, lastProcessTime: Thu Mar 10 07:01:05 UTC 2022, created: Thu Mar 10 07:01:05 UTC 2022} 2022-03-10 07:01:05,610 DEBUG [o.a.c.f.j.i.AsyncJobManagerImpl] (AsyncJobMgr-Heartbeat-1:ctx-36d318a2) (logid:a1ee22c4) Schedule queued job-79 2022-03-10 07:01:05,614 INFO [o.a.c.f.j.i.AsyncJobMonitor] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79) (logid:26532d03) Add job-79 into job monitoring 2022-03-10 07:01:05,618 DEBUG [o.a.c.f.j.i.AsyncJobManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79) (logid:a882c7d6) Executing AsyncJobVO: {id:79, userId: 2, accountId: 2, instanceType: null, instanceId: null, cmd: com.cloud.vm.VmWorkStart, cmdInfo: rO0ABXNyABhjb20uY2xvdWQudm0uVm1Xb3JrU3RhcnR9cMGsvxz73gIAC0oABGRjSWRMAAZhdm9pZHN0ADBMY29tL2Nsb3VkL2RlcGxveS9EZXBsb3ltZW50UGxhbm5lciRFeGNsdWRlTGlzdDtMAAljbHVzdGVySWR0ABBMamF2YS9sYW5nL0xvbmc7TAAGaG9zdElkcQB-AAJMAAtqb3VybmFsTmFtZXQAEkxqYXZhL2xhbmcvU3RyaW5nO0wAEXBoeXNpY2FsTmV0d29ya0lkcQB-AAJMAAdwbGFubmVycQB-AANMAAVwb2RJZHEAfgACTAAGcG9vbElkcQB-AAJMAAlyYXdQYXJhbXN0AA9MamF2YS91dGlsL01hcDtMAA1yZXNlcnZhdGlvbklkcQB-AAN4cgATY29tLmNsb3VkLnZtLlZtV29ya5-ZtlbwJWdrAgAESgAJYWNjb3VudElkSgAGdXNlcklkSgAEdm1JZEwAC2hhbmRsZXJOYW1lcQB-AAN4cAAAAAAAAAACAAAAAAAAAAIAAAAAAAAADHQAGVZpcnR1YWxNYWNoaW5lTWFuYWdlckltcGwAAAAAAAAAAXBzcgAOamF2YS5sYW5nLkxvbmc7i-SQzI8j3wIAAUoABXZhbHVleHIAEGphdmEubGFuZy5OdW1iZXKGrJUdC5TgiwIAAHhwAAAAAAAAAAFzcQB-AAgAAAAAAAAAAnBwcHEAfgAKcHNyABFqYXZhLnV0aWwuSGFzaE1hcAUH2sHDFmDRAwACRgAKbG9hZEZhY3RvckkACXRocmVzaG9sZHhwP0AAAAAAAAx3CAAAABAAAAABdAAKVm1QYXNzd29yZHQAHHJPMEFCWFFBRG5OaGRtVmtYM0JoYzNOM2IzSmt4cA, cmdVersion: 0, status: IN_PROGRESS, processStatus: 0, resultCode: 0, result: null, initMsid: 32988737831924, completeMsid: null, lastUpdated: null, lastPolled: null, created: Thu Mar 10 07:01:05 UTC 2022, removed: null} 2022-03-10 07:01:05,619 DEBUG [c.c.v.VmWorkJobDispatcher] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79) (logid:a882c7d6) Run VM work job: com.cloud.vm.VmWorkStart for VM 12, job origin: 78 2022-03-10 07:01:05,621 DEBUG [c.c.v.VmWorkJobHandlerProxy] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Execute VM work job: com.cloud.vm.VmWorkStart{"dcId":1,"podId":1,"clusterId":1,"hostId":2,"rawParams":{"VmPassword":"rO0ABXQADnNhdmVkX3Bhc3N3b3Jk"},"userId":2,"accountId":2,"vmId":12,"handlerName":"VirtualMachineManagerImpl"} 2022-03-10 07:01:05,627 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) VM instance {id: "12", name: "i-2-12-VM", uuid: "fe006474-e8c7-4f78-8982-e72c058c6a3e", type="User"} state transited from [Stopped] to [Starting] with event [StartRequested]. VM's original host: null, new host: null, host before state transition: null 2022-03-10 07:01:05,627 DEBUG [c.c.v.VirtualMachineManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Successfully transitioned to start state for VM instance {id: "12", name: "i-2-12-VM", uuid: "fe006474-e8c7-4f78-8982-e72c058c6a3e", type="User"} reservation id = 0000487c-2fc2-48f9-a691-7c714c7b0d6e 2022-03-10 07:01:05,632 DEBUG [c.c.v.VirtualMachineManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) advanceStart: DeploymentPlan is provided, using dcId:1, podId: 1, clusterId: 1, hostId: 2, poolId: null 2022-03-10 07:01:05,632 DEBUG [c.c.v.VirtualMachineManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Deploy avoids pods: null, clusters: null, hosts: null 2022-03-10 07:01:05,633 DEBUG [c.c.v.VirtualMachineManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) VM start attempt #1 2022-03-10 07:01:05,637 DEBUG [c.c.d.DeploymentPlanningManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Adding pods to avoid lists for non-explicit VM deployment: [] 2022-03-10 07:01:05,637 DEBUG [c.c.d.DeploymentPlanningManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Adding clusters to avoid lists for non-explicit VM deployment: [] 2022-03-10 07:01:05,637 DEBUG [c.c.d.DeploymentPlanningManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Adding hosts to avoid lists for non-explicit VM deployment: [] 2022-03-10 07:01:05,637 DEBUG [c.c.d.DeploymentPlanningManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) DeploymentPlanner allocation algorithm: null 2022-03-10 07:01:05,637 DEBUG [c.c.d.DeploymentPlanningManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Trying to allocate a host and storage pools from dc:1, pod:1,cluster:1, requested cpu: 500, requested ram: (512.00 MB) 536870912 2022-03-10 07:01:05,637 DEBUG [c.c.d.DeploymentPlanningManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Is ROOT volume READY (pool already allocated)?: No 2022-03-10 07:01:05,640 DEBUG [c.c.d.DeploymentPlanningManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) DeploymentPlan has host_id specified, choosing this host and making no checks on this host: 2 2022-03-10 07:01:05,640 DEBUG [c.c.d.DeploymentPlanningManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Looking for suitable pools for this host under zone: 1, pod: 1, cluster: 1 2022-03-10 07:01:05,642 DEBUG [c.c.d.DeploymentPlanningManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Checking suitable pools for volume (Id, Type): (12,ROOT) 2022-03-10 07:01:05,642 DEBUG [c.c.d.DeploymentPlanningManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) We need to allocate new storagepool for this volume 2022-03-10 07:01:05,643 DEBUG [c.c.d.DeploymentPlanningManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Calling StoragePoolAllocators to find suitable pools 2022-03-10 07:01:05,643 DEBUG [o.a.c.s.a.LocalStoragePoolAllocator] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) LocalStoragePoolAllocator trying to find storage pool to fit the vm 2022-03-10 07:01:05,643 DEBUG [o.a.c.s.a.ClusterScopeStoragePoolAllocator] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) ClusterScopeStoragePoolAllocator looking for storage pool 2022-03-10 07:01:05,643 DEBUG [o.a.c.s.a.ClusterScopeStoragePoolAllocator] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Looking for pools in dc: 1 pod:1 cluster:1. Disabled pools will be ignored. 2022-03-10 07:01:05,644 DEBUG [o.a.c.s.a.ClusterScopeStoragePoolAllocator] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Found pools matching tags: [Pool[1|NetworkFilesystem]] 2022-03-10 07:01:05,645 DEBUG [o.a.c.s.a.AbstractStoragePoolAllocator] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Checking if storage pool is suitable, name: trl-283-v-M7-boris-stoyanov-esxi-pri1 ,poolId: 1 2022-03-10 07:01:05,648 INFO [c.c.s.StorageManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Storage pool trl-283-v-M7-boris-stoyanov-esxi-pri1 (1) does not supply IOPS capacity, assuming enough capacity 2022-03-10 07:01:05,649 DEBUG [c.c.s.StorageManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Checking pool 1 for storage, totalSize: 2197949513728, usedBytes: 1326611099648, usedPct: 0.6035675939607457, disable threshold: 0.85 2022-03-10 07:01:05,649 DEBUG [c.c.s.StorageManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Destination pool id: 1 2022-03-10 07:01:05,652 DEBUG [c.c.s.StorageManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Pool ID for the volume with ID 12 is null 2022-03-10 07:01:05,655 DEBUG [c.c.s.StorageManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Found storage pool trl-283-v-M7-boris-stoyanov-esxi-pri1 of type NetworkFilesystem with over-provisioning factor 2 2022-03-10 07:01:05,655 DEBUG [c.c.s.StorageManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Total over-provisioned capacity calculated is 2 * (1.9990 TB) 2197949513728 2022-03-10 07:01:05,655 DEBUG [c.c.s.StorageManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Total capacity of the pool trl-283-v-M7-boris-stoyanov-esxi-pri1 with ID 1 is (3.9980 TB) 4395899027456 2022-03-10 07:01:05,656 DEBUG [c.c.s.StorageManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Checking pool: 1 for storage allocation , maxSize : (3.9980 TB) 4395899027456, totalAllocatedSize : (5.91 GB) 6341787648, askingSize : (2.00 GB) 2147483648, allocated disable threshold: 0.85 2022-03-10 07:01:05,656 DEBUG [o.a.c.s.a.ClusterScopeStoragePoolAllocator] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) ClusterScopeStoragePoolAllocator returning 1 suitable storage pools 2022-03-10 07:01:05,658 DEBUG [c.c.d.DeploymentPlanningManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Trying to find a potenial host and associated storage pools from the suitable host/pool lists for this VM 2022-03-10 07:01:05,658 DEBUG [c.c.d.DeploymentPlanningManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Checking if host: 2 can access any suitable storage pool for volume: ROOT 2022-03-10 07:01:05,659 DEBUG [c.c.d.DeploymentPlanningManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Host: 2 can access pool: 1 2022-03-10 07:01:05,659 DEBUG [c.c.d.DeploymentPlanningManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Found a potential host id: 2 name: 10.0.34.192 and associated storage pools for this VM 2022-03-10 07:01:05,659 DEBUG [c.c.d.DeploymentPlanningManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Returning Deployment Destination: Dest[Zone(Id)-Pod(Id)-Cluster(Id)-Host(Id)-Storage(Volume(Id|Type-->Pool(Id))] : Dest[Zone(1)-Pod(1)-Cluster(1)-Host(2)-Storage(Volume(12|ROOT-->Pool(1))] 2022-03-10 07:01:05,660 DEBUG [c.c.v.VirtualMachineManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Deployment found - Attempt #1 - P0=VM instance {id: "12", name: "i-2-12-VM", uuid: "fe006474-e8c7-4f78-8982-e72c058c6a3e", type="User"}, P0=Dest[Zone(Id)-Pod(Id)-Cluster(Id)-Host(Id)-Storage(Volume(Id|Type-->Pool(Id))] : Dest[Zone(1)-Pod(1)-Cluster(1)-Host(2)-Storage(Volume(12|ROOT-->Pool(1))] 2022-03-10 07:01:05,672 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) VM instance {id: "12", name: "i-2-12-VM", uuid: "fe006474-e8c7-4f78-8982-e72c058c6a3e", type="User"} state transited from [Starting] to [Starting] with event [OperationRetry]. VM's original host: null, new host: Host {"id": "2", "name": "10.0.34.192", "uuid": "3980320d-7451-49cd-b253-7086d4ad4e51", "type"="Routing"}, host before state transition: null 2022-03-10 07:01:05,677 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Hosts's actual total CPU: 6300 and CPU after applying overprovisioning: 12600 2022-03-10 07:01:05,677 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) We are allocating VM, increasing the used capacity of this host:2 2022-03-10 07:01:05,677 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Current Used CPU: 0 , Free CPU:12600 ,Requested CPU: 500 2022-03-10 07:01:05,677 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Current Used RAM: (0 bytes) 0 , Free RAM:(8.00 GB) 8589328384 ,Requested RAM: (512.00 MB) 536870912 2022-03-10 07:01:05,677 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) CPU STATS after allocation: for host: 2, old used: 0, old reserved: 0, actual total: 6300, total with overprovisioning: 12600; new used:500, reserved:0; requested cpu:500,alloc_from_last:false 2022-03-10 07:01:05,677 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) RAM STATS after allocation: for host: 2, old used: (0 bytes) 0, old reserved: (0 bytes) 0, total: (8.00 GB) 8589328384; new used: (512.00 MB) 536870912, reserved: (0 bytes) 0; requested mem: (512.00 MB) 536870912,alloc_from_last:false 2022-03-10 07:01:05,679 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Host: 2 has cpu capability (cpu:3, speed:2100) to support requested CPU: 1 and requested speed: 500 2022-03-10 07:01:05,679 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Checking if host: 2 has enough capacity for requested CPU: 500 and requested RAM: (512.00 MB) 536870912 , cpuOverprovisioningFactor: 2.0 2022-03-10 07:01:05,680 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Hosts's actual total CPU: 6300 and CPU after applying overprovisioning: 12600 2022-03-10 07:01:05,680 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) We need to allocate to the last host again, so checking if there is enough reserved capacity 2022-03-10 07:01:05,680 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Reserved CPU: 0 , Requested CPU: 500 2022-03-10 07:01:05,680 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Reserved RAM: (0 bytes) 0 , Requested RAM: (512.00 MB) 536870912 2022-03-10 07:01:05,680 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) STATS: Failed to alloc resource from host: 2 reservedCpu: 0, requested cpu: 500, reservedMem: (0 bytes) 0, requested mem: (512.00 MB) 536870912 2022-03-10 07:01:05,680 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Host does not have enough reserved CPU available, cannot allocate to this host. 2022-03-10 07:01:05,680 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Checking if host: 2 has enough capacity for requested CPU: 500 and requested RAM: (512.00 MB) 536870912 , cpuOverprovisioningFactor: 2.0 2022-03-10 07:01:05,682 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Hosts's actual total CPU: 6300 and CPU after applying overprovisioning: 12600 2022-03-10 07:01:05,682 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Free CPU: 12600 , Requested CPU: 500 2022-03-10 07:01:05,682 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Free RAM: (8.00 GB) 8589328384 , Requested RAM: (512.00 MB) 536870912 2022-03-10 07:01:05,682 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Host has enough CPU and RAM available 2022-03-10 07:01:05,682 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) STATS: Can alloc CPU from host: 2, used: 0, reserved: 0, actual total: 6300, total with overprovisioning: 12600; requested cpu:500,alloc_from_last_host?:false ,considerReservedCapacity?: true 2022-03-10 07:01:05,682 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) STATS: Can alloc MEM from host: 2, used: (0 bytes) 0, reserved: (0 bytes) 0, total: (8.00 GB) 8589328384; requested mem: (512.00 MB) 536870912, alloc_from_last_host?: false , considerReservedCapacity?: true 2022-03-10 07:01:05,700 DEBUG [o.a.c.e.o.NetworkOrchestrator] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Lock is acquired for network id 209 as a part of network implement 2022-03-10 07:01:05,701 DEBUG [o.a.c.e.o.NetworkOrchestrator] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Asking ExternalGuestNetworkGuru to implement Ntwk[209|Guest|8] 2022-03-10 07:01:05,719 DEBUG [o.a.c.e.o.NetworkOrchestrator] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Creating a source nat ip for network Ntwk[209|Guest|8] 2022-03-10 07:01:05,724 DEBUG [c.c.n.IpAddressManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) lock account 2 is acquired 2022-03-10 07:01:05,732 DEBUG [c.c.r.ResourceLimitManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Updating resource Type = public_ip count for Account = 2 Operation = increasing Amount = 1 2022-03-10 07:01:05,740 DEBUG [c.c.n.IpAddressManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Releasing lock account 2 2022-03-10 07:01:05,744 DEBUG [o.a.c.e.o.NetworkOrchestrator] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Asking VirtualRouter to implemenet Ntwk[209|Guest|8] 2022-03-10 07:01:05,762 DEBUG [c.c.n.r.NetworkHelperImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Allocating the VR with id=13 in datacenter Zone {"id": "1", "name": "trl-283-v-M7-boris-stoyanov", "uuid": "fa30df44-1240-499d-a0a2-0e8307a2c565"} with the hypervisor type VMware 2022-03-10 07:01:05,768 DEBUG [c.c.n.r.NetworkHelperImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Adding nic for Virtual Router in Guest network Ntwk[209|Guest|8] 2022-03-10 07:01:05,769 DEBUG [c.c.n.r.NetworkHelperImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Adding nic for Virtual Router in Control network 2022-03-10 07:01:05,773 DEBUG [o.a.c.e.o.NetworkOrchestrator] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Found existing network configuration for offering [Network Offering [3-Control-System-Control-Network]: Ntwk[202|Control|3] 2022-03-10 07:01:05,773 DEBUG [o.a.c.e.o.NetworkOrchestrator] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Releasing lock for Acct[11c573bb-9fc4-11ec-81d4-1e00ca0003f4-system] -- Account {"id": 1, "name": "system", "uuid": "11c573bb-9fc4-11ec-81d4-1e00ca0003f4"} 2022-03-10 07:01:05,776 DEBUG [c.c.n.r.NetworkHelperImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Adding nic for Virtual Router in Public network 2022-03-10 07:01:05,780 DEBUG [o.a.c.e.o.NetworkOrchestrator] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Found existing network configuration for offering [Network Offering [1-Public-System-Public-Network]: Ntwk[200|Public|1] 2022-03-10 07:01:05,780 DEBUG [o.a.c.e.o.NetworkOrchestrator] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Releasing lock for Acct[11c573bb-9fc4-11ec-81d4-1e00ca0003f4-system] -- Account {"id": 1, "name": "system", "uuid": "11c573bb-9fc4-11ec-81d4-1e00ca0003f4"} 2022-03-10 07:01:05,784 INFO [c.c.v.VirtualMachineManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) allocating virtual machine from template:d78624ff-069c-4907-a463-9a0ed1127c78 with hostname:r-13-VM and 3 networks 2022-03-10 07:01:05,786 DEBUG [c.c.v.VirtualMachineManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Allocating entries for VM: VM instance {id: "13", name: "r-13-VM", uuid: "dd2c7b06-cfd8-4dc6-953c-7efebd963ed3", type="DomainRouter"} 2022-03-10 07:01:05,789 DEBUG [c.c.v.VirtualMachineManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Allocating nics for VM instance {id: "13", name: "r-13-VM", uuid: "dd2c7b06-cfd8-4dc6-953c-7efebd963ed3", type="DomainRouter"} 2022-03-10 07:01:05,790 DEBUG [o.a.c.e.o.NetworkOrchestrator] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Allocating nic for vm VM instance {id: "13", name: "r-13-VM", uuid: "dd2c7b06-cfd8-4dc6-953c-7efebd963ed3", type="DomainRouter"} in network Ntwk[209|Guest|8] with requested profile NicProfile[0-0-null-10.1.1.1-vlan://2025] 2022-03-10 07:01:05,799 DEBUG [c.c.n.NetworkModelImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Service SecurityGroup is not supported in the network id=209 2022-03-10 07:01:05,801 DEBUG [o.a.c.e.o.NetworkOrchestrator] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Allocating nic for vm VM instance {id: "13", name: "r-13-VM", uuid: "dd2c7b06-cfd8-4dc6-953c-7efebd963ed3", type="DomainRouter"} in network Ntwk[202|Control|3] with requested profile null 2022-03-10 07:01:05,806 DEBUG [o.a.c.e.o.NetworkOrchestrator] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Allocating nic for vm VM instance {id: "13", name: "r-13-VM", uuid: "dd2c7b06-cfd8-4dc6-953c-7efebd963ed3", type="DomainRouter"} in network Ntwk[200|Public|1] with requested profile NicProfile[0-0-null-10.0.55.224-vlan://51] 2022-03-10 07:01:05,812 DEBUG [c.c.v.VirtualMachineManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Allocating disks for VM instance {id: "13", name: "r-13-VM", uuid: "dd2c7b06-cfd8-4dc6-953c-7efebd963ed3", type="DomainRouter"} 2022-03-10 07:01:05,813 INFO [o.a.c.e.o.VolumeOrchestrator] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) adding disk object ROOT-13 to r-13-VM 2022-03-10 07:01:05,819 DEBUG [c.c.v.VirtualMachineManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Allocation completed for VM: VM instance {id: "13", name: "r-13-VM", uuid: "dd2c7b06-cfd8-4dc6-953c-7efebd963ed3", type="DomainRouter"} 2022-03-10 07:01:05,829 DEBUG [c.c.n.r.NetworkHelperImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Starting router VM instance {id: "13", name: "r-13-VM", uuid: "dd2c7b06-cfd8-4dc6-953c-7efebd963ed3", type="DomainRouter"} 2022-03-10 07:01:05,838 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) VM instance {id: "13", name: "r-13-VM", uuid: "dd2c7b06-cfd8-4dc6-953c-7efebd963ed3", type="DomainRouter"} state transited from [Stopped] to [Starting] with event [StartRequested]. VM's original host: null, new host: null, host before state transition: null 2022-03-10 07:01:05,839 DEBUG [c.c.v.VirtualMachineManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Successfully transitioned to start state for VM instance {id: "13", name: "r-13-VM", uuid: "dd2c7b06-cfd8-4dc6-953c-7efebd963ed3", type="DomainRouter"} reservation id = 404d1c4c-dd87-4f85-9314-3007d4503fed 2022-03-10 07:01:05,842 DEBUG [c.c.v.VirtualMachineManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Deploy avoids pods: null, clusters: null, hosts: null 2022-03-10 07:01:05,843 DEBUG [c.c.v.VirtualMachineManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) VM start attempt #1 2022-03-10 07:01:05,848 DEBUG [c.c.d.DeploymentPlanningManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Adding pods to avoid lists: [] 2022-03-10 07:01:05,848 DEBUG [c.c.d.DeploymentPlanningManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Adding clusters to avoid lists: [] 2022-03-10 07:01:05,848 DEBUG [c.c.d.DeploymentPlanningManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Adding hosts to avoid lists: [] 2022-03-10 07:01:05,848 DEBUG [c.c.d.DeploymentPlanningManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) DeploymentPlanner allocation algorithm: null 2022-03-10 07:01:05,849 DEBUG [c.c.d.DeploymentPlanningManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Trying to allocate a host and storage pools from dc:1, pod:null,cluster:null, requested cpu: 500, requested ram: (254.00 MB) 266338304 2022-03-10 07:01:05,849 DEBUG [c.c.d.DeploymentPlanningManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Is ROOT volume READY (pool already allocated)?: No 2022-03-10 07:01:05,853 DEBUG [c.c.d.DeploymentPlanningManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Deploy avoids pods: [], clusters: [], hosts: [] 2022-03-10 07:01:05,854 DEBUG [c.c.d.FirstFitPlanner] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Searching all possible resources under this Zone: 1 2022-03-10 07:01:05,855 DEBUG [c.c.d.FirstFitPlanner] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Listing clusters in order of aggregate capacity, that have (at least one host with) enough CPU and RAM capacity under this Zone: 1 2022-03-10 07:01:05,858 DEBUG [c.c.d.FirstFitPlanner] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Removing from the clusterId list these clusters from avoid set: [] 2022-03-10 07:01:05,863 DEBUG [c.c.d.FirstFitPlanner] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) The clusterId list for the given offering tag: [2] 2022-03-10 07:01:05,867 DEBUG [c.c.d.DeploymentPlanningManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Cluster: 2 has HyperVisorType that does not match the VM, skipping this cluster 2022-03-10 07:01:05,867 DEBUG [c.c.d.DeploymentPlanningManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Could not find suitable Deployment Destination for this VM under any clusters, returning. 2022-03-10 07:01:05,868 DEBUG [c.c.d.FirstFitPlanner] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Searching all possible resources under this Zone: 1 2022-03-10 07:01:05,869 DEBUG [c.c.d.FirstFitPlanner] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Listing clusters in order of aggregate capacity, that have (at least one host with) enough CPU and RAM capacity under this Zone: 1 2022-03-10 07:01:05,870 DEBUG [c.c.d.FirstFitPlanner] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Removing from the clusterId list these clusters from avoid set: [2] 2022-03-10 07:01:05,872 DEBUG [c.c.d.FirstFitPlanner] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) The clusterId list for the given offering tag: [] 2022-03-10 07:01:05,873 DEBUG [c.c.d.FirstFitPlanner] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) No clusters found after removing disabled clusters and clusters in avoid list, returning. 2022-03-10 07:01:05,880 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) VM instance {id: "13", name: "r-13-VM", uuid: "dd2c7b06-cfd8-4dc6-953c-7efebd963ed3", type="DomainRouter"} state transited from [Starting] to [Stopped] with event [OperationFailed]. VM's original host: null, new host: null, host before state transition: null 2022-03-10 07:01:05,884 DEBUG [o.a.c.e.o.NetworkOrchestrator] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Cleaning up because we're unable to implement the network Ntwk[209|Guest|8] 2022-03-10 07:01:05,893 DEBUG [o.a.c.e.o.NetworkOrchestrator] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Lock is acquired for network Ntwk[209|Guest|8] as a part of network shutdown 2022-03-10 07:01:05,895 DEBUG [o.a.c.e.o.NetworkOrchestrator] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Sending network shutdown to VirtualRouter 2022-03-10 07:01:05,898 DEBUG [c.c.n.r.VirtualNetworkApplianceManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Stopping router VM instance {id: "13", name: "r-13-VM", uuid: "dd2c7b06-cfd8-4dc6-953c-7efebd963ed3", type="DomainRouter"} 2022-03-10 07:01:05,904 DEBUG [c.c.v.VirtualMachineManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) VM is already stopped: VM instance {id: "13", name: "r-13-VM", uuid: "dd2c7b06-cfd8-4dc6-953c-7efebd963ed3", type="DomainRouter"} 2022-03-10 07:01:05,910 DEBUG [o.a.c.e.o.NetworkOrchestrator] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Network id=209 is shutdown successfully, cleaning up corresponding resources now. 2022-03-10 07:01:05,912 DEBUG [c.c.n.g.GuestNetworkGuru] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Releasing vnet for the network id=209 2022-03-10 07:01:05,923 DEBUG [o.a.c.e.o.NetworkOrchestrator] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Lock is released for network Ntwk[209|Guest|8] as a part of network shutdown 2022-03-10 07:01:05,927 DEBUG [o.a.c.e.o.NetworkOrchestrator] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Lock is released for network id 209 as a part of network implement 2022-03-10 07:01:05,927 INFO [c.c.v.VirtualMachineManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Insufficient capacity com.cloud.exception.InsufficientServerCapacityException: Unable to create a deployment for VM instance {id: "13", name: "r-13-VM", uuid: "dd2c7b06-cfd8-4dc6-953c-7efebd963ed3", type="DomainRouter"}Scope=interface com.cloud.dc.DataCenter; id=1 at com.cloud.vm.VirtualMachineManagerImpl.orchestrateStart(VirtualMachineManagerImpl.java:1160) at com.cloud.vm.VirtualMachineManagerImpl.advanceStart(VirtualMachineManagerImpl.java:966) at com.cloud.network.router.NetworkHelperImpl.start(NetworkHelperImpl.java:288) at com.cloud.network.router.NetworkHelperImpl.startVirtualRouter(NetworkHelperImpl.java:367) at com.cloud.network.router.NetworkHelperImpl.startRouters(NetworkHelperImpl.java:352) at org.cloud.network.router.deployment.RouterDeploymentDefinition.deployVirtualRouter(RouterDeploymentDefinition.java:206) at com.cloud.network.element.VirtualRouterElement.implement(VirtualRouterElement.java:239) at org.apache.cloudstack.engine.orchestration.NetworkOrchestrator.implementNetworkElements(NetworkOrchestrator.java:1488) at org.apache.cloudstack.engine.orchestration.NetworkOrchestrator.implementNetworkElementsAndResources(NetworkOrchestrator.java:1423) at org.apache.cloudstack.engine.orchestration.NetworkOrchestrator.implementNetwork(NetworkOrchestrator.java:1338) at org.apache.cloudstack.engine.orchestration.NetworkOrchestrator.implementNetwork(NetworkOrchestrator.java:1189) at org.apache.cloudstack.engine.orchestration.NetworkOrchestrator.prepare(NetworkOrchestrator.java:1863) at com.cloud.vm.VirtualMachineManagerImpl.orchestrateStart(VirtualMachineManagerImpl.java:1199) at com.cloud.vm.VirtualMachineManagerImpl.orchestrateStart(VirtualMachineManagerImpl.java:5853) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:566) at com.cloud.vm.VmWorkJobHandlerProxy.handleVmWorkJob(VmWorkJobHandlerProxy.java:107) at com.cloud.vm.VirtualMachineManagerImpl.handleVmWorkJob(VirtualMachineManagerImpl.java:6018) at com.cloud.vm.VmWorkJobDispatcher.runJob(VmWorkJobDispatcher.java:102) at org.apache.cloudstack.framework.jobs.impl.AsyncJobManagerImpl$5.runInContext(AsyncJobManagerImpl.java:620) at org.apache.cloudstack.managed.context.ManagedContextRunnable$1.run(ManagedContextRunnable.java:48) at org.apache.cloudstack.managed.context.impl.DefaultManagedContext$1.call(DefaultManagedContext.java:55) at org.apache.cloudstack.managed.context.impl.DefaultManagedContext.callWithContext(DefaultManagedContext.java:102) at org.apache.cloudstack.managed.context.impl.DefaultManagedContext.runWithContext(DefaultManagedContext.java:52) at org.apache.cloudstack.managed.context.ManagedContextRunnable.run(ManagedContextRunnable.java:45) at org.apache.cloudstack.framework.jobs.impl.AsyncJobManagerImpl$5.run(AsyncJobManagerImpl.java:568) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) 2022-03-10 07:01:05,938 DEBUG [c.c.v.VirtualMachineManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Cleaning up resources for the vm VM instance {id: "12", name: "i-2-12-VM", uuid: "fe006474-e8c7-4f78-8982-e72c058c6a3e", type="User"} in Starting state 2022-03-10 07:01:05,953 DEBUG [c.c.a.t.Request] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Seq 2-3538984882183077896: Sending { Cmd , MgmtId: 32988737831924, via: 2(10.0.34.192), Ver: v1, Flags: 100011, [{"com.cloud.agent.api.StopCommand":{"isProxy":"false","checkBeforeCleanup":"false","forceStop":"false","volumesToDisconnect":[],"vmName":"i-2-12-VM","executeInSequence":"false","wait":"0","bypassHostMaintenance":"false"}}] } 2022-03-10 07:01:05,954 DEBUG [c.c.a.t.Request] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Seq 2-3538984882183077896: Executing: { Cmd , MgmtId: 32988737831924, via: 2(10.0.34.192), Ver: v1, Flags: 100011, [{"com.cloud.agent.api.StopCommand":{"isProxy":"false","checkBeforeCleanup":"false","forceStop":"false","volumesToDisconnect":[],"vmName":"i-2-12-VM","executeInSequence":"false","wait":"0","bypassHostMaintenance":"false"}}] } 2022-03-10 07:01:05,954 DEBUG [c.c.a.m.DirectAgentAttache] (DirectAgent-25:ctx-14d6ce7a) (logid:aba040b3) Seq 2-3538984882183077896: Executing request 2022-03-10 07:01:05,955 INFO [c.c.h.v.r.VmwareResource] (DirectAgent-25:ctx-14d6ce7a 10.0.34.192, job-78/job-79, cmd: StopCommand) (logid:a882c7d6) Executing resource StopCommand: {"isProxy":false,"checkBeforeCleanup":false,"forceStop":false,"volumesToDisconnect":[],"vmName":"i-2-12-VM","executeInSequence":false,"wait":0,"bypassHostMaintenance":false} 2022-03-10 07:01:05,973 DEBUG [c.c.h.v.m.HostMO] (DirectAgent-25:ctx-14d6ce7a 10.0.34.192, job-78/job-79, cmd: StopCommand) (logid:a882c7d6) find VM i-2-12-VM on host 2022-03-10 07:01:05,973 INFO [c.c.h.v.m.HostMO] (DirectAgent-25:ctx-14d6ce7a 10.0.34.192, job-78/job-79, cmd: StopCommand) (logid:a882c7d6) VM i-2-12-VM not found in host cache 2022-03-10 07:01:05,973 DEBUG [c.c.h.v.m.HostMO] (DirectAgent-25:ctx-14d6ce7a 10.0.34.192, job-78/job-79, cmd: StopCommand) (logid:a882c7d6) load VM cache on host 2022-03-10 07:01:05,985 INFO [c.c.h.v.r.VmwareResource] (DirectAgent-25:ctx-14d6ce7a 10.0.34.192, job-78/job-79, cmd: StopCommand) (logid:a882c7d6) VM i-2-12-VM is no longer on the expected host in vSphere 2022-03-10 07:01:05,985 DEBUG [c.c.a.m.DirectAgentAttache] (DirectAgent-25:ctx-14d6ce7a) (logid:a882c7d6) Seq 2-3538984882183077896: Response Received: 2022-03-10 07:01:05,986 DEBUG [c.c.a.t.Request] (DirectAgent-25:ctx-14d6ce7a) (logid:a882c7d6) Seq 2-3538984882183077896: Processing: { Ans: , MgmtId: 32988737831924, via: 2(10.0.34.192), Ver: v1, Flags: 10, [{"com.cloud.agent.api.StopAnswer":{"result":"true","details":"VM i-2-12-VM is no longer on the expected host in vSphere","wait":"0","bypassHostMaintenance":"false"}}] } 2022-03-10 07:01:05,986 DEBUG [c.c.a.t.Request] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Seq 2-3538984882183077896: Received: { Ans: , MgmtId: 32988737831924, via: 2(10.0.34.192), Ver: v1, Flags: 10, { StopAnswer } } 2022-03-10 07:01:06,000 DEBUG [c.c.v.VirtualMachineManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Successfully released network resources for the VM VM instance {id: "12", name: "i-2-12-VM", uuid: "fe006474-e8c7-4f78-8982-e72c058c6a3e", type="User"} in Starting state 2022-03-10 07:01:06,003 DEBUG [c.c.v.VirtualMachineManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Successfully released storage resources for the VM VM instance {id: "12", name: "i-2-12-VM", uuid: "fe006474-e8c7-4f78-8982-e72c058c6a3e", type="User"} in Starting state 2022-03-10 07:01:06,003 DEBUG [c.c.v.VirtualMachineManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Successfully cleaned up resources for the VM VM instance {id: "12", name: "i-2-12-VM", uuid: "fe006474-e8c7-4f78-8982-e72c058c6a3e", type="User"} in Starting state 2022-03-10 07:01:06,004 DEBUG [c.c.v.VirtualMachineManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) VM start attempt #2 2022-03-10 07:01:06,007 DEBUG [c.c.d.DeploymentPlanningManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Adding pods to avoid lists for non-explicit VM deployment: [] 2022-03-10 07:01:06,008 DEBUG [c.c.d.DeploymentPlanningManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Adding clusters to avoid lists for non-explicit VM deployment: [] 2022-03-10 07:01:06,008 DEBUG [c.c.d.DeploymentPlanningManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Adding hosts to avoid lists for non-explicit VM deployment: [] 2022-03-10 07:01:06,008 DEBUG [c.c.d.DeploymentPlanningManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) DeploymentPlanner allocation algorithm: null 2022-03-10 07:01:06,008 DEBUG [c.c.d.DeploymentPlanningManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Trying to allocate a host and storage pools from dc:1, pod:1,cluster:1, requested cpu: 500, requested ram: (512.00 MB) 536870912 2022-03-10 07:01:06,008 DEBUG [c.c.d.DeploymentPlanningManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Is ROOT volume READY (pool already allocated)?: No 2022-03-10 07:01:06,011 DEBUG [c.c.d.DeploymentPlanningManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) DeploymentPlan has host_id specified, choosing this host and making no checks on this host: 2 2022-03-10 07:01:06,012 DEBUG [c.c.d.DeploymentPlanningManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) The specified host is in avoid set 2022-03-10 07:01:06,012 DEBUG [c.c.d.DeploymentPlanningManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Cannot deploy to specified host, returning. 2022-03-10 07:01:06,021 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) VM instance {id: "12", name: "i-2-12-VM", uuid: "fe006474-e8c7-4f78-8982-e72c058c6a3e", type="User"} state transited from [Starting] to [Stopped] with event [OperationFailed]. VM's original host: null, new host: null, host before state transition: Host {"id": "2", "name": "10.0.34.192", "uuid": "3980320d-7451-49cd-b253-7086d4ad4e51", "type"="Routing"} 2022-03-10 07:01:06,028 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Hosts's actual total CPU: 6300 and CPU after applying overprovisioning: 12600 2022-03-10 07:01:06,028 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Hosts's actual total RAM: (8.00 GB) 8589328384 and RAM after applying overprovisioning: (8.00 GB) 8589328384 2022-03-10 07:01:06,035 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) release cpu from host: 2, old used: 500,reserved: 0, actual total: 6300, total with overprovisioning: 12600; new used: 0,reserved:0; movedfromreserved: false,moveToReserveredfalse 2022-03-10 07:01:06,044 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) release mem from host: 2, old used: (512.00 MB) 536870912,reserved: (0 bytes) 0, total: (8.00 GB) 8589328384; new used: (0 bytes) 0,reserved:(0 bytes) 0; movedfromreserved: false,moveToReserveredfalse 2022-03-10 07:01:06,059 ERROR [c.c.v.VmWorkJobHandlerProxy] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Invocation exception, caused by: com.cloud.exception.InsufficientServerCapacityException: Unable to create a deployment for VM instance {id: "12", name: "i-2-12-VM", uuid: "fe006474-e8c7-4f78-8982-e72c058c6a3e", type="User"}Scope=interface com.cloud.dc.DataCenter; id=1 2022-03-10 07:01:06,059 INFO [c.c.v.VmWorkJobHandlerProxy] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79 ctx-2c66a675) (logid:a882c7d6) Rethrow exception com.cloud.exception.InsufficientServerCapacityException: Unable to create a deployment for VM instance {id: "12", name: "i-2-12-VM", uuid: "fe006474-e8c7-4f78-8982-e72c058c6a3e", type="User"}Scope=interface com.cloud.dc.DataCenter; id=1 2022-03-10 07:01:06,060 DEBUG [c.c.v.VmWorkJobDispatcher] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79) (logid:a882c7d6) Done with run of VM work job: com.cloud.vm.VmWorkStart for VM 12, job origin: 78 2022-03-10 07:01:06,060 ERROR [c.c.v.VmWorkJobDispatcher] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79) (logid:a882c7d6) Unable to complete AsyncJobVO: {id:79, userId: 2, accountId: 2, instanceType: null, instanceId: null, cmd: com.cloud.vm.VmWorkStart, cmdInfo: rO0ABXNyABhjb20uY2xvdWQudm0uVm1Xb3JrU3RhcnR9cMGsvxz73gIAC0oABGRjSWRMAAZhdm9pZHN0ADBMY29tL2Nsb3VkL2RlcGxveS9EZXBsb3ltZW50UGxhbm5lciRFeGNsdWRlTGlzdDtMAAljbHVzdGVySWR0ABBMamF2YS9sYW5nL0xvbmc7TAAGaG9zdElkcQB-AAJMAAtqb3VybmFsTmFtZXQAEkxqYXZhL2xhbmcvU3RyaW5nO0wAEXBoeXNpY2FsTmV0d29ya0lkcQB-AAJMAAdwbGFubmVycQB-AANMAAVwb2RJZHEAfgACTAAGcG9vbElkcQB-AAJMAAlyYXdQYXJhbXN0AA9MamF2YS91dGlsL01hcDtMAA1yZXNlcnZhdGlvbklkcQB-AAN4cgATY29tLmNsb3VkLnZtLlZtV29ya5-ZtlbwJWdrAgAESgAJYWNjb3VudElkSgAGdXNlcklkSgAEdm1JZEwAC2hhbmRsZXJOYW1lcQB-AAN4cAAAAAAAAAACAAAAAAAAAAIAAAAAAAAADHQAGVZpcnR1YWxNYWNoaW5lTWFuYWdlckltcGwAAAAAAAAAAXBzcgAOamF2YS5sYW5nLkxvbmc7i-SQzI8j3wIAAUoABXZhbHVleHIAEGphdmEubGFuZy5OdW1iZXKGrJUdC5TgiwIAAHhwAAAAAAAAAAFzcQB-AAgAAAAAAAAAAnBwcHEAfgAKcHNyABFqYXZhLnV0aWwuSGFzaE1hcAUH2sHDFmDRAwACRgAKbG9hZEZhY3RvckkACXRocmVzaG9sZHhwP0AAAAAAAAx3CAAAABAAAAABdAAKVm1QYXNzd29yZHQAHHJPMEFCWFFBRG5OaGRtVmtYM0JoYzNOM2IzSmt4cA, cmdVersion: 0, status: IN_PROGRESS, processStatus: 0, resultCode: 0, result: null, initMsid: 32988737831924, completeMsid: null, lastUpdated: null, lastPolled: null, created: Thu Mar 10 07:01:05 UTC 2022, removed: null}, job origin:78 com.cloud.exception.InsufficientServerCapacityException: Unable to create a deployment for VM instance {id: "12", name: "i-2-12-VM", uuid: "fe006474-e8c7-4f78-8982-e72c058c6a3e", type="User"}Scope=interface com.cloud.dc.DataCenter; id=1 at com.cloud.vm.VirtualMachineManagerImpl.orchestrateStart(VirtualMachineManagerImpl.java:1160) at com.cloud.vm.VirtualMachineManagerImpl.orchestrateStart(VirtualMachineManagerImpl.java:5853) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:566) at com.cloud.vm.VmWorkJobHandlerProxy.handleVmWorkJob(VmWorkJobHandlerProxy.java:107) at com.cloud.vm.VirtualMachineManagerImpl.handleVmWorkJob(VirtualMachineManagerImpl.java:6018) at com.cloud.vm.VmWorkJobDispatcher.runJob(VmWorkJobDispatcher.java:102) at org.apache.cloudstack.framework.jobs.impl.AsyncJobManagerImpl$5.runInContext(AsyncJobManagerImpl.java:620) at org.apache.cloudstack.managed.context.ManagedContextRunnable$1.run(ManagedContextRunnable.java:48) at org.apache.cloudstack.managed.context.impl.DefaultManagedContext$1.call(DefaultManagedContext.java:55) at org.apache.cloudstack.managed.context.impl.DefaultManagedContext.callWithContext(DefaultManagedContext.java:102) at org.apache.cloudstack.managed.context.impl.DefaultManagedContext.runWithContext(DefaultManagedContext.java:52) at org.apache.cloudstack.managed.context.ManagedContextRunnable.run(ManagedContextRunnable.java:45) at org.apache.cloudstack.framework.jobs.impl.AsyncJobManagerImpl$5.run(AsyncJobManagerImpl.java:568) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) 2022-03-10 07:01:06,078 DEBUG [o.a.c.f.j.i.AsyncJobManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79) (logid:a882c7d6) Complete async job-79, jobStatus: FAILED, resultCode: 0, result: rO0ABXNyADdjb20uY2xvdWQuZXhjZXB0aW9uLkluc3VmZmljaWVudFNlcnZlckNhcGFjaXR5RXhjZXB0aW9uAAAAAFZNT3ACAAFaABVhZmZpbml0eUdyb3Vwc0FwcGxpZWR4cgAxY29tLmNsb3VkLmV4Y2VwdGlvbi5JbnN1ZmZpY2llbnRDYXBhY2l0eUV4Y2VwdGlvbgAAAABWTU91AgACTAACaWR0ABBMamF2YS9sYW5nL0xvbmc7TAAFc2NvcGV0ABFMamF2YS9sYW5nL0NsYXNzO3hyACJjb20uY2xvdWQuZXhjZXB0aW9uLkNsb3VkRXhjZXB0aW9ueeiOG_HkRMcCAAJMAAtjc0Vycm9yQ29kZXQAE0xqYXZhL2xhbmcvSW50ZWdlcjtMAAZpZExpc3R0ABVMamF2YS91dGlsL0FycmF5TGlzdDt4cgATamF2YS5sYW5nLkV4Y2VwdGlvbtD9Hz4aOxzEAgAAeHIAE2phdmEubGFuZy5UaHJvd2FibGXVxjUnOXe4ywMABEwABWNhdXNldAAVTGphdmEvbGFuZy9UaHJvd2FibGU7TAANZGV0YWlsTWVzc2FnZXQAEkxqYXZhL2xhbmcvU3RyaW5nO1sACnN0YWNrVHJhY2V0AB5bTGphdmEvbGFuZy9TdGFja1RyYWNlRWxlbWVudDtMABRzdXBwcmVzc2VkRXhjZXB0aW9uc3QAEExqYXZhL3V0aWwvTGlzdDt4cHEAfgANdACGVW5hYmxlIHRvIGNyZWF0ZSBhIGRlcGxveW1lbnQgZm9yIFZNIGluc3RhbmNlIHtpZDogIjEyIiwgbmFtZTogImktMi0xMi1WTSIsIHV1aWQ6ICJmZTAwNjQ3NC1lOGM3LTRmNzgtODk4Mi1lNzJjMDU4YzZhM2UiLCB0eXBlPSJVc2VyIn11cgAeW0xqYXZhLmxhbmcuU3RhY2tUcmFjZUVsZW1lbnQ7AkYqPDz9IjkCAAB4cAAAABVzcgAbamF2YS5sYW5nLlN0YWNrVHJhY2VFbGVtZW50YQnFmiY23YUCAAhCAAZmb3JtYXRJAApsaW5lTnVtYmVyTAAPY2xhc3NMb2FkZXJOYW1lcQB-AApMAA5kZWNsYXJpbmdDbGFzc3EAfgAKTAAIZmlsZU5hbWVxAH4ACkwACm1ldGhvZE5hbWVxAH4ACkwACm1vZHVsZU5hbWVxAH4ACkwADW1vZHVsZVZlcnNpb25xAH4ACnhwAQAABIh0AANhcHB0ACZjb20uY2xvdWQudm0uVmlydHVhbE1hY2hpbmVNYW5hZ2VySW1wbHQAHlZpcnR1YWxNYWNoaW5lTWFuYWdlckltcGwuamF2YXQAEG9yY2hlc3RyYXRlU3RhcnRwcHNxAH4AEQEAABbdcQB-ABNxAH4AFHEAfgAVcQB-ABZwcHNxAH4AEQL____-cHQALWpkay5pbnRlcm5hbC5yZWZsZWN0Lk5hdGl2ZU1ldGhvZEFjY2Vzc29ySW1wbHQAHU5hdGl2ZU1ldGhvZEFjY2Vzc29ySW1wbC5qYXZhdAAHaW52b2tlMHQACWphdmEuYmFzZXQACTExLjAuMTQuMXNxAH4AEQIAAAA-cHEAfgAZcQB-ABp0AAZpbnZva2VxAH4AHHEAfgAdc3EAfgARAgAAACtwdAAxamRrLmludGVybmFsLnJlZmxlY3QuRGVsZWdhdGluZ01ldGhvZEFjY2Vzc29ySW1wbHQAIURlbGVnYXRpbmdNZXRob2RBY2Nlc3NvckltcGwuamF2YXEAfgAfcQB-ABxxAH4AHXNxAH4AEQIAAAI2cHQAGGphdmEubGFuZy5yZWZsZWN0Lk1ldGhvZHQAC01ldGhvZC5qYXZhcQB-AB9xAH4AHHEAfgAdc3EAfgARAQAAAGtxAH4AE3QAImNvbS5jbG91ZC52bS5WbVdvcmtKb2JIYW5kbGVyUHJveHl0ABpWbVdvcmtKb2JIYW5kbGVyUHJveHkuamF2YXQAD2hhbmRsZVZtV29ya0pvYnBwc3EAfgARAQAAF4JxAH4AE3EAfgAUcQB-ABVxAH4AKXBwc3EAfgARAQAAAGZxAH4AE3QAIGNvbS5jbG91ZC52bS5WbVdvcmtKb2JEaXNwYXRjaGVydAAYVm1Xb3JrSm9iRGlzcGF0Y2hlci5qYXZhdAAGcnVuSm9icHBzcQB-ABEBAAACbHEAfgATdAA_b3JnLmFwYWNoZS5jbG91ZHN0YWNrLmZyYW1ld29yay5qb2JzLmltcGwuQXN5bmNKb2JNYW5hZ2VySW1wbCQ1dAAYQXN5bmNKb2JNYW5hZ2VySW1wbC5qYXZhdAAMcnVuSW5Db250ZXh0cHBzcQB-ABEBAAAAMHEAfgATdAA-b3JnLmFwYWNoZS5jbG91ZHN0YWNrLm1hbmFnZWQuY29udGV4dC5NYW5hZ2VkQ29udGV4dFJ1bm5hYmxlJDF0ABtNYW5hZ2VkQ29udGV4dFJ1bm5hYmxlLmphdmF0AANydW5wcHNxAH4AEQEAAAA3cQB-ABN0AEJvcmcuYXBhY2hlLmNsb3Vkc3RhY2subWFuYWdlZC5jb250ZXh0LmltcGwuRGVmYXVsdE1hbmFnZWRDb250ZXh0JDF0ABpEZWZhdWx0TWFuYWdlZENvbnRleHQuamF2YXQABGNhbGxwcHNxAH4AEQEAAABmcQB-ABN0AEBvcmcuYXBhY2hlLmNsb3Vkc3RhY2subWFuYWdlZC5jb250ZXh0LmltcGwuRGVmYXVsdE1hbmFnZWRDb250ZXh0cQB-ADl0AA9jYWxsV2l0aENvbnRleHRwcHNxAH4AEQEAAAA0cQB-ABNxAH4APHEAfgA5dAAOcnVuV2l0aENvbnRleHRwcHNxAH4AEQEAAAAtcQB-ABN0ADxvcmcuYXBhY2hlLmNsb3Vkc3RhY2subWFuYWdlZC5jb250ZXh0Lk1hbmFnZWRDb250ZXh0UnVubmFibGVxAH4ANXEAfgA2cHBzcQB-ABEBAAACOHEAfgATcQB-ADBxAH4AMXEAfgA2cHBzcQB-ABECAAACA3B0AC5qYXZhLnV0aWwuY29uY3VycmVudC5FeGVjdXRvcnMkUnVubmFibGVBZGFwdGVydAAORXhlY3V0b3JzLmphdmFxAH4AOnEAfgAccQB-AB1zcQB-ABECAAABCHB0AB9qYXZhLnV0aWwuY29uY3VycmVudC5GdXR1cmVUYXNrdAAPRnV0dXJlVGFzay5qYXZhcQB-ADZxAH4AHHEAfgAdc3EAfgARAgAABGhwdAAnamF2YS51dGlsLmNvbmN1cnJlbnQuVGhyZWFkUG9vbEV4ZWN1dG9ydAAXVGhyZWFkUG9vbEV4ZWN1dG9yLmphdmF0AAlydW5Xb3JrZXJxAH4AHHEAfgAdc3EAfgARAgAAAnRwdAAuamF2YS51dGlsLmNvbmN1cnJlbnQuVGhyZWFkUG9vbEV4ZWN1dG9yJFdvcmtlcnEAfgBLcQB-ADZxAH4AHHEAfgAdc3EAfgARAgAAAz1wdAAQamF2YS5sYW5nLlRocmVhZHQAC1RocmVhZC5qYXZhcQB-ADZxAH4AHHEAfgAdc3IAH2phdmEudXRpbC5Db2xsZWN0aW9ucyRFbXB0eUxpc3R6uBe0PKee3gIAAHhweHNyABFqYXZhLmxhbmcuSW50ZWdlchLioKT3gYc4AgABSQAFdmFsdWV4cgAQamF2YS5sYW5nLk51bWJlcoaslR0LlOCLAgAAeHAAABDvc3IAE2phdmEudXRpbC5BcnJheUxpc3R4gdIdmcdhnQMAAUkABHNpemV4cAAAAAB3BAAAAAB4c3IADmphdmEubGFuZy5Mb25nO4vkkMyPI98CAAFKAAV2YWx1ZXhxAH4AVQAAAAAAAAABdnIAF2NvbS5jbG91ZC5kYy5EYXRhQ2VudGVyU-gkGtLTtlwCAAB4cAA 2022-03-10 07:01:06,079 DEBUG [o.a.c.f.j.i.AsyncJobManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79) (logid:a882c7d6) Publish async job-79 complete on message bus 2022-03-10 07:01:06,079 DEBUG [o.a.c.f.j.i.AsyncJobManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79) (logid:a882c7d6) Wake up jobs related to job-79 2022-03-10 07:01:06,079 DEBUG [o.a.c.f.j.i.AsyncJobManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79) (logid:a882c7d6) Update db status for job-79 2022-03-10 07:01:06,080 DEBUG [o.a.c.f.j.i.AsyncJobManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79) (logid:a882c7d6) Wake up jobs joined with job-79 and disjoin all subjobs created from job- 79 2022-03-10 07:01:06,086 DEBUG [o.a.c.f.j.i.AsyncJobManagerImpl] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79) (logid:a882c7d6) Done executing com.cloud.vm.VmWorkStart for job-79 2022-03-10 07:01:06,087 INFO [o.a.c.f.j.i.AsyncJobMonitor] (Work-Job-Executor-2:ctx-f447dfc9 job-78/job-79) (logid:a882c7d6) Remove job-79 from job monitoring 2022-03-10 07:01:06,105 DEBUG [o.a.c.f.j.i.AsyncJobManagerImpl] (API-Job-Executor-2:ctx-949eff6a job-78 ctx-7b924ad9) (logid:a882c7d6) Sync job-82 execution on object VmWorkJobQueue.12 2022-03-10 07:01:06,235 INFO [c.c.h.v.r.VmwareResource] (DirectAgentCronJob-9:ctx-40c054da) (logid:4d34735a) Scan hung worker VM to recycle 2022-03-10 07:01:06,287 DEBUG [c.c.a.m.DirectAgentAttache] (DirectAgentCronJob-9:ctx-40c054da) (logid:4d34735a) Ping from 1(10.0.33.183) 2022-03-10 07:01:06,287 DEBUG [c.c.v.VirtualMachinePowerStateSyncImpl] (DirectAgentCronJob-9:ctx-40c054da) (logid:4d34735a) Process host VM state report from ping process. host: 1 2022-03-10 07:01:06,289 DEBUG [c.c.v.VirtualMachinePowerStateSyncImpl] (DirectAgentCronJob-9:ctx-40c054da) (logid:4d34735a) Process VM state report. host: 1, number of records in report: 1 2022-03-10 07:01:06,289 DEBUG [c.c.v.VirtualMachinePowerStateSyncImpl] (DirectAgentCronJob-9:ctx-40c054da) (logid:4d34735a) VM state report. host: 1, vm id: 2, power state: PowerOn 2022-03-10 07:01:06,294 DEBUG [c.c.v.VirtualMachinePowerStateSyncImpl] (DirectAgentCronJob-9:ctx-40c054da) (logid:4d34735a) VM state report is updated. host: 1, vm id: 2, power state: PowerOn 2022-03-10 07:01:06,301 DEBUG [c.c.c.CapacityManagerImpl] (DirectAgentCronJob-9:ctx-40c054da) (logid:4d34735a) VM instance {id: "2", name: "v-2-VM", uuid: "f7dcfdda-d005-445f-b55b-6388ddad1773", type="ConsoleProxy"} state transited from [Running] to [Running] with event [FollowAgentPowerOnReport]. VM's original host: Host {"id": "1", "name": "10.0.33.183", "uuid": "14c10bb4-0403-4eab-8d02-b37bdafaad25", "type"="Routing"}, new host: Host {"id": "1", "name": "10.0.33.183", "uuid": "14c10bb4-0403-4eab-8d02-b37bdafaad25", "type"="Routing"}, host before state transition: Host {"id": "1", "name": "10.0.33.183", "uuid": "14c10bb4-0403-4eab-8d02-b37bdafaad25", "type"="Routing"} 2022-03-10 07:01:06,303 DEBUG [c.c.v.VirtualMachinePowerStateSyncImpl] (DirectAgentCronJob-9:ctx-40c054da) (logid:4d34735a) Done with process of VM state report. host: 1 2022-03-10 07:01:06,441 INFO [c.c.h.v.r.VmwareResource] (DirectAgentCronJob-10:ctx-dd059fc1) (logid:389207c4) Scan hung worker VM to recycle 2022-03-10 07:01:06,490 DEBUG [c.c.a.m.DirectAgentAttache] (DirectAgentCronJob-10:ctx-dd059fc1) (logid:389207c4) Ping from 2(10.0.34.192) 2022-03-10 07:01:06,490 DEBUG [c.c.v.VirtualMachinePowerStateSyncImpl] (DirectAgentCronJob-10:ctx-dd059fc1) (logid:389207c4) Process host VM state report from ping process. host: 2 2022-03-10 07:01:06,490 DEBUG [c.c.v.VirtualMachinePowerStateSyncImpl] (DirectAgentCronJob-10:ctx-dd059fc1) (logid:389207c4) Process VM state report. host: 2, number of records in report: 0 2022-03-10 07:01:06,491 DEBUG [c.c.v.VirtualMachinePowerStateSyncImpl] (DirectAgentCronJob-10:ctx-dd059fc1) (logid:389207c4) Done with process of VM state report. host: 2 2022-03-10 07:01:06,843 DEBUG [o.a.c.h.HAManagerImpl] (BackgroundTaskPollManager-4:ctx-19ce1ca4) (logid:31d33593) HA health check task is running... 2022-03-10 07:01:07,598 DEBUG [o.a.c.f.j.i.AsyncJobManagerImpl] (AsyncJobMgr-Heartbeat-1:ctx-14f1478c) (logid:53f8d590) Execute sync-queue item: SyncQueueItemVO {id:28, queueId: 27, contentType: AsyncJob, contentId: 82, lastProcessMsid: 32988737831924, lastprocessNumber: 2, lastProcessTime: Thu Mar 10 07:01:07 UTC 2022, created: Thu Mar 10 07:01:06 UTC 2022} 2022-03-10 07:01:07,598 DEBUG [o.a.c.f.j.i.AsyncJobManagerImpl] (AsyncJobMgr-Heartbeat-1:ctx-14f1478c) (logid:53f8d590) Schedule queued job-82 2022-03-10 07:01:07,604 INFO [o.a.c.f.j.i.AsyncJobMonitor] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82) (logid:fd1f8055) Add job-82 into job monitoring 2022-03-10 07:01:07,612 DEBUG [o.a.c.f.j.i.AsyncJobManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82) (logid:a882c7d6) Executing AsyncJobVO: {id:82, userId: 2, accountId: 2, instanceType: null, instanceId: null, cmd: com.cloud.vm.VmWorkStart, cmdInfo: rO0ABXNyABhjb20uY2xvdWQudm0uVm1Xb3JrU3RhcnR9cMGsvxz73gIAC0oABGRjSWRMAAZhdm9pZHN0ADBMY29tL2Nsb3VkL2RlcGxveS9EZXBsb3ltZW50UGxhbm5lciRFeGNsdWRlTGlzdDtMAAljbHVzdGVySWR0ABBMamF2YS9sYW5nL0xvbmc7TAAGaG9zdElkcQB-AAJMAAtqb3VybmFsTmFtZXQAEkxqYXZhL2xhbmcvU3RyaW5nO0wAEXBoeXNpY2FsTmV0d29ya0lkcQB-AAJMAAdwbGFubmVycQB-AANMAAVwb2RJZHEAfgACTAAGcG9vbElkcQB-AAJMAAlyYXdQYXJhbXN0AA9MamF2YS91dGlsL01hcDtMAA1yZXNlcnZhdGlvbklkcQB-AAN4cgATY29tLmNsb3VkLnZtLlZtV29ya5-ZtlbwJWdrAgAESgAJYWNjb3VudElkSgAGdXNlcklkSgAEdm1JZEwAC2hhbmRsZXJOYW1lcQB-AAN4cAAAAAAAAAACAAAAAAAAAAIAAAAAAAAADHQAGVZpcnR1YWxNYWNoaW5lTWFuYWdlckltcGwAAAAAAAAAAHBwcHBwcHBwc3IAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA_QAAAAAAADHcIAAAAEAAAAAF0AApWbVBhc3N3b3JkdAAcck8wQUJYUUFEbk5oZG1Wa1gzQmhjM04zYjNKa3hw, cmdVersion: 0, status: IN_PROGRESS, processStatus: 0, resultCode: 0, result: null, initMsid: 32988737831924, completeMsid: null, lastUpdated: null, lastPolled: null, created: Thu Mar 10 07:01:06 UTC 2022, removed: null} 2022-03-10 07:01:07,613 DEBUG [c.c.v.VmWorkJobDispatcher] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82) (logid:a882c7d6) Run VM work job: com.cloud.vm.VmWorkStart for VM 12, job origin: 78 2022-03-10 07:01:07,615 DEBUG [c.c.v.VmWorkJobHandlerProxy] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) Execute VM work job: com.cloud.vm.VmWorkStart{"dcId":0,"rawParams":{"VmPassword":"rO0ABXQADnNhdmVkX3Bhc3N3b3Jk"},"userId":2,"accountId":2,"vmId":12,"handlerName":"VirtualMachineManagerImpl"} 2022-03-10 07:01:07,624 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) VM instance {id: "12", name: "i-2-12-VM", uuid: "fe006474-e8c7-4f78-8982-e72c058c6a3e", type="User"} state transited from [Stopped] to [Starting] with event [StartRequested]. VM's original host: null, new host: null, host before state transition: null 2022-03-10 07:01:07,625 DEBUG [c.c.v.VirtualMachineManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) Successfully transitioned to start state for VM instance {id: "12", name: "i-2-12-VM", uuid: "fe006474-e8c7-4f78-8982-e72c058c6a3e", type="User"} reservation id = 94934246-b55a-48e8-ab8f-2b9582c2f36d 2022-03-10 07:01:07,628 DEBUG [c.c.v.VirtualMachineManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) Deploy avoids pods: null, clusters: null, hosts: null 2022-03-10 07:01:07,629 DEBUG [c.c.v.VirtualMachineManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) VM start attempt #1 2022-03-10 07:01:07,633 DEBUG [c.c.d.DeploymentPlanningManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) Adding pods to avoid lists for non-explicit VM deployment: [] 2022-03-10 07:01:07,633 DEBUG [c.c.d.DeploymentPlanningManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) Adding clusters to avoid lists for non-explicit VM deployment: [] 2022-03-10 07:01:07,633 DEBUG [c.c.d.DeploymentPlanningManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) Adding hosts to avoid lists for non-explicit VM deployment: [] 2022-03-10 07:01:07,633 DEBUG [c.c.d.DeploymentPlanningManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) DeploymentPlanner allocation algorithm: null 2022-03-10 07:01:07,633 DEBUG [c.c.d.DeploymentPlanningManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) Trying to allocate a host and storage pools from dc:1, pod:1,cluster:null, requested cpu: 500, requested ram: (512.00 MB) 536870912 2022-03-10 07:01:07,633 DEBUG [c.c.d.DeploymentPlanningManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) Is ROOT volume READY (pool already allocated)?: No 2022-03-10 07:01:07,637 DEBUG [c.c.d.DeploymentPlanningManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) Deploy avoids pods: [], clusters: [], hosts: [] 2022-03-10 07:01:07,638 DEBUG [c.c.d.FirstFitPlanner] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) Searching resources only under specified Pod: 1 2022-03-10 07:01:07,639 DEBUG [c.c.d.FirstFitPlanner] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) Listing clusters in order of aggregate capacity, that have (at least one host with) enough CPU and RAM capacity under this Pod: 1 2022-03-10 07:01:07,641 DEBUG [c.c.d.FirstFitPlanner] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) Removing from the clusterId list these clusters from avoid set: [] 2022-03-10 07:01:07,654 DEBUG [c.c.d.DeploymentPlanningManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) Checking resources in Cluster: 1 under Pod: 1 2022-03-10 07:01:07,656 INFO [c.c.a.m.a.i.FirstFitAllocator] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22 FirstFitRoutingAllocator) (logid:a882c7d6) Guest VM is requested with Custom[UEFI] Boot Type false 2022-03-10 07:01:07,656 DEBUG [c.c.a.m.a.i.FirstFitAllocator] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22 FirstFitRoutingAllocator) (logid:a882c7d6) Looking for hosts in dc: 1 pod:1 cluster:1 2022-03-10 07:01:07,659 DEBUG [c.c.a.m.a.i.FirstFitAllocator] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22 FirstFitRoutingAllocator) (logid:a882c7d6) FirstFitAllocator has 2 hosts to check for allocation: [Host {"id": "2", "name": "10.0.34.192", "uuid": "3980320d-7451-49cd-b253-7086d4ad4e51", "type"="Routing"}, Host {"id": "1", "name": "10.0.33.183", "uuid": "14c10bb4-0403-4eab-8d02-b37bdafaad25", "type"="Routing"}] 2022-03-10 07:01:07,663 DEBUG [c.c.a.m.a.i.FirstFitAllocator] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22 FirstFitRoutingAllocator) (logid:a882c7d6) Found 2 hosts for allocation after prioritization: [Host {"id": "2", "name": "10.0.34.192", "uuid": "3980320d-7451-49cd-b253-7086d4ad4e51", "type"="Routing"}, Host {"id": "1", "name": "10.0.33.183", "uuid": "14c10bb4-0403-4eab-8d02-b37bdafaad25", "type"="Routing"}] 2022-03-10 07:01:07,663 DEBUG [c.c.a.m.a.i.FirstFitAllocator] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22 FirstFitRoutingAllocator) (logid:a882c7d6) Looking for speed=500Mhz, Ram=512 MB 2022-03-10 07:01:07,667 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22 FirstFitRoutingAllocator) (logid:a882c7d6) Host: 2 has cpu capability (cpu:3, speed:2100) to support requested CPU: 1 and requested speed: 500 2022-03-10 07:01:07,667 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22 FirstFitRoutingAllocator) (logid:a882c7d6) Checking if host: 2 has enough capacity for requested CPU: 500 and requested RAM: (512.00 MB) 536870912 , cpuOverprovisioningFactor: 2.0 2022-03-10 07:01:07,669 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22 FirstFitRoutingAllocator) (logid:a882c7d6) Hosts's actual total CPU: 6300 and CPU after applying overprovisioning: 12600 2022-03-10 07:01:07,669 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22 FirstFitRoutingAllocator) (logid:a882c7d6) Free CPU: 12600 , Requested CPU: 500 2022-03-10 07:01:07,669 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22 FirstFitRoutingAllocator) (logid:a882c7d6) Free RAM: (8.00 GB) 8589328384 , Requested RAM: (512.00 MB) 536870912 2022-03-10 07:01:07,669 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22 FirstFitRoutingAllocator) (logid:a882c7d6) Host has enough CPU and RAM available 2022-03-10 07:01:07,669 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22 FirstFitRoutingAllocator) (logid:a882c7d6) STATS: Can alloc CPU from host: 2, used: 0, reserved: 0, actual total: 6300, total with overprovisioning: 12600; requested cpu:500,alloc_from_last_host?:false ,considerReservedCapacity?: true 2022-03-10 07:01:07,669 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22 FirstFitRoutingAllocator) (logid:a882c7d6) STATS: Can alloc MEM from host: 2, used: (0 bytes) 0, reserved: (0 bytes) 0, total: (8.00 GB) 8589328384; requested mem: (512.00 MB) 536870912, alloc_from_last_host?: false , considerReservedCapacity?: true 2022-03-10 07:01:07,669 DEBUG [c.c.a.m.a.i.FirstFitAllocator] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22 FirstFitRoutingAllocator) (logid:a882c7d6) Found a suitable host, adding to list: 2 2022-03-10 07:01:07,677 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22 FirstFitRoutingAllocator) (logid:a882c7d6) Host: 1 has cpu capability (cpu:3, speed:2100) to support requested CPU: 1 and requested speed: 500 2022-03-10 07:01:07,677 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22 FirstFitRoutingAllocator) (logid:a882c7d6) Checking if host: 1 has enough capacity for requested CPU: 500 and requested RAM: (512.00 MB) 536870912 , cpuOverprovisioningFactor: 2.0 2022-03-10 07:01:07,679 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22 FirstFitRoutingAllocator) (logid:a882c7d6) Hosts's actual total CPU: 6300 and CPU after applying overprovisioning: 12600 2022-03-10 07:01:07,679 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22 FirstFitRoutingAllocator) (logid:a882c7d6) Free CPU: 12100 , Requested CPU: 500 2022-03-10 07:01:07,679 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22 FirstFitRoutingAllocator) (logid:a882c7d6) Free RAM: (7.00 GB) 7515586560 , Requested RAM: (512.00 MB) 536870912 2022-03-10 07:01:07,679 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22 FirstFitRoutingAllocator) (logid:a882c7d6) Host has enough CPU and RAM available 2022-03-10 07:01:07,679 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22 FirstFitRoutingAllocator) (logid:a882c7d6) STATS: Can alloc CPU from host: 1, used: 500, reserved: 0, actual total: 6300, total with overprovisioning: 12600; requested cpu:500,alloc_from_last_host?:false ,considerReservedCapacity?: true 2022-03-10 07:01:07,679 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22 FirstFitRoutingAllocator) (logid:a882c7d6) STATS: Can alloc MEM from host: 1, used: (1.00 GB) 1073741824, reserved: (0 bytes) 0, total: (8.00 GB) 8589328384; requested mem: (512.00 MB) 536870912, alloc_from_last_host?: false , considerReservedCapacity?: true 2022-03-10 07:01:07,679 DEBUG [c.c.a.m.a.i.FirstFitAllocator] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22 FirstFitRoutingAllocator) (logid:a882c7d6) Found a suitable host, adding to list: 1 2022-03-10 07:01:07,679 DEBUG [c.c.a.m.a.i.FirstFitAllocator] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22 FirstFitRoutingAllocator) (logid:a882c7d6) Host Allocator returning 2 suitable hosts 2022-03-10 07:01:07,681 DEBUG [c.c.d.DeploymentPlanningManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) Checking suitable pools for volume (Id, Type): (12,ROOT) 2022-03-10 07:01:07,681 DEBUG [c.c.d.DeploymentPlanningManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) We need to allocate new storagepool for this volume 2022-03-10 07:01:07,681 DEBUG [c.c.d.DeploymentPlanningManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) Calling StoragePoolAllocators to find suitable pools 2022-03-10 07:01:07,682 DEBUG [o.a.c.s.a.LocalStoragePoolAllocator] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) LocalStoragePoolAllocator trying to find storage pool to fit the vm 2022-03-10 07:01:07,682 DEBUG [o.a.c.s.a.ClusterScopeStoragePoolAllocator] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) ClusterScopeStoragePoolAllocator looking for storage pool 2022-03-10 07:01:07,682 DEBUG [o.a.c.s.a.ClusterScopeStoragePoolAllocator] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) Looking for pools in dc: 1 pod:1 cluster:1. Disabled pools will be ignored. 2022-03-10 07:01:07,682 DEBUG [o.a.c.s.a.ClusterScopeStoragePoolAllocator] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) Found pools matching tags: [Pool[1|NetworkFilesystem]] 2022-03-10 07:01:07,684 DEBUG [o.a.c.s.a.AbstractStoragePoolAllocator] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) Checking if storage pool is suitable, name: trl-283-v-M7-boris-stoyanov-esxi-pri1 ,poolId: 1 2022-03-10 07:01:07,687 INFO [c.c.s.StorageManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) Storage pool trl-283-v-M7-boris-stoyanov-esxi-pri1 (1) does not supply IOPS capacity, assuming enough capacity 2022-03-10 07:01:07,688 DEBUG [c.c.s.StorageManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) Checking pool 1 for storage, totalSize: 2197949513728, usedBytes: 1326611099648, usedPct: 0.6035675939607457, disable threshold: 0.85 2022-03-10 07:01:07,688 DEBUG [c.c.s.StorageManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) Destination pool id: 1 2022-03-10 07:01:07,691 DEBUG [c.c.s.StorageManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) Pool ID for the volume with ID 12 is null 2022-03-10 07:01:07,693 DEBUG [c.c.s.StorageManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) Found storage pool trl-283-v-M7-boris-stoyanov-esxi-pri1 of type NetworkFilesystem with over-provisioning factor 2 2022-03-10 07:01:07,693 DEBUG [c.c.s.StorageManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) Total over-provisioned capacity calculated is 2 * (1.9990 TB) 2197949513728 2022-03-10 07:01:07,693 DEBUG [c.c.s.StorageManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) Total capacity of the pool trl-283-v-M7-boris-stoyanov-esxi-pri1 with ID 1 is (3.9980 TB) 4395899027456 2022-03-10 07:01:07,694 DEBUG [c.c.s.StorageManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) Checking pool: 1 for storage allocation , maxSize : (3.9980 TB) 4395899027456, totalAllocatedSize : (5.91 GB) 6341787648, askingSize : (2.00 GB) 2147483648, allocated disable threshold: 0.85 2022-03-10 07:01:07,694 DEBUG [o.a.c.s.a.ClusterScopeStoragePoolAllocator] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) ClusterScopeStoragePoolAllocator returning 1 suitable storage pools 2022-03-10 07:01:07,696 DEBUG [c.c.d.DeploymentPlanningManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) Trying to find a potenial host and associated storage pools from the suitable host/pool lists for this VM 2022-03-10 07:01:07,696 DEBUG [c.c.d.DeploymentPlanningManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) Checking if host: 2 can access any suitable storage pool for volume: ROOT 2022-03-10 07:01:07,696 DEBUG [c.c.d.DeploymentPlanningManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) Host: 2 can access pool: 1 2022-03-10 07:01:07,697 DEBUG [c.c.d.DeploymentPlanningManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) Found a potential host id: 2 name: 10.0.34.192 and associated storage pools for this VM 2022-03-10 07:01:07,697 DEBUG [c.c.d.DeploymentPlanningManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) Returning Deployment Destination: Dest[Zone(Id)-Pod(Id)-Cluster(Id)-Host(Id)-Storage(Volume(Id|Type-->Pool(Id))] : Dest[Zone(1)-Pod(1)-Cluster(1)-Host(2)-Storage(Volume(12|ROOT-->Pool(1))] 2022-03-10 07:01:07,697 DEBUG [c.c.v.VirtualMachineManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) Deployment found - Attempt #1 - P0=VM instance {id: "12", name: "i-2-12-VM", uuid: "fe006474-e8c7-4f78-8982-e72c058c6a3e", type="User"}, P0=Dest[Zone(Id)-Pod(Id)-Cluster(Id)-Host(Id)-Storage(Volume(Id|Type-->Pool(Id))] : Dest[Zone(1)-Pod(1)-Cluster(1)-Host(2)-Storage(Volume(12|ROOT-->Pool(1))] 2022-03-10 07:01:07,714 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) VM instance {id: "12", name: "i-2-12-VM", uuid: "fe006474-e8c7-4f78-8982-e72c058c6a3e", type="User"} state transited from [Starting] to [Starting] with event [OperationRetry]. VM's original host: null, new host: Host {"id": "2", "name": "10.0.34.192", "uuid": "3980320d-7451-49cd-b253-7086d4ad4e51", "type"="Routing"}, host before state transition: null 2022-03-10 07:01:07,719 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) Hosts's actual total CPU: 6300 and CPU after applying overprovisioning: 12600 2022-03-10 07:01:07,720 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) We are allocating VM, increasing the used capacity of this host:2 2022-03-10 07:01:07,720 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) Current Used CPU: 0 , Free CPU:12600 ,Requested CPU: 500 2022-03-10 07:01:07,720 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) Current Used RAM: (0 bytes) 0 , Free RAM:(8.00 GB) 8589328384 ,Requested RAM: (512.00 MB) 536870912 2022-03-10 07:01:07,720 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) CPU STATS after allocation: for host: 2, old used: 0, old reserved: 0, actual total: 6300, total with overprovisioning: 12600; new used:500, reserved:0; requested cpu:500,alloc_from_last:false 2022-03-10 07:01:07,720 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) RAM STATS after allocation: for host: 2, old used: (0 bytes) 0, old reserved: (0 bytes) 0, total: (8.00 GB) 8589328384; new used: (512.00 MB) 536870912, reserved: (0 bytes) 0; requested mem: (512.00 MB) 536870912,alloc_from_last:false 2022-03-10 07:01:07,721 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) Host: 2 has cpu capability (cpu:3, speed:2100) to support requested CPU: 1 and requested speed: 500 2022-03-10 07:01:07,721 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) Checking if host: 2 has enough capacity for requested CPU: 500 and requested RAM: (512.00 MB) 536870912 , cpuOverprovisioningFactor: 2.0 2022-03-10 07:01:07,722 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) Hosts's actual total CPU: 6300 and CPU after applying overprovisioning: 12600 2022-03-10 07:01:07,722 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) We need to allocate to the last host again, so checking if there is enough reserved capacity 2022-03-10 07:01:07,722 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) Reserved CPU: 0 , Requested CPU: 500 2022-03-10 07:01:07,722 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) Reserved RAM: (0 bytes) 0 , Requested RAM: (512.00 MB) 536870912 2022-03-10 07:01:07,723 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) STATS: Failed to alloc resource from host: 2 reservedCpu: 0, requested cpu: 500, reservedMem: (0 bytes) 0, requested mem: (512.00 MB) 536870912 2022-03-10 07:01:07,723 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) Host does not have enough reserved CPU available, cannot allocate to this host. 2022-03-10 07:01:07,723 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) Checking if host: 2 has enough capacity for requested CPU: 500 and requested RAM: (512.00 MB) 536870912 , cpuOverprovisioningFactor: 2.0 2022-03-10 07:01:07,724 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) Hosts's actual total CPU: 6300 and CPU after applying overprovisioning: 12600 2022-03-10 07:01:07,724 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) Free CPU: 12600 , Requested CPU: 500 2022-03-10 07:01:07,724 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) Free RAM: (8.00 GB) 8589328384 , Requested RAM: (512.00 MB) 536870912 2022-03-10 07:01:07,724 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) Host has enough CPU and RAM available 2022-03-10 07:01:07,724 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) STATS: Can alloc CPU from host: 2, used: 0, reserved: 0, actual total: 6300, total with overprovisioning: 12600; requested cpu:500,alloc_from_last_host?:false ,considerReservedCapacity?: true 2022-03-10 07:01:07,724 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) STATS: Can alloc MEM from host: 2, used: (0 bytes) 0, reserved: (0 bytes) 0, total: (8.00 GB) 8589328384; requested mem: (512.00 MB) 536870912, alloc_from_last_host?: false , considerReservedCapacity?: true 2022-03-10 07:01:07,733 DEBUG [o.a.c.e.o.NetworkOrchestrator] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) Lock is acquired for network id 209 as a part of network implement 2022-03-10 07:01:07,734 DEBUG [o.a.c.e.o.NetworkOrchestrator] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) Asking ExternalGuestNetworkGuru to implement Ntwk[209|Guest|8] 2022-03-10 07:01:07,752 DEBUG [o.a.c.e.o.NetworkOrchestrator] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) Asking VirtualRouter to implemenet Ntwk[209|Guest|8] 2022-03-10 07:01:07,754 DEBUG [c.c.n.r.NetworkHelperImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) Starting router VM instance {id: "13", name: "r-13-VM", uuid: "dd2c7b06-cfd8-4dc6-953c-7efebd963ed3", type="DomainRouter"} 2022-03-10 07:01:07,764 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) VM instance {id: "13", name: "r-13-VM", uuid: "dd2c7b06-cfd8-4dc6-953c-7efebd963ed3", type="DomainRouter"} state transited from [Stopped] to [Starting] with event [StartRequested]. VM's original host: null, new host: null, host before state transition: null 2022-03-10 07:01:07,764 DEBUG [c.c.v.VirtualMachineManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) Successfully transitioned to start state for VM instance {id: "13", name: "r-13-VM", uuid: "dd2c7b06-cfd8-4dc6-953c-7efebd963ed3", type="DomainRouter"} reservation id = ccd26c03-8d95-4235-9c0f-ce8a796eaecb 2022-03-10 07:01:07,769 DEBUG [c.c.v.VirtualMachineManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) Deploy avoids pods: null, clusters: null, hosts: null 2022-03-10 07:01:07,770 DEBUG [c.c.v.VirtualMachineManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) VM start attempt #1 2022-03-10 07:01:07,775 DEBUG [c.c.d.DeploymentPlanningManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) Adding pods to avoid lists: [] 2022-03-10 07:01:07,775 DEBUG [c.c.d.DeploymentPlanningManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) Adding clusters to avoid lists: [] 2022-03-10 07:01:07,775 DEBUG [c.c.d.DeploymentPlanningManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) Adding hosts to avoid lists: [] 2022-03-10 07:01:07,775 DEBUG [c.c.d.DeploymentPlanningManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) DeploymentPlanner allocation algorithm: null 2022-03-10 07:01:07,775 DEBUG [c.c.d.DeploymentPlanningManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) Trying to allocate a host and storage pools from dc:1, pod:null,cluster:null, requested cpu: 500, requested ram: (254.00 MB) 266338304 2022-03-10 07:01:07,775 DEBUG [c.c.d.DeploymentPlanningManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) Is ROOT volume READY (pool already allocated)?: No 2022-03-10 07:01:07,779 DEBUG [c.c.d.DeploymentPlanningManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) Deploy avoids pods: [], clusters: [], hosts: [] 2022-03-10 07:01:07,780 DEBUG [c.c.d.FirstFitPlanner] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) Searching all possible resources under this Zone: 1 2022-03-10 07:01:07,781 DEBUG [c.c.d.FirstFitPlanner] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) Listing clusters in order of aggregate capacity, that have (at least one host with) enough CPU and RAM capacity under this Zone: 1 2022-03-10 07:01:07,785 DEBUG [c.c.d.FirstFitPlanner] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) Removing from the clusterId list these clusters from avoid set: [] 2022-03-10 07:01:07,790 DEBUG [c.c.d.FirstFitPlanner] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) The clusterId list for the given offering tag: [2] 2022-03-10 07:01:07,794 DEBUG [c.c.d.DeploymentPlanningManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) Cluster: 2 has HyperVisorType that does not match the VM, skipping this cluster 2022-03-10 07:01:07,795 DEBUG [c.c.d.DeploymentPlanningManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) Could not find suitable Deployment Destination for this VM under any clusters, returning. 2022-03-10 07:01:07,795 DEBUG [c.c.d.FirstFitPlanner] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) Searching all possible resources under this Zone: 1 2022-03-10 07:01:07,796 DEBUG [c.c.d.FirstFitPlanner] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) Listing clusters in order of aggregate capacity, that have (at least one host with) enough CPU and RAM capacity under this Zone: 1 2022-03-10 07:01:07,798 DEBUG [c.c.d.FirstFitPlanner] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) Removing from the clusterId list these clusters from avoid set: [2] 2022-03-10 07:01:07,801 DEBUG [c.c.d.FirstFitPlanner] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) The clusterId list for the given offering tag: [] 2022-03-10 07:01:07,801 DEBUG [c.c.d.FirstFitPlanner] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) No clusters found after removing disabled clusters and clusters in avoid list, returning. 2022-03-10 07:01:07,808 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) VM instance {id: "13", name: "r-13-VM", uuid: "dd2c7b06-cfd8-4dc6-953c-7efebd963ed3", type="DomainRouter"} state transited from [Starting] to [Stopped] with event [OperationFailed]. VM's original host: null, new host: null, host before state transition: null 2022-03-10 07:01:07,812 DEBUG [o.a.c.e.o.NetworkOrchestrator] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) Cleaning up because we're unable to implement the network Ntwk[209|Guest|8] 2022-03-10 07:01:07,819 DEBUG [o.a.c.e.o.NetworkOrchestrator] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) Lock is acquired for network Ntwk[209|Guest|8] as a part of network shutdown 2022-03-10 07:01:07,823 DEBUG [o.a.c.e.o.NetworkOrchestrator] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) Sending network shutdown to VirtualRouter 2022-03-10 07:01:07,825 DEBUG [c.c.n.r.VirtualNetworkApplianceManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) Stopping router VM instance {id: "13", name: "r-13-VM", uuid: "dd2c7b06-cfd8-4dc6-953c-7efebd963ed3", type="DomainRouter"} 2022-03-10 07:01:07,832 DEBUG [c.c.v.VirtualMachineManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) VM is already stopped: VM instance {id: "13", name: "r-13-VM", uuid: "dd2c7b06-cfd8-4dc6-953c-7efebd963ed3", type="DomainRouter"} 2022-03-10 07:01:07,836 DEBUG [o.a.c.e.o.NetworkOrchestrator] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) Network id=209 is shutdown successfully, cleaning up corresponding resources now. 2022-03-10 07:01:07,838 DEBUG [c.c.n.g.GuestNetworkGuru] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) Releasing vnet for the network id=209 2022-03-10 07:01:07,846 DEBUG [o.a.c.e.o.NetworkOrchestrator] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) Lock is released for network Ntwk[209|Guest|8] as a part of network shutdown 2022-03-10 07:01:07,849 DEBUG [o.a.c.e.o.NetworkOrchestrator] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) Lock is released for network id 209 as a part of network implement 2022-03-10 07:01:07,849 INFO [c.c.v.VirtualMachineManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) Insufficient capacity com.cloud.exception.InsufficientServerCapacityException: Unable to create a deployment for VM instance {id: "13", name: "r-13-VM", uuid: "dd2c7b06-cfd8-4dc6-953c-7efebd963ed3", type="DomainRouter"}Scope=interface com.cloud.dc.DataCenter; id=1 at com.cloud.vm.VirtualMachineManagerImpl.orchestrateStart(VirtualMachineManagerImpl.java:1160) at com.cloud.vm.VirtualMachineManagerImpl.advanceStart(VirtualMachineManagerImpl.java:966) at com.cloud.network.router.NetworkHelperImpl.start(NetworkHelperImpl.java:288) at com.cloud.network.router.NetworkHelperImpl.startVirtualRouter(NetworkHelperImpl.java:367) at com.cloud.network.router.NetworkHelperImpl.startRouters(NetworkHelperImpl.java:352) at org.cloud.network.router.deployment.RouterDeploymentDefinition.deployVirtualRouter(RouterDeploymentDefinition.java:206) at com.cloud.network.element.VirtualRouterElement.implement(VirtualRouterElement.java:239) at org.apache.cloudstack.engine.orchestration.NetworkOrchestrator.implementNetworkElements(NetworkOrchestrator.java:1488) at org.apache.cloudstack.engine.orchestration.NetworkOrchestrator.implementNetworkElementsAndResources(NetworkOrchestrator.java:1423) at org.apache.cloudstack.engine.orchestration.NetworkOrchestrator.implementNetwork(NetworkOrchestrator.java:1338) at org.apache.cloudstack.engine.orchestration.NetworkOrchestrator.implementNetwork(NetworkOrchestrator.java:1189) at org.apache.cloudstack.engine.orchestration.NetworkOrchestrator.prepare(NetworkOrchestrator.java:1863) at com.cloud.vm.VirtualMachineManagerImpl.orchestrateStart(VirtualMachineManagerImpl.java:1199) at com.cloud.vm.VirtualMachineManagerImpl.orchestrateStart(VirtualMachineManagerImpl.java:5853) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:566) at com.cloud.vm.VmWorkJobHandlerProxy.handleVmWorkJob(VmWorkJobHandlerProxy.java:107) at com.cloud.vm.VirtualMachineManagerImpl.handleVmWorkJob(VirtualMachineManagerImpl.java:6018) at com.cloud.vm.VmWorkJobDispatcher.runJob(VmWorkJobDispatcher.java:102) at org.apache.cloudstack.framework.jobs.impl.AsyncJobManagerImpl$5.runInContext(AsyncJobManagerImpl.java:620) at org.apache.cloudstack.managed.context.ManagedContextRunnable$1.run(ManagedContextRunnable.java:48) at org.apache.cloudstack.managed.context.impl.DefaultManagedContext$1.call(DefaultManagedContext.java:55) at org.apache.cloudstack.managed.context.impl.DefaultManagedContext.callWithContext(DefaultManagedContext.java:102) at org.apache.cloudstack.managed.context.impl.DefaultManagedContext.runWithContext(DefaultManagedContext.java:52) at org.apache.cloudstack.managed.context.ManagedContextRunnable.run(ManagedContextRunnable.java:45) at org.apache.cloudstack.framework.jobs.impl.AsyncJobManagerImpl$5.run(AsyncJobManagerImpl.java:568) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) 2022-03-10 07:01:07,859 DEBUG [c.c.v.VirtualMachineManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) Cleaning up resources for the vm VM instance {id: "12", name: "i-2-12-VM", uuid: "fe006474-e8c7-4f78-8982-e72c058c6a3e", type="User"} in Starting state 2022-03-10 07:01:07,866 DEBUG [c.c.a.t.Request] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) Seq 2-3538984882183077897: Sending { Cmd , MgmtId: 32988737831924, via: 2(10.0.34.192), Ver: v1, Flags: 100011, [{"com.cloud.agent.api.StopCommand":{"isProxy":"false","checkBeforeCleanup":"false","forceStop":"false","volumesToDisconnect":[],"vmName":"i-2-12-VM","executeInSequence":"false","wait":"0","bypassHostMaintenance":"false"}}] } 2022-03-10 07:01:07,866 DEBUG [c.c.a.t.Request] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) Seq 2-3538984882183077897: Executing: { Cmd , MgmtId: 32988737831924, via: 2(10.0.34.192), Ver: v1, Flags: 100011, [{"com.cloud.agent.api.StopCommand":{"isProxy":"false","checkBeforeCleanup":"false","forceStop":"false","volumesToDisconnect":[],"vmName":"i-2-12-VM","executeInSequence":"false","wait":"0","bypassHostMaintenance":"false"}}] } 2022-03-10 07:01:07,867 DEBUG [c.c.a.m.DirectAgentAttache] (DirectAgent-49:ctx-624cb031) (logid:6a0c0c40) Seq 2-3538984882183077897: Executing request 2022-03-10 07:01:07,867 INFO [c.c.h.v.r.VmwareResource] (DirectAgent-49:ctx-624cb031 10.0.34.192, job-78/job-82, cmd: StopCommand) (logid:a882c7d6) Executing resource StopCommand: {"isProxy":false,"checkBeforeCleanup":false,"forceStop":false,"volumesToDisconnect":[],"vmName":"i-2-12-VM","executeInSequence":false,"wait":0,"bypassHostMaintenance":false} 2022-03-10 07:01:07,876 DEBUG [c.c.h.v.m.HostMO] (DirectAgent-49:ctx-624cb031 10.0.34.192, job-78/job-82, cmd: StopCommand) (logid:a882c7d6) find VM i-2-12-VM on host 2022-03-10 07:01:07,876 INFO [c.c.h.v.m.HostMO] (DirectAgent-49:ctx-624cb031 10.0.34.192, job-78/job-82, cmd: StopCommand) (logid:a882c7d6) VM i-2-12-VM not found in host cache 2022-03-10 07:01:07,876 DEBUG [c.c.h.v.m.HostMO] (DirectAgent-49:ctx-624cb031 10.0.34.192, job-78/job-82, cmd: StopCommand) (logid:a882c7d6) load VM cache on host 2022-03-10 07:01:07,887 INFO [c.c.h.v.r.VmwareResource] (DirectAgent-49:ctx-624cb031 10.0.34.192, job-78/job-82, cmd: StopCommand) (logid:a882c7d6) VM i-2-12-VM is no longer on the expected host in vSphere 2022-03-10 07:01:07,887 DEBUG [c.c.a.m.DirectAgentAttache] (DirectAgent-49:ctx-624cb031) (logid:a882c7d6) Seq 2-3538984882183077897: Response Received: 2022-03-10 07:01:07,887 DEBUG [c.c.a.t.Request] (DirectAgent-49:ctx-624cb031) (logid:a882c7d6) Seq 2-3538984882183077897: Processing: { Ans: , MgmtId: 32988737831924, via: 2(10.0.34.192), Ver: v1, Flags: 10, [{"com.cloud.agent.api.StopAnswer":{"result":"true","details":"VM i-2-12-VM is no longer on the expected host in vSphere","wait":"0","bypassHostMaintenance":"false"}}] } 2022-03-10 07:01:07,888 DEBUG [c.c.a.t.Request] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) Seq 2-3538984882183077897: Received: { Ans: , MgmtId: 32988737831924, via: 2(10.0.34.192), Ver: v1, Flags: 10, { StopAnswer } } 2022-03-10 07:01:07,895 DEBUG [c.c.v.VirtualMachineManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) Successfully released network resources for the VM VM instance {id: "12", name: "i-2-12-VM", uuid: "fe006474-e8c7-4f78-8982-e72c058c6a3e", type="User"} in Starting state 2022-03-10 07:01:07,897 DEBUG [c.c.v.VirtualMachineManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) Successfully released storage resources for the VM VM instance {id: "12", name: "i-2-12-VM", uuid: "fe006474-e8c7-4f78-8982-e72c058c6a3e", type="User"} in Starting state 2022-03-10 07:01:07,898 DEBUG [c.c.v.VirtualMachineManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) Successfully cleaned up resources for the VM VM instance {id: "12", name: "i-2-12-VM", uuid: "fe006474-e8c7-4f78-8982-e72c058c6a3e", type="User"} in Starting state 2022-03-10 07:01:07,898 DEBUG [c.c.v.VirtualMachineManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) VM start attempt #2 2022-03-10 07:01:07,901 DEBUG [c.c.d.DeploymentPlanningManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) Adding pods to avoid lists for non-explicit VM deployment: [] 2022-03-10 07:01:07,901 DEBUG [c.c.d.DeploymentPlanningManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) Adding clusters to avoid lists for non-explicit VM deployment: [] 2022-03-10 07:01:07,901 DEBUG [c.c.d.DeploymentPlanningManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) Adding hosts to avoid lists for non-explicit VM deployment: [] 2022-03-10 07:01:07,901 DEBUG [c.c.d.DeploymentPlanningManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) DeploymentPlanner allocation algorithm: null 2022-03-10 07:01:07,901 DEBUG [c.c.d.DeploymentPlanningManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) Trying to allocate a host and storage pools from dc:1, pod:1,cluster:null, requested cpu: 500, requested ram: (512.00 MB) 536870912 2022-03-10 07:01:07,901 DEBUG [c.c.d.DeploymentPlanningManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) Is ROOT volume READY (pool already allocated)?: No 2022-03-10 07:01:07,904 DEBUG [c.c.d.DeploymentPlanningManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) Deploy avoids pods: [], clusters: [], hosts: [2] 2022-03-10 07:01:07,905 DEBUG [c.c.d.DeploymentPlanningManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) DataCenter id = '1' provided is in avoid set, DeploymentPlanner cannot allocate the VM, returning. 2022-03-10 07:01:07,913 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) VM instance {id: "12", name: "i-2-12-VM", uuid: "fe006474-e8c7-4f78-8982-e72c058c6a3e", type="User"} state transited from [Starting] to [Stopped] with event [OperationFailed]. VM's original host: null, new host: null, host before state transition: Host {"id": "2", "name": "10.0.34.192", "uuid": "3980320d-7451-49cd-b253-7086d4ad4e51", "type"="Routing"} 2022-03-10 07:01:07,918 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) Hosts's actual total CPU: 6300 and CPU after applying overprovisioning: 12600 2022-03-10 07:01:07,918 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) Hosts's actual total RAM: (8.00 GB) 8589328384 and RAM after applying overprovisioning: (8.00 GB) 8589328384 2022-03-10 07:01:07,918 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) release cpu from host: 2, old used: 500,reserved: 0, actual total: 6300, total with overprovisioning: 12600; new used: 0,reserved:0; movedfromreserved: false,moveToReserveredfalse 2022-03-10 07:01:07,918 DEBUG [c.c.c.CapacityManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) release mem from host: 2, old used: (512.00 MB) 536870912,reserved: (0 bytes) 0, total: (8.00 GB) 8589328384; new used: (0 bytes) 0,reserved:(0 bytes) 0; movedfromreserved: false,moveToReserveredfalse 2022-03-10 07:01:07,929 ERROR [c.c.v.VmWorkJobHandlerProxy] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) Invocation exception, caused by: com.cloud.exception.InsufficientServerCapacityException: Unable to create a deployment for VM instance {id: "12", name: "i-2-12-VM", uuid: "fe006474-e8c7-4f78-8982-e72c058c6a3e", type="User"}Scope=interface com.cloud.dc.DataCenter; id=1 2022-03-10 07:01:07,929 INFO [c.c.v.VmWorkJobHandlerProxy] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82 ctx-24a58e22) (logid:a882c7d6) Rethrow exception com.cloud.exception.InsufficientServerCapacityException: Unable to create a deployment for VM instance {id: "12", name: "i-2-12-VM", uuid: "fe006474-e8c7-4f78-8982-e72c058c6a3e", type="User"}Scope=interface com.cloud.dc.DataCenter; id=1 2022-03-10 07:01:07,929 DEBUG [c.c.v.VmWorkJobDispatcher] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82) (logid:a882c7d6) Done with run of VM work job: com.cloud.vm.VmWorkStart for VM 12, job origin: 78 2022-03-10 07:01:07,929 ERROR [c.c.v.VmWorkJobDispatcher] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82) (logid:a882c7d6) Unable to complete AsyncJobVO: {id:82, userId: 2, accountId: 2, instanceType: null, instanceId: null, cmd: com.cloud.vm.VmWorkStart, cmdInfo: rO0ABXNyABhjb20uY2xvdWQudm0uVm1Xb3JrU3RhcnR9cMGsvxz73gIAC0oABGRjSWRMAAZhdm9pZHN0ADBMY29tL2Nsb3VkL2RlcGxveS9EZXBsb3ltZW50UGxhbm5lciRFeGNsdWRlTGlzdDtMAAljbHVzdGVySWR0ABBMamF2YS9sYW5nL0xvbmc7TAAGaG9zdElkcQB-AAJMAAtqb3VybmFsTmFtZXQAEkxqYXZhL2xhbmcvU3RyaW5nO0wAEXBoeXNpY2FsTmV0d29ya0lkcQB-AAJMAAdwbGFubmVycQB-AANMAAVwb2RJZHEAfgACTAAGcG9vbElkcQB-AAJMAAlyYXdQYXJhbXN0AA9MamF2YS91dGlsL01hcDtMAA1yZXNlcnZhdGlvbklkcQB-AAN4cgATY29tLmNsb3VkLnZtLlZtV29ya5-ZtlbwJWdrAgAESgAJYWNjb3VudElkSgAGdXNlcklkSgAEdm1JZEwAC2hhbmRsZXJOYW1lcQB-AAN4cAAAAAAAAAACAAAAAAAAAAIAAAAAAAAADHQAGVZpcnR1YWxNYWNoaW5lTWFuYWdlckltcGwAAAAAAAAAAHBwcHBwcHBwc3IAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA_QAAAAAAADHcIAAAAEAAAAAF0AApWbVBhc3N3b3JkdAAcck8wQUJYUUFEbk5oZG1Wa1gzQmhjM04zYjNKa3hw, cmdVersion: 0, status: IN_PROGRESS, processStatus: 0, resultCode: 0, result: null, initMsid: 32988737831924, completeMsid: null, lastUpdated: null, lastPolled: null, created: Thu Mar 10 07:01:06 UTC 2022, removed: null}, job origin:78 com.cloud.exception.InsufficientServerCapacityException: Unable to create a deployment for VM instance {id: "12", name: "i-2-12-VM", uuid: "fe006474-e8c7-4f78-8982-e72c058c6a3e", type="User"}Scope=interface com.cloud.dc.DataCenter; id=1 at com.cloud.vm.VirtualMachineManagerImpl.orchestrateStart(VirtualMachineManagerImpl.java:1160) at com.cloud.vm.VirtualMachineManagerImpl.orchestrateStart(VirtualMachineManagerImpl.java:5853) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:566) at com.cloud.vm.VmWorkJobHandlerProxy.handleVmWorkJob(VmWorkJobHandlerProxy.java:107) at com.cloud.vm.VirtualMachineManagerImpl.handleVmWorkJob(VirtualMachineManagerImpl.java:6018) at com.cloud.vm.VmWorkJobDispatcher.runJob(VmWorkJobDispatcher.java:102) at org.apache.cloudstack.framework.jobs.impl.AsyncJobManagerImpl$5.runInContext(AsyncJobManagerImpl.java:620) at org.apache.cloudstack.managed.context.ManagedContextRunnable$1.run(ManagedContextRunnable.java:48) at org.apache.cloudstack.managed.context.impl.DefaultManagedContext$1.call(DefaultManagedContext.java:55) at org.apache.cloudstack.managed.context.impl.DefaultManagedContext.callWithContext(DefaultManagedContext.java:102) at org.apache.cloudstack.managed.context.impl.DefaultManagedContext.runWithContext(DefaultManagedContext.java:52) at org.apache.cloudstack.managed.context.ManagedContextRunnable.run(ManagedContextRunnable.java:45) at org.apache.cloudstack.framework.jobs.impl.AsyncJobManagerImpl$5.run(AsyncJobManagerImpl.java:568) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) 2022-03-10 07:01:07,939 DEBUG [o.a.c.f.j.i.AsyncJobManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82) (logid:a882c7d6) Complete async job-82, jobStatus: FAILED, resultCode: 0, result: rO0ABXNyADdjb20uY2xvdWQuZXhjZXB0aW9uLkluc3VmZmljaWVudFNlcnZlckNhcGFjaXR5RXhjZXB0aW9uAAAAAFZNT3ACAAFaABVhZmZpbml0eUdyb3Vwc0FwcGxpZWR4cgAxY29tLmNsb3VkLmV4Y2VwdGlvbi5JbnN1ZmZpY2llbnRDYXBhY2l0eUV4Y2VwdGlvbgAAAABWTU91AgACTAACaWR0ABBMamF2YS9sYW5nL0xvbmc7TAAFc2NvcGV0ABFMamF2YS9sYW5nL0NsYXNzO3hyACJjb20uY2xvdWQuZXhjZXB0aW9uLkNsb3VkRXhjZXB0aW9ueeiOG_HkRMcCAAJMAAtjc0Vycm9yQ29kZXQAE0xqYXZhL2xhbmcvSW50ZWdlcjtMAAZpZExpc3R0ABVMamF2YS91dGlsL0FycmF5TGlzdDt4cgATamF2YS5sYW5nLkV4Y2VwdGlvbtD9Hz4aOxzEAgAAeHIAE2phdmEubGFuZy5UaHJvd2FibGXVxjUnOXe4ywMABEwABWNhdXNldAAVTGphdmEvbGFuZy9UaHJvd2FibGU7TAANZGV0YWlsTWVzc2FnZXQAEkxqYXZhL2xhbmcvU3RyaW5nO1sACnN0YWNrVHJhY2V0AB5bTGphdmEvbGFuZy9TdGFja1RyYWNlRWxlbWVudDtMABRzdXBwcmVzc2VkRXhjZXB0aW9uc3QAEExqYXZhL3V0aWwvTGlzdDt4cHEAfgANdACGVW5hYmxlIHRvIGNyZWF0ZSBhIGRlcGxveW1lbnQgZm9yIFZNIGluc3RhbmNlIHtpZDogIjEyIiwgbmFtZTogImktMi0xMi1WTSIsIHV1aWQ6ICJmZTAwNjQ3NC1lOGM3LTRmNzgtODk4Mi1lNzJjMDU4YzZhM2UiLCB0eXBlPSJVc2VyIn11cgAeW0xqYXZhLmxhbmcuU3RhY2tUcmFjZUVsZW1lbnQ7AkYqPDz9IjkCAAB4cAAAABVzcgAbamF2YS5sYW5nLlN0YWNrVHJhY2VFbGVtZW50YQnFmiY23YUCAAhCAAZmb3JtYXRJAApsaW5lTnVtYmVyTAAPY2xhc3NMb2FkZXJOYW1lcQB-AApMAA5kZWNsYXJpbmdDbGFzc3EAfgAKTAAIZmlsZU5hbWVxAH4ACkwACm1ldGhvZE5hbWVxAH4ACkwACm1vZHVsZU5hbWVxAH4ACkwADW1vZHVsZVZlcnNpb25xAH4ACnhwAQAABIh0AANhcHB0ACZjb20uY2xvdWQudm0uVmlydHVhbE1hY2hpbmVNYW5hZ2VySW1wbHQAHlZpcnR1YWxNYWNoaW5lTWFuYWdlckltcGwuamF2YXQAEG9yY2hlc3RyYXRlU3RhcnRwcHNxAH4AEQEAABbdcQB-ABNxAH4AFHEAfgAVcQB-ABZwcHNxAH4AEQL____-cHQALWpkay5pbnRlcm5hbC5yZWZsZWN0Lk5hdGl2ZU1ldGhvZEFjY2Vzc29ySW1wbHQAHU5hdGl2ZU1ldGhvZEFjY2Vzc29ySW1wbC5qYXZhdAAHaW52b2tlMHQACWphdmEuYmFzZXQACTExLjAuMTQuMXNxAH4AEQIAAAA-cHEAfgAZcQB-ABp0AAZpbnZva2VxAH4AHHEAfgAdc3EAfgARAgAAACtwdAAxamRrLmludGVybmFsLnJlZmxlY3QuRGVsZWdhdGluZ01ldGhvZEFjY2Vzc29ySW1wbHQAIURlbGVnYXRpbmdNZXRob2RBY2Nlc3NvckltcGwuamF2YXEAfgAfcQB-ABxxAH4AHXNxAH4AEQIAAAI2cHQAGGphdmEubGFuZy5yZWZsZWN0Lk1ldGhvZHQAC01ldGhvZC5qYXZhcQB-AB9xAH4AHHEAfgAdc3EAfgARAQAAAGtxAH4AE3QAImNvbS5jbG91ZC52bS5WbVdvcmtKb2JIYW5kbGVyUHJveHl0ABpWbVdvcmtKb2JIYW5kbGVyUHJveHkuamF2YXQAD2hhbmRsZVZtV29ya0pvYnBwc3EAfgARAQAAF4JxAH4AE3EAfgAUcQB-ABVxAH4AKXBwc3EAfgARAQAAAGZxAH4AE3QAIGNvbS5jbG91ZC52bS5WbVdvcmtKb2JEaXNwYXRjaGVydAAYVm1Xb3JrSm9iRGlzcGF0Y2hlci5qYXZhdAAGcnVuSm9icHBzcQB-ABEBAAACbHEAfgATdAA_b3JnLmFwYWNoZS5jbG91ZHN0YWNrLmZyYW1ld29yay5qb2JzLmltcGwuQXN5bmNKb2JNYW5hZ2VySW1wbCQ1dAAYQXN5bmNKb2JNYW5hZ2VySW1wbC5qYXZhdAAMcnVuSW5Db250ZXh0cHBzcQB-ABEBAAAAMHEAfgATdAA-b3JnLmFwYWNoZS5jbG91ZHN0YWNrLm1hbmFnZWQuY29udGV4dC5NYW5hZ2VkQ29udGV4dFJ1bm5hYmxlJDF0ABtNYW5hZ2VkQ29udGV4dFJ1bm5hYmxlLmphdmF0AANydW5wcHNxAH4AEQEAAAA3cQB-ABN0AEJvcmcuYXBhY2hlLmNsb3Vkc3RhY2subWFuYWdlZC5jb250ZXh0LmltcGwuRGVmYXVsdE1hbmFnZWRDb250ZXh0JDF0ABpEZWZhdWx0TWFuYWdlZENvbnRleHQuamF2YXQABGNhbGxwcHNxAH4AEQEAAABmcQB-ABN0AEBvcmcuYXBhY2hlLmNsb3Vkc3RhY2subWFuYWdlZC5jb250ZXh0LmltcGwuRGVmYXVsdE1hbmFnZWRDb250ZXh0cQB-ADl0AA9jYWxsV2l0aENvbnRleHRwcHNxAH4AEQEAAAA0cQB-ABNxAH4APHEAfgA5dAAOcnVuV2l0aENvbnRleHRwcHNxAH4AEQEAAAAtcQB-ABN0ADxvcmcuYXBhY2hlLmNsb3Vkc3RhY2subWFuYWdlZC5jb250ZXh0Lk1hbmFnZWRDb250ZXh0UnVubmFibGVxAH4ANXEAfgA2cHBzcQB-ABEBAAACOHEAfgATcQB-ADBxAH4AMXEAfgA2cHBzcQB-ABECAAACA3B0AC5qYXZhLnV0aWwuY29uY3VycmVudC5FeGVjdXRvcnMkUnVubmFibGVBZGFwdGVydAAORXhlY3V0b3JzLmphdmFxAH4AOnEAfgAccQB-AB1zcQB-ABECAAABCHB0AB9qYXZhLnV0aWwuY29uY3VycmVudC5GdXR1cmVUYXNrdAAPRnV0dXJlVGFzay5qYXZhcQB-ADZxAH4AHHEAfgAdc3EAfgARAgAABGhwdAAnamF2YS51dGlsLmNvbmN1cnJlbnQuVGhyZWFkUG9vbEV4ZWN1dG9ydAAXVGhyZWFkUG9vbEV4ZWN1dG9yLmphdmF0AAlydW5Xb3JrZXJxAH4AHHEAfgAdc3EAfgARAgAAAnRwdAAuamF2YS51dGlsLmNvbmN1cnJlbnQuVGhyZWFkUG9vbEV4ZWN1dG9yJFdvcmtlcnEAfgBLcQB-ADZxAH4AHHEAfgAdc3EAfgARAgAAAz1wdAAQamF2YS5sYW5nLlRocmVhZHQAC1RocmVhZC5qYXZhcQB-ADZxAH4AHHEAfgAdc3IAH2phdmEudXRpbC5Db2xsZWN0aW9ucyRFbXB0eUxpc3R6uBe0PKee3gIAAHhweHNyABFqYXZhLmxhbmcuSW50ZWdlchLioKT3gYc4AgABSQAFdmFsdWV4cgAQamF2YS5sYW5nLk51bWJlcoaslR0LlOCLAgAAeHAAABDvc3IAE2phdmEudXRpbC5BcnJheUxpc3R4gdIdmcdhnQMAAUkABHNpemV4cAAAAAB3BAAAAAB4c3IADmphdmEubGFuZy5Mb25nO4vkkMyPI98CAAFKAAV2YWx1ZXhxAH4AVQAAAAAAAAABdnIAF2NvbS5jbG91ZC5kYy5EYXRhQ2VudGVyU-gkGtLTtlwCAAB4cAA 2022-03-10 07:01:07,940 DEBUG [o.a.c.f.j.i.AsyncJobManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82) (logid:a882c7d6) Publish async job-82 complete on message bus 2022-03-10 07:01:07,940 DEBUG [o.a.c.f.j.i.AsyncJobManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82) (logid:a882c7d6) Wake up jobs related to job-82 2022-03-10 07:01:07,940 DEBUG [o.a.c.f.j.i.AsyncJobManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82) (logid:a882c7d6) Update db status for job-82 2022-03-10 07:01:07,941 DEBUG [o.a.c.f.j.i.AsyncJobManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82) (logid:a882c7d6) Wake up jobs joined with job-82 and disjoin all subjobs created from job- 82 2022-03-10 07:01:08,049 DEBUG [o.a.c.f.j.i.AsyncJobManagerImpl] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82) (logid:a882c7d6) Done executing com.cloud.vm.VmWorkStart for job-82 2022-03-10 07:01:08,051 INFO [o.a.c.f.j.i.AsyncJobMonitor] (Work-Job-Executor-3:ctx-938cc33b job-78/job-82) (logid:a882c7d6) Remove job-82 from job monitoring 2022-03-10 07:01:08,059 DEBUG [c.c.v.UserVmManagerImpl] (API-Job-Executor-2:ctx-949eff6a job-78 ctx-7b924ad9) (logid:a882c7d6) Destroying vm VM instance {id: "12", name: "i-2-12-VM", uuid: "fe006474-e8c7-4f78-8982-e72c058c6a3e", type="User"} as it failed to create on Host with Id:null 2022-03-10 07:01:08,065 DEBUG [c.c.c.CapacityManagerImpl] (API-Job-Executor-2:ctx-949eff6a job-78 ctx-7b924ad9) (logid:a882c7d6) VM instance {id: "12", name: "i-2-12-VM", uuid: "fe006474-e8c7-4f78-8982-e72c058c6a3e", type="User"} state transited from [Stopped] to [Error] with event [OperationFailedToError]. VM's original host: null, new host: null, host before state transition: null 2022-03-10 07:01:08,078 DEBUG [c.c.r.ResourceLimitManagerImpl] (API-Job-Executor-2:ctx-949eff6a job-78 ctx-7b924ad9) (logid:a882c7d6) Updating resource Type = volume count for Account = 2 Operation = decreasing Amount = 1 2022-03-10 07:01:08,082 DEBUG [c.c.r.ResourceLimitManagerImpl] (API-Job-Executor-2:ctx-949eff6a job-78 ctx-7b924ad9) (logid:a882c7d6) Updating resource Type = primary_storage count for Account = 2 Operation = decreasing Amount = (2.00 GB) 2147483648 2022-03-10 07:01:08,092 WARN [c.c.a.AlertManagerImpl] (API-Job-Executor-2:ctx-949eff6a job-78 ctx-7b924ad9) (logid:a882c7d6) alertType=[8] dataCenterId=[1] podId=[1] clusterId=[null] message=[Failed to deploy Vm with Id: 12, on Host with Id: null]. 2022-03-10 07:01:08,097 WARN [c.c.a.AlertManagerImpl] (API-Job-Executor-2:ctx-949eff6a job-78 ctx-7b924ad9) (logid:a882c7d6) No recipients set in 'alert.email.addresses', skipping sending alert with subject: Failed to deploy Vm with Id: 12, on Host with Id: null and content: Failed to deploy Vm with Id: 12, on Host with Id: null 2022-03-10 07:01:08,098 DEBUG [c.c.r.ResourceLimitManagerImpl] (API-Job-Executor-2:ctx-949eff6a job-78 ctx-7b924ad9) (logid:a882c7d6) Updating resource Type = user_vm count for Account = 2 Operation = decreasing Amount = 1 2022-03-10 07:01:08,102 DEBUG [c.c.r.ResourceLimitManagerImpl] (API-Job-Executor-2:ctx-949eff6a job-78 ctx-7b924ad9) (logid:a882c7d6) Updating resource Type = cpu count for Account = 2 Operation = decreasing Amount = 1 2022-03-10 07:01:08,106 DEBUG [c.c.r.ResourceLimitManagerImpl] (API-Job-Executor-2:ctx-949eff6a job-78 ctx-7b924ad9) (logid:a882c7d6) Updating resource Type = memory count for Account = 2 Operation = decreasing Amount = 512 2022-03-10 07:01:08,114 ERROR [c.c.a.ApiAsyncJobDispatcher] (API-Job-Executor-2:ctx-949eff6a job-78) (logid:a882c7d6) Unexpected exception while executing org.apache.cloudstack.api.command.admin.vm.DeployVMCmdByAdmin com.cloud.utils.exception.CloudRuntimeException: Unable to start a VM due to insufficient capacity at com.cloud.vm.VirtualMachineManagerImpl.start(VirtualMachineManagerImpl.java:791) at org.apache.cloudstack.engine.cloud.entity.api.VMEntityManagerImpl.deployVirtualMachine(VMEntityManagerImpl.java:255) at org.apache.cloudstack.engine.cloud.entity.api.VirtualMachineEntityImpl.deploy(VirtualMachineEntityImpl.java:214) at com.cloud.vm.UserVmManagerImpl.startVirtualMachine(UserVmManagerImpl.java:5251) at com.cloud.vm.UserVmManagerImpl.startVirtualMachine(UserVmManagerImpl.java:5102) at com.cloud.vm.UserVmManagerImpl.startVirtualMachine(UserVmManagerImpl.java:4729) at com.cloud.vm.UserVmManagerImpl.startVirtualMachine(UserVmManagerImpl.java:4718) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:566) at org.springframework.aop.support.AopUtils.invokeJoinpointUsingReflection(AopUtils.java:344) at org.springframework.aop.framework.ReflectiveMethodInvocation.invokeJoinpoint(ReflectiveMethodInvocation.java:198) at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:163) at org.apache.cloudstack.network.contrail.management.EventUtils$EventInterceptor.invoke(EventUtils.java:107) at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:175) at com.cloud.event.ActionEventInterceptor.invoke(ActionEventInterceptor.java:51) at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:175) at org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke(ExposeInvocationInterceptor.java:97) at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:186) at org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:215) at com.sun.proxy.$Proxy127.startVirtualMachine(Unknown Source) at org.apache.cloudstack.api.command.user.vm.DeployVMCmd.execute(DeployVMCmd.java:696) at com.cloud.api.ApiDispatcher.dispatch(ApiDispatcher.java:156) at com.cloud.api.ApiAsyncJobDispatcher.runJob(ApiAsyncJobDispatcher.java:108) at org.apache.cloudstack.framework.jobs.impl.AsyncJobManagerImpl$5.runInContext(AsyncJobManagerImpl.java:620) at org.apache.cloudstack.managed.context.ManagedContextRunnable$1.run(ManagedContextRunnable.java:48) at org.apache.cloudstack.managed.context.impl.DefaultManagedContext$1.call(DefaultManagedContext.java:55) at org.apache.cloudstack.managed.context.impl.DefaultManagedContext.callWithContext(DefaultManagedContext.java:102) at org.apache.cloudstack.managed.context.impl.DefaultManagedContext.runWithContext(DefaultManagedContext.java:52) at org.apache.cloudstack.managed.context.ManagedContextRunnable.run(ManagedContextRunnable.java:45) at org.apache.cloudstack.framework.jobs.impl.AsyncJobManagerImpl$5.run(AsyncJobManagerImpl.java:568) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) Caused by: com.cloud.exception.InsufficientServerCapacityException: Unable to create a deployment for VM instance {id: "12", name: "i-2-12-VM", uuid: "fe006474-e8c7-4f78-8982-e72c058c6a3e", type="User"}Scope=interface com.cloud.dc.DataCenter; id=1 at com.cloud.vm.VirtualMachineManagerImpl.orchestrateStart(VirtualMachineManagerImpl.java:1160) at com.cloud.vm.VirtualMachineManagerImpl.orchestrateStart(VirtualMachineManagerImpl.java:5853) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ... 18 more 2022-03-10 07:01:08,116 DEBUG [o.a.c.f.j.i.AsyncJobManagerImpl] (API-Job-Executor-2:ctx-949eff6a job-78) (logid:a882c7d6) Complete async job-78, jobStatus: FAILED, resultCode: 530, result: org.apache.cloudstack.api.response.ExceptionResponse/null/{"uuidList":[],"errorcode":"530","errortext":"Unable to start a VM due to insufficient capacity"} 2022-03-10 07:01:08,117 DEBUG [o.a.c.f.j.i.AsyncJobManagerImpl] (API-Job-Executor-2:ctx-949eff6a job-78) (logid:a882c7d6) Publish async job-78 complete on message bus 2022-03-10 07:01:08,117 DEBUG [o.a.c.f.j.i.AsyncJobManagerImpl] (API-Job-Executor-2:ctx-949eff6a job-78) (logid:a882c7d6) Wake up jobs related to job-78 2022-03-10 07:01:08,117 DEBUG [o.a.c.f.j.i.AsyncJobManagerImpl] (API-Job-Executor-2:ctx-949eff6a job-78) (logid:a882c7d6) Update db status for job-78 2022-03-10 07:01:08,118 DEBUG [o.a.c.f.j.i.AsyncJobManagerImpl] (API-Job-Executor-2:ctx-949eff6a job-78) (logid:a882c7d6) Wake up jobs joined with job-78 and disjoin all subjobs created from job- 78 2022-03-10 07:01:08,128 DEBUG [o.a.c.f.j.i.AsyncJobManagerImpl] (API-Job-Executor-2:ctx-949eff6a job-78) (logid:a882c7d6) Done executing org.apache.cloudstack.api.command.admin.vm.DeployVMCmdByAdmin for job-78 2022-03-10 07:01:08,128 INFO [o.a.c.f.j.i.AsyncJobMonitor] (API-Job-Executor-2:ctx-949eff6a job-78) (logid:a882c7d6) Remove job-78 from job monitoring 2022-03-10 07:01:08,258 DEBUG [c.c.a.ApiServlet] (qtp1747352992-13:ctx-5a566825) (logid:a45531f9) ===START=== 10.0.3.151 -- GET listall=true&page=1&pagesize=20&showIcon=true&command=listVirtualMachinesMetrics&response=json 2022-03-10 07:01:08,263 DEBUG [c.c.a.ApiServer] (qtp1747352992-13:ctx-5a566825 ctx-47e31cd2) (logid:a45531f9) CIDRs from which account 'Acct[11c652b3-9fc4-11ec-81d4-1e00ca0003f4-admin] -- Account {"id": 2, "name": "admin", "uuid": "11c652b3-9fc4-11ec-81d4-1e00ca0003f4"}' is allowed to perform API calls: 0.0.0.0/0,::/0 2022-03-10 07:01:08,286 DEBUG [c.c.a.ApiServlet] (qtp1747352992-13:ctx-5a566825 ctx-47e31cd2) (logid:a45531f9) ===END=== 10.0.3.151 -- GET listall=true&page=1&pagesize=20&showIcon=true&command=listVirtualMachinesMetrics&response=json 2022-03-10 07:01:08,441 DEBUG [c.c.a.ApiServlet] (qtp1747352992-21:ctx-6571544a) (logid:222a46c9) ===START=== 10.0.3.151 -- GET jobId=a882c7d6-b61b-4684-8285-079287e3c320&command=queryAsyncJobResult&response=json 2022-03-10 07:01:08,446 DEBUG [c.c.a.ApiServer] (qtp1747352992-21:ctx-6571544a ctx-29ab395e) (logid:222a46c9) CIDRs from which account 'Acct[11c652b3-9fc4-11ec-81d4-1e00ca0003f4-admin] -- Account {"id": 2, "name": "admin", "uuid": "11c652b3-9fc4-11ec-81d4-1e00ca0003f4"}' is allowed to perform API calls: 0.0.0.0/0,::/0 2022-03-10 07:01:08,456 DEBUG [c.c.a.ApiServlet] (qtp1747352992-21:ctx-6571544a ctx-29ab395e) (logid:222a46c9) ===END=== 10.0.3.151 -- GET jobId=a882c7d6-b61b-4684-8285-079287e3c320&command=queryAsyncJobResult&response=json 2022-03-10 07:01:08,714 DEBUG [c.c.s.StatsCollector] (StatsCollector-1:ctx-ec5c77a9) (logid:b433b002) AutoScaling Monitor is running... 2022-03-10 07:01:08,722 DEBUG [c.c.a.m.DirectAgentAttache] (DirectAgent-14:ctx-c45bd9c1) (logid:9e064a46) Seq 4-3240621406869782545: Executing request 2022-03-10 07:01:08,761 DEBUG [c.c.h.x.r.CitrixResourceBase] (DirectAgent-14:ctx-c45bd9c1) (logid:89744db9) Vm cpu utilization 72.75 2022-03-10 07:01:08,761 DEBUG [c.c.a.m.DirectAgentAttache] (DirectAgent-14:ctx-c45bd9c1) (logid:89744db9) Seq 4-3240621406869782545: Response Received: 2022-03-10 07:01:08,769 DEBUG [c.c.a.t.Request] (StatsCollector-6:ctx-2ab49b88) (logid:89744db9) Seq 4-3240621406869782545: Received: { Ans: , MgmtId: 32988737831924, via: 4(trl-283-v-M7-boris-stoyanov-xs2), Ver: v1, Flags: 10, { GetVmStatsAnswer } } 2022-03-10 07:01:09,195 DEBUG [c.c.s.StatsCollector] (StatsCollector-3:ctx-b2e21041) (logid:e28f3be5) HostStatsCollector is running... 2022-03-10 07:01:09,199 DEBUG [c.c.a.m.DirectAgentAttache] (DirectAgent-26:ctx-93652a6f) (logid:e4ab144e) Seq 1-5082593654464315404: Executing request 2022-03-10 07:01:09,223 DEBUG [c.c.a.m.DirectAgentAttache] (DirectAgent-26:ctx-93652a6f) (logid:e28f3be5) Seq 1-5082593654464315404: Response Received: 2022-03-10 07:01:09,224 DEBUG [c.c.a.t.Request] (StatsCollector-3:ctx-b2e21041) (logid:e28f3be5) Seq 1-5082593654464315404: Received: { Ans: , MgmtId: 32988737831924, via: 1(10.0.33.183), Ver: v1, Flags: 10, { GetHostStatsAnswer } } 2022-03-10 07:01:09,227 DEBUG [c.c.a.m.DirectAgentAttache] (DirectAgent-1:ctx-e1ed62fb) (logid:27e242ed) Seq 2-3538984882183077898: Executing request 2022-03-10 07:01:09,234 DEBUG [c.c.a.m.DirectAgentAttache] (DirectAgentCronJob-1:ctx-47dda4de) (logid:64697338) Ping from 3(trl-283-v-M7-boris-stoyanov-xs1) 2022-03-10 07:01:09,234 DEBUG [c.c.v.VirtualMachinePowerStateSyncImpl] (DirectAgentCronJob-1:ctx-47dda4de) (logid:64697338) Process host VM state report from ping process. host: 3 2022-03-10 07:01:09,238 DEBUG [c.c.v.VirtualMachinePowerStateSyncImpl] (DirectAgentCronJob-1:ctx-47dda4de) (logid:64697338) Process VM state report. host: 3, number of records in report: 1 2022-03-10 07:01:09,238 DEBUG [c.c.v.VirtualMachinePowerStateSyncImpl] (DirectAgentCronJob-1:ctx-47dda4de) (logid:64697338) VM state report. host: 3, vm id: 11, power state: PowerOn 2022-03-10 07:01:09,245 DEBUG [c.c.v.VirtualMachinePowerStateSyncImpl] (DirectAgentCronJob-1:ctx-47dda4de) (logid:64697338) VM state report is updated. host: 3, vm id: 11, power state: PowerOn 2022-03-10 07:01:09,248 DEBUG [c.c.a.m.DirectAgentAttache] (DirectAgent-1:ctx-e1ed62fb) (logid:e28f3be5) Seq 2-3538984882183077898: Response Received: 2022-03-10 07:01:09,248 DEBUG [c.c.a.t.Request] (StatsCollector-3:ctx-b2e21041) (logid:e28f3be5) Seq 2-3538984882183077898: Received: { Ans: , MgmtId: 32988737831924, via: 2(10.0.34.192), Ver: v1, Flags: 10, { GetHostStatsAnswer } } 2022-03-10 07:01:09,252 DEBUG [c.c.a.m.DirectAgentAttache] (DirectAgent-27:ctx-30a92daf) (logid:7f84691c) Seq 3-4849250898771181588: Executing request 2022-03-10 07:01:09,259 DEBUG [c.c.c.CapacityManagerImpl] (DirectAgentCronJob-1:ctx-47dda4de) (logid:64697338) VM instance {id: "11", name: "r-11-VM", uuid: "d27d7dfa-db2b-4726-8965-4a3b873d70b6", type="DomainRouter"} state transited from [Running] to [Running] with event [FollowAgentPowerOnReport]. VM's original host: Host {"id": "3", "name": "trl-283-v-M7-boris-stoyanov-xs1", "uuid": "bd640d4a-608f-4bdf-8d0f-b95bb3310dff", "type"="Routing"}, new host: Host {"id": "3", "name": "trl-283-v-M7-boris-stoyanov-xs1", "uuid": "bd640d4a-608f-4bdf-8d0f-b95bb3310dff", "type"="Routing"}, host before state transition: Host {"id": "3", "name": "trl-283-v-M7-boris-stoyanov-xs1", "uuid": "bd640d4a-608f-4bdf-8d0f-b95bb3310dff", "type"="Routing"} 2022-03-10 07:01:09,261 DEBUG [c.c.v.VirtualMachinePowerStateSyncImpl] (DirectAgentCronJob-1:ctx-47dda4de) (logid:64697338) Done with process of VM state report. host: 3 2022-03-10 07:01:09,266 DEBUG [c.c.a.m.DirectAgentAttache] (DirectAgent-27:ctx-30a92daf) (logid:e28f3be5) Seq 3-4849250898771181588: Response Received: 2022-03-10 07:01:09,267 DEBUG [c.c.a.t.Request] (StatsCollector-3:ctx-b2e21041) (logid:e28f3be5) Seq 3-4849250898771181588: Received: { Ans: , MgmtId: 32988737831924, via: 3(trl-283-v-M7-boris-stoyanov-xs1), Ver: v1, Flags: 10, { GetHostStatsAnswer } } 2022-03-10 07:01:09,273 DEBUG [c.c.a.m.DirectAgentAttache] (DirectAgent-8:ctx-52d24ccf) (logid:593d91f3) Seq 4-3240621406869782546: Executing request 2022-03-10 07:01:09,319 DEBUG [c.c.a.m.DirectAgentAttache] (DirectAgent-8:ctx-52d24ccf) (logid:e28f3be5) Seq 4-3240621406869782546: Response Received: 2022-03-10 07:01:09,319 DEBUG [c.c.a.t.Request] (StatsCollector-3:ctx-b2e21041) (logid:e28f3be5) Seq 4-3240621406869782546: Received: { Ans: , MgmtId: 32988737831924, via: 4(trl-283-v-M7-boris-stoyanov-xs2), Ver: v1, Flags: 10, { GetHostStatsAnswer } } 2022-03-10 07:01:09,325 DEBUG [c.c.a.m.DirectAgentAttache] (DirectAgent-55:ctx-8253ff56) (logid:cc2abf88) Seq 3-4849250898771181589: Executing request 2022-03-10 07:01:09,352 DEBUG [c.c.a.m.DirectAgentAttache] (DirectAgent-55:ctx-8253ff56) (logid:e28f3be5) Seq 3-4849250898771181589: Response Received: 2022-03-10 07:01:09,352 DEBUG [c.c.a.t.Request] (StatsCollector-3:ctx-b2e21041) (logid:e28f3be5) Seq 3-4849250898771181589: Received: { Ans: , MgmtId: 32988737831924, via: 3(trl-283-v-M7-boris-stoyanov-xs1), Ver: v1, Flags: 10, { GetGPUStatsAnswer } } 2022-03-10 07:01:09,355 DEBUG [c.c.a.m.DirectAgentAttache] (DirectAgent-56:ctx-d8357b13) (logid:0697e327) Seq 4-3240621406869782547: Executing request 2022-03-10 07:01:09,373 DEBUG [c.c.a.m.DirectAgentAttache] (DirectAgent-56:ctx-d8357b13) (logid:e28f3be5) Seq 4-3240621406869782547: Response Received: 2022-03-10 07:01:09,373 DEBUG [c.c.a.t.Request] (StatsCollector-3:ctx-b2e21041) (logid:e28f3be5) Seq 4-3240621406869782547: Received: { Ans: , MgmtId: 32988737831924, via: 4(trl-283-v-M7-boris-stoyanov-xs2), Ver: v1, Flags: 10, { GetGPUStatsAnswer } } 2022-03-10 07:01:10,801 DEBUG [c.c.a.m.DirectAgentAttache] (DirectAgentCronJob-11:ctx-8ef5db12) (logid:9c2e17ac) Ping from 4(trl-283-v-M7-boris-stoyanov-xs2) 2022-03-10 07:01:10,801 DEBUG [c.c.v.VirtualMachinePowerStateSyncImpl] (DirectAgentCronJob-11:ctx-8ef5db12) (logid:9c2e17ac) Process host VM state report from ping process. host: 4 2022-03-10 07:01:10,805 DEBUG [c.c.v.VirtualMachinePowerStateSyncImpl] (DirectAgentCronJob-11:ctx-8ef5db12) (logid:9c2e17ac) Process VM state report. host: 4, number of records in report: 2 2022-03-10 07:01:10,805 DEBUG [c.c.v.VirtualMachinePowerStateSyncImpl] (DirectAgentCronJob-11:ctx-8ef5db12) (logid:9c2e17ac) VM state report. host: 4, vm id: 1, power state: PowerOn 2022-03-10 07:01:10,808 DEBUG [c.c.v.VirtualMachinePowerStateSyncImpl] (DirectAgentCronJob-11:ctx-8ef5db12) (logid:9c2e17ac) VM state report is updated. host: 4, vm id: 1, power state: PowerOn 2022-03-10 07:01:10,813 DEBUG [c.c.c.CapacityManagerImpl] (DirectAgentCronJob-11:ctx-8ef5db12) (logid:9c2e17ac) VM instance {id: "1", name: "s-1-VM", uuid: "4712c16f-9926-4f02-9fd2-126a59f33070", type="SecondaryStorageVm"} state transited from [Running] to [Running] with event [FollowAgentPowerOnReport]. VM's original host: Host {"id": "4", "name": "trl-283-v-M7-boris-stoyanov-xs2", "uuid": "1416da17-9f06-48cb-9fec-d5c70a978569", "type"="Routing"}, new host: Host {"id": "4", "name": "trl-283-v-M7-boris-stoyanov-xs2", "uuid": "1416da17-9f06-48cb-9fec-d5c70a978569", "type"="Routing"}, host before state transition: Host {"id": "4", "name": "trl-283-v-M7-boris-stoyanov-xs2", "uuid": "1416da17-9f06-48cb-9fec-d5c70a978569", "type"="Routing"} 2022-03-10 07:01:10,814 DEBUG [c.c.v.VirtualMachinePowerStateSyncImpl] (DirectAgentCronJob-11:ctx-8ef5db12) (logid:9c2e17ac) VM state report. host: 4, vm id: 10, power state: PowerOn 2022-03-10 07:01:10,817 DEBUG [c.c.v.VirtualMachinePowerStateSyncImpl] (DirectAgentCronJob-11:ctx-8ef5db12) (logid:9c2e17ac) VM state report is updated. host: 4, vm id: 10, power state: PowerOn 2022-03-10 07:01:10,822 DEBUG [c.c.c.CapacityManagerImpl] (DirectAgentCronJob-11:ctx-8ef5db12) (logid:9c2e17ac) VM instance {id: "10", name: "i-2-10-VM", uuid: "ec8b2959-a7f3-4d65-9bca-bda421d885cb", type="User"} state transited from [Running] to [Running] with event [FollowAgentPowerOnReport]. VM's original host: Host {"id": "4", "name": "trl-283-v-M7-boris-stoyanov-xs2", "uuid": "1416da17-9f06-48cb-9fec-d5c70a978569", "type"="Routing"}, new host: Host {"id": "4", "name": "trl-283-v-M7-boris-stoyanov-xs2", "uuid": "1416da17-9f06-48cb-9fec-d5c70a978569", "type"="Routing"}, host before state transition: Host {"id": "4", "name": "trl-283-v-M7-boris-stoyanov-xs2", "uuid": "1416da17-9f06-48cb-9fec-d5c70a978569", "type"="Routing"} 2022-03-10 07:01:10,825 DEBUG [c.c.v.VirtualMachinePowerStateSyncImpl] (DirectAgentCronJob-11:ctx-8ef5db12) (logid:9c2e17ac) Done with process of VM state report. host: 4 2022-03-10 07:01:10,844 DEBUG [o.a.c.h.HAManagerImpl] (BackgroundTaskPollManager-6:ctx-ee697285) (logid:97da1f62) HA health check task is running... 2022-03-10 07:01:11,107 DEBUG [c.c.s.StatsCollector] (StatsCollector-5:ctx-4be0d3a4) (logid:39d26d00) StorageCollector is running... 2022-03-10 07:01:11,112 DEBUG [c.c.h.o.r.Ovm3HypervisorGuru] (StatsCollector-5:ctx-4be0d3a4) (logid:39d26d00) getCommandHostDelegation: class com.cloud.agent.api.GetStorageStatsCommand 2022-03-10 07:01:11,112 DEBUG [c.c.h.XenServerGuru] (StatsCollector-5:ctx-4be0d3a4) (logid:39d26d00) We are returning the default host to execute commands because the command is not of Copy type. 2022-03-10 07:01:11,165 DEBUG [c.c.a.t.Request] (StatsCollector-5:ctx-4be0d3a4) (logid:39d26d00) Seq 6-8975955532326109195: Received: { Ans: , MgmtId: 32988737831924, via: 6(s-1-VM), Ver: v1, Flags: 10, { GetStorageStatsAnswer } } 2022-03-10 07:01:11,169 DEBUG [c.c.h.o.r.Ovm3HypervisorGuru] (StatsCollector-5:ctx-4be0d3a4) (logid:39d26d00) getCommandHostDelegation: class com.cloud.agent.api.GetStorageStatsCommand 2022-03-10 07:01:11,169 DEBUG [c.c.h.XenServerGuru] (StatsCollector-5:ctx-4be0d3a4) (logid:39d26d00) We are returning the default host to execute commands because the command is not of Copy type. 2022-03-10 07:01:11,171 DEBUG [c.c.a.m.DirectAgentAttache] (DirectAgent-15:ctx-072e421f) (logid:e40bcc7d) Seq 2-3538984882183077899: Executing request 2022-03-10 07:01:11,203 DEBUG [c.c.h.v.r.VmwareResource] (DirectAgent-15:ctx-072e421f 10.0.34.192, cmd: GetStorageStatsCommand) (logid:39d26d00) Datastore summary info, storageId: e7d754b3-ee65-3b09-b36d-2c1b23298ffb, localPath: /acs/primary/trl-283-v-M7-boris-stoyanov/trl-283-v-M7-boris-stoyanov-esxi-pri1, poolType: NetworkFilesystem, capacity: (1.9990 TB) 2197949513728, free: (811.50 GB) 871338414080, used: (1.2065 TB) 1326611099648 2022-03-10 07:01:11,203 DEBUG [c.c.a.m.DirectAgentAttache] (DirectAgent-15:ctx-072e421f) (logid:39d26d00) Seq 2-3538984882183077899: Response Received: 2022-03-10 07:01:11,204 DEBUG [c.c.a.t.Request] (StatsCollector-5:ctx-4be0d3a4) (logid:39d26d00) Seq 2-3538984882183077899: Received: { Ans: , MgmtId: 32988737831924, via: 2(10.0.34.192), Ver: v1, Flags: 10, { GetStorageStatsAnswer } } 2022-03-10 07:01:11,204 DEBUG [c.c.h.o.r.Ovm3HypervisorGuru] (StatsCollector-5:ctx-4be0d3a4) (logid:39d26d00) getCommandHostDelegation: class com.cloud.agent.api.GetStorageStatsCommand 2022-03-10 07:01:11,205 DEBUG [c.c.h.XenServerGuru] (StatsCollector-5:ctx-4be0d3a4) (logid:39d26d00) We are returning the default host to execute commands because the command is not of Copy type. 2022-03-10 07:01:11,206 DEBUG [c.c.a.m.DirectAgentAttache] (DirectAgent-58:ctx-29816828) (logid:96ed06a8) Seq 4-3240621406869782548: Executing request 2022-03-10 07:01:11,700 DEBUG [c.c.a.m.AgentManagerImpl] (AgentManager-Handler-8:null) (logid:) SeqA 5-6801: Processing Seq 5-6801: { Cmd , MgmtId: -1, via: 5, Ver: v1, Flags: 11, [{"com.cloud.agent.api.ConsoleProxyLoadReportCommand":{"_proxyVmId":"2","_loadInfo":"{ "connections": [] }","wait":"0","bypassHostMaintenance":"false"}}] } 2022-03-10 07:01:11,705 DEBUG [c.c.a.m.AgentManagerImpl] (AgentManager-Handler-8:null) (logid:) SeqA 5-6801: Sending Seq 5-6801: { Ans: , MgmtId: 32988737831924, via: 5, Ver: v1, Flags: 100010, [{"com.cloud.agent.api.AgentControlAnswer":{"result":"true","wait":"0","bypassHostMaintenance":"false"}}] } 2022-03-10 07:01:12,077 DEBUG [c.c.a.m.DirectAgentAttache] (DirectAgent-58:ctx-29816828) (logid:39d26d00) Seq 4-3240621406869782548: Response Received: 2022-03-10 07:01:12,078 DEBUG [c.c.a.t.Request] (StatsCollector-5:ctx-4be0d3a4) (logid:39d26d00) Seq 4-3240621406869782548: Received: { Ans: , MgmtId: 32988737831924, via: 4(trl-283-v-M7-boris-stoyanov-xs2), Ver: v1, Flags: 10, { GetStorageStatsAnswer } } 2022-03-10 07:01:13,291 DEBUG [c.c.a.m.DirectAgentAttache] (DirectAgentCronJob-2:ctx-6300a9c7) (logid:010cb78e) Seq 3-4849250898771181573: Executing request 2022-03-10 07:01:13,332 DEBUG [c.c.a.m.DirectAgentAttache] (DirectAgentCronJob-2:ctx-6300a9c7) (logid:66797afc) Seq 3-4849250898771181573: Response Received: 2022-03-10 07:01:13,333 DEBUG [c.c.a.t.Request] (DirectAgentCronJob-2:ctx-6300a9c7) (logid:66797afc) Seq 3-4849250898771181573: Processing: { Ans: , MgmtId: 32988737831924, via: 3(trl-283-v-M7-boris-stoyanov-xs1), Ver: v1, Flags: 10, [{"com.cloud.agent.api.ClusterVMMetaDataSyncAnswer":{"_clusterId":"2","_vmMetaDatum":{"r-11-VM":"device-model:qemu-upstream-compat;apic:true;viridian:true;timeoffset:0;pae:true;acpi:1;hpet:true;secureboot:false;nx:true","s-1-VM":"device-model:qemu-upstream-compat;apic:true;viridian:true;timeoffset:1;pae:true;acpi:1;hpet:true;secureboot:false;nx:true","i-2-10-VM":"device-model:qemu-upstream-compat;vga:std;videoram:8;apic:true;viridian:false;timeoffset:0;pae:true;acpi:1;hpet:true;secureboot:false;nx:true"},"_isExecuted":"false","result":"true","wait":"0","bypassHostMaintenance":"false"}}] } 2022-03-10 07:01:14,846 DEBUG [o.a.c.h.HAManagerImpl] (BackgroundTaskPollManager-3:ctx-21161efc) (logid:ba707286) HA health check task is running... 2022-03-10 07:01:15,102 DEBUG [c.c.s.StatsCollector] (secstorage-1:ctx-181008a4) (logid:283d29d3) Verifying image storage [1]. Capacity: total=[1 TB], used=[1 TB], threshold=[89.99999761581421%]. 2022-03-10 07:01:15,106 DEBUG [o.a.c.s.SecondaryStorageManagerImpl] (secstorage-1:ctx-181008a4) (logid:283d29d3) Zone [1] is ready to launch secondary storage VM. ^C [root@trl-283-v-M7-boris-stoyanov-mgmt1 ~]#