NSX: Add retry logic with sleep to delete segments (#8554)

* NSX: Add retry logic with sleep to delete segments

* add logs

Author: Pearl1594

plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxApiClient.java

@@ -453,19 +453,42 @@ protected void removeSegment(String segmentName) {
         String siteId = getDefaultSiteId();
         String enforcementPointPath = getDefaultEnforcementPointPath(siteId);
         SegmentPorts segmentPortsService = (SegmentPorts) nsxService.apply(SegmentPorts.class);
-        PolicyGroupMembersListResult segmentPortsList = segmentPortsService.list(DEFAULT_DOMAIN, segmentName, null, enforcementPointPath,
-                false, null, 50L, false, null);
-        if (segmentPortsList.getResultCount() == 0L) {
+        PolicyGroupMembersListResult segmentPortsList = getSegmentPortList(segmentPortsService, segmentName, enforcementPointPath);
+        Long portCount = segmentPortsList.getResultCount();
+        portCount = retrySegmentDeletion(segmentPortsService, portCount, segmentName, enforcementPointPath);
+        LOGGER.info("Port count: " + portCount);
+        if (portCount == 0L) {
             LOGGER.debug(String.format("Removing the segment with ID %s", segmentName));
             removeGroupForSegment(segmentName);
             segmentService.delete(segmentName);
         } else {
-            String msg = String.format("Cannot remove the NSX segment %s because there are still %s port group(s) attached to it", segmentName, segmentPortsList.getResultCount());
+            String msg = String.format("Cannot remove the NSX segment %s because there are still %s port group(s) attached to it", segmentName, portCount);
             LOGGER.debug(msg);
             throw new CloudRuntimeException(msg);
         }
     }
 
+    private PolicyGroupMembersListResult getSegmentPortList(SegmentPorts segmentPortsService, String segmentName, String enforcementPointPath) {
+        return segmentPortsService.list(DEFAULT_DOMAIN, segmentName, null, enforcementPointPath,
+                false, null, 50L, false, null);
+    }
+
+    private Long retrySegmentDeletion(SegmentPorts segmentPortsService, Long portCount, String segmentName, String enforcementPointPath) {
+        int retries = 20;
+        int count = 1;
+        do {
+            try {
+                LOGGER.info("Waiting for all port groups to be unlinked from the segment - Attempt: " + count++ + " Waiting for 5 secs");
+                Thread.sleep(5000);
+                portCount = getSegmentPortList(segmentPortsService, segmentName, enforcementPointPath).getResultCount();
+                retries--;
+            } catch (InterruptedException e) {
+                throw new CloudRuntimeException(String.format("Unable to delete segment %s due to: %s", segmentName, e.getLocalizedMessage()));
+            }
+        } while (retries > 0 && portCount > 0);
+        return portCount;
+    }
+
     public void createStaticNatRule(String vpcName, String tier1GatewayName,
                                     String ruleName, String publicIp, String vmIp) {
         try {