--- /dev/null
+from __future__ import unicode_literals
+
+from datetime import datetime
+
+from os_connector import OpenStackActions
+from logger import logger
+
+try:
+ import prehooks
+except ImportError:
+ pass
+
+
+class Cleaner:
+ DEFAULT_LIFETIME_HOURS = 48
+
+ def __init__(self, lifetime=None, os_auth_url=None,
+ os_user=None, os_password=None, os_project_name=None,
+ os_project_domain_name=None,
+ os_user_domain_name=None):
+ self.os_conn = OpenStackActions(
+ auth_url=os_auth_url,
+ user=os_user,
+ password=os_password,
+ project_name=os_project_name,
+ project_domain_name=os_project_domain_name,
+ user_domain_name=os_user_domain_name)
+
+ self.lifetime = lifetime if lifetime is not None \
+ else self.DEFAULT_LIFETIME_HOURS
+ self.heat_resources_cache = {}
+ self.non_heat_resources_cache = {}
+
+ @property
+ def heat_resources(self):
+ if not self.heat_resources_cache:
+ self.heat_resources_cache = self.collect_resources_under_heat()
+ return self.heat_resources_cache
+
+ @property
+ def non_heat_resources(self):
+ if not self.non_heat_resources_cache:
+ self.non_heat_resources_cache = \
+ self.collect_os_non_stack_resources()
+ return self.non_heat_resources_cache
+
+ def get_stacks(self):
+ return list(self.os_conn.get_stacks())
+
+ def get_os_stack_resources(self, stack):
+ if stack.stack_status == "CREATE_IN_PROGRESS":
+ return {}
+ client = stack.manager.client
+ req = "/stacks/{}/{}/resources?nested_depth=99".format(
+ stack.stack_name, stack.id)
+ data = client.get(req)
+ content = data.json()
+ resources = content['resources']
+ result = {}
+ for resource in resources:
+ if hasattr(resource["resource_type"], "startswith") and \
+ resource["resource_type"].startswith("OS::"):
+ result[resource["physical_resource_id"]] = resource
+ return result
+
+ def collect_os_non_stack_resources(self):
+ result = {"servers": [],
+ "networks": [],
+ "routers": []}
+ for server in self.get_servers():
+ result["servers"].append(server)
+ for net in self.get_networks():
+ result["networks"].append(net)
+ for router in self.get_routers():
+ result["routers"].append(router)
+
+ return result
+
+ @staticmethod
+ def get_creation_date(os_obj):
+ return datetime.strptime(os_obj, "%Y-%m-%dT%H:%M:%SZ")
+
+ def check_lifetime(self, creation_time):
+ delta = datetime.utcnow() - creation_time
+ return delta.seconds // 3600 > self.lifetime
+
+ def collect_resources_under_heat(self):
+ stacks = self.get_stacks()
+ resources = {}
+ for stack in stacks:
+ resources.update(self.get_os_stack_resources(stack))
+ return resources
+
+ def get_servers(self):
+ return self.os_conn.get_servers()
+
+ def get_networks(self):
+ return self.os_conn.get_networks()
+
+ def get_routers(self):
+ return self.os_conn.get_routers()
+
+ def cleanup_stack_one_by_one(self, stack):
+ resources = self.get_os_stack_resources(stack)
+ server_uuids = []
+ router_uuids = []
+ network_uuids = []
+ secgroup_uuids = []
+ floating_ip_uuids = []
+ volumes_uuids = []
+ for uuid, resource in resources.items():
+ if resource["resource_type"] == "OS::Nova::Server":
+ server_uuids.append(uuid)
+ elif resource["resource_type"] == "OS::Neutron::Router":
+ router_uuids.append(uuid)
+ elif resource["resource_type"] == "OS::Neutron::Net":
+ network_uuids.append(uuid)
+ elif resource["resource_type"] == "OS::Neutron::SecurityGroup":
+ secgroup_uuids.append(uuid)
+ elif resource["resource_type"] == "OS::Neutron::FloatingIP":
+ floating_ip_uuids.append(uuid)
+ elif resource["resource_type"] == "OS::Cinder::Volume":
+ volumes_uuids.append(uuid)
+
+ for uuid in floating_ip_uuids:
+ self.os_conn.cleanup_floating_ip(uuid)
+ for uuid in volumes_uuids:
+ self.os_conn.cleanup_volume(uuid)
+ for uuid in server_uuids:
+ self.os_conn.cleanup_instance(uuid)
+ # time.sleep(60)
+ for uuid in router_uuids:
+ self.os_conn.cleanup_router(uuid)
+ for uuid in network_uuids:
+ self.os_conn.cleanup_network(uuid)
+ for uuid in secgroup_uuids:
+ self.os_conn.cleanup_secgroup(uuid)
+
+ def cleanup_stack_parallel(self, stack):
+ if not hasattr(stack, 'id'):
+ stack = list(self.os_conn.heat.stacks.list(id=stack))
+ if not stack:
+ logger.info("Can not find stack {}".format(stack))
+ stack = stack[0]
+
+ resources = self.get_os_stack_resources(stack)
+ server_uuids = []
+ router_uuids = []
+ network_uuids = []
+ secgroup_uuids = []
+ floating_ip_uuids = []
+ volumes_uuids = []
+ temp = set()
+ for uuid, resource in resources.items():
+ temp.add(resource["resource_type"])
+ if resource["resource_type"] == "OS::Nova::Server":
+ server_uuids.append(uuid)
+ elif resource["resource_type"] == "OS::Neutron::Router":
+ router_uuids.append(uuid)
+ elif resource["resource_type"] == "OS::Neutron::Net":
+ network_uuids.append(uuid)
+ elif resource["resource_type"] == "OS::Neutron::SecurityGroup":
+ secgroup_uuids.append(uuid)
+ elif resource["resource_type"] == "OS::Neutron::FloatingIP":
+ floating_ip_uuids.append(uuid)
+ elif resource["resource_type"] == "OS::Cinder::Volume":
+ volumes_uuids.append(uuid)
+
+ self.os_conn.cleanup_floating_batch(floating_ip_uuids)
+ self.os_conn.cleanup_servers_batch(server_uuids)
+ self.os_conn.cleanup_volumes_batch(volumes_uuids)
+ self.os_conn.cleanup_router_batch(router_uuids)
+ self.os_conn.cleanup_network_batch(network_uuids)
+ self.os_conn.cleanup_stack(stack.id, ignore_err=True)
+
+ def process_resource(self, uuid):
+ arg = [uuid]
+ if self.os_conn.check_floating_exists(uuid):
+ self.os_conn.cleanup_floating_batch(arg)
+ elif self.os_conn.check_server_exists(uuid):
+ self.os_conn.cleanup_servers_batch(arg)
+ elif self.os_conn.check_volume_exists(uuid):
+ self.os_conn.cleanup_volumes_batch(arg)
+ elif self.os_conn.check_router_exists(uuid):
+ self.os_conn.cleanup_router_batch(arg)
+ elif self.os_conn.check_network_exists(arg):
+ self.os_conn.cleanup_network_batch(arg)
+ elif self.os_conn.check_stack_exists(uuid):
+ self.cleanup_stack_parallel(uuid)
--- /dev/null
+import signal
+import time
+
+
+class TimeoutError(Exception):
+ pass
+
+
+def wait(predicate, interval=5, timeout=60,
+ predicate_args=None, predicate_kwargs=None):
+
+ predicate_args = predicate_args or []
+ predicate_kwargs = predicate_kwargs or {}
+ _check_wait_args(predicate, predicate_args, predicate_kwargs,
+ interval, timeout)
+
+ start_time = time.time()
+ while True:
+ result = predicate(*predicate_args, **predicate_kwargs)
+ if result:
+ return result
+
+ if start_time + timeout < time.time():
+ raise TimeoutError()
+
+ time.sleep(interval)
+
+
+def wait_false(predicate, interval=3, timeout=60,
+ predicate_args=None, predicate_kwargs=None):
+ predicate_args = predicate_args or []
+ predicate_kwargs = predicate_kwargs or {}
+ _check_wait_args(predicate, predicate_args, predicate_kwargs,
+ interval, timeout)
+
+ start_time = time.time()
+ while True:
+ result = predicate(*predicate_args, **predicate_kwargs)
+ if not result:
+ return result
+
+ if start_time + timeout < time.time():
+ raise TimeoutError()
+
+ time.sleep(interval)
+
+
+
+def wait_err(raising_predicate, expected=Exception,
+ interval=5, timeout=60,
+ predicate_args=None, predicate_kwargs=None):
+ """Wait for successful return from predicate ignoring expected exception
+
+ Options:
+
+ :param interval: - seconds between checks.
+ :param timeout: - raise TimeoutError if predicate still throwing expected
+ exception after this amount of seconds.
+ :param timeout_msg: - text of the TimeoutError
+ :param predicate_args: - positional arguments for given predicate wrapped
+ in list or tuple
+ :param predicate_kwargs: - dict with named arguments for the predicate
+ :param expected_exc: Exception that can be ignored while waiting (its
+ possible to pass several using list/tuple
+
+ """
+
+ predicate_args = predicate_args or []
+ predicate_kwargs = predicate_kwargs or {}
+ _check_wait_args(raising_predicate, predicate_args, predicate_kwargs,
+ interval, timeout)
+
+ start_time = time.time()
+
+ while True:
+ try:
+ result = raising_predicate(*predicate_args, **predicate_kwargs)
+ time.sleep(interval)
+ except expected as e:
+ return result
+ if start_time + timeout < time.time():
+ raise TimeoutError()
+
+class RunLimit(object):
+ def __init__(self, timeout=60, timeout_msg='Timeout'):
+ self.seconds = int(timeout)
+ self.error_message = timeout_msg
+
+ def handle_timeout(self, signum, frame):
+ raise TimeoutError(self.error_message.format(spent=self.seconds))
+
+ def __enter__(self):
+ signal.signal(signal.SIGALRM, self.handle_timeout)
+ signal.alarm(self.seconds)
+
+ def __exit__(self, exc_type, value, traceback):
+ time_remained = signal.alarm(0)
+
+
+def _check_wait_args(predicate,
+ predicate_args,
+ predicate_kwargs,
+ interval,
+ timeout):
+
+ if not callable(predicate):
+ raise TypeError("Not callable raising_predicate has been posted: '{0}'"
+ .format(predicate))
+ if not isinstance(predicate_args, (list, tuple)):
+ raise TypeError("Incorrect predicate_args type for '{0}', should be "
+ "list or tuple, got '{1}'"
+ .format(predicate, type(predicate_args)))
+ if not isinstance(predicate_kwargs, dict):
+ raise TypeError("Incorrect predicate_kwargs type, should be dict, "
+ "got {}".format(type(predicate_kwargs)))
+ if interval <= 0:
+ raise ValueError("For '{0}(*{1}, **{2})', waiting interval '{3}'sec is"
+ " wrong".format(predicate,
+ predicate_args,
+ predicate_kwargs,
+ interval))
+ if timeout <= 0:
+ raise ValueError("For '{0}(*{1}, **{2})', timeout '{3}'sec is "
+ "wrong".format(predicate,
+ predicate_args,
+ predicate_kwargs,
+ timeout))
--- /dev/null
+from __future__ import unicode_literals
+
+import time
+
+from cinderclient.exceptions import NotFound
+from cinderclient.v2.client import Client as CinderClient
+from heatclient.v1.client import Client as HeatClient
+from keystoneauth1.identity import V3Password
+from keystoneauth1.session import Session as KeystoneSession
+from keystoneclient.v2_0 import Client as KeystoneClient
+from novaclient.client import Client as NovaClient
+from novaclient.exceptions import NotFound as NovaNotFoundException
+from neutronclient.common.exceptions import NotFound as NeutronNotFoundException
+from neutronclient.v2_0.client import Client as NeutronClient
+
+import helpers
+
+from logger import logger
+
+
+class OpenStackActions(object):
+ def __init__(self, auth_url, user, password, project_name,
+ project_domain_name, user_domain_name):
+ self.keystone_session = None
+
+ self.auth_url = auth_url
+
+ self.__keystone_auth = V3Password(
+ auth_url=auth_url,
+ username=user,
+ password=password,
+ project_name=project_name,
+ project_domain_name=project_domain_name,
+ user_domain_name=user_domain_name)
+
+ self.keystone_session = KeystoneSession(
+ auth=self.__keystone_auth)
+
+ self.keystone_session.get_auth_headers()
+ self.clients_cache = {
+ "keystone": KeystoneClient(session=self.keystone_session),
+ # "glance": GlanceClient(
+ # version='1',
+ # session=self.keystone_session,
+ # endpoint_override=self._get_url_for_svc(service_type='image')),
+ "neutron": NeutronClient(
+ session=self.keystone_session,
+ endpoint_override=self._get_url_for_svc(
+ service_type='network')),
+ "nova": NovaClient(
+ version='2',
+ session=self.keystone_session,
+ endpoint_override=self._get_url_for_svc(
+ service_type='compute')),
+ "cinder": CinderClient(
+ version='2',
+ session=self.keystone_session,
+ endpoint_override=self._get_url_for_svc(
+ service_type='volumev2')),
+ "heat":
+ HeatClient(
+ session=self.keystone_session,
+ endpoint=self._get_url_for_svc(
+ service_type='orchestration'))
+
+ }
+
+ @property
+ def keystone(self):
+ return self.clients_cache.get("keystone")
+
+ @property
+ def glance(self):
+ return self.clients_cache.get("glance")
+
+ @property
+ def neutron(self):
+ return self.clients_cache.get("neutron")
+
+ @property
+ def nova(self):
+ return self.clients_cache.get("nova")
+
+ @property
+ def cinder(self):
+ return self.clients_cache.get("cinder")
+
+ @property
+ def heat(self):
+ return self.clients_cache.get("heat")
+
+ @property
+ def keystone_access(self):
+ return self.__keystone_auth.get_access(session=self.keystone_session)
+
+ def _get_url_for_svc(
+ self, service_type=None, interface='public',
+ region_name=None, service_name=None,
+ service_id=None, endpoint_id=None):
+ return self.keystone_access.service_catalog.url_for(
+ service_type=service_type, interface=interface,
+ region_name=region_name, service_name=service_name,
+ service_id=service_id, endpoint_id=endpoint_id)
+
+ def get_floating_ip_uuid(self, floating_ip):
+ neutron_data = self.neutron.list_floatingips(
+ floating_ip_address=floating_ip)
+ for f_ip_data in neutron_data['floatingips']:
+ if f_ip_data['floating_ip_address'] == floating_ip:
+ return f_ip_data['id']
+
+ def get_router_ports(self, router_uuid):
+ response = self.neutron.list_ports(device_id=router_uuid)
+ ports = response['ports']
+ return [port['id'] for port in ports]
+
+ def get_servers(self):
+ servers = self.nova.servers.list()
+ return servers
+
+ def get_keystone_endpoints(self):
+ endpoints = self.keystone.endpoints.list()
+ return endpoints
+
+ def get_stacks(self):
+ stacks = self.heat.stacks.list()
+ return stacks
+
+ def get_stacks_by_name_or_uuid(self, stack_id):
+ stacks = list(self.heat.stacks.list(id=stack_id))
+ if not stacks:
+ stacks = list(self.heat.stacks.list(stack_name=stack_id))
+ return stacks
+
+ def get_networks(self):
+ net_list = self.neutron.list_networks()
+ return net_list['networks']
+
+ def get_routers(self):
+ router_list = self.neutron.list_routers()
+ return router_list['routers']
+
+ def cleanup_floating_ip(self, uuid):
+ if self.neutron.list_floatingips(id=uuid)['floatingips']:
+ print "DELETE SEPARATED FLOATING IP DRYRUN"
+ self.neutron.delete_floatingip(uuid)
+
+ def cleanup_volume(self, uuid):
+ if not self.check_volume_exists(uuid):
+ return
+ self.cinder.volumes.detach(uuid)
+ helpers.wait_false(self.check_volume_attached,
+ interval=3,
+ timeout=30,
+ predicate_args=[uuid])
+ self.cinder.volumes.delete(uuid)
+ helpers.wait_false(self.check_volume_exists,
+ interval=3,
+ timeout=60,
+ predicate_args=[uuid])
+
+ def cleanup_instance(self, server_uuid):
+ if not self.check_server_exists(server_uuid):
+ logger.info("Server {} does not exist, do nothing".format(
+ server_uuid))
+
+ server = self.nova.servers.get(server_uuid)
+ floating_ips_uuid = []
+ for net in server.addresses.values():
+ for ip in net:
+ if ip.get('OS-EXT-IPS:type') and \
+ ip['OS-EXT-IPS:type'] == "floating":
+ logger.debug("Queuing floating id {} to delete "
+ "queue".format(ip['addr']))
+ floating_ips_uuid.append(
+ self.get_floating_ip_uuid(ip['addr']))
+
+ self.cleanup_floating_batch(floating_ips_uuid)
+
+ volumes_uuids = []
+ for volume in self.nova.volumes.get_server_volumes(server_uuid):
+ volumes_uuids.append(volume.id)
+ self.cleanup_volumes_batch(volumes_uuids)
+
+ self.nova.servers.delete(server)
+ helpers.wait_false(self.check_volume_exists,
+ interval=3,
+ timeout=60,
+ predicate_args=[server_uuid])
+
+ def check_router_ports_exists(self, router_uuid):
+ if self.get_router_ports(router_uuid):
+ return True
+ return False
+
+ def check_port_exists(self, port_uuid):
+ response = self.neutron.list_ports(id=port_uuid)
+ if response['ports']:
+ return True
+ return False
+
+ def check_any_port_exists(self, uuids):
+ return any(
+ [self.check_port_exists(uuid) for uuid in uuids]
+ )
+
+ def check_network_exists(self, network_uuid):
+ resp = self.neutron.list_networks(id=network_uuid)['networks']
+ if resp:
+ return True
+ return False
+
+ def check_any_network_exists(self, uuids):
+ return any(
+ [self.check_network_exists(uuid) for uuid in uuids]
+ )
+
+ def check_subnet_exists(self, subnet_uuid):
+ resp = self.neutron.list_subnets(id=subnet_uuid)
+ if resp['subnets']:
+ return True
+ return False
+
+ def check_any_subnet_exists(self, uuids):
+ return any(
+ [self.check_subnet_exists(uuid) for uuid in uuids]
+ )
+
+ def cleanup_router(self, router_uuid):
+ if not self.neutron.list_routers(id=router_uuid)['routers']:
+ return
+ ports = self.get_router_ports(router_uuid)
+ for uuid in ports:
+ print "DELETE PORT DRYRUN"
+ self.neutron.remove_interface_router(router_uuid,
+ {'port_id': uuid})
+ print "DROP GATEWAY PORT DRYRUN"
+ self.neutron.remove_gateway_router(router_uuid)
+ time.sleep(10)
+ print "DELETE ROUTER DRYRUN"
+ self.neutron.delete_router(router_uuid)
+
+ def cleanup_network(self, network_uuid):
+ if not self.neutron.list_networks(id=network_uuid)['networks']:
+ return
+ net_ports_resp = self.neutron.list_ports(network_id=network_uuid)
+ for net_port in net_ports_resp['ports']:
+ self.neutron.delete_port(net_port['id'])
+ time.sleep(10)
+ subnets_resp = self.neutron.list_subnets(network_id=network_uuid)
+ print "DELETE SUBNETS DRYRUN"
+ for subnet in subnets_resp['subnets']:
+ self.neutron.delete_subnet(subnet['id'])
+ print "DELETE NET DRYRUN"
+ self.neutron.delete_network(network_uuid)
+
+ def cleanup_secgroup(self, secgroup_uuid):
+ if self.neutron.list_security_groups(
+ id=secgroup_uuid)["security_groups"]:
+ print "DELETE SECGROUPS DRYRUN"
+ self.neutron.delete_security_group(secgroup_uuid)
+
+ def check_floating_exists(self, floating_uuid):
+ resp = self.neutron.list_floatingips(id=floating_uuid)
+ if resp['floatingips']:
+ return True
+ return False
+
+ def check_any_floating_exists(self, uuids):
+ return any(
+ [self.check_floating_exists(uuid) for uuid in uuids]
+ )
+
+ def check_volume_exists(self, volume_uuid):
+ try:
+ self.cinder.volumes.get(volume_uuid)
+ return True
+ except NotFound:
+ return False
+
+ def check_any_volume_exists(self, uuids):
+ return any(
+ [self.check_volume_exists(uuid) for uuid in uuids]
+ )
+
+ def check_volume_attached(self, volume_uuid):
+ volume = self.cinder.volumes.get(volume_uuid)
+ if volume.attachments:
+ return True
+ return False
+
+ def check_any_volume_attached(self, uuids):
+ return any(
+ [self.check_volume_attached(uuid) for uuid in uuids]
+ )
+
+ def check_server_exists(self, server_uuid):
+ try:
+ self.nova.servers.get(server_uuid)
+ return True
+ except NovaNotFoundException:
+ return False
+
+ def check_any_server_exists(self, uuids):
+ return any(
+ [self.check_server_exists(uuid) for uuid in uuids]
+ )
+
+ def check_router_exists(self, router_uuid):
+ if self.neutron.list_routers(id=router_uuid)['routers']:
+ return True
+ return False
+
+ def check_any_router_exists(self, uuids):
+ return any(
+ [self.check_router_exists(uuid) for uuid in uuids]
+ )
+
+ def check_stack_exists(self, stack_uuid):
+ resp = list(self.heat.stacks.list(id=stack_uuid))
+ if resp:
+ return True
+ return False
+
+ def cleanup_floating_batch(self, uuid_list):
+ logger.info("Cleanup floatings list: {}".format(uuid_list))
+ existing_floating = []
+ for uuid in uuid_list:
+ if self.check_floating_exists(uuid):
+ logger.debug("Floating {} found, queued".format(uuid))
+ existing_floating.append(uuid)
+ else:
+ logger.info(
+ "Given floating ip {} does not exist, skipping".format(
+ uuid))
+
+ for uuid in existing_floating:
+ logger.debug("Deleting floating {}".format(uuid))
+ self.neutron.delete_floatingip(uuid)
+
+ try:
+ helpers.wait_false(self.check_any_floating_exists,
+ timeout=20,
+ predicate_kwargs={
+ "uuids": existing_floating})
+ logger.info("All floating ips deleted!")
+ except helpers.TimeoutError:
+ for uuid in existing_floating:
+ if self.check_floating_exists(uuid):
+ logger.error("Floating {} stuck!".format(uuid))
+ raise helpers.TimeoutError
+
+ def cleanup_servers_batch(self, servers_uuids, reset_state=False,
+ is_recursive_call=False):
+ logger.info("Cleaning up servers: {}".format(servers_uuids))
+ existing_servers = []
+ for uuid in servers_uuids:
+ if self.check_server_exists(uuid):
+ logger.debug("Server {} found, queued".format(uuid))
+ existing_servers.append(uuid)
+ else:
+ logger.info("Server {} not found, ignoring".format(uuid))
+
+ if reset_state:
+ for uuid in existing_servers:
+ logger.info("Resetting state of {}".format(uuid))
+ self.nova.servers.reset_state(uuid, state='active')
+
+ for uuid in existing_servers:
+ logger.info("Deleting server {}".format(uuid))
+ self.nova.servers.delete(uuid)
+
+ logger.info("Waiting servers deletion...")
+
+ try:
+ helpers.wait_false(self.check_any_server_exists,
+ interval=5, timeout=120,
+ predicate_kwargs={"uuids": existing_servers})
+ logger.info("All servers deleted!")
+ except helpers.TimeoutError:
+ failed_servers = []
+ for uuid in existing_servers:
+ if self.check_server_exists(uuid):
+ logger.error("Server {} stuck!".format(uuid))
+ failed_servers.append(uuid)
+ if is_recursive_call:
+ logger.error("Servers are completely stuck, "
+ "even after reset-state")
+ raise
+ else:
+ logger.info("Trying to reset state of stuck servers")
+ self.cleanup_servers_batch(failed_servers, reset_state=True,
+ is_recursive_call=True)
+
+ def cleanup_volumes_batch(self, volumes_uuids,
+ force_delete=False,
+ is_recursive_call=False):
+ logger.info("Cleaning up volumes: {}".format(volumes_uuids))
+ existing_volumes = []
+ for uuid in volumes_uuids:
+ if self.check_volume_exists(uuid):
+ existing_volumes.append(uuid)
+ for uuid in existing_volumes:
+ if self.check_volume_attached(uuid):
+ logger.info("Detaching volume {}".format(uuid))
+ self.cinder.volumes.detach(uuid)
+ try:
+ helpers.wait_false(self.check_any_volume_attached,
+ interval=3,
+ timeout=30,
+ predicate_kwargs={"uuids": existing_volumes})
+ except helpers.TimeoutError:
+ for uuid in existing_volumes:
+ if self.check_volume_attached(uuid):
+ logger.error("Volume {} still attached!".format(uuid))
+ raise
+
+ for uuid in existing_volumes:
+ if force_delete:
+ logger.info("Force-deleting volume {}".format(uuid))
+ self.cinder.volumes.force_delete(uuid)
+ else:
+ logger.info("Deleting volume {}".format(uuid))
+ self.cinder.volumes.delete(uuid)
+
+ try:
+ helpers.wait_false(self.check_any_volume_exists,
+ interval=3,
+ timeout=60,
+ predicate_kwargs={"uuids": existing_volumes})
+ logger.info("All volumes deleted!")
+ except helpers.TimeoutError:
+ failed_volumes = []
+ for uuid in existing_volumes:
+ if self.check_volume_exists(uuid):
+ logger.error("Volume {} is stuck!".format(uuid))
+ failed_volumes.append(uuid)
+ if is_recursive_call:
+ logger.error("Volumes are stuck even after force-delete!")
+ raise
+ else:
+ logger.info("Trying to force-delete stuck volumes")
+ self.cleanup_volumes_batch(failed_volumes,
+ force_delete=True,
+ is_recursive_call=True)
+
+ def cleanup_router_batch(self, router_uuids):
+ logger.info("Cleaning up routers {}".format(router_uuids))
+ existing_routers = []
+ existing_ports = []
+ for uuid in router_uuids:
+ if self.check_router_exists(uuid):
+ logger.debug("Router {} found, queued".format(uuid))
+ existing_routers.append(uuid)
+ else:
+ logger.info("Router {} does not exist, ignoring".format(uuid))
+
+ for router_uuid in existing_routers:
+ logger.info(
+ "Removing inner router components from {}".format(router_uuid))
+ ports = self.get_router_ports(router_uuid)
+ existing_ports.extend(ports)
+ self.neutron.update_router(router_uuid,
+ {'router': {"routes": None}})
+ for port_uuid in ports:
+ try:
+ self.neutron.remove_interface_router(
+ router_uuid,
+ {'port_id': port_uuid})
+ except NeutronNotFoundException:
+ logger.info("Port {} not found at router {}, ignoring"
+ "".format(port_uuid, router_uuid))
+ self.neutron.remove_gateway_router(router_uuid)
+
+ try:
+ helpers.wait_false(self.check_any_port_exists,
+ interval=3,
+ timeout=30,
+ predicate_kwargs={"uuids": existing_ports})
+ except helpers.TimeoutError:
+ for uuid in existing_ports:
+ if self.check_port_exists(uuid):
+ logger.error("Port {} is stuck!".format(uuid))
+ raise
+ for router_uuid in existing_routers:
+ logger.info("Deleting router {}".format(router_uuid))
+ self.neutron.delete_router(router_uuid)
+ try:
+ helpers.wait_false(self.check_any_router_exists,
+ interval=3,
+ timeout=30,
+ predicate_kwargs={"uuids": existing_routers})
+ except helpers.TimeoutError:
+ for router_uuid in existing_routers:
+ if self.check_router_exists(router_uuid):
+ logger.error("Router {} is stuck!".format(router_uuid))
+ raise
+
+ def cleanup_network_batch(self, network_uuids):
+ logger.info("Cleaning up networks {}".format(network_uuids))
+ existing_networks = []
+ existing_subnets = []
+ existing_ports = []
+ for net_uuid in network_uuids:
+ if self.check_network_exists(net_uuid):
+ existing_networks.append(net_uuid)
+
+ for net_uuid in existing_networks:
+ logger.info("Removing ports from network {}".format(net_uuid))
+ net_ports_resp = self.neutron.list_ports(network_id=net_uuid)
+ for net_port in net_ports_resp['ports']:
+ existing_ports.append(net_port['id'])
+ self.neutron.delete_port(net_port['id'])
+
+ try:
+ helpers.wait_false(self.check_any_port_exists,
+ interval=3,
+ timeout=30,
+ predicate_kwargs={"uuids": existing_ports})
+ except helpers.TimeoutError:
+ for port_uuid in existing_ports:
+ if self.check_port_exists(port_uuid):
+ logger.error("Port {} was not removed!".format(port_uuid))
+ raise
+
+ for net_uuid in existing_networks:
+ logger.info("Processing subnets of network {}".format(net_uuid))
+ resp = self.neutron.list_subnets(network_id=net_uuid)
+ for subnet in resp['subnets']:
+ existing_subnets.append(subnet['id'])
+ self.neutron.delete_subnet(subnet['id'])
+
+ try:
+ helpers.wait_false(self.check_any_subnet_exists,
+ predicate_kwargs={"uuids": existing_subnets})
+ except helpers.TimeoutError:
+ for subnet_uuid in existing_subnets:
+ if self.check_subnet_exists(subnet_uuid):
+ logger.error("Can not delete subnet {}".format(subnet_uuid))
+ raise
+
+ for net_uuid in existing_networks:
+ logger.info("Deleting network {}".format(net_uuid))
+ self.neutron.delete_network(net_uuid)
+ try:
+ helpers.wait_false(self.check_any_network_exists,
+ predicate_kwargs={"uuids": existing_subnets})
+ except helpers.TimeoutError:
+ for net_uuid in existing_networks:
+ if self.check_network_exists(net_uuid):
+ logger.error("Network {} is stuck!".format(net_uuid))
+ raise
+
+ def delete_stack(self, stack_uuid):
+ self.heat.stacks.delete(stack_uuid)
+
+ def cleanup_stack(self, stack_uuid, ignore_err=False):
+ if not self.check_stack_exists(stack_uuid):
+ return True
+ logger.info("Deleting stack {}".format(stack_uuid))
+ self.delete_stack(stack_uuid)
+ try:
+ helpers.wait_false(self.check_stack_exists,
+ predicate_args=[stack_uuid])
+ except helpers.TimeoutError:
+ logger.error("Stack {} is stuck!".format(stack_uuid))
+ if ignore_err:
+ return
+ else:
+ raise
Code Review / tools / sustaining.git / commitdiff