From: Vladimir Khlyunev Date: Wed, 23 Dec 2020 10:35:47 +0000 (+0400) Subject: Initial commit of cloud cleaner X-Git-Url: https://review.fuel-infra.org/gitweb?a=commitdiff_plain;h=e33f2917ed6995f046116230a6f4299748820ef7;p=tools%2Fsustaining.git Initial commit of cloud cleaner Very base version, right now it's able to cleanup stacks Usage: shell.py cleanup stack %names_or_ids% It should be able to handle stuck resources except stacks itself (heat is so heat) Change-Id: Idc31f8c68a3748c6187f5aa38ed49ff7660c5d1d --- diff --git a/os_cloud_cleaner/cleaner.py b/os_cloud_cleaner/cleaner.py new file mode 100644 index 0000000..5678c46 --- /dev/null +++ b/os_cloud_cleaner/cleaner.py @@ -0,0 +1,189 @@ +from __future__ import unicode_literals + +from datetime import datetime + +from os_connector import OpenStackActions +from logger import logger + +try: + import prehooks +except ImportError: + pass + + +class Cleaner: + DEFAULT_LIFETIME_HOURS = 48 + + def __init__(self, lifetime=None, os_auth_url=None, + os_user=None, os_password=None, os_project_name=None, + os_project_domain_name=None, + os_user_domain_name=None): + self.os_conn = OpenStackActions( + auth_url=os_auth_url, + user=os_user, + password=os_password, + project_name=os_project_name, + project_domain_name=os_project_domain_name, + user_domain_name=os_user_domain_name) + + self.lifetime = lifetime if lifetime is not None \ + else self.DEFAULT_LIFETIME_HOURS + self.heat_resources_cache = {} + self.non_heat_resources_cache = {} + + @property + def heat_resources(self): + if not self.heat_resources_cache: + self.heat_resources_cache = self.collect_resources_under_heat() + return self.heat_resources_cache + + @property + def non_heat_resources(self): + if not self.non_heat_resources_cache: + self.non_heat_resources_cache = \ + self.collect_os_non_stack_resources() + return self.non_heat_resources_cache + + def get_stacks(self): + return list(self.os_conn.get_stacks()) + + def get_os_stack_resources(self, stack): + if stack.stack_status == "CREATE_IN_PROGRESS": + return {} + client = stack.manager.client + req = "/stacks/{}/{}/resources?nested_depth=99".format( + stack.stack_name, stack.id) + data = client.get(req) + content = data.json() + resources = content['resources'] + result = {} + for resource in resources: + if hasattr(resource["resource_type"], "startswith") and \ + resource["resource_type"].startswith("OS::"): + result[resource["physical_resource_id"]] = resource + return result + + def collect_os_non_stack_resources(self): + result = {"servers": [], + "networks": [], + "routers": []} + for server in self.get_servers(): + result["servers"].append(server) + for net in self.get_networks(): + result["networks"].append(net) + for router in self.get_routers(): + result["routers"].append(router) + + return result + + @staticmethod + def get_creation_date(os_obj): + return datetime.strptime(os_obj, "%Y-%m-%dT%H:%M:%SZ") + + def check_lifetime(self, creation_time): + delta = datetime.utcnow() - creation_time + return delta.seconds // 3600 > self.lifetime + + def collect_resources_under_heat(self): + stacks = self.get_stacks() + resources = {} + for stack in stacks: + resources.update(self.get_os_stack_resources(stack)) + return resources + + def get_servers(self): + return self.os_conn.get_servers() + + def get_networks(self): + return self.os_conn.get_networks() + + def get_routers(self): + return self.os_conn.get_routers() + + def cleanup_stack_one_by_one(self, stack): + resources = self.get_os_stack_resources(stack) + server_uuids = [] + router_uuids = [] + network_uuids = [] + secgroup_uuids = [] + floating_ip_uuids = [] + volumes_uuids = [] + for uuid, resource in resources.items(): + if resource["resource_type"] == "OS::Nova::Server": + server_uuids.append(uuid) + elif resource["resource_type"] == "OS::Neutron::Router": + router_uuids.append(uuid) + elif resource["resource_type"] == "OS::Neutron::Net": + network_uuids.append(uuid) + elif resource["resource_type"] == "OS::Neutron::SecurityGroup": + secgroup_uuids.append(uuid) + elif resource["resource_type"] == "OS::Neutron::FloatingIP": + floating_ip_uuids.append(uuid) + elif resource["resource_type"] == "OS::Cinder::Volume": + volumes_uuids.append(uuid) + + for uuid in floating_ip_uuids: + self.os_conn.cleanup_floating_ip(uuid) + for uuid in volumes_uuids: + self.os_conn.cleanup_volume(uuid) + for uuid in server_uuids: + self.os_conn.cleanup_instance(uuid) + # time.sleep(60) + for uuid in router_uuids: + self.os_conn.cleanup_router(uuid) + for uuid in network_uuids: + self.os_conn.cleanup_network(uuid) + for uuid in secgroup_uuids: + self.os_conn.cleanup_secgroup(uuid) + + def cleanup_stack_parallel(self, stack): + if not hasattr(stack, 'id'): + stack = list(self.os_conn.heat.stacks.list(id=stack)) + if not stack: + logger.info("Can not find stack {}".format(stack)) + stack = stack[0] + + resources = self.get_os_stack_resources(stack) + server_uuids = [] + router_uuids = [] + network_uuids = [] + secgroup_uuids = [] + floating_ip_uuids = [] + volumes_uuids = [] + temp = set() + for uuid, resource in resources.items(): + temp.add(resource["resource_type"]) + if resource["resource_type"] == "OS::Nova::Server": + server_uuids.append(uuid) + elif resource["resource_type"] == "OS::Neutron::Router": + router_uuids.append(uuid) + elif resource["resource_type"] == "OS::Neutron::Net": + network_uuids.append(uuid) + elif resource["resource_type"] == "OS::Neutron::SecurityGroup": + secgroup_uuids.append(uuid) + elif resource["resource_type"] == "OS::Neutron::FloatingIP": + floating_ip_uuids.append(uuid) + elif resource["resource_type"] == "OS::Cinder::Volume": + volumes_uuids.append(uuid) + + self.os_conn.cleanup_floating_batch(floating_ip_uuids) + self.os_conn.cleanup_servers_batch(server_uuids) + self.os_conn.cleanup_volumes_batch(volumes_uuids) + self.os_conn.cleanup_router_batch(router_uuids) + self.os_conn.cleanup_network_batch(network_uuids) + self.os_conn.cleanup_stack(stack.id, ignore_err=True) + + def process_resource(self, uuid): + arg = [uuid] + if self.os_conn.check_floating_exists(uuid): + self.os_conn.cleanup_floating_batch(arg) + elif self.os_conn.check_server_exists(uuid): + self.os_conn.cleanup_servers_batch(arg) + elif self.os_conn.check_volume_exists(uuid): + self.os_conn.cleanup_volumes_batch(arg) + elif self.os_conn.check_router_exists(uuid): + self.os_conn.cleanup_router_batch(arg) + elif self.os_conn.check_network_exists(arg): + self.os_conn.cleanup_network_batch(arg) + elif self.os_conn.check_stack_exists(uuid): + self.cleanup_stack_parallel(uuid) diff --git a/os_cloud_cleaner/helpers.py b/os_cloud_cleaner/helpers.py new file mode 100644 index 0000000..d209d63 --- /dev/null +++ b/os_cloud_cleaner/helpers.py @@ -0,0 +1,127 @@ +import signal +import time + + +class TimeoutError(Exception): + pass + + +def wait(predicate, interval=5, timeout=60, + predicate_args=None, predicate_kwargs=None): + + predicate_args = predicate_args or [] + predicate_kwargs = predicate_kwargs or {} + _check_wait_args(predicate, predicate_args, predicate_kwargs, + interval, timeout) + + start_time = time.time() + while True: + result = predicate(*predicate_args, **predicate_kwargs) + if result: + return result + + if start_time + timeout < time.time(): + raise TimeoutError() + + time.sleep(interval) + + +def wait_false(predicate, interval=3, timeout=60, + predicate_args=None, predicate_kwargs=None): + predicate_args = predicate_args or [] + predicate_kwargs = predicate_kwargs or {} + _check_wait_args(predicate, predicate_args, predicate_kwargs, + interval, timeout) + + start_time = time.time() + while True: + result = predicate(*predicate_args, **predicate_kwargs) + if not result: + return result + + if start_time + timeout < time.time(): + raise TimeoutError() + + time.sleep(interval) + + + +def wait_err(raising_predicate, expected=Exception, + interval=5, timeout=60, + predicate_args=None, predicate_kwargs=None): + """Wait for successful return from predicate ignoring expected exception + + Options: + + :param interval: - seconds between checks. + :param timeout: - raise TimeoutError if predicate still throwing expected + exception after this amount of seconds. + :param timeout_msg: - text of the TimeoutError + :param predicate_args: - positional arguments for given predicate wrapped + in list or tuple + :param predicate_kwargs: - dict with named arguments for the predicate + :param expected_exc: Exception that can be ignored while waiting (its + possible to pass several using list/tuple + + """ + + predicate_args = predicate_args or [] + predicate_kwargs = predicate_kwargs or {} + _check_wait_args(raising_predicate, predicate_args, predicate_kwargs, + interval, timeout) + + start_time = time.time() + + while True: + try: + result = raising_predicate(*predicate_args, **predicate_kwargs) + time.sleep(interval) + except expected as e: + return result + if start_time + timeout < time.time(): + raise TimeoutError() + +class RunLimit(object): + def __init__(self, timeout=60, timeout_msg='Timeout'): + self.seconds = int(timeout) + self.error_message = timeout_msg + + def handle_timeout(self, signum, frame): + raise TimeoutError(self.error_message.format(spent=self.seconds)) + + def __enter__(self): + signal.signal(signal.SIGALRM, self.handle_timeout) + signal.alarm(self.seconds) + + def __exit__(self, exc_type, value, traceback): + time_remained = signal.alarm(0) + + +def _check_wait_args(predicate, + predicate_args, + predicate_kwargs, + interval, + timeout): + + if not callable(predicate): + raise TypeError("Not callable raising_predicate has been posted: '{0}'" + .format(predicate)) + if not isinstance(predicate_args, (list, tuple)): + raise TypeError("Incorrect predicate_args type for '{0}', should be " + "list or tuple, got '{1}'" + .format(predicate, type(predicate_args))) + if not isinstance(predicate_kwargs, dict): + raise TypeError("Incorrect predicate_kwargs type, should be dict, " + "got {}".format(type(predicate_kwargs))) + if interval <= 0: + raise ValueError("For '{0}(*{1}, **{2})', waiting interval '{3}'sec is" + " wrong".format(predicate, + predicate_args, + predicate_kwargs, + interval)) + if timeout <= 0: + raise ValueError("For '{0}(*{1}, **{2})', timeout '{3}'sec is " + "wrong".format(predicate, + predicate_args, + predicate_kwargs, + timeout)) diff --git a/os_cloud_cleaner/logger.py b/os_cloud_cleaner/logger.py new file mode 100644 index 0000000..bd1b466 --- /dev/null +++ b/os_cloud_cleaner/logger.py @@ -0,0 +1,69 @@ +from __future__ import unicode_literals + +import logging.config +import os +import warnings + +LOGS_DIR = os.environ.get("LOGS_DIR", False) or os.curdir + +_log_config = { + 'version': 1, + 'disable_existing_loggers': False, + 'formatters': { + 'default': { + 'format': '%(asctime)s - %(levelname)s %(filename)s:' + '%(lineno)d -- %(message)s', + 'datefmt': '%Y-%m-%d %H:%M:%S' + }, + }, + 'handlers': { + 'console': { + 'level': 'INFO', + 'class': 'logging.StreamHandler', + 'formatter': 'default' + }, + 'logfile': { + 'level': 'DEBUG', + 'class': 'logging.FileHandler', + 'formatter': 'default', + 'filename': os.path.join(LOGS_DIR, 'cleaner.log'), + 'mode': 'w', + 'encoding': 'utf8', + }, + 'null': { + 'level': 'CRITICAL', + 'class': 'logging.NullHandler', + }, + }, + 'loggers': { + # Log all to log file , but by default only warnings. + '': { + 'handlers': ['logfile'], + 'level': 'WARNING', + }, + 'cleaner_logger': { + 'handlers': ['console', 'logfile'], + 'level': 'DEBUG', + 'propagate': True + }, + # py.warnings is changed by Django -> do not propagate + 'py.warnings': { + 'handlers': ['console', 'logfile'], + 'level': 'WARNING', + 'propagate': False + }, + 'paramiko': {'level': 'WARNING'}, + 'iso8601': {'level': 'WARNING'}, + 'keystoneauth': {'level': 'WARNING'}, + } +} + +logging.config.dictConfig(_log_config) +logging.captureWarnings(True) # Log warnings +# Filter deprecation warnings: log only when deletion announced +warnings.filterwarnings( + 'default', + message=r'.*(drop|remove)+.*', + category=DeprecationWarning) + +logger = logging.getLogger('cleaner_logger'.format(__name__)) \ No newline at end of file diff --git a/os_cloud_cleaner/os_connector.py b/os_cloud_cleaner/os_connector.py new file mode 100644 index 0000000..efe84ed --- /dev/null +++ b/os_cloud_cleaner/os_connector.py @@ -0,0 +1,570 @@ +from __future__ import unicode_literals + +import time + +from cinderclient.exceptions import NotFound +from cinderclient.v2.client import Client as CinderClient +from heatclient.v1.client import Client as HeatClient +from keystoneauth1.identity import V3Password +from keystoneauth1.session import Session as KeystoneSession +from keystoneclient.v2_0 import Client as KeystoneClient +from novaclient.client import Client as NovaClient +from novaclient.exceptions import NotFound as NovaNotFoundException +from neutronclient.common.exceptions import NotFound as NeutronNotFoundException +from neutronclient.v2_0.client import Client as NeutronClient + +import helpers + +from logger import logger + + +class OpenStackActions(object): + def __init__(self, auth_url, user, password, project_name, + project_domain_name, user_domain_name): + self.keystone_session = None + + self.auth_url = auth_url + + self.__keystone_auth = V3Password( + auth_url=auth_url, + username=user, + password=password, + project_name=project_name, + project_domain_name=project_domain_name, + user_domain_name=user_domain_name) + + self.keystone_session = KeystoneSession( + auth=self.__keystone_auth) + + self.keystone_session.get_auth_headers() + self.clients_cache = { + "keystone": KeystoneClient(session=self.keystone_session), + # "glance": GlanceClient( + # version='1', + # session=self.keystone_session, + # endpoint_override=self._get_url_for_svc(service_type='image')), + "neutron": NeutronClient( + session=self.keystone_session, + endpoint_override=self._get_url_for_svc( + service_type='network')), + "nova": NovaClient( + version='2', + session=self.keystone_session, + endpoint_override=self._get_url_for_svc( + service_type='compute')), + "cinder": CinderClient( + version='2', + session=self.keystone_session, + endpoint_override=self._get_url_for_svc( + service_type='volumev2')), + "heat": + HeatClient( + session=self.keystone_session, + endpoint=self._get_url_for_svc( + service_type='orchestration')) + + } + + @property + def keystone(self): + return self.clients_cache.get("keystone") + + @property + def glance(self): + return self.clients_cache.get("glance") + + @property + def neutron(self): + return self.clients_cache.get("neutron") + + @property + def nova(self): + return self.clients_cache.get("nova") + + @property + def cinder(self): + return self.clients_cache.get("cinder") + + @property + def heat(self): + return self.clients_cache.get("heat") + + @property + def keystone_access(self): + return self.__keystone_auth.get_access(session=self.keystone_session) + + def _get_url_for_svc( + self, service_type=None, interface='public', + region_name=None, service_name=None, + service_id=None, endpoint_id=None): + return self.keystone_access.service_catalog.url_for( + service_type=service_type, interface=interface, + region_name=region_name, service_name=service_name, + service_id=service_id, endpoint_id=endpoint_id) + + def get_floating_ip_uuid(self, floating_ip): + neutron_data = self.neutron.list_floatingips( + floating_ip_address=floating_ip) + for f_ip_data in neutron_data['floatingips']: + if f_ip_data['floating_ip_address'] == floating_ip: + return f_ip_data['id'] + + def get_router_ports(self, router_uuid): + response = self.neutron.list_ports(device_id=router_uuid) + ports = response['ports'] + return [port['id'] for port in ports] + + def get_servers(self): + servers = self.nova.servers.list() + return servers + + def get_keystone_endpoints(self): + endpoints = self.keystone.endpoints.list() + return endpoints + + def get_stacks(self): + stacks = self.heat.stacks.list() + return stacks + + def get_stacks_by_name_or_uuid(self, stack_id): + stacks = list(self.heat.stacks.list(id=stack_id)) + if not stacks: + stacks = list(self.heat.stacks.list(stack_name=stack_id)) + return stacks + + def get_networks(self): + net_list = self.neutron.list_networks() + return net_list['networks'] + + def get_routers(self): + router_list = self.neutron.list_routers() + return router_list['routers'] + + def cleanup_floating_ip(self, uuid): + if self.neutron.list_floatingips(id=uuid)['floatingips']: + print "DELETE SEPARATED FLOATING IP DRYRUN" + self.neutron.delete_floatingip(uuid) + + def cleanup_volume(self, uuid): + if not self.check_volume_exists(uuid): + return + self.cinder.volumes.detach(uuid) + helpers.wait_false(self.check_volume_attached, + interval=3, + timeout=30, + predicate_args=[uuid]) + self.cinder.volumes.delete(uuid) + helpers.wait_false(self.check_volume_exists, + interval=3, + timeout=60, + predicate_args=[uuid]) + + def cleanup_instance(self, server_uuid): + if not self.check_server_exists(server_uuid): + logger.info("Server {} does not exist, do nothing".format( + server_uuid)) + + server = self.nova.servers.get(server_uuid) + floating_ips_uuid = [] + for net in server.addresses.values(): + for ip in net: + if ip.get('OS-EXT-IPS:type') and \ + ip['OS-EXT-IPS:type'] == "floating": + logger.debug("Queuing floating id {} to delete " + "queue".format(ip['addr'])) + floating_ips_uuid.append( + self.get_floating_ip_uuid(ip['addr'])) + + self.cleanup_floating_batch(floating_ips_uuid) + + volumes_uuids = [] + for volume in self.nova.volumes.get_server_volumes(server_uuid): + volumes_uuids.append(volume.id) + self.cleanup_volumes_batch(volumes_uuids) + + self.nova.servers.delete(server) + helpers.wait_false(self.check_volume_exists, + interval=3, + timeout=60, + predicate_args=[server_uuid]) + + def check_router_ports_exists(self, router_uuid): + if self.get_router_ports(router_uuid): + return True + return False + + def check_port_exists(self, port_uuid): + response = self.neutron.list_ports(id=port_uuid) + if response['ports']: + return True + return False + + def check_any_port_exists(self, uuids): + return any( + [self.check_port_exists(uuid) for uuid in uuids] + ) + + def check_network_exists(self, network_uuid): + resp = self.neutron.list_networks(id=network_uuid)['networks'] + if resp: + return True + return False + + def check_any_network_exists(self, uuids): + return any( + [self.check_network_exists(uuid) for uuid in uuids] + ) + + def check_subnet_exists(self, subnet_uuid): + resp = self.neutron.list_subnets(id=subnet_uuid) + if resp['subnets']: + return True + return False + + def check_any_subnet_exists(self, uuids): + return any( + [self.check_subnet_exists(uuid) for uuid in uuids] + ) + + def cleanup_router(self, router_uuid): + if not self.neutron.list_routers(id=router_uuid)['routers']: + return + ports = self.get_router_ports(router_uuid) + for uuid in ports: + print "DELETE PORT DRYRUN" + self.neutron.remove_interface_router(router_uuid, + {'port_id': uuid}) + print "DROP GATEWAY PORT DRYRUN" + self.neutron.remove_gateway_router(router_uuid) + time.sleep(10) + print "DELETE ROUTER DRYRUN" + self.neutron.delete_router(router_uuid) + + def cleanup_network(self, network_uuid): + if not self.neutron.list_networks(id=network_uuid)['networks']: + return + net_ports_resp = self.neutron.list_ports(network_id=network_uuid) + for net_port in net_ports_resp['ports']: + self.neutron.delete_port(net_port['id']) + time.sleep(10) + subnets_resp = self.neutron.list_subnets(network_id=network_uuid) + print "DELETE SUBNETS DRYRUN" + for subnet in subnets_resp['subnets']: + self.neutron.delete_subnet(subnet['id']) + print "DELETE NET DRYRUN" + self.neutron.delete_network(network_uuid) + + def cleanup_secgroup(self, secgroup_uuid): + if self.neutron.list_security_groups( + id=secgroup_uuid)["security_groups"]: + print "DELETE SECGROUPS DRYRUN" + self.neutron.delete_security_group(secgroup_uuid) + + def check_floating_exists(self, floating_uuid): + resp = self.neutron.list_floatingips(id=floating_uuid) + if resp['floatingips']: + return True + return False + + def check_any_floating_exists(self, uuids): + return any( + [self.check_floating_exists(uuid) for uuid in uuids] + ) + + def check_volume_exists(self, volume_uuid): + try: + self.cinder.volumes.get(volume_uuid) + return True + except NotFound: + return False + + def check_any_volume_exists(self, uuids): + return any( + [self.check_volume_exists(uuid) for uuid in uuids] + ) + + def check_volume_attached(self, volume_uuid): + volume = self.cinder.volumes.get(volume_uuid) + if volume.attachments: + return True + return False + + def check_any_volume_attached(self, uuids): + return any( + [self.check_volume_attached(uuid) for uuid in uuids] + ) + + def check_server_exists(self, server_uuid): + try: + self.nova.servers.get(server_uuid) + return True + except NovaNotFoundException: + return False + + def check_any_server_exists(self, uuids): + return any( + [self.check_server_exists(uuid) for uuid in uuids] + ) + + def check_router_exists(self, router_uuid): + if self.neutron.list_routers(id=router_uuid)['routers']: + return True + return False + + def check_any_router_exists(self, uuids): + return any( + [self.check_router_exists(uuid) for uuid in uuids] + ) + + def check_stack_exists(self, stack_uuid): + resp = list(self.heat.stacks.list(id=stack_uuid)) + if resp: + return True + return False + + def cleanup_floating_batch(self, uuid_list): + logger.info("Cleanup floatings list: {}".format(uuid_list)) + existing_floating = [] + for uuid in uuid_list: + if self.check_floating_exists(uuid): + logger.debug("Floating {} found, queued".format(uuid)) + existing_floating.append(uuid) + else: + logger.info( + "Given floating ip {} does not exist, skipping".format( + uuid)) + + for uuid in existing_floating: + logger.debug("Deleting floating {}".format(uuid)) + self.neutron.delete_floatingip(uuid) + + try: + helpers.wait_false(self.check_any_floating_exists, + timeout=20, + predicate_kwargs={ + "uuids": existing_floating}) + logger.info("All floating ips deleted!") + except helpers.TimeoutError: + for uuid in existing_floating: + if self.check_floating_exists(uuid): + logger.error("Floating {} stuck!".format(uuid)) + raise helpers.TimeoutError + + def cleanup_servers_batch(self, servers_uuids, reset_state=False, + is_recursive_call=False): + logger.info("Cleaning up servers: {}".format(servers_uuids)) + existing_servers = [] + for uuid in servers_uuids: + if self.check_server_exists(uuid): + logger.debug("Server {} found, queued".format(uuid)) + existing_servers.append(uuid) + else: + logger.info("Server {} not found, ignoring".format(uuid)) + + if reset_state: + for uuid in existing_servers: + logger.info("Resetting state of {}".format(uuid)) + self.nova.servers.reset_state(uuid, state='active') + + for uuid in existing_servers: + logger.info("Deleting server {}".format(uuid)) + self.nova.servers.delete(uuid) + + logger.info("Waiting servers deletion...") + + try: + helpers.wait_false(self.check_any_server_exists, + interval=5, timeout=120, + predicate_kwargs={"uuids": existing_servers}) + logger.info("All servers deleted!") + except helpers.TimeoutError: + failed_servers = [] + for uuid in existing_servers: + if self.check_server_exists(uuid): + logger.error("Server {} stuck!".format(uuid)) + failed_servers.append(uuid) + if is_recursive_call: + logger.error("Servers are completely stuck, " + "even after reset-state") + raise + else: + logger.info("Trying to reset state of stuck servers") + self.cleanup_servers_batch(failed_servers, reset_state=True, + is_recursive_call=True) + + def cleanup_volumes_batch(self, volumes_uuids, + force_delete=False, + is_recursive_call=False): + logger.info("Cleaning up volumes: {}".format(volumes_uuids)) + existing_volumes = [] + for uuid in volumes_uuids: + if self.check_volume_exists(uuid): + existing_volumes.append(uuid) + for uuid in existing_volumes: + if self.check_volume_attached(uuid): + logger.info("Detaching volume {}".format(uuid)) + self.cinder.volumes.detach(uuid) + try: + helpers.wait_false(self.check_any_volume_attached, + interval=3, + timeout=30, + predicate_kwargs={"uuids": existing_volumes}) + except helpers.TimeoutError: + for uuid in existing_volumes: + if self.check_volume_attached(uuid): + logger.error("Volume {} still attached!".format(uuid)) + raise + + for uuid in existing_volumes: + if force_delete: + logger.info("Force-deleting volume {}".format(uuid)) + self.cinder.volumes.force_delete(uuid) + else: + logger.info("Deleting volume {}".format(uuid)) + self.cinder.volumes.delete(uuid) + + try: + helpers.wait_false(self.check_any_volume_exists, + interval=3, + timeout=60, + predicate_kwargs={"uuids": existing_volumes}) + logger.info("All volumes deleted!") + except helpers.TimeoutError: + failed_volumes = [] + for uuid in existing_volumes: + if self.check_volume_exists(uuid): + logger.error("Volume {} is stuck!".format(uuid)) + failed_volumes.append(uuid) + if is_recursive_call: + logger.error("Volumes are stuck even after force-delete!") + raise + else: + logger.info("Trying to force-delete stuck volumes") + self.cleanup_volumes_batch(failed_volumes, + force_delete=True, + is_recursive_call=True) + + def cleanup_router_batch(self, router_uuids): + logger.info("Cleaning up routers {}".format(router_uuids)) + existing_routers = [] + existing_ports = [] + for uuid in router_uuids: + if self.check_router_exists(uuid): + logger.debug("Router {} found, queued".format(uuid)) + existing_routers.append(uuid) + else: + logger.info("Router {} does not exist, ignoring".format(uuid)) + + for router_uuid in existing_routers: + logger.info( + "Removing inner router components from {}".format(router_uuid)) + ports = self.get_router_ports(router_uuid) + existing_ports.extend(ports) + self.neutron.update_router(router_uuid, + {'router': {"routes": None}}) + for port_uuid in ports: + try: + self.neutron.remove_interface_router( + router_uuid, + {'port_id': port_uuid}) + except NeutronNotFoundException: + logger.info("Port {} not found at router {}, ignoring" + "".format(port_uuid, router_uuid)) + self.neutron.remove_gateway_router(router_uuid) + + try: + helpers.wait_false(self.check_any_port_exists, + interval=3, + timeout=30, + predicate_kwargs={"uuids": existing_ports}) + except helpers.TimeoutError: + for uuid in existing_ports: + if self.check_port_exists(uuid): + logger.error("Port {} is stuck!".format(uuid)) + raise + for router_uuid in existing_routers: + logger.info("Deleting router {}".format(router_uuid)) + self.neutron.delete_router(router_uuid) + try: + helpers.wait_false(self.check_any_router_exists, + interval=3, + timeout=30, + predicate_kwargs={"uuids": existing_routers}) + except helpers.TimeoutError: + for router_uuid in existing_routers: + if self.check_router_exists(router_uuid): + logger.error("Router {} is stuck!".format(router_uuid)) + raise + + def cleanup_network_batch(self, network_uuids): + logger.info("Cleaning up networks {}".format(network_uuids)) + existing_networks = [] + existing_subnets = [] + existing_ports = [] + for net_uuid in network_uuids: + if self.check_network_exists(net_uuid): + existing_networks.append(net_uuid) + + for net_uuid in existing_networks: + logger.info("Removing ports from network {}".format(net_uuid)) + net_ports_resp = self.neutron.list_ports(network_id=net_uuid) + for net_port in net_ports_resp['ports']: + existing_ports.append(net_port['id']) + self.neutron.delete_port(net_port['id']) + + try: + helpers.wait_false(self.check_any_port_exists, + interval=3, + timeout=30, + predicate_kwargs={"uuids": existing_ports}) + except helpers.TimeoutError: + for port_uuid in existing_ports: + if self.check_port_exists(port_uuid): + logger.error("Port {} was not removed!".format(port_uuid)) + raise + + for net_uuid in existing_networks: + logger.info("Processing subnets of network {}".format(net_uuid)) + resp = self.neutron.list_subnets(network_id=net_uuid) + for subnet in resp['subnets']: + existing_subnets.append(subnet['id']) + self.neutron.delete_subnet(subnet['id']) + + try: + helpers.wait_false(self.check_any_subnet_exists, + predicate_kwargs={"uuids": existing_subnets}) + except helpers.TimeoutError: + for subnet_uuid in existing_subnets: + if self.check_subnet_exists(subnet_uuid): + logger.error("Can not delete subnet {}".format(subnet_uuid)) + raise + + for net_uuid in existing_networks: + logger.info("Deleting network {}".format(net_uuid)) + self.neutron.delete_network(net_uuid) + try: + helpers.wait_false(self.check_any_network_exists, + predicate_kwargs={"uuids": existing_subnets}) + except helpers.TimeoutError: + for net_uuid in existing_networks: + if self.check_network_exists(net_uuid): + logger.error("Network {} is stuck!".format(net_uuid)) + raise + + def delete_stack(self, stack_uuid): + self.heat.stacks.delete(stack_uuid) + + def cleanup_stack(self, stack_uuid, ignore_err=False): + if not self.check_stack_exists(stack_uuid): + return True + logger.info("Deleting stack {}".format(stack_uuid)) + self.delete_stack(stack_uuid) + try: + helpers.wait_false(self.check_stack_exists, + predicate_args=[stack_uuid]) + except helpers.TimeoutError: + logger.error("Stack {} is stuck!".format(stack_uuid)) + if ignore_err: + return + else: + raise diff --git a/os_cloud_cleaner/requirements.txt b/os_cloud_cleaner/requirements.txt new file mode 100644 index 0000000..d769f03 --- /dev/null +++ b/os_cloud_cleaner/requirements.txt @@ -0,0 +1,4 @@ +python-openstackclient==4.0.0 +openstacksdk<0.44.0 +python-heatclient==1.18.0 +python-neutronclient>=2.2.6,<2.3.0 \ No newline at end of file diff --git a/os_cloud_cleaner/shell.py b/os_cloud_cleaner/shell.py new file mode 100644 index 0000000..834624a --- /dev/null +++ b/os_cloud_cleaner/shell.py @@ -0,0 +1,62 @@ +import argparse +import os + +from cleaner import Cleaner +from logger import logger + +try: + import prehooks +except ImportError: + pass + +parser = argparse.ArgumentParser() +parser.add_argument('--os-auth-url', type=str) +parser.add_argument('--os-username', type=str) +parser.add_argument('--os-password', type=str) +parser.add_argument('--os-project-name', type=str) +parser.add_argument('--os-project-domain-name', type=str) +parser.add_argument('--os-user-domain-name', type=str) + +subparsers = parser.add_subparsers(dest='action') + +cleanup_subparser = subparsers.add_parser('cleanup') +cleanup_subparser.add_argument('resource_type', type=str, + help="allowed: stack") +cleanup_subparser.add_argument('ids', nargs='*', type=str, + help="uuids or names, error if name duplicates") +# cleanup_subparser.add_argument('--force', '-f', default=False, +# action='store_true') +search_subparser = subparsers.add_parser('search') + +# args = parser.parse_args("cleanup stack released-heat-cicd-queens-dvr-sl".split(" ")) +args = parser.parse_args() + +auth_url = args.os_auth_url or os.environ.get('OS_AUTH_URL') +username = args.os_username or os.environ.get('OS_USERNAME') +password = args.os_password or os.environ.get('OS_PASSWORD') +project_name = args.os_project_name or os.environ.get('OS_PROJECT_NAME') +project_domain_name = args.os_project_domain_name or os.environ.get( + 'OS_PROJECT_DOMAIN_NAME') or os.environ.get('OS_PROJECT_DOMAIN_ID') +user_domain_name = args.os_user_domain_name or os.environ.get( + 'OS_USER_DOMAIN_NAME') + +cleaner = Cleaner(os_auth_url=auth_url, + os_user=username, + os_password=password, + os_project_name=project_name, + os_project_domain_name=project_domain_name, + os_user_domain_name=user_domain_name) + +if args.action == "cleanup": + if args.resource_type == "stack": + for os_id in args.ids: + stacks = cleaner.os_conn.get_stacks_by_name_or_uuid(os_id) + if len(stacks) == 1: + stack = stacks[0] + cleaner.cleanup_stack_parallel(stack.id) + elif len(stacks) > 1: + logger.error( + "More than one stack found for given name '{}', do nothing".format( + os_id)) + else: + logger.info("Stack not found, nothing to delete")