From 484c830ff5d01a8048291f8590b2dd9555fda073 Mon Sep 17 00:00:00 2001 From: Vladimir Khlyunev Date: Wed, 13 Jan 2021 17:52:29 +0400 Subject: [PATCH] Add ignorelist behavior to cleaner Should be populated with uuids later Change-Id: I582505054f3404ef39b59588ef121eee685ca839 --- os_cloud_cleaner/cleaner.py | 73 ++++++++----------- os_cloud_cleaner/ignorelist.txt | 4 + os_cloud_cleaner/os_connector.py | 121 ++++++++++--------------------- os_cloud_cleaner/shell.py | 7 +- 4 files changed, 77 insertions(+), 128 deletions(-) create mode 100644 os_cloud_cleaner/ignorelist.txt diff --git a/os_cloud_cleaner/cleaner.py b/os_cloud_cleaner/cleaner.py index 5678c46..ab48a6c 100644 --- a/os_cloud_cleaner/cleaner.py +++ b/os_cloud_cleaner/cleaner.py @@ -1,5 +1,6 @@ from __future__ import unicode_literals +import os from datetime import datetime from os_connector import OpenStackActions @@ -17,20 +18,35 @@ class Cleaner: def __init__(self, lifetime=None, os_auth_url=None, os_user=None, os_password=None, os_project_name=None, os_project_domain_name=None, - os_user_domain_name=None): + os_user_domain_name=None, + ignorelist_file=None): + + self.ignorelist_uuids = [] + if ignorelist_file is not None and os.path.exists(ignorelist_file): + with open(ignorelist_file) as f: + for line in f.readlines(): + if line and not line.startswith("#"): + self.ignorelist_uuids.append(line.split(" ")[0]) + self.os_conn = OpenStackActions( auth_url=os_auth_url, user=os_user, password=os_password, project_name=os_project_name, project_domain_name=os_project_domain_name, - user_domain_name=os_user_domain_name) + user_domain_name=os_user_domain_name, + ignorelist_uuids=self.ignorelist_uuids + ) self.lifetime = lifetime if lifetime is not None \ else self.DEFAULT_LIFETIME_HOURS self.heat_resources_cache = {} self.non_heat_resources_cache = {} + def reset_cache(self): + self.heat_resources_cache = {} + self.non_heat_resources_cache = {} + @property def heat_resources(self): if not self.heat_resources_cache: @@ -66,14 +82,20 @@ class Cleaner: def collect_os_non_stack_resources(self): result = {"servers": [], "networks": [], - "routers": []} + "routers": [], + "volumes": []} for server in self.get_servers(): - result["servers"].append(server) + if server.id not in self.heat_resources.keys(): + result["servers"].append(server) for net in self.get_networks(): - result["networks"].append(net) + if net['id'] not in self.heat_resources.keys(): + result['networks'].append(net) for router in self.get_routers(): - result["routers"].append(router) - + if router['id'] not in self.heat_resources.keys(): + result["routers"].append(router) + for volume in self.get_volumes(): + if volume.id not in self.heat_resources.keys(): + result['volumes'].append(volume) return result @staticmethod @@ -100,41 +122,8 @@ class Cleaner: def get_routers(self): return self.os_conn.get_routers() - def cleanup_stack_one_by_one(self, stack): - resources = self.get_os_stack_resources(stack) - server_uuids = [] - router_uuids = [] - network_uuids = [] - secgroup_uuids = [] - floating_ip_uuids = [] - volumes_uuids = [] - for uuid, resource in resources.items(): - if resource["resource_type"] == "OS::Nova::Server": - server_uuids.append(uuid) - elif resource["resource_type"] == "OS::Neutron::Router": - router_uuids.append(uuid) - elif resource["resource_type"] == "OS::Neutron::Net": - network_uuids.append(uuid) - elif resource["resource_type"] == "OS::Neutron::SecurityGroup": - secgroup_uuids.append(uuid) - elif resource["resource_type"] == "OS::Neutron::FloatingIP": - floating_ip_uuids.append(uuid) - elif resource["resource_type"] == "OS::Cinder::Volume": - volumes_uuids.append(uuid) - - for uuid in floating_ip_uuids: - self.os_conn.cleanup_floating_ip(uuid) - for uuid in volumes_uuids: - self.os_conn.cleanup_volume(uuid) - for uuid in server_uuids: - self.os_conn.cleanup_instance(uuid) - # time.sleep(60) - for uuid in router_uuids: - self.os_conn.cleanup_router(uuid) - for uuid in network_uuids: - self.os_conn.cleanup_network(uuid) - for uuid in secgroup_uuids: - self.os_conn.cleanup_secgroup(uuid) + def get_volumes(self): + return self.os_conn.get_volumes() def cleanup_stack_parallel(self, stack): if not hasattr(stack, 'id'): diff --git a/os_cloud_cleaner/ignorelist.txt b/os_cloud_cleaner/ignorelist.txt new file mode 100644 index 0000000..4d0a57a --- /dev/null +++ b/os_cloud_cleaner/ignorelist.txt @@ -0,0 +1,4 @@ +# Servers +c207a259-0448-4191-a5df-a4859adfce74 # dstremkovski-vw-jump +# Networks +# Routers \ No newline at end of file diff --git a/os_cloud_cleaner/os_connector.py b/os_cloud_cleaner/os_connector.py index efe84ed..b0b613f 100644 --- a/os_cloud_cleaner/os_connector.py +++ b/os_cloud_cleaner/os_connector.py @@ -17,10 +17,18 @@ import helpers from logger import logger +ignored_msg = "{} {} is in ignored list! Do nothing" + class OpenStackActions(object): def __init__(self, auth_url, user, password, project_name, - project_domain_name, user_domain_name): + project_domain_name, user_domain_name, + ignorelist_uuids=None): + if ignorelist_uuids is not None: + self.ignorelist_uuids = ignorelist_uuids + else: + self.ignorelist_uuids = [] + self.keystone_session = None self.auth_url = auth_url @@ -126,6 +134,10 @@ class OpenStackActions(object): stacks = self.heat.stacks.list() return stacks + def get_volumes(self): + volumes = self.cinder.volumes.list() + return volumes + def get_stacks_by_name_or_uuid(self, stack_id): stacks = list(self.heat.stacks.list(id=stack_id)) if not stacks: @@ -140,54 +152,6 @@ class OpenStackActions(object): router_list = self.neutron.list_routers() return router_list['routers'] - def cleanup_floating_ip(self, uuid): - if self.neutron.list_floatingips(id=uuid)['floatingips']: - print "DELETE SEPARATED FLOATING IP DRYRUN" - self.neutron.delete_floatingip(uuid) - - def cleanup_volume(self, uuid): - if not self.check_volume_exists(uuid): - return - self.cinder.volumes.detach(uuid) - helpers.wait_false(self.check_volume_attached, - interval=3, - timeout=30, - predicate_args=[uuid]) - self.cinder.volumes.delete(uuid) - helpers.wait_false(self.check_volume_exists, - interval=3, - timeout=60, - predicate_args=[uuid]) - - def cleanup_instance(self, server_uuid): - if not self.check_server_exists(server_uuid): - logger.info("Server {} does not exist, do nothing".format( - server_uuid)) - - server = self.nova.servers.get(server_uuid) - floating_ips_uuid = [] - for net in server.addresses.values(): - for ip in net: - if ip.get('OS-EXT-IPS:type') and \ - ip['OS-EXT-IPS:type'] == "floating": - logger.debug("Queuing floating id {} to delete " - "queue".format(ip['addr'])) - floating_ips_uuid.append( - self.get_floating_ip_uuid(ip['addr'])) - - self.cleanup_floating_batch(floating_ips_uuid) - - volumes_uuids = [] - for volume in self.nova.volumes.get_server_volumes(server_uuid): - volumes_uuids.append(volume.id) - self.cleanup_volumes_batch(volumes_uuids) - - self.nova.servers.delete(server) - helpers.wait_false(self.check_volume_exists, - interval=3, - timeout=60, - predicate_args=[server_uuid]) - def check_router_ports_exists(self, router_uuid): if self.get_router_ports(router_uuid): return True @@ -226,40 +190,6 @@ class OpenStackActions(object): [self.check_subnet_exists(uuid) for uuid in uuids] ) - def cleanup_router(self, router_uuid): - if not self.neutron.list_routers(id=router_uuid)['routers']: - return - ports = self.get_router_ports(router_uuid) - for uuid in ports: - print "DELETE PORT DRYRUN" - self.neutron.remove_interface_router(router_uuid, - {'port_id': uuid}) - print "DROP GATEWAY PORT DRYRUN" - self.neutron.remove_gateway_router(router_uuid) - time.sleep(10) - print "DELETE ROUTER DRYRUN" - self.neutron.delete_router(router_uuid) - - def cleanup_network(self, network_uuid): - if not self.neutron.list_networks(id=network_uuid)['networks']: - return - net_ports_resp = self.neutron.list_ports(network_id=network_uuid) - for net_port in net_ports_resp['ports']: - self.neutron.delete_port(net_port['id']) - time.sleep(10) - subnets_resp = self.neutron.list_subnets(network_id=network_uuid) - print "DELETE SUBNETS DRYRUN" - for subnet in subnets_resp['subnets']: - self.neutron.delete_subnet(subnet['id']) - print "DELETE NET DRYRUN" - self.neutron.delete_network(network_uuid) - - def cleanup_secgroup(self, secgroup_uuid): - if self.neutron.list_security_groups( - id=secgroup_uuid)["security_groups"]: - print "DELETE SECGROUPS DRYRUN" - self.neutron.delete_security_group(secgroup_uuid) - def check_floating_exists(self, floating_uuid): resp = self.neutron.list_floatingips(id=floating_uuid) if resp['floatingips']: @@ -326,6 +256,9 @@ class OpenStackActions(object): logger.info("Cleanup floatings list: {}".format(uuid_list)) existing_floating = [] for uuid in uuid_list: + if uuid in self.ignorelist_uuids: + logger.warning(ignored_msg.format("Floating ip", uuid)) + continue if self.check_floating_exists(uuid): logger.debug("Floating {} found, queued".format(uuid)) existing_floating.append(uuid) @@ -352,9 +285,14 @@ class OpenStackActions(object): def cleanup_servers_batch(self, servers_uuids, reset_state=False, is_recursive_call=False): + if not isinstance(servers_uuids, list): + servers_uuids = [servers_uuids] logger.info("Cleaning up servers: {}".format(servers_uuids)) existing_servers = [] for uuid in servers_uuids: + if uuid in self.ignorelist_uuids: + logger.warning(ignored_msg.format("Server", uuid)) + continue if self.check_server_exists(uuid): logger.debug("Server {} found, queued".format(uuid)) existing_servers.append(uuid) @@ -398,6 +336,9 @@ class OpenStackActions(object): logger.info("Cleaning up volumes: {}".format(volumes_uuids)) existing_volumes = [] for uuid in volumes_uuids: + if uuid in self.ignorelist_uuids: + logger.warning(ignored_msg.format("Volume", uuid)) + continue if self.check_volume_exists(uuid): existing_volumes.append(uuid) for uuid in existing_volumes: @@ -449,6 +390,9 @@ class OpenStackActions(object): existing_routers = [] existing_ports = [] for uuid in router_uuids: + if uuid in self.ignorelist_uuids: + logger.warning(ignored_msg.format("Router", uuid)) + continue if self.check_router_exists(uuid): logger.debug("Router {} found, queued".format(uuid)) existing_routers.append(uuid) @@ -502,6 +446,9 @@ class OpenStackActions(object): existing_subnets = [] existing_ports = [] for net_uuid in network_uuids: + if net_uuid in self.ignorelist_uuids: + logger.warning(ignored_msg.format("Network", net_uuid)) + continue if self.check_network_exists(net_uuid): existing_networks.append(net_uuid) @@ -552,9 +499,15 @@ class OpenStackActions(object): raise def delete_stack(self, stack_uuid): - self.heat.stacks.delete(stack_uuid) + if stack_uuid in self.ignorelist_uuids: + logger.warning(ignored_msg.format("Stack", stack_uuid)) + else: + self.heat.stacks.delete(stack_uuid) def cleanup_stack(self, stack_uuid, ignore_err=False): + if stack_uuid in self.ignorelist_uuids: + logger.warning(ignored_msg.format("Stack", stack_uuid)) + return if not self.check_stack_exists(stack_uuid): return True logger.info("Deleting stack {}".format(stack_uuid)) diff --git a/os_cloud_cleaner/shell.py b/os_cloud_cleaner/shell.py index 834624a..c600fb3 100644 --- a/os_cloud_cleaner/shell.py +++ b/os_cloud_cleaner/shell.py @@ -16,6 +16,7 @@ parser.add_argument('--os-password', type=str) parser.add_argument('--os-project-name', type=str) parser.add_argument('--os-project-domain-name', type=str) parser.add_argument('--os-user-domain-name', type=str) +parser.add_argument('--ignorelist-file', default='ignorelist.txt') subparsers = parser.add_subparsers(dest='action') @@ -28,7 +29,8 @@ cleanup_subparser.add_argument('ids', nargs='*', type=str, # action='store_true') search_subparser = subparsers.add_parser('search') -# args = parser.parse_args("cleanup stack released-heat-cicd-queens-dvr-sl".split(" ")) +# args = parser.parse_args("cleanup stack rl-queens-dsgn-2k19-10".split(" ")) +# args = parser.parse_args("guess released-heat-cicd-queens-contrail41-sl".split(" ")) args = parser.parse_args() auth_url = args.os_auth_url or os.environ.get('OS_AUTH_URL') @@ -45,7 +47,8 @@ cleaner = Cleaner(os_auth_url=auth_url, os_password=password, os_project_name=project_name, os_project_domain_name=project_domain_name, - os_user_domain_name=user_domain_name) + os_user_domain_name=user_domain_name, + ignorelist_file=args.ignorelist_file) if args.action == "cleanup": if args.resource_type == "stack": -- 2.45.2