From: Vladimir Khlyunev Date: Thu, 14 Jan 2021 11:24:55 +0000 (+0400) Subject: Add search and enhance cleanup feature X-Git-Url: https://review.fuel-infra.org/gitweb?a=commitdiff_plain;h=refs%2Fchanges%2F72%2F41672%2F1;p=tools%2Fsustaining.git Add search and enhance cleanup feature Change-Id: I7c2d639074865af11f5ae19644b44feeac93af7b --- diff --git a/os_cloud_cleaner/cleaner.py b/os_cloud_cleaner/cleaner.py index ab48a6c..438fab0 100644 --- a/os_cloud_cleaner/cleaner.py +++ b/os_cloud_cleaner/cleaner.py @@ -83,13 +83,17 @@ class Cleaner: result = {"servers": [], "networks": [], "routers": [], - "volumes": []} + "volumes": [], + "subnets": []} for server in self.get_servers(): if server.id not in self.heat_resources.keys(): result["servers"].append(server) for net in self.get_networks(): if net['id'] not in self.heat_resources.keys(): result['networks'].append(net) + for subnet in self.get_subnets(): + if subnet['id'] not in self.heat_resources.keys(): + result['subnets'].append(subnet) for router in self.get_routers(): if router['id'] not in self.heat_resources.keys(): result["routers"].append(router) @@ -113,12 +117,41 @@ class Cleaner: resources.update(self.get_os_stack_resources(stack)) return resources + def search_non_heat_resources_by_name(self, name): + self.reset_cache() + resources = self.collect_os_non_stack_resources() + servers = {server.id: server.name for server in resources['servers'] + if name in server.name and + server.id not in self.ignorelist_uuids} + volumes = {volume.id: volume.name for volume in resources['volumes'] + if name in volume.name and + volume.id not in self.ignorelist_uuids} + networks = {net['id']: net['name'] for net in resources['networks'] + if name in net['name'] and + net['id'] not in self.ignorelist_uuids} + subnets = {subnet['id']: subnet['name'] for subnet in + resources['subnets'] + if name in subnet['name'] and + subnet['id'] not in self.ignorelist_uuids} + routers = {router['id']: router['name'] for router in + resources['routers'] if name in router['name'] and + router['id'] not in self.ignorelist_uuids} + + return {'servers': servers, + 'volumes': volumes, + 'networks': networks, + 'routers': routers, + 'subnets': subnets} + def get_servers(self): return self.os_conn.get_servers() def get_networks(self): return self.os_conn.get_networks() + def get_subnets(self): + return self.os_conn.get_subnets() + def get_routers(self): return self.os_conn.get_routers() diff --git a/os_cloud_cleaner/ignorelist.txt b/os_cloud_cleaner/ignorelist.txt index 4d0a57a..800b503 100644 --- a/os_cloud_cleaner/ignorelist.txt +++ b/os_cloud_cleaner/ignorelist.txt @@ -1,4 +1,44 @@ # Servers c207a259-0448-4191-a5df-a4859adfce74 # dstremkovski-vw-jump +b950305a-20db-45c9-a957-d9332913a486 # dstremkouski-jumphost-eu +11df29b5-f931-4d16-b710-011c7a0f9a27 # dstremkouski-k8s-master +ba308971-abd7-4630-a1f1-06d2caa3be37 # dstremkouski-k8s-node01 +3fb4af2b-40d4-4ec3-88a0-17c3edd8ed67 # dstremkouski-k8s-node02 +1589214e-43bf-4d57-80ac-68b28030c536 # sre-monitoring +5b222dd9-aeb9-42bd-8fac-e8f3874238d9 # dench-desktop # Networks -# Routers \ No newline at end of file +0563f790-92a3-4c92-ab41-74478442a75e # physnet1-402 +b7b8b2e1-fb2e-4edb-9540-500363155fe3 # cvp-internal +4e7df785-0e23-44c9-8095-c07db7b15954 # system-phys-2406 +fadbd9d3-d569-49fa-baad-3d8fbe25fd8e # maintenance-ci-network +0f417039-c9e5-4441-8194-119134d12602 # system-phys-2402 +fa371330-d802-4b5d-a36e-7590f05ddfb1 # system-phys-2401 +86f15097-630d-493d-8636-c91ca2cbe8b5 # cvp-internal-2 +1ba652fc-5b03-46df-87b4-e044121317f4 # vdrok-unittest +e977057f-ab64-466f-b953-1d9e44ac9f36 # system-phys-2404 +260ad854-9927-401d-9d4a-e6a278067d44 # system-phys-410 +57002ca1-2c3c-4279-8a0e-472b95d51217 # baremetal-flat-network +16ed86ab-d128-4c5c-aa69-e2e1e419b6a5 # system-phys-2403 +55e9a5f5-0f61-4502-be82-7dc2806f2f05 # system-phys-2405 +5d4a2029-6c32-4bee-89dc-855829cc2233 # physnet1-432 +c11bf78a-de07-4e67-8047-a28b247c27bb # sre-internal +bf6b85a1-39db-4582-b0d1-f4291dddb9cf # public +# Routers +d92fc584-371b-4c0b-aa85-9c5c49292129 # sre-router +808a47c2-7ee4-46c2-b232-b625c70bd5d7 # vdrok-unittest +# Volumes +83bada1a-3a04-413f-b3ef-d0f26adf69e2 # dstremkouski-convert-qcow2-raw +0d0665c2-c684-4a66-a224-72462ccb016d # dstremkouski-500G-stripe +0601a251-940f-4489-b5d3-4ed58d9d2727 # dstremkouski-VW-jumphost +d0eb9a28-8ea1-4728-9fcc-5e89d1fd773d # dstremkouski-500G +d32349f4-74eb-4acb-a7a4-b9ae32029000 # windows-vw-jump +# Subnets +4e248805-d684-4411-bb3f-143ec8da5bad # cvp-subnet-internal-2 +7d7e65ba-d6cf-458f-b963-50614fac5e00 # baremetal-subnet +0822b2c1-4676-46f1-a382-54d3c9ab1458 # physnet1-432-subnet +8431ab7d-b000-4660-b71a-36c97321e8a2 # system-phys-410-subnet +3409347e-718c-45c2-ac62-e1a260e89575 # sre-internal +428265dc-4073-47b5-947d-d7b59d6f8dbf # maintenance-ci-subnet +a9054669-1d2a-4341-b6f5-51d7dbc36dca # physnet1-402-subnet +125cf702-d82f-4e59-b3ea-f1f0b2f263b4 # vdrok-unittest +37ca76fd-fadc-4d8e-a52e-4a42d3cc7306 # cvp-subnet-internal \ No newline at end of file diff --git a/os_cloud_cleaner/os_connector.py b/os_cloud_cleaner/os_connector.py index b0b613f..4c2c93e 100644 --- a/os_cloud_cleaner/os_connector.py +++ b/os_cloud_cleaner/os_connector.py @@ -148,6 +148,10 @@ class OpenStackActions(object): net_list = self.neutron.list_networks() return net_list['networks'] + def get_subnets(self): + net_list = self.neutron.list_subnets() + return net_list['subnets'] + def get_routers(self): router_list = self.neutron.list_routers() return router_list['routers'] diff --git a/os_cloud_cleaner/shell.py b/os_cloud_cleaner/shell.py index c600fb3..8686f51 100644 --- a/os_cloud_cleaner/shell.py +++ b/os_cloud_cleaner/shell.py @@ -1,5 +1,6 @@ import argparse import os +from prettytable import PrettyTable from cleaner import Cleaner from logger import logger @@ -9,6 +10,31 @@ try: except ImportError: pass + +def do_search(name, only_uuids): + result = cleaner.search_non_heat_resources_by_name(name) + keys_ordered = ['servers', 'subnets', 'networks', 'routers', 'volumes'] + if only_uuids: + uuids = [] + for k in keys_ordered: + uuids.extend(result[k].keys()) + print " ".join(uuids) + else: + for key in keys_ordered: + if not result[key]: + continue + print key + table = PrettyTable() + table.field_names = ["UUID", "Name"] + table.align = 'l' + # table.vertical_char = "#" + # table.horizontal_char = "#" + # table.left_padding_width = 0 + for uuid in result[key]: + table.add_row([uuid, result[key][uuid]]) + print table + + parser = argparse.ArgumentParser() parser.add_argument('--os-auth-url', type=str) parser.add_argument('--os-username', type=str) @@ -28,9 +54,18 @@ cleanup_subparser.add_argument('ids', nargs='*', type=str, # cleanup_subparser.add_argument('--force', '-f', default=False, # action='store_true') search_subparser = subparsers.add_parser('search') +search_subparser.add_argument('--name', '-n', type=str, + help='part of name to search for', + default='') +search_subparser.add_argument('--only-uuids', '-i', + help='print only uuids for chain xargs call', + default=False, action='store_true') -# args = parser.parse_args("cleanup stack rl-queens-dsgn-2k19-10".split(" ")) -# args = parser.parse_args("guess released-heat-cicd-queens-contrail41-sl".split(" ")) +# args = parser.parse_args("cleanup stack pike-ci-dvr-tarask-test".split(" ")) +# args = parser.parse_args("cleanup misc c32eade8-78a2-4cb2-a6a5-9b692a979620 0e429566-4566-41b3-8767-4d157aa90e38 37a8cdf7-0684-4fe8-860a-4ed7eedbe07c".split(" ")) +# args = parser.parse_args("search -i -n pike-ci-dvr-tarask-test".split(" ")) +# args = parser.parse_args("search -i".split(" ")) +# args = parser.parse_args("search".split(" ")) args = parser.parse_args() auth_url = args.os_auth_url or os.environ.get('OS_AUTH_URL') @@ -56,10 +91,21 @@ if args.action == "cleanup": stacks = cleaner.os_conn.get_stacks_by_name_or_uuid(os_id) if len(stacks) == 1: stack = stacks[0] + stack_name = stack.stack_name cleaner.cleanup_stack_parallel(stack.id) + logger.info("Stack cleanup done, looking for non-deleted OS " + "resources by stack-name") + do_search(stack_name, only_uuids=False) elif len(stacks) > 1: logger.error( "More than one stack found for given name '{}', do nothing".format( os_id)) else: logger.info("Stack not found, nothing to delete") + if args.resource_type == "misc": + for os_id in args.ids: + if os_id: + logger.info("Processing {}...".format(os_id)) + cleaner.process_resource(os_id) +elif args.action == "search": + do_search(args.name, args.only_uuids)