From 7587e194a05f43cb2d654c0287bfdd00dd28302e Mon Sep 17 00:00:00 2001 From: Abel Lopez Date: Mon, 12 Jan 2015 18:50:00 -0800 Subject: [PATCH] Purge deleted rows Adds the ability to clean up rows that are already marked as deleted of a certain specified age. Age is calculated as timedelta from now() in days, which are given at command line DocImpact Change-Id: Ia50ab0dc4aa1547a5a6a2430f7941aab194e4baf Implements: blueprint database-purge --- cinder/cmd/manage.py | 11 ++++ cinder/db/api.py | 9 +++ cinder/db/sqlalchemy/api.py | 54 ++++++++++++++++- cinder/tests/db/test_purge.py | 105 ++++++++++++++++++++++++++++++++++ 4 files changed, 178 insertions(+), 1 deletion(-) create mode 100644 cinder/tests/db/test_purge.py diff --git a/cinder/cmd/manage.py b/cinder/cmd/manage.py index 05b491746..5681278c6 100644 --- a/cinder/cmd/manage.py +++ b/cinder/cmd/manage.py @@ -228,6 +228,17 @@ class DbCommands(object): db_migration.MIGRATE_REPO_PATH, db_migration.INIT_VERSION)) + @args('age_in_days', type=int, + help='Purge deleted rows older than age in days') + def purge(self, age_in_days): + """Purge deleted rows older than a given age from cinder tables.""" + age_in_days = int(age_in_days) + if age_in_days <= 0: + print(_("Must supply a positive, non-zero value for age")) + exit(1) + ctxt = context.get_admin_context() + db.purge_deleted_rows(ctxt, age_in_days) + class VersionCommands(object): """Class for exposing the codebase version.""" diff --git a/cinder/db/api.py b/cinder/db/api.py index 4a393e60d..f6e318e21 100644 --- a/cinder/db/api.py +++ b/cinder/db/api.py @@ -935,3 +935,12 @@ def cgsnapshot_update(context, cgsnapshot_id, values): def cgsnapshot_destroy(context, cgsnapshot_id): """Destroy the cgsnapshot or raise if it does not exist.""" return IMPL.cgsnapshot_destroy(context, cgsnapshot_id) + + +def purge_deleted_rows(context, age_in_days): + """Purge deleted rows older than given age from cinder tables + + Raises InvalidParameterValue if age_in_days is incorrect. + :returns: number of deleted rows + """ + return IMPL.purge_deleted_rows(context, age_in_days=age_in_days) diff --git a/cinder/db/sqlalchemy/api.py b/cinder/db/sqlalchemy/api.py index fd771a9b8..cbc0f7e15 100644 --- a/cinder/db/sqlalchemy/api.py +++ b/cinder/db/sqlalchemy/api.py @@ -19,6 +19,8 @@ """Implementation of SQLAlchemy backend.""" +from datetime import datetime +from datetime import timedelta import functools import sys import threading @@ -34,9 +36,11 @@ from oslo.utils import timeutils import osprofiler.sqlalchemy import six import sqlalchemy +from sqlalchemy import MetaData from sqlalchemy import or_ from sqlalchemy.orm import joinedload, joinedload_all from sqlalchemy.orm import RelationshipProperty +from sqlalchemy.schema import Table from sqlalchemy.sql.expression import literal_column from sqlalchemy.sql.expression import true from sqlalchemy.sql import func @@ -44,7 +48,7 @@ from sqlalchemy.sql import func from cinder.common import sqlalchemyutils from cinder.db.sqlalchemy import models from cinder import exception -from cinder.i18n import _, _LW +from cinder.i18n import _, _LW, _LE, _LI from cinder.openstack.common import log as logging from cinder.openstack.common import uuidutils @@ -3285,3 +3289,51 @@ def cgsnapshot_destroy(context, cgsnapshot_id): 'deleted': True, 'deleted_at': timeutils.utcnow(), 'updated_at': literal_column('updated_at')}) + + +@require_admin_context +def purge_deleted_rows(context, age_in_days): + """Purge deleted rows older than age from cinder tables.""" + try: + age_in_days = int(age_in_days) + except ValueError: + msg = _LE('Invalid valude for age, %(age)s') + LOG.exception(msg, {'age': age_in_days}) + raise exception.InvalidParameterValue(msg % {'age': age_in_days}) + if age_in_days <= 0: + msg = _LE('Must supply a positive value for age') + LOG.exception(msg) + raise exception.InvalidParameterValue(msg) + + engine = get_engine() + session = get_session() + metadata = MetaData() + metadata.bind = engine + tables = [] + + for model_class in models.__dict__.itervalues(): + if hasattr(model_class, "__tablename__"): + tables.append(model_class.__tablename__) + + # Reorder the list so the volumes table is last to avoid FK constraints + tables.remove("volumes") + tables.append("volumes") + for table in tables: + t = Table(table, metadata, autoload=True) + LOG.info(_LI('Purging deleted rows older than age=%(age)d days ' + 'from table=%(table)s'), {'age': age_in_days, + 'table': table}) + deleted_age = datetime.now() - timedelta(days=age_in_days) + try: + with session.begin(): + result = session.execute( + t.delete() + .where(t.c.deleted_at < deleted_age)) + except db_exc.DBReferenceError: + LOG.exception(_LE('DBError detected when purging from ' + 'table=%(table)s'), {'table': table}) + raise + + rows_purged = result.rowcount + LOG.info(_LI("Deleted %(row)d rows from table=%(table)s"), + {'row': rows_purged, 'table': table}) diff --git a/cinder/tests/db/test_purge.py b/cinder/tests/db/test_purge.py new file mode 100644 index 000000000..211b38900 --- /dev/null +++ b/cinder/tests/db/test_purge.py @@ -0,0 +1,105 @@ +# Copyright (C) 2015 OpenStack Foundation +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Tests for db purge.""" + +from datetime import datetime +from datetime import timedelta +import uuid + +from cinder import context +from cinder import db +from cinder.db.sqlalchemy import api as db_api +from cinder import exception +from cinder.openstack.common import log as logging +from cinder import test + +from oslo_db.sqlalchemy import utils as sqlalchemyutils + + +LOG = logging.getLogger(__name__) + + +class PurgeDeletedTest(test.TestCase): + + def setUp(self): + super(PurgeDeletedTest, self).setUp() + self.context = context.get_admin_context() + self.engine = db_api.get_engine() + self.session = db_api.get_session() + self.conn = self.engine.connect() + self.volumes = sqlalchemyutils.get_table( + self.engine, "volumes") + # The volume_metadata table has a FK of volume_id + self.vm = sqlalchemyutils.get_table( + self.engine, "volume_metadata") + self.uuidstrs = [] + for unused in range(6): + self.uuidstrs.append(uuid.uuid4().hex) + # Add 6 rows to table + for uuidstr in self.uuidstrs: + ins_stmt = self.volumes.insert().values(id=uuidstr) + self.conn.execute(ins_stmt) + ins_stmt = self.vm.insert().values(volume_id=uuidstr) + self.conn.execute(ins_stmt) + # Set 4 of them deleted, 2 are 60 days ago, 2 are 20 days ago + old = datetime.now() - timedelta(days=20) + older = datetime.now() - timedelta(days=60) + make_old = self.volumes.update().\ + where(self.volumes.c.id.in_(self.uuidstrs[1:3]))\ + .values(deleted_at=old) + make_older = self.volumes.update().\ + where(self.volumes.c.id.in_(self.uuidstrs[4:6]))\ + .values(deleted_at=older) + make_meta_old = self.vm.update().\ + where(self.vm.c.volume_id.in_(self.uuidstrs[1:3]))\ + .values(deleted_at=old) + make_meta_older = self.vm.update().\ + where(self.vm.c.volume_id.in_(self.uuidstrs[4:6]))\ + .values(deleted_at=older) + self.conn.execute(make_old) + self.conn.execute(make_older) + self.conn.execute(make_meta_old) + self.conn.execute(make_meta_older) + + def test_purge_deleted_rows_old(self): + # Purge at 30 days old, should only delete 2 rows + db.purge_deleted_rows(self.context, age_in_days=30) + rows = self.session.query(self.volumes).count() + meta_rows = self.session.query(self.vm).count() + # Verify that we only deleted 2 + self.assertEqual(4, rows) + self.assertEqual(4, meta_rows) + + def test_purge_deleted_rows_older(self): + # Purge at 10 days old now, should delete 2 more rows + db.purge_deleted_rows(self.context, age_in_days=10) + rows = self.session.query(self.volumes).count() + meta_rows = self.session.query(self.vm).count() + # Verify that we only have 2 rows now + self.assertEqual(2, rows) + self.assertEqual(2, meta_rows) + + def test_purge_deleted_rows_bad_args(self): + # Test with no age argument + self.assertRaises(TypeError, db.purge_deleted_rows, self.context) + # Test purge with non-integer + self.assertRaises(exception.InvalidParameterValue, + db.purge_deleted_rows, self.context, + age_in_days='ten') + # Test with negative value + self.assertRaises(exception.InvalidParameterValue, + db.purge_deleted_rows, self.context, + age_in_days=-1) -- 2.45.2