Purge deleted rows

Adds the ability to clean up rows that are already marked as
deleted of a certain specified age. Age is calculated as timedelta
from now() in days, which are given at command line

DocImpact
Change-Id: Ia50ab0dc4aa1547a5a6a2430f7941aab194e4baf
Implements: blueprint database-purge

diff --git a/cinder/cmd/manage.py b/cinder/cmd/manage.py
index 05b491746a260a5a8a0f31a5552883c612fd9bf5..5681278c6ef224f45c0b982aed3ac960c2a3667a 100644 (file)
--- a/cinder/cmd/manage.py
+++ b/cinder/cmd/manage.py
@@ -228,6 +228,17 @@ class DbCommands(object):
                                    db_migration.MIGRATE_REPO_PATH,
                                    db_migration.INIT_VERSION))
 
+    @args('age_in_days', type=int,
+          help='Purge deleted rows older than age in days')
+    def purge(self, age_in_days):
+        """Purge deleted rows older than a given age from cinder tables."""
+        age_in_days = int(age_in_days)
+        if age_in_days <= 0:
+            print(_("Must supply a positive, non-zero value for age"))
+            exit(1)
+        ctxt = context.get_admin_context()
+        db.purge_deleted_rows(ctxt, age_in_days)
+
 
 class VersionCommands(object):
     """Class for exposing the codebase version."""

diff --git a/cinder/db/api.py b/cinder/db/api.py
index 4a393e60d752dd2f8741da9edbb5bf17c4771db3..f6e318e21e796ccbc59b5e16b8bacffa70160447 100644 (file)
--- a/cinder/db/api.py
+++ b/cinder/db/api.py
@@ -935,3 +935,12 @@ def cgsnapshot_update(context, cgsnapshot_id, values):
 def cgsnapshot_destroy(context, cgsnapshot_id):
     """Destroy the cgsnapshot or raise if it does not exist."""
     return IMPL.cgsnapshot_destroy(context, cgsnapshot_id)
+
+
+def purge_deleted_rows(context, age_in_days):
+    """Purge deleted rows older than given age from cinder tables
+
+    Raises InvalidParameterValue if age_in_days is incorrect.
+    :returns: number of deleted rows
+    """
+    return IMPL.purge_deleted_rows(context, age_in_days=age_in_days)

diff --git a/cinder/db/sqlalchemy/api.py b/cinder/db/sqlalchemy/api.py
index fd771a9b8155692b4bccd14fd6d38db720e5087f..cbc0f7e15db188ac58d52b7ee93296bf51e12ad8 100644 (file)
--- a/cinder/db/sqlalchemy/api.py
+++ b/cinder/db/sqlalchemy/api.py
@@ -19,6 +19,8 @@
 """Implementation of SQLAlchemy backend."""
 
 
+from datetime import datetime
+from datetime import timedelta
 import functools
 import sys
 import threading
@@ -34,9 +36,11 @@ from oslo.utils import timeutils
 import osprofiler.sqlalchemy
 import six
 import sqlalchemy
+from sqlalchemy import MetaData
 from sqlalchemy import or_
 from sqlalchemy.orm import joinedload, joinedload_all
 from sqlalchemy.orm import RelationshipProperty
+from sqlalchemy.schema import Table
 from sqlalchemy.sql.expression import literal_column
 from sqlalchemy.sql.expression import true
 from sqlalchemy.sql import func
@@ -44,7 +48,7 @@ from sqlalchemy.sql import func
 from cinder.common import sqlalchemyutils
 from cinder.db.sqlalchemy import models
 from cinder import exception
-from cinder.i18n import _, _LW
+from cinder.i18n import _, _LW, _LE, _LI
 from cinder.openstack.common import log as logging
 from cinder.openstack.common import uuidutils
 
@@ -3285,3 +3289,51 @@ def cgsnapshot_destroy(context, cgsnapshot_id):
                     'deleted': True,
                     'deleted_at': timeutils.utcnow(),
                     'updated_at': literal_column('updated_at')})
+
+
+@require_admin_context
+def purge_deleted_rows(context, age_in_days):
+    """Purge deleted rows older than age from cinder tables."""
+    try:
+        age_in_days = int(age_in_days)
+    except ValueError:
+        msg = _LE('Invalid valude for age, %(age)s')
+        LOG.exception(msg, {'age': age_in_days})
+        raise exception.InvalidParameterValue(msg % {'age': age_in_days})
+    if age_in_days <= 0:
+        msg = _LE('Must supply a positive value for age')
+        LOG.exception(msg)
+        raise exception.InvalidParameterValue(msg)
+
+    engine = get_engine()
+    session = get_session()
+    metadata = MetaData()
+    metadata.bind = engine
+    tables = []
+
+    for model_class in models.__dict__.itervalues():
+        if hasattr(model_class, "__tablename__"):
+            tables.append(model_class.__tablename__)
+
+    # Reorder the list so the volumes table is last to avoid FK constraints
+    tables.remove("volumes")
+    tables.append("volumes")
+    for table in tables:
+        t = Table(table, metadata, autoload=True)
+        LOG.info(_LI('Purging deleted rows older than age=%(age)d days '
+                     'from table=%(table)s'), {'age': age_in_days,
+                                               'table': table})
+        deleted_age = datetime.now() - timedelta(days=age_in_days)
+        try:
+            with session.begin():
+                result = session.execute(
+                    t.delete()
+                    .where(t.c.deleted_at < deleted_age))
+        except db_exc.DBReferenceError:
+            LOG.exception(_LE('DBError detected when purging from '
+                              'table=%(table)s'), {'table': table})
+            raise
+
+        rows_purged = result.rowcount
+        LOG.info(_LI("Deleted %(row)d rows from table=%(table)s"),
+                 {'row': rows_purged, 'table': table})

diff --git a/cinder/tests/db/test_purge.py b/cinder/tests/db/test_purge.py
new file mode 100644 (file)
index 0000000..211b389
--- /dev/null
+++ b/cinder/tests/db/test_purge.py
@@ -0,0 +1,105 @@
+# Copyright (C) 2015 OpenStack Foundation
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Tests for db purge."""
+
+from datetime import datetime
+from datetime import timedelta
+import uuid
+
+from cinder import context
+from cinder import db
+from cinder.db.sqlalchemy import api as db_api
+from cinder import exception
+from cinder.openstack.common import log as logging
+from cinder import test
+
+from oslo_db.sqlalchemy import utils as sqlalchemyutils
+
+
+LOG = logging.getLogger(__name__)
+
+
+class PurgeDeletedTest(test.TestCase):
+
+    def setUp(self):
+        super(PurgeDeletedTest, self).setUp()
+        self.context = context.get_admin_context()
+        self.engine = db_api.get_engine()
+        self.session = db_api.get_session()
+        self.conn = self.engine.connect()
+        self.volumes = sqlalchemyutils.get_table(
+            self.engine, "volumes")
+        # The volume_metadata table has a FK of volume_id
+        self.vm = sqlalchemyutils.get_table(
+            self.engine, "volume_metadata")
+        self.uuidstrs = []
+        for unused in range(6):
+            self.uuidstrs.append(uuid.uuid4().hex)
+        # Add 6 rows to table
+        for uuidstr in self.uuidstrs:
+            ins_stmt = self.volumes.insert().values(id=uuidstr)
+            self.conn.execute(ins_stmt)
+            ins_stmt = self.vm.insert().values(volume_id=uuidstr)
+            self.conn.execute(ins_stmt)
+        # Set 4 of them deleted, 2 are 60 days ago, 2 are 20 days ago
+        old = datetime.now() - timedelta(days=20)
+        older = datetime.now() - timedelta(days=60)
+        make_old = self.volumes.update().\
+            where(self.volumes.c.id.in_(self.uuidstrs[1:3]))\
+            .values(deleted_at=old)
+        make_older = self.volumes.update().\
+            where(self.volumes.c.id.in_(self.uuidstrs[4:6]))\
+            .values(deleted_at=older)
+        make_meta_old = self.vm.update().\
+            where(self.vm.c.volume_id.in_(self.uuidstrs[1:3]))\
+            .values(deleted_at=old)
+        make_meta_older = self.vm.update().\
+            where(self.vm.c.volume_id.in_(self.uuidstrs[4:6]))\
+            .values(deleted_at=older)
+        self.conn.execute(make_old)
+        self.conn.execute(make_older)
+        self.conn.execute(make_meta_old)
+        self.conn.execute(make_meta_older)
+
+    def test_purge_deleted_rows_old(self):
+        # Purge at 30 days old, should only delete 2 rows
+        db.purge_deleted_rows(self.context, age_in_days=30)
+        rows = self.session.query(self.volumes).count()
+        meta_rows = self.session.query(self.vm).count()
+        # Verify that we only deleted 2
+        self.assertEqual(4, rows)
+        self.assertEqual(4, meta_rows)
+
+    def test_purge_deleted_rows_older(self):
+        # Purge at 10 days old now, should delete 2 more rows
+        db.purge_deleted_rows(self.context, age_in_days=10)
+        rows = self.session.query(self.volumes).count()
+        meta_rows = self.session.query(self.vm).count()
+        # Verify that we only have 2 rows now
+        self.assertEqual(2, rows)
+        self.assertEqual(2, meta_rows)
+
+    def test_purge_deleted_rows_bad_args(self):
+        # Test with no age argument
+        self.assertRaises(TypeError, db.purge_deleted_rows, self.context)
+        # Test purge with non-integer
+        self.assertRaises(exception.InvalidParameterValue,
+                          db.purge_deleted_rows, self.context,
+                          age_in_days='ten')
+        # Test with negative value
+        self.assertRaises(exception.InvalidParameterValue,
+                          db.purge_deleted_rows, self.context,
+                          age_in_days=-1)