From 1e0c5d7d44bca0e4e45f53299fdc84e2b100a4f2 Mon Sep 17 00:00:00 2001 From: Liang Chen Date: Wed, 7 Aug 2013 08:08:22 +0800 Subject: [PATCH] Enable stack soft delete for event persistence Instead of removing database records when deleting data from stack, table, mark them as deleted through the deleted_at field. And allow access to soft deleted records by stack Id. blueprint event-persistence Change-Id: Iaddd9ba683deab65a9b2be3cc2e727c34e816dd4 --- heat/db/api.py | 4 +- heat/db/sqlalchemy/api.py | 45 ++++++--- heat/engine/parser.py | 13 ++- heat/engine/service.py | 13 +-- heat/tests/test_engine_service.py | 12 ++- heat/tests/test_sqlalchemy_api.py | 161 +++++++++++++++++++++++++++++- 6 files changed, 211 insertions(+), 37 deletions(-) diff --git a/heat/db/api.py b/heat/db/api.py index 39d479a9..0120e1ea 100644 --- a/heat/db/api.py +++ b/heat/db/api.py @@ -108,8 +108,8 @@ def resource_get_by_physical_resource_id(context, physical_resource_id): physical_resource_id) -def stack_get(context, stack_id, admin=False): - return IMPL.stack_get(context, stack_id, admin) +def stack_get(context, stack_id, admin=False, show_deleted=False): + return IMPL.stack_get(context, stack_id, admin, show_deleted=show_deleted) def stack_get_by_name(context, stack_name): diff --git a/heat/db/sqlalchemy/api.py b/heat/db/sqlalchemy/api.py index 72aa18a8..dd1b91c9 100644 --- a/heat/db/sqlalchemy/api.py +++ b/heat/db/sqlalchemy/api.py @@ -29,6 +29,21 @@ def model_query(context, *args): return query +def soft_delete_aware_query(context, *args, **kwargs): + """Stack query helper that accounts for context's `show_deleted` field. + + :param show_deleted: if present, overrides context's show_deleted field. + """ + + query = model_query(context, *args) + show_deleted = kwargs.get('show_deleted') + + if not show_deleted: + query = query.filter_by(deleted_at=None) + + return query + + def _session(context): return (context and context.session) or get_session() @@ -150,7 +165,7 @@ def resource_get_all_by_stack(context, stack_id): def stack_get_by_name(context, stack_name, owner_id=None): - query = model_query(context, models.Stack).\ + query = soft_delete_aware_query(context, models.Stack).\ filter_by(tenant=context.tenant_id).\ filter_by(name=stack_name).\ filter_by(owner_id=owner_id) @@ -158,8 +173,11 @@ def stack_get_by_name(context, stack_name, owner_id=None): return query.first() -def stack_get(context, stack_id, admin=False): - result = model_query(context, models.Stack).get(stack_id) +def stack_get(context, stack_id, admin=False, show_deleted=False): + result = soft_delete_aware_query(context, + models.Stack, + show_deleted=show_deleted).\ + filter_by(id=stack_id).first() # If the admin flag is True, we allow retrieval of a specific # stack without the tenant scoping @@ -174,13 +192,13 @@ def stack_get(context, stack_id, admin=False): def stack_get_all(context): - results = model_query(context, models.Stack).\ + results = soft_delete_aware_query(context, models.Stack).\ filter_by(owner_id=None).all() return results def stack_get_all_by_tenant(context): - results = model_query(context, models.Stack).\ + results = soft_delete_aware_query(context, models.Stack).\ filter_by(owner_id=None).\ filter_by(tenant=context.tenant_id).all() return results @@ -222,18 +240,10 @@ def stack_delete(context, stack_id): session = Session.object_session(s) - for e in s.events: - session.delete(e) - for r in s.resources: session.delete(r) - rt = s.raw_template - uc = s.user_creds - - session.delete(s) - session.delete(rt) - session.delete(uc) + s.soft_delete(session=session) session.flush() @@ -265,13 +275,16 @@ def event_get(context, event_id): def event_get_all(context): - results = model_query(context, models.Event).all() + stacks = soft_delete_aware_query(context, models.Stack) + stack_ids = [stack.id for stack in stacks] + results = model_query(context, models.Event).\ + filter(models.Event.stack_id.in_(stack_ids)).all() return results def event_get_all_by_tenant(context): - stacks = model_query(context, models.Stack).\ + stacks = soft_delete_aware_query(context, models.Stack).\ filter_by(tenant=context.tenant_id).all() results = [] for stack in stacks: diff --git a/heat/engine/parser.py b/heat/engine/parser.py index a0b40711..4d779911 100644 --- a/heat/engine/parser.py +++ b/heat/engine/parser.py @@ -51,8 +51,12 @@ class Stack(object): STATUSES = (IN_PROGRESS, FAILED, COMPLETE ) = ('IN_PROGRESS', 'FAILED', 'COMPLETE') - created_time = timestamp.Timestamp(db_api.stack_get, 'created_at') - updated_time = timestamp.Timestamp(db_api.stack_get, 'updated_at') + created_time = timestamp.Timestamp(functools.partial(db_api.stack_get, + show_deleted=True), + 'created_at') + updated_time = timestamp.Timestamp(functools.partial(db_api.stack_get, + show_deleted=True), + 'updated_at') _zones = None @@ -129,10 +133,11 @@ class Stack(object): @classmethod def load(cls, context, stack_id=None, stack=None, resolve_data=True, - parent_resource=None): + parent_resource=None, show_deleted=True): '''Retrieve a Stack from the database.''' if stack is None: - stack = db_api.stack_get(context, stack_id) + stack = db_api.stack_get(context, stack_id, + show_deleted=show_deleted) if stack is None: message = 'No stack exists with id "%s"' % str(stack_id) raise exception.NotFound(message) diff --git a/heat/engine/service.py b/heat/engine/service.py index fed7d3bf..81916b94 100644 --- a/heat/engine/service.py +++ b/heat/engine/service.py @@ -137,7 +137,7 @@ class EngineService(service.Service): arg2 -> Name or UUID of the stack to look up. """ if uuidutils.is_uuid_like(stack_name): - s = db_api.stack_get(cnxt, stack_name) + s = db_api.stack_get(cnxt, stack_name, show_deleted=True) else: s = db_api.stack_get_by_name(cnxt, stack_name) if s: @@ -146,14 +146,15 @@ class EngineService(service.Service): else: raise exception.StackNotFound(stack_name=stack_name) - def _get_stack(self, cnxt, stack_identity): + def _get_stack(self, cnxt, stack_identity, show_deleted=False): identity = identifier.HeatIdentifier(**stack_identity) if identity.tenant != cnxt.tenant_id: raise exception.InvalidTenant(target=identity.tenant, actual=cnxt.tenant_id) - s = db_api.stack_get(cnxt, identity.stack_id) + s = db_api.stack_get(cnxt, identity.stack_id, + show_deleted=show_deleted) if s is None: raise exception.StackNotFound(stack_name=identity.stack_name) @@ -171,7 +172,7 @@ class EngineService(service.Service): arg2 -> Name of the stack you want to show, or None to show all """ if stack_identity is not None: - stacks = [self._get_stack(cnxt, stack_identity)] + stacks = [self._get_stack(cnxt, stack_identity, show_deleted=True)] else: stacks = db_api.stack_get_all_by_tenant(cnxt) or [] @@ -368,7 +369,7 @@ class EngineService(service.Service): arg1 -> RPC context. arg2 -> Name of the stack you want to see. """ - s = self._get_stack(cnxt, stack_identity) + s = self._get_stack(cnxt, stack_identity, show_deleted=True) if s: return s.raw_template.template return None @@ -425,7 +426,7 @@ class EngineService(service.Service): """ if stack_identity is not None: - st = self._get_stack(cnxt, stack_identity) + st = self._get_stack(cnxt, stack_identity, show_deleted=True) events = db_api.event_get_all_by_stack(cnxt, st.id) else: diff --git a/heat/tests/test_engine_service.py b/heat/tests/test_engine_service.py index d9867125..82021e04 100644 --- a/heat/tests/test_engine_service.py +++ b/heat/tests/test_engine_service.py @@ -704,7 +704,8 @@ class StackServiceTest(HeatTestCase): self.m.StubOutWithMock(service.EngineService, '_get_stack') s = db_api.stack_get(self.ctx, self.stack.id) service.EngineService._get_stack(self.ctx, - self.stack.identifier()).AndReturn(s) + self.stack.identifier(), + show_deleted=True).AndReturn(s) self.m.ReplayAll() events = self.eng.list_events(self.ctx, self.stack.identifier()) @@ -823,7 +824,8 @@ class StackServiceTest(HeatTestCase): self.m.StubOutWithMock(service.EngineService, '_get_stack') service.EngineService._get_stack( - self.ctx, non_exist_identifier).AndRaise(exception.StackNotFound) + self.ctx, non_exist_identifier, + show_deleted=True).AndRaise(exception.StackNotFound) self.m.ReplayAll() self.assertRaises(exception.StackNotFound, @@ -838,7 +840,8 @@ class StackServiceTest(HeatTestCase): self.m.StubOutWithMock(service.EngineService, '_get_stack') service.EngineService._get_stack( - self.ctx, non_exist_identifier).AndRaise(exception.InvalidTenant) + self.ctx, non_exist_identifier, + show_deleted=True).AndRaise(exception.InvalidTenant) self.m.ReplayAll() self.assertRaises(exception.InvalidTenant, @@ -852,7 +855,8 @@ class StackServiceTest(HeatTestCase): self.m.StubOutWithMock(service.EngineService, '_get_stack') s = db_api.stack_get(self.ctx, self.stack.id) service.EngineService._get_stack(self.ctx, - self.stack.identifier()).AndReturn(s) + self.stack.identifier(), + show_deleted=True).AndReturn(s) self.m.ReplayAll() sl = self.eng.show_stack(self.ctx, self.stack.identifier()) diff --git a/heat/tests/test_sqlalchemy_api.py b/heat/tests/test_sqlalchemy_api.py index 2ae81e93..37eb98a9 100644 --- a/heat/tests/test_sqlalchemy_api.py +++ b/heat/tests/test_sqlalchemy_api.py @@ -10,17 +10,23 @@ # License for the specific language governing permissions and limitations # under the License. +import mox + from heat.db.sqlalchemy import api as db_api from heat.engine import environment from heat.tests.v1_1 import fakes from heat.engine.resource import Resource from heat.common import template_format +from heat.engine.resources import instance as instances from heat.engine import parser from heat.openstack.common import uuidutils from heat.tests.common import HeatTestCase +from heat.tests import utils from heat.tests.utils import setup_dummy_db from heat.tests.utils import dummy_context +from heat.tests.utils import reset_dummy_db +from heat.engine.clients import novaclient wp_template = ''' { @@ -47,6 +53,8 @@ wp_template = ''' } ''' +UUIDs = (UUID1, UUID2) = sorted([uuidutils.generate_uuid() for x in range(2)]) + class MyResource(Resource): properties_schema = { @@ -71,16 +79,46 @@ class SqlAlchemyTest(HeatTestCase): super(SqlAlchemyTest, self).setUp() self.fc = fakes.FakeClient() setup_dummy_db() + reset_dummy_db() + self.ctx = dummy_context() + + def tearDown(self): + super(SqlAlchemyTest, self).tearDown() - def _setup_test_stack(self, stack_name): + def _setup_test_stack(self, stack_name, stack_id=None): t = template_format.parse(wp_template) template = parser.Template(t) - ctx = dummy_context() - stack = parser.Stack(ctx, stack_name, template, - environment.Environment({'KeyName': 'test'}), - stack_id=uuidutils.generate_uuid()) + stack_id = stack_id or uuidutils.generate_uuid() + stack = parser.Stack(self.ctx, stack_name, template, + environment.Environment({'KeyName': 'test'})) + with utils.UUIDStub(stack_id): + stack.store() return (t, stack) + def _mock_create(self, mocks): + fc = fakes.FakeClient() + mocks.StubOutWithMock(instances.Instance, 'nova') + instances.Instance.nova().MultipleTimes().AndReturn(fc) + + mocks.StubOutWithMock(fc.servers, 'create') + fc.servers.create(image=744, flavor=3, key_name='test', + name=mox.IgnoreArg(), + security_groups=None, + userdata=mox.IgnoreArg(), scheduler_hints=None, + meta=None, nics=None, + availability_zone=None).MultipleTimes().AndReturn( + fc.servers.list()[-1]) + return fc + + def _mock_delete(self, mocks): + fc = fakes.FakeClient() + mocks.StubOutWithMock(instances.Instance, 'nova') + instances.Instance.nova().MultipleTimes().AndReturn(fc) + + mocks.StubOutWithMock(fc.client, 'get_servers_9999') + get = fc.client.get_servers_9999 + get().MultipleTimes().AndRaise(novaclient.exceptions.NotFound(404)) + def test_encryption(self): stack_name = 'test_encryption' (t, stack) = self._setup_test_stack(stack_name) @@ -99,3 +137,116 @@ class SqlAlchemyTest(HeatTestCase): self.assertNotEqual(encrypted_key, "fake secret") decrypted_key = cs.my_secret self.assertEqual(decrypted_key, "fake secret") + cs.destroy() + + def test_stack_get_by_name(self): + stack = self._setup_test_stack('stack', UUID1)[1] + + st = db_api.stack_get_by_name(self.ctx, 'stack') + self.assertEqual(UUID1, st.id) + + stack.delete() + + st = db_api.stack_get_by_name(self.ctx, 'stack') + self.assertIsNone(st) + + def test_stack_get(self): + stack = self._setup_test_stack('stack', UUID1)[1] + + st = db_api.stack_get(self.ctx, UUID1, show_deleted=False) + self.assertEqual(UUID1, st.id) + + stack.delete() + st = db_api.stack_get(self.ctx, UUID1, show_deleted=False) + self.assertIsNone(st) + + st = db_api.stack_get(self.ctx, UUID1, show_deleted=True) + self.assertEqual(UUID1, st.id) + + def test_stack_get_all(self): + stacks = [self._setup_test_stack('stack', x)[1] for x in UUIDs] + + st_db = db_api.stack_get_all(self.ctx) + self.assertEqual(2, len(st_db)) + + stacks[0].delete() + st_db = db_api.stack_get_all(self.ctx) + self.assertEqual(1, len(st_db)) + + stacks[1].delete() + st_db = db_api.stack_get_all(self.ctx) + self.assertEqual(0, len(st_db)) + + def test_stack_get_all_by_tenant(self): + stacks = [self._setup_test_stack('stack', x)[1] for x in UUIDs] + + st_db = db_api.stack_get_all_by_tenant(self.ctx) + self.assertEqual(2, len(st_db)) + + stacks[0].delete() + st_db = db_api.stack_get_all_by_tenant(self.ctx) + self.assertEqual(1, len(st_db)) + + stacks[1].delete() + st_db = db_api.stack_get_all_by_tenant(self.ctx) + self.assertEqual(0, len(st_db)) + + def test_event_get_all_by_stack(self): + stack = self._setup_test_stack('stack', UUID1)[1] + + self._mock_create(self.m) + self.m.ReplayAll() + stack.create() + self.m.UnsetStubs() + + events = db_api.event_get_all_by_stack(self.ctx, UUID1) + self.assertEqual(2, len(events)) + + self._mock_delete(self.m) + self.m.ReplayAll() + stack.delete() + + events = db_api.event_get_all_by_stack(self.ctx, UUID1) + self.assertEqual(4, len(events)) + + self.m.VerifyAll() + + def test_event_get_all_by_tenant(self): + stacks = [self._setup_test_stack('stack', x)[1] for x in UUIDs] + + self._mock_create(self.m) + self.m.ReplayAll() + [s.create() for s in stacks] + self.m.UnsetStubs() + + events = db_api.event_get_all_by_tenant(self.ctx) + self.assertEqual(4, len(events)) + + self._mock_delete(self.m) + self.m.ReplayAll() + [s.delete() for s in stacks] + + events = db_api.event_get_all_by_tenant(self.ctx) + self.assertEqual(0, len(events)) + + self.m.VerifyAll() + + def test_event_get_all(self): + stacks = [self._setup_test_stack('stack', x)[1] for x in UUIDs] + + self._mock_create(self.m) + self.m.ReplayAll() + [s.create() for s in stacks] + self.m.UnsetStubs() + + events = db_api.event_get_all(self.ctx) + self.assertEqual(4, len(events)) + + self._mock_delete(self.m) + self.m.ReplayAll() + stacks[0].delete() + + events = db_api.event_get_all(self.ctx) + self.assertEqual(2, len(events)) + + self.m.VerifyAll() -- 2.45.2