1 # Copyright 2012 New Dream Network, LLC (DreamHost)
3 # Licensed under the Apache License, Version 2.0 (the "License"); you may
4 # not use this file except in compliance with the License. You may obtain
5 # a copy of the License at
7 # http://www.apache.org/licenses/LICENSE-2.0
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12 # License for the specific language governing permissions and limitations
17 from alembic import command as alembic_command
18 from alembic import config as alembic_config
19 from alembic import environment
20 from alembic import script as alembic_script
21 from alembic import util as alembic_util
23 from oslo_config import cfg
24 from oslo_utils import fileutils
25 from oslo_utils import importutils
29 from neutron._i18n import _
30 from neutron.common import utils
31 from neutron.db import migration
34 HEAD_FILENAME = 'HEAD'
35 HEADS_FILENAME = 'HEADS'
36 CONTRACT_HEAD_FILENAME = 'CONTRACT_HEAD'
37 EXPAND_HEAD_FILENAME = 'EXPAND_HEAD'
39 CURRENT_RELEASE = migration.MITAKA
45 EXPAND_BRANCH = 'expand'
46 CONTRACT_BRANCH = 'contract'
47 MIGRATION_BRANCHES = (EXPAND_BRANCH, CONTRACT_BRANCH)
49 MIGRATION_ENTRYPOINTS = 'neutron.db.alembic_migrations'
50 migration_entrypoints = {
51 entrypoint.name: entrypoint
52 for entrypoint in pkg_resources.iter_entry_points(MIGRATION_ENTRYPOINTS)
56 BRANCHLESS_WARNING = 'Branchless migration chains are deprecated as of Mitaka.'
59 neutron_alembic_ini = os.path.join(os.path.dirname(__file__), 'alembic.ini')
61 VALID_SERVICES = ['fwaas', 'lbaas', 'vpnaas']
62 INSTALLED_SERVICES = [service_ for service_ in VALID_SERVICES
63 if 'neutron-%s' % service_ in migration_entrypoints]
64 INSTALLED_SUBPROJECTS = [project_ for project_ in migration_entrypoints]
67 cfg.StrOpt('core_plugin',
69 help=_('Neutron plugin provider module'),
70 deprecated_for_removal=True),
72 choices=INSTALLED_SERVICES,
73 help=(_("(Deprecated. Use '--subproject neutron-SERVICE' "
74 "instead.) The advanced service to execute the "
76 deprecated_for_removal=True),
77 cfg.StrOpt('subproject',
78 choices=INSTALLED_SUBPROJECTS,
79 help=(_("The subproject to execute the command against. "
80 "Can be one of: '%s'.")
81 % "', '".join(INSTALLED_SUBPROJECTS))),
82 cfg.BoolOpt('split_branches',
84 help=_("Enforce using split branches file structure."))
88 cfg.StrOpt('quota_driver',
90 help=_('Neutron quota driver class'),
91 deprecated_for_removal=True),
95 cfg.StrOpt('connection',
96 deprecated_name='sql_connection',
99 help=_('URL to database')),
102 help=_('Database engine for which script will be generated '
103 'when using offline migration.')),
106 CONF = cfg.ConfigOpts()
107 CONF.register_cli_opts(_core_opts)
108 CONF.register_cli_opts(_db_opts, 'database')
109 CONF.register_opts(_quota_opts, 'QUOTAS')
112 def do_alembic_command(config, cmd, revision=None, desc=None, **kwargs):
115 args.append(revision)
117 project = config.get_main_option('neutron_project')
119 alembic_util.msg(_('Running %(cmd)s (%(desc)s) for %(project)s ...') %
120 {'cmd': cmd, 'desc': desc, 'project': project})
122 alembic_util.msg(_('Running %(cmd)s for %(project)s ...') %
123 {'cmd': cmd, 'project': project})
125 getattr(alembic_command, cmd)(config, *args, **kwargs)
126 except alembic_util.CommandError as e:
127 alembic_util.err(six.text_type(e))
128 alembic_util.msg(_('OK'))
131 def _get_alembic_entrypoint(project):
132 if project not in migration_entrypoints:
133 alembic_util.err(_('Sub-project %s not installed.') % project)
134 return migration_entrypoints[project]
137 def do_generic_show(config, cmd):
138 kwargs = {'verbose': CONF.command.verbose}
139 do_alembic_command(config, cmd, **kwargs)
142 def do_check_migration(config, cmd):
143 do_alembic_command(config, 'branches')
144 validate_revisions(config)
145 validate_head_file(config)
148 def add_alembic_subparser(sub, cmd):
149 return sub.add_parser(cmd, help=getattr(alembic_command, cmd).__doc__)
152 def add_branch_options(parser):
153 group = parser.add_mutually_exclusive_group()
154 group.add_argument('--expand', action='store_true')
155 group.add_argument('--contract', action='store_true')
158 def _find_milestone_revisions(config, milestone, branch=None):
159 """Return the revision(s) for a given milestone."""
160 script = alembic_script.ScriptDirectory.from_config(config)
163 for m in _get_revisions(script)
164 for label in (m.branch_labels or [None])
165 if milestone in getattr(m.module, 'neutron_milestone', []) and
166 (branch is None or branch in m.branch_labels)
170 def do_upgrade(config, cmd):
173 if ((CONF.command.revision or CONF.command.delta) and
174 (CONF.command.expand or CONF.command.contract)):
176 'Phase upgrade options do not accept revision specification'))
178 if CONF.command.expand:
179 branch = EXPAND_BRANCH
180 revision = _get_branch_head(EXPAND_BRANCH)
182 elif CONF.command.contract:
183 branch = CONTRACT_BRANCH
184 revision = _get_branch_head(CONTRACT_BRANCH)
186 elif not CONF.command.revision and not CONF.command.delta:
187 raise SystemExit(_('You must provide a revision or relative delta'))
190 revision = CONF.command.revision or ''
192 raise SystemExit(_('Negative relative revision (downgrade) not '
195 delta = CONF.command.delta
198 raise SystemExit(_('Use either --delta or relative revision, '
201 raise SystemExit(_('Negative delta (downgrade) not supported'))
202 revision = '%s+%d' % (revision, delta)
204 # leave branchless 'head' revision request backward compatible by
205 # applying all heads in all available branches.
206 if revision == 'head':
209 if revision in migration.NEUTRON_MILESTONES:
210 revisions = _find_milestone_revisions(config, revision, branch)
212 revisions = [(revision, branch)]
214 for revision, branch in revisions:
215 if not CONF.command.sql:
216 run_sanity_checks(config, revision)
217 do_alembic_command(config, cmd, revision=revision,
218 desc=branch, sql=CONF.command.sql)
221 def no_downgrade(config, cmd):
222 raise SystemExit(_("Downgrade no longer supported"))
225 def do_stamp(config, cmd):
226 do_alembic_command(config, cmd,
227 revision=CONF.command.revision,
228 sql=CONF.command.sql)
231 def _get_branch_head(branch):
232 '''Get the latest @head specification for a branch.'''
233 return '%s@head' % branch
236 def _check_bootstrap_new_branch(branch, version_path, addn_kwargs):
237 addn_kwargs['version_path'] = version_path
238 addn_kwargs['head'] = _get_branch_head(branch)
239 if not os.path.exists(version_path):
240 # Bootstrap initial directory structure
241 utils.ensure_dir(version_path)
244 def do_revision(config, cmd):
246 'message': CONF.command.message,
247 'autogenerate': CONF.command.autogenerate,
248 'sql': CONF.command.sql,
250 if CONF.command.expand:
251 kwargs['head'] = 'expand@head'
252 elif CONF.command.contract:
253 kwargs['head'] = 'contract@head'
255 do_alembic_command(config, cmd, **kwargs)
256 if _use_separate_migration_branches(config):
257 update_head_files(config)
259 update_head_file(config)
262 def _get_release_labels(labels):
265 # release labels were introduced Liberty for a short time and dropped
266 # in that same release cycle
267 result.add('%s_%s' % (migration.LIBERTY, label))
271 def _compare_labels(revision, expected_labels):
272 # validate that the script has expected labels only
273 bad_labels = revision.branch_labels - expected_labels
275 # NOTE(ihrachyshka): this hack is temporary to accommodate those
276 # projects that already initialized their branches with liberty_*
277 # labels. Let's notify them about the deprecation for now and drop it
279 bad_labels_with_release = (revision.branch_labels -
280 _get_release_labels(expected_labels))
281 if not bad_labels_with_release:
283 _('Release aware branch labels (%s) are deprecated. '
284 'Please switch to expand@ and contract@ '
285 'labels.') % bad_labels)
288 script_name = os.path.basename(revision.path)
290 _('Unexpected label for script %(script_name)s: %(labels)s') %
291 {'script_name': script_name,
292 'labels': bad_labels}
296 def _validate_single_revision_labels(script_dir, revision, label=None):
297 expected_labels = set()
298 if label is not None:
299 expected_labels.add(label)
301 _compare_labels(revision, expected_labels)
303 # if it's not the root element of the branch, expect the parent of the
304 # script to have the same label
305 if revision.down_revision is not None:
306 down_revision = script_dir.get_revision(revision.down_revision)
307 _compare_labels(down_revision, expected_labels)
310 def _validate_revision(script_dir, revision):
311 for branch in MIGRATION_BRANCHES:
312 if branch in revision.path:
313 _validate_single_revision_labels(
314 script_dir, revision, label=branch)
317 # validate script from branchless part of migration rules
318 _validate_single_revision_labels(script_dir, revision)
321 def validate_revisions(config):
322 script_dir = alembic_script.ScriptDirectory.from_config(config)
323 revisions = _get_revisions(script_dir)
325 for revision in revisions:
326 _validate_revision(script_dir, revision)
328 branchpoints = _get_branch_points(script_dir)
329 if len(branchpoints) > 1:
330 branchpoints = ', '.join(p.revision for p in branchpoints)
332 _('Unexpected number of alembic branch points: %(branchpoints)s') %
333 {'branchpoints': branchpoints}
337 def _get_revisions(script):
338 return list(script.walk_revisions(base='base', head='heads'))
341 def _get_branch_points(script):
343 for revision in _get_revisions(script):
344 if revision.is_branch_point:
345 branchpoints.append(revision)
349 def validate_head_file(config):
350 '''Check that HEAD file contains the latest head for the branch.'''
351 if _use_separate_migration_branches(config):
352 _validate_head_files(config)
354 _validate_head_file(config)
357 @debtcollector.removals.remove(message=BRANCHLESS_WARNING)
358 def _validate_head_file(config):
359 '''Check that HEAD file contains the latest head for the branch.'''
360 script = alembic_script.ScriptDirectory.from_config(config)
361 expected_head = script.get_heads()
362 head_path = _get_head_file_path(config)
364 with open(head_path) as file_:
365 observed_head = file_.read().split()
366 if observed_head == expected_head:
371 _('HEAD file does not match migration timeline head, expected: %s')
375 def _get_heads_map(config):
376 script = alembic_script.ScriptDirectory.from_config(config)
377 heads = script.get_heads()
380 if CONTRACT_BRANCH in script.get_revision(head).branch_labels:
381 head_map[CONTRACT_BRANCH] = head
383 head_map[EXPAND_BRANCH] = head
387 def _check_head(branch_name, head_file, head):
389 with open(head_file) as file_:
390 observed_head = file_.read().strip()
394 if observed_head != head:
396 _('%(branch)s HEAD file does not match migration timeline '
397 'head, expected: %(head)s') % {'branch': branch_name.title(),
401 def _validate_head_files(config):
402 '''Check that HEAD files contain the latest head for the branch.'''
403 contract_head = _get_contract_head_file_path(config)
404 expand_head = _get_expand_head_file_path(config)
405 if not os.path.exists(contract_head) or not os.path.exists(expand_head):
406 alembic_util.warn(_("Repository does not contain HEAD files for "
407 "contract and expand branches."))
409 head_map = _get_heads_map(config)
410 _check_head(CONTRACT_BRANCH, contract_head, head_map[CONTRACT_BRANCH])
411 _check_head(EXPAND_BRANCH, expand_head, head_map[EXPAND_BRANCH])
414 def update_head_files(config):
415 '''Update HEAD files with the latest branch heads.'''
416 head_map = _get_heads_map(config)
417 contract_head = _get_contract_head_file_path(config)
418 expand_head = _get_expand_head_file_path(config)
419 with open(contract_head, 'w+') as f:
420 f.write(head_map[CONTRACT_BRANCH] + '\n')
421 with open(expand_head, 'w+') as f:
422 f.write(head_map[EXPAND_BRANCH] + '\n')
424 old_head_file = _get_head_file_path(config)
425 old_heads_file = _get_heads_file_path(config)
426 for file_ in (old_head_file, old_heads_file):
427 fileutils.delete_if_exists(file_)
430 @debtcollector.removals.remove(message=BRANCHLESS_WARNING)
431 def update_head_file(config):
432 script = alembic_script.ScriptDirectory.from_config(config)
433 head = script.get_heads()
434 with open(_get_head_file_path(config), 'w+') as f:
435 f.write('\n'.join(head))
438 def add_command_parsers(subparsers):
439 for name in ['current', 'history', 'branches', 'heads']:
440 parser = add_alembic_subparser(subparsers, name)
441 parser.set_defaults(func=do_generic_show)
442 parser.add_argument('--verbose',
444 help='Display more verbose output for the '
447 help_text = (getattr(alembic_command, 'branches').__doc__ +
448 ' and validate head file')
449 parser = subparsers.add_parser('check_migration', help=help_text)
450 parser.set_defaults(func=do_check_migration)
452 parser = add_alembic_subparser(subparsers, 'upgrade')
453 parser.add_argument('--delta', type=int)
454 parser.add_argument('--sql', action='store_true')
455 parser.add_argument('revision', nargs='?')
456 parser.add_argument('--mysql-engine',
458 help='Change MySQL storage engine of current '
460 add_branch_options(parser)
462 parser.set_defaults(func=do_upgrade)
464 parser = subparsers.add_parser('downgrade', help="(No longer supported)")
465 parser.add_argument('None', nargs='?', help="Downgrade not supported")
466 parser.set_defaults(func=no_downgrade)
468 parser = add_alembic_subparser(subparsers, 'stamp')
469 parser.add_argument('--sql', action='store_true')
470 parser.add_argument('revision')
471 parser.set_defaults(func=do_stamp)
473 parser = add_alembic_subparser(subparsers, 'revision')
474 parser.add_argument('-m', '--message')
475 parser.add_argument('--autogenerate', action='store_true')
476 parser.add_argument('--sql', action='store_true')
477 add_branch_options(parser)
478 parser.set_defaults(func=do_revision)
481 command_opt = cfg.SubCommandOpt('command',
483 help=_('Available commands'),
484 handler=add_command_parsers)
486 CONF.register_cli_opt(command_opt)
489 def _get_project_base(config):
490 '''Return the base python namespace name for a project.'''
491 script_location = config.get_main_option('script_location')
492 return script_location.split(':')[0].split('.')[0]
495 def _get_package_root_dir(config):
496 root_module = importutils.try_import(_get_project_base(config))
498 project = config.get_main_option('neutron_project')
499 alembic_util.err(_("Failed to locate source for %s.") % project)
500 # The root_module.__file__ property is a path like
501 # '/opt/stack/networking-foo/networking_foo/__init__.py'
503 # '/opt/stack/networking-foo'
504 return os.path.dirname(os.path.dirname(root_module.__file__))
507 def _get_root_versions_dir(config):
508 '''Return root directory that contains all migration rules.'''
509 root_dir = _get_package_root_dir(config)
510 script_location = config.get_main_option('script_location')
511 # Script location is something like:
512 # 'project_base.db.migration:alembic_migrations'
514 # 'project_base/db/migration/alembic_migrations/versions'
515 part1, part2 = script_location.split(':')
516 parts = part1.split('.') + part2.split('.') + ['versions']
517 # Return the absolute path to the versions dir
518 return os.path.join(root_dir, *parts)
521 def _get_head_file_path(config):
522 '''Return the path of the file that contains single head.'''
524 _get_root_versions_dir(config),
528 def _get_heads_file_path(config):
530 Return the path of the file that was once used to maintain the list of
534 _get_root_versions_dir(config),
538 def _get_contract_head_file_path(config):
540 Return the path of the file that is used to maintain contract head
543 _get_root_versions_dir(config),
544 CONTRACT_HEAD_FILENAME)
547 def _get_expand_head_file_path(config):
549 Return the path of the file that is used to maintain expand head
552 _get_root_versions_dir(config),
553 EXPAND_HEAD_FILENAME)
556 def _get_version_branch_path(config, release=None, branch=None):
557 version_path = _get_root_versions_dir(config)
558 if branch and release:
559 return os.path.join(version_path, release, branch)
563 def _use_separate_migration_branches(config):
564 '''Detect whether split migration branches should be used.'''
565 if CONF.split_branches:
568 script_dir = alembic_script.ScriptDirectory.from_config(config)
569 if _get_branch_points(script_dir):
575 def _set_version_locations(config):
576 '''Make alembic see all revisions in all migration branches.'''
577 split_branches = False
578 version_paths = [_get_version_branch_path(config)]
579 for release in RELEASES:
580 for branch in MIGRATION_BRANCHES:
581 version_path = _get_version_branch_path(config, release, branch)
582 if split_branches or os.path.exists(version_path):
583 split_branches = True
584 version_paths.append(version_path)
586 config.set_main_option('version_locations', ' '.join(version_paths))
589 def _get_installed_entrypoint(subproject):
590 '''Get the entrypoint for the subproject, which must be installed.'''
591 if subproject not in migration_entrypoints:
592 alembic_util.err(_('Package %s not installed') % subproject)
593 return migration_entrypoints[subproject]
596 def _get_subproject_script_location(subproject):
597 '''Get the script location for the installed subproject.'''
598 entrypoint = _get_installed_entrypoint(subproject)
599 return ':'.join([entrypoint.module_name, entrypoint.attrs[0]])
602 def _get_service_script_location(service):
603 '''Get the script location for the service, which must be installed.'''
604 return _get_subproject_script_location('neutron-%s' % service)
607 def _get_subproject_base(subproject):
608 '''Get the import base name for the installed subproject.'''
609 entrypoint = _get_installed_entrypoint(subproject)
610 return entrypoint.module_name.split('.')[0]
613 def get_alembic_configs():
614 '''Return a list of alembic configs, one per project.
617 # Get the script locations for the specified or installed projects.
618 # Which projects to get script locations for is determined by the CLI
619 # options as follows:
620 # --service X # only subproject neutron-X (deprecated)
621 # --subproject Y # only subproject Y (where Y can be neutron)
622 # (none specified) # neutron and all installed subprojects
623 script_locations = {}
625 script_location = _get_service_script_location(CONF.service)
626 script_locations['neutron-%s' % CONF.service] = script_location
627 elif CONF.subproject:
628 script_location = _get_subproject_script_location(CONF.subproject)
629 script_locations[CONF.subproject] = script_location
631 for subproject, ep in migration_entrypoints.items():
632 script_locations[subproject] = _get_subproject_script_location(
635 # Return a list of alembic configs from the projects in the
636 # script_locations dict. If neutron is in the list it is first.
638 project_seq = sorted(script_locations.keys())
639 # Core neutron must be the first project if there is more than one
640 if len(project_seq) > 1 and 'neutron' in project_seq:
641 project_seq.insert(0, project_seq.pop(project_seq.index('neutron')))
642 for project in project_seq:
643 config = alembic_config.Config(neutron_alembic_ini)
644 config.set_main_option('neutron_project', project)
645 script_location = script_locations[project]
646 config.set_main_option('script_location', script_location)
647 _set_version_locations(config)
648 config.neutron_config = CONF
649 configs.append(config)
654 def get_neutron_config():
655 # Neutron's alembic config is always the first one
656 return get_alembic_configs()[0]
659 def run_sanity_checks(config, revision):
660 script_dir = alembic_script.ScriptDirectory.from_config(config)
662 def check_sanity(rev, context):
663 # TODO(ihrachyshka): here we use internal API for alembic; we may need
664 # alembic to expose implicit_base= argument into public
665 # iterate_revisions() call
666 for script in script_dir.revision_map.iterate_revisions(
667 revision, rev, implicit_base=True):
668 if hasattr(script.module, 'check_sanity'):
669 script.module.check_sanity(context.connection)
672 with environment.EnvironmentContext(config, script_dir,
675 destination_rev=revision):
679 def validate_cli_options():
680 if CONF.subproject and CONF.service:
681 alembic_util.err(_("Cannot specify both --service and --subproject."))
684 def get_engine_config():
685 return [obj for obj in _db_opts if obj.name == 'engine']
689 CONF(project='neutron')
690 validate_cli_options()
691 for config in get_alembic_configs():
692 #TODO(gongysh) enable logging
693 CONF.command.func(config, CONF.command.name)