From 603c0d03aed5fdeefb471086c0aef879938f9641 Mon Sep 17 00:00:00 2001 From: Henry Gessau Date: Sun, 5 Jul 2015 03:29:38 -0400 Subject: [PATCH] Support for independent alembic branches in sub-projects Sub-projects shall now register their independent alembic migrations via entrypoints in setup.cfg, and neutron-db-manage will discover them and run them automatically. If a service or sub-project is specified explicitly, then neutron-db-manage will run on only that service or sub-project. The advanced services project are just special cases of sub-projects. For example, specifying the CLI option '--service lbaas' is the same as specifying '--subproject neutron-lbaas'. Specifying no service or sub-project will cause neutron-db-manage to run the command on neutron and all installed sub-projects. Added and consolidated documentation into devref for alembic migrations. Partial-Bug: #1471333 Partial-Bug: #1470625 Change-Id: I9a06de64ce35675af28adf819de6f22dc832390d --- doc/source/devref/alembic_migrations.rst | 313 ++++++++++++++++++ doc/source/devref/db_layer.rst | 147 +------- doc/source/devref/index.rst | 1 + neutron/db/migration/README | 90 +---- neutron/db/migration/cli.py | 219 ++++++++---- .../tests/functional/db/test_migrations.py | 6 +- neutron/tests/unit/db/test_migration.py | 109 +++++- setup.cfg | 2 + 8 files changed, 580 insertions(+), 307 deletions(-) create mode 100644 doc/source/devref/alembic_migrations.rst diff --git a/doc/source/devref/alembic_migrations.rst b/doc/source/devref/alembic_migrations.rst new file mode 100644 index 000000000..245bf2fe9 --- /dev/null +++ b/doc/source/devref/alembic_migrations.rst @@ -0,0 +1,313 @@ +.. + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + + + Convention for heading levels in Neutron devref: + ======= Heading 0 (reserved for the title in a document) + ------- Heading 1 + ~~~~~~~ Heading 2 + +++++++ Heading 3 + ''''''' Heading 4 + (Avoid deeper levels because they do not render well.) + + +Alembic Migrations +================== + +Introduction +------------ + +The migrations in the alembic/versions contain the changes needed to migrate +from older Neutron releases to newer versions. A migration occurs by executing +a script that details the changes needed to upgrade the database. The migration +scripts are ordered so that multiple scripts can run sequentially to update the +database. + + +The Migration Wrapper +--------------------- + +The scripts are executed by Neutron's migration wrapper ``neutron-db-manage`` +which uses the Alembic library to manage the migration. Pass the ``--help`` +option to the wrapper for usage information. + +The wrapper takes some options followed by some commands:: + + neutron-db-manage + +The wrapper needs to be provided with the database connection string, which is +usually provided in the ``neutron.conf`` configuration file in an installation. +The wrapper automatically reads from ``/etc/neutron/neutron.conf`` if it is +present. If the configuration is in a different location:: + + neutron-db-manage --config-file /path/to/neutron.conf + +Multiple ``--config-file`` options can be passed if needed. + +Instead of reading the DB connection from the configuration file(s) the +``--database-connection`` option can be used:: + + neutron-db-manage --database-connection mysql+pymysql://root:secret@127.0.0.1/neutron?charset=utf8 + +For some commands the wrapper needs to know the entrypoint of the core plugin +for the installation. This can be read from the configuration file(s) or +specified using the ``--core_plugin`` option:: + + neutron-db-manage --core_plugin neutron.plugins.ml2.plugin.Ml2Plugin + +When giving examples below of using the wrapper the options will not be shown. +It is assumed you will use the options that you need for your environment. + +For new deployments you will start with an empty database. You then upgrade +to the latest database version via:: + + neutron-db-manage upgrade heads + +For existing deployments the database will already be at some version. To +check the current database version:: + + neutron-db-manage current + +After installing a new version of Neutron server, upgrading the database is +the same command:: + + neutron-db-manage upgrade heads + +To create a script to run the migration offline:: + + neutron-db-manage upgrade heads --sql + +To run the offline migration between specific migration versions:: + + neutron-db-manage upgrade : --sql + +Upgrade the database incrementally:: + + neutron-db-manage upgrade --delta <# of revs> + +**NOTE:** Database downgrade is not supported. + + +Migration Branches +------------------ + +Neutron makes use of alembic branches for two purposes. + +1. Indepedent Sub-Project Tables +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Various `sub-projects `_ can be installed with Neutron. Each +sub-project registers its own alembic branch which is responsible for migrating +the schemas of the tables owned by the sub-project. + +The neutron-db-manage script detects which sub-projects have been installed by +enumerating the ``neutron.db.alembic_migrations`` entrypoints. For more details +see the `Entry Points section of Contributing extensions to Neutron +`_. + +The neutron-db-manage script runs the given alembic command against all +installed sub-projects. (An exception is the ``revision`` command, which is +discussed in the `Developers`_ section below.) + +2. Offline/Online Migrations +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Since Liberty, Neutron maintains two parallel alembic migration branches. + +The first one, called 'expand', is used to store expansion-only migration +rules. Those rules are strictly additive and can be applied while +neutron-server is running. Examples of additive database schema changes are: +creating a new table, adding a new table column, adding a new index, etc. + +The second branch, called 'contract', is used to store those migration rules +that are not safe to apply while neutron-server is running. Those include: +column or table removal, moving data from one part of the database into another +(renaming a column, transforming single table into multiple, etc.), introducing +or modifying constraints, etc. + +The intent of the split is to allow invoking those safe migrations from +'expand' branch while neutron-server is running, reducing downtime needed to +upgrade the service. + +For more details, see the `Expand and Contract Scripts`_ section below. + + +Developers +---------- + +A database migration script is required when you submit a change to Neutron or +a sub-project that alters the database model definition. The migration script +is a special python file that includes code to upgrade the database to match +the changes in the model definition. Alembic will execute these scripts in +order to provide a linear migration path between revisions. The +neutron-db-manage command can be used to generate migration scripts for you to +complete. The operations in the template are those supported by the Alembic +migration library. + + +Script Auto-generation +~~~~~~~~~~~~~~~~~~~~~~ + +:: + + neutron-db-manage revision -m "description of revision" --autogenerate + +This generates a prepopulated template with the changes needed to match the +database state with the models. You should inspect the autogenerated template +to ensure that the proper models have been altered. + +In rare circumstances, you may want to start with an empty migration template +and manually author the changes necessary for an upgrade. You can create a +blank file via:: + + neutron-db-manage revision -m "description of revision" + +The timeline on each alembic branch should remain linear and not interleave +with other branches, so that there is a clear path when upgrading. To verify +that alembic branches maintain linear timelines, you can run this command:: + + neutron-db-manage check_migration + +If this command reports an error, you can troubleshoot by showing the migration +timelines using the ``history`` command:: + + neutron-db-manage history + + +Expand and Contract Scripts +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The obsolete "branchless" design of a migration script included that it +indicates a specific "version" of the schema, and includes directives that +apply all necessary changes to the database at once. If we look for example at +the script ``2d2a8a565438_hierarchical_binding.py``, we will see:: + + # .../alembic_migrations/versions/2d2a8a565438_hierarchical_binding.py + + def upgrade(): + + # .. inspection code ... + + op.create_table( + 'ml2_port_binding_levels', + sa.Column('port_id', sa.String(length=36), nullable=False), + sa.Column('host', sa.String(length=255), nullable=False), + # ... more columns ... + ) + + for table in port_binding_tables: + op.execute(( + "INSERT INTO ml2_port_binding_levels " + "SELECT port_id, host, 0 AS level, driver, segment AS segment_id " + "FROM %s " + "WHERE host <> '' " + "AND driver <> '';" + ) % table) + + op.drop_constraint(fk_name_dvr[0], 'ml2_dvr_port_bindings', 'foreignkey') + op.drop_column('ml2_dvr_port_bindings', 'cap_port_filter') + op.drop_column('ml2_dvr_port_bindings', 'segment') + op.drop_column('ml2_dvr_port_bindings', 'driver') + + # ... more DROP instructions ... + +The above script contains directives that are both under the "expand" +and "contract" categories, as well as some data migrations. the ``op.create_table`` +directive is an "expand"; it may be run safely while the old version of the +application still runs, as the old code simply doesn't look for this table. +The ``op.drop_constraint`` and ``op.drop_column`` directives are +"contract" directives (the drop column moreso than the drop constraint); running +at least the ``op.drop_column`` directives means that the old version of the +application will fail, as it will attempt to access these columns which no longer +exist. + +The data migrations in this script are adding new +rows to the newly added ``ml2_port_binding_levels`` table. + +Under the new migration script directory structure, the above script would be +stated as two scripts; an "expand" and a "contract" script:: + + # expansion operations + # .../alembic_migrations/versions/liberty/expand/2bde560fc638_hierarchical_binding.py + + def upgrade(): + + op.create_table( + 'ml2_port_binding_levels', + sa.Column('port_id', sa.String(length=36), nullable=False), + sa.Column('host', sa.String(length=255), nullable=False), + # ... more columns ... + ) + + + # contraction operations + # .../alembic_migrations/versions/liberty/contract/4405aedc050e_hierarchical_binding.py + + def upgrade(): + + for table in port_binding_tables: + op.execute(( + "INSERT INTO ml2_port_binding_levels " + "SELECT port_id, host, 0 AS level, driver, segment AS segment_id " + "FROM %s " + "WHERE host <> '' " + "AND driver <> '';" + ) % table) + + op.drop_constraint(fk_name_dvr[0], 'ml2_dvr_port_bindings', 'foreignkey') + op.drop_column('ml2_dvr_port_bindings', 'cap_port_filter') + op.drop_column('ml2_dvr_port_bindings', 'segment') + op.drop_column('ml2_dvr_port_bindings', 'driver') + + # ... more DROP instructions ... + +The two scripts would be present in different subdirectories and also part of +entirely separate versioning streams. The "expand" operations are in the +"expand" script, and the "contract" operations are in the "contract" script. + +For the time being, data migration rules also belong to contract branch. There +is expectation that eventually live data migrations move into middleware that +will be aware about different database schema elements to converge on, but +Neutron is still not there. + +Scripts that contain only expansion or contraction rules do not require a split +into two parts. + +If a contraction script depends on a script from expansion stream, the +following directive should be added in the contraction script:: + + depends_on = ('',) + + +Applying database migration rules +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +To apply just expansion rules, execute:: + + neutron-db-manage upgrade liberty_expand@head + +After the first step is done, you can stop neutron-server, apply remaining +non-expansive migration rules, if any:: + + neutron-db-manage upgrade liberty_contract@head + +and finally, start your neutron-server again. + +If you are not interested in applying safe migration rules while the service is +running, you can still upgrade database the old way, by stopping the service, +and then applying all available rules:: + + neutron-db-manage upgrade head[s] + +It will apply all the rules from both the expand and the contract branches, in +proper order. diff --git a/doc/source/devref/db_layer.rst b/doc/source/devref/db_layer.rst index 2b6ded3fa..248c85e0b 100644 --- a/doc/source/devref/db_layer.rst +++ b/doc/source/devref/db_layer.rst @@ -23,150 +23,11 @@ should also be added in model. If default value in database is not needed, business logic. -How we manage database migration rules --------------------------------------- +Database migrations +------------------- -Since Liberty, Neutron maintains two parallel alembic migration branches. - -The first one, called 'expand', is used to store expansion-only migration -rules. Those rules are strictly additive and can be applied while -neutron-server is running. Examples of additive database schema changes are: -creating a new table, adding a new table column, adding a new index, etc. - -The second branch, called 'contract', is used to store those migration rules -that are not safe to apply while neutron-server is running. Those include: -column or table removal, moving data from one part of the database into another -(renaming a column, transforming single table into multiple, etc.), introducing -or modifying constraints, etc. - -The intent of the split is to allow invoking those safe migrations from -'expand' branch while neutron-server is running, reducing downtime needed to -upgrade the service. - -To apply just expansion rules, execute: - -- neutron-db-manage upgrade liberty_expand@head - -After the first step is done, you can stop neutron-server, apply remaining -non-expansive migration rules, if any: - -- neutron-db-manage upgrade liberty_contract@head - -and finally, start your neutron-server again. - -If you are not interested in applying safe migration rules while the service is -running, you can still upgrade database the old way, by stopping the service, -and then applying all available rules: - -- neutron-db-manage upgrade head[s] - -It will apply all the rules from both the expand and the contract branches, in -proper order. - - -Expand and Contract Scripts ---------------------------- - -The obsolete "branchless" design of a migration script included that it -indicates a specific "version" of the schema, and includes directives that -apply all necessary changes to the database at once. If we look for example at -the script ``2d2a8a565438_hierarchical_binding.py``, we will see:: - - # .../alembic_migrations/versions/2d2a8a565438_hierarchical_binding.py - - def upgrade(): - - # .. inspection code ... - - op.create_table( - 'ml2_port_binding_levels', - sa.Column('port_id', sa.String(length=36), nullable=False), - sa.Column('host', sa.String(length=255), nullable=False), - # ... more columns ... - ) - - for table in port_binding_tables: - op.execute(( - "INSERT INTO ml2_port_binding_levels " - "SELECT port_id, host, 0 AS level, driver, segment AS segment_id " - "FROM %s " - "WHERE host <> '' " - "AND driver <> '';" - ) % table) - - op.drop_constraint(fk_name_dvr[0], 'ml2_dvr_port_bindings', 'foreignkey') - op.drop_column('ml2_dvr_port_bindings', 'cap_port_filter') - op.drop_column('ml2_dvr_port_bindings', 'segment') - op.drop_column('ml2_dvr_port_bindings', 'driver') - - # ... more DROP instructions ... - -The above script contains directives that are both under the "expand" -and "contract" categories, as well as some data migrations. the ``op.create_table`` -directive is an "expand"; it may be run safely while the old version of the -application still runs, as the old code simply doesn't look for this table. -The ``op.drop_constraint`` and ``op.drop_column`` directives are -"contract" directives (the drop column moreso than the drop constraint); running -at least the ``op.drop_column`` directives means that the old version of the -application will fail, as it will attempt to access these columns which no longer -exist. - -The data migrations in this script are adding new -rows to the newly added ``ml2_port_binding_levels`` table. - -Under the new migration script directory structure, the above script would be -stated as two scripts; an "expand" and a "contract" script:: - - # expansion operations - # .../alembic_migrations/versions/liberty/expand/2bde560fc638_hierarchical_binding.py - - def upgrade(): - - op.create_table( - 'ml2_port_binding_levels', - sa.Column('port_id', sa.String(length=36), nullable=False), - sa.Column('host', sa.String(length=255), nullable=False), - # ... more columns ... - ) - - - # contraction operations - # .../alembic_migrations/versions/liberty/contract/4405aedc050e_hierarchical_binding.py - - def upgrade(): - - for table in port_binding_tables: - op.execute(( - "INSERT INTO ml2_port_binding_levels " - "SELECT port_id, host, 0 AS level, driver, segment AS segment_id " - "FROM %s " - "WHERE host <> '' " - "AND driver <> '';" - ) % table) - - op.drop_constraint(fk_name_dvr[0], 'ml2_dvr_port_bindings', 'foreignkey') - op.drop_column('ml2_dvr_port_bindings', 'cap_port_filter') - op.drop_column('ml2_dvr_port_bindings', 'segment') - op.drop_column('ml2_dvr_port_bindings', 'driver') - - # ... more DROP instructions ... - -The two scripts would be present in different subdirectories and also part of -entirely separate versioning streams. The "expand" operations are in the -"expand" script, and the "contract" operations are in the "contract" script. - -For the time being, data migration rules also belong to contract branch. There -is expectation that eventually live data migrations move into middleware that -will be aware about different database schema elements to converge on, but -Neutron is still not there. - -Scripts that contain only expansion or contraction rules do not require a split -into two parts. - -If a contraction script depends on a script from expansion stream, the -following directive should be added in the contraction script:: - - depends_on = ('',) +For details on the neutron-db-manage wrapper and alembic migrations, see +`Alembic Migrations `_. Tests to verify that database migrations and models are in sync diff --git a/doc/source/devref/index.rst b/doc/source/devref/index.rst index aa541bfca..bdb0634b1 100644 --- a/doc/source/devref/index.rst +++ b/doc/source/devref/index.rst @@ -44,6 +44,7 @@ Programming HowTos and Tutorials neutron_api sub_projects client_command_extensions + alembic_migrations Neutron Internals diff --git a/neutron/db/migration/README b/neutron/db/migration/README index e6e513887..18a126cb2 100644 --- a/neutron/db/migration/README +++ b/neutron/db/migration/README @@ -1,88 +1,4 @@ -# Copyright 2012 New Dream Network, LLC (DreamHost) -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. +See doc/source/devref/alembic_migrations.rst -The migrations in the alembic/versions contain the changes needed to migrate -from older Neutron releases to newer versions. A migration occurs by executing -a script that details the changes needed to upgrade the database. The migration -scripts are ordered so that multiple scripts can run sequentially to update the -database. The scripts are executed by Neutron's migration wrapper which uses -the Alembic library to manage the migration. Neutron supports migration from -Havana or later. - - -If you are a deployer or developer and want to migrate from Folsom to Grizzly -or later you must first add version tracking to the database: - -$ neutron-db-manage --config-file /path/to/neutron.conf \ - --config-file /path/to/plugin/config.ini stamp folsom - -You can then upgrade to the latest database version via: -$ neutron-db-manage --config-file /path/to/neutron.conf \ - --config-file /path/to/plugin/config.ini upgrade head - -To check the current database version: -$ neutron-db-manage --config-file /path/to/neutron.conf \ - --config-file /path/to/plugin/config.ini current - -To create a script to run the migration offline: -$ neutron-db-manage --config-file /path/to/neutron.conf \ - --config-file /path/to/plugin/config.ini upgrade head --sql - -To run the offline migration between specific migration versions: -$ neutron-db-manage --config-file /path/to/neutron.conf \ ---config-file /path/to/plugin/config.ini upgrade \ -: --sql - -Upgrade the database incrementally: -$ neutron-db-manage --config-file /path/to/neutron.conf \ ---config-file /path/to/plugin/config.ini upgrade --delta <# of revs> - -NOTE: Database downgrade is not supported. - - -DEVELOPERS: - -A database migration script is required when you submit a change to Neutron -that alters the database model definition. The migration script is a special -python file that includes code to upgrade the database to match the changes in -the model definition. Alembic will execute these scripts in order to provide a -linear migration path between revision. The neutron-db-manage command can be -used to generate migration template for you to complete. The operations in the -template are those supported by the Alembic migration library. - -$ neutron-db-manage --config-file /path/to/neutron.conf \ ---config-file /path/to/plugin/config.ini revision \ --m "description of revision" \ ---autogenerate - -This generates a prepopulated template with the changes needed to match the -database state with the models. You should inspect the autogenerated template -to ensure that the proper models have been altered. - -In rare circumstances, you may want to start with an empty migration template -and manually author the changes necessary for an upgrade. You can create a -blank file via: - -$ neutron-db-manage --config-file /path/to/neutron.conf \ ---config-file /path/to/plugin/config.ini revision \ --m "description of revision" - -The migration timeline should remain linear so that there is a clear path when -upgrading. To verify that the timeline does branch, you can run this command: -$ neutron-db-manage --config-file /path/to/neutron.conf \ ---config-file /path/to/plugin/config.ini check_migration - -If the migration path does branch, you can find the branch point via: -$ neutron-db-manage --config-file /path/to/neutron.conf \ ---config-file /path/to/plugin/config.ini history +Rendered at +http://docs.openstack.org/developer/neutron/devref/alembic_migrations.html diff --git a/neutron/db/migration/cli.py b/neutron/db/migration/cli.py index 0881c7211..53c5393bd 100644 --- a/neutron/db/migration/cli.py +++ b/neutron/db/migration/cli.py @@ -22,8 +22,8 @@ from alembic import script as alembic_script from alembic import util as alembic_util from oslo_config import cfg from oslo_utils import importutils +import pkg_resources -from neutron.common import repos from neutron.common import utils @@ -33,22 +33,40 @@ HEADS_FILENAME = 'HEADS' CURRENT_RELEASE = "liberty" MIGRATION_BRANCHES = ('expand', 'contract') +MIGRATION_ENTRYPOINTS = 'neutron.db.alembic_migrations' +migration_entrypoints = { + entrypoint.name: entrypoint + for entrypoint in pkg_resources.iter_entry_points(MIGRATION_ENTRYPOINTS) +} -mods = repos.NeutronModules() -VALID_SERVICES = list(map(mods.alembic_name, mods.installed_list())) +neutron_alembic_ini = os.path.join(os.path.dirname(__file__), 'alembic.ini') +VALID_SERVICES = ['fwaas', 'lbaas', 'vpnaas'] +INSTALLED_SERVICES = [service_ for service_ in VALID_SERVICES + if 'neutron-%s' % service_ in migration_entrypoints] +INSTALLED_SERVICE_PROJECTS = ['neutron-%s' % service_ + for service_ in INSTALLED_SERVICES] +INSTALLED_SUBPROJECTS = [project_ for project_ in migration_entrypoints + if project_ not in INSTALLED_SERVICE_PROJECTS] + +service_help = ( + _("Can be one of '%s'.") % "', '".join(INSTALLED_SERVICES) + if INSTALLED_SERVICES else + _("(No services are currently installed).") +) _core_opts = [ cfg.StrOpt('core_plugin', default='', help=_('Neutron plugin provider module')), - cfg.ListOpt('service_plugins', - default=[], - help=_("The service plugins Neutron will use")), cfg.StrOpt('service', - choices=VALID_SERVICES, - help=_("The advanced service to execute the command against. " - "Can be one of '%s'.") % "', '".join(VALID_SERVICES)), + choices=INSTALLED_SERVICES, + help=(_("The advanced service to execute the command against. ") + + service_help)), + cfg.StrOpt('subproject', + choices=INSTALLED_SUBPROJECTS, + help=(_("The subproject to execute the command against. " + "Can be one of %s.") % INSTALLED_SUBPROJECTS)), cfg.BoolOpt('split_branches', default=False, help=_("Enforce using split branches file structure.")) @@ -78,10 +96,20 @@ CONF.register_opts(_quota_opts, 'QUOTAS') def do_alembic_command(config, cmd, *args, **kwargs): + project = config.get_main_option('neutron_project') + alembic_util.msg(_('Running %(cmd)s for %(project)s ...') % + {'cmd': cmd, 'project': project}) try: getattr(alembic_command, cmd)(config, *args, **kwargs) except alembic_util.CommandError as e: alembic_util.err(six.text_type(e)) + alembic_util.msg(_('OK')) + + +def _get_alembic_entrypoint(project): + if project not in migration_entrypoints: + alembic_util.err(_('Sub-project %s not installed.') % project) + return migration_entrypoints[project] def do_check_migration(config, cmd): @@ -148,9 +176,9 @@ def do_revision(config, cmd): 'sql': CONF.command.sql, } - if _use_separate_migration_branches(CONF): + if _use_separate_migration_branches(config): for branch in MIGRATION_BRANCHES: - version_path = _get_version_branch_path(CONF, branch) + version_path = _get_version_branch_path(config, branch) addn_kwargs['version_path'] = version_path if not os.path.exists(version_path): @@ -187,7 +215,7 @@ def validate_heads_file(config): '''Check that HEADS file contains the latest heads for each branch.''' script = alembic_script.ScriptDirectory.from_config(config) expected_heads = _get_sorted_heads(script) - heads_path = _get_active_head_file_path(CONF) + heads_path = _get_active_head_file_path(config) try: with open(heads_path) as file_: observed_heads = file_.read().split() @@ -204,7 +232,7 @@ def update_heads_file(config): '''Update HEADS file with the latest branch heads.''' script = alembic_script.ScriptDirectory.from_config(config) heads = _get_sorted_heads(script) - heads_path = _get_active_head_file_path(CONF) + heads_path = _get_active_head_file_path(config) with open(heads_path, 'w+') as f: f.write('\n'.join(heads)) @@ -253,88 +281,153 @@ command_opt = cfg.SubCommandOpt('command', CONF.register_cli_opt(command_opt) -def _get_neutron_service_base(neutron_config): - '''Return base python namespace name for a service.''' - if neutron_config.service: - validate_service_installed(neutron_config.service) - return "neutron_%s" % neutron_config.service - return "neutron" +def _get_project_base(config): + '''Return the base python namespace name for a project.''' + script_location = config.get_main_option('script_location') + return script_location.split(':')[0].split('.')[0] -def _get_root_versions_dir(neutron_config): - '''Return root directory that contains all migration rules.''' - service_base = _get_neutron_service_base(neutron_config) - root_module = importutils.import_module(service_base) - return os.path.join( - os.path.dirname(root_module.__file__), - 'db/migration/alembic_migrations/versions') +def _get_package_root_dir(config): + root_module = importutils.try_import(_get_project_base(config)) + if not root_module: + project = config.get_main_option('neutron_project') + alembic_util.err(_("Failed to locate source for %s.") % project) + # The root_module.__file__ property is a path like + # '/opt/stack/networking-foo/networking_foo/__init__.py' + # We return just + # '/opt/stack/networking-foo' + return os.path.dirname(os.path.dirname(root_module.__file__)) -def _get_head_file_path(neutron_config): +def _get_root_versions_dir(config): + '''Return root directory that contains all migration rules.''' + root_dir = _get_package_root_dir(config) + script_location = config.get_main_option('script_location') + # Script location is something like: + # 'project_base.db.migration:alembic_migrations' + # Convert it to: + # 'project_base/db/migration/alembic_migrations/versions' + part1, part2 = script_location.split(':') + parts = part1.split('.') + part2.split('.') + ['versions'] + # Return the absolute path to the versions dir + return os.path.join(root_dir, *parts) + + +def _get_head_file_path(config): '''Return the path of the file that contains single head.''' return os.path.join( - _get_root_versions_dir(neutron_config), + _get_root_versions_dir(config), HEAD_FILENAME) -def _get_heads_file_path(neutron_config): +def _get_heads_file_path(config): '''Return the path of the file that contains all latest heads, sorted.''' return os.path.join( - _get_root_versions_dir(neutron_config), + _get_root_versions_dir(config), HEADS_FILENAME) -def _get_active_head_file_path(neutron_config): +def _get_active_head_file_path(config): '''Return the path of the file that contains latest head(s), depending on whether multiple branches are used. ''' - if _use_separate_migration_branches(neutron_config): - return _get_heads_file_path(neutron_config) - return _get_head_file_path(neutron_config) + if _use_separate_migration_branches(config): + return _get_heads_file_path(config) + return _get_head_file_path(config) -def _get_version_branch_path(neutron_config, branch=None): - version_path = _get_root_versions_dir(neutron_config) +def _get_version_branch_path(config, branch=None): + version_path = _get_root_versions_dir(config) if branch: return os.path.join(version_path, CURRENT_RELEASE, branch) return version_path -def _use_separate_migration_branches(neutron_config): +def _use_separate_migration_branches(config): '''Detect whether split migration branches should be used.''' - return (neutron_config.split_branches or + return (CONF.split_branches or # Use HEADS file to indicate the new, split migration world - os.path.exists(_get_heads_file_path(neutron_config))) + os.path.exists(_get_heads_file_path(config))) def _set_version_locations(config): '''Make alembic see all revisions in all migration branches.''' - version_paths = [] - - version_paths.append(_get_version_branch_path(CONF)) - if _use_separate_migration_branches(CONF): + version_paths = [_get_version_branch_path(config)] + if _use_separate_migration_branches(config): for branch in MIGRATION_BRANCHES: - version_paths.append(_get_version_branch_path(CONF, branch)) + version_paths.append(_get_version_branch_path(config, branch)) config.set_main_option('version_locations', ' '.join(version_paths)) -def validate_service_installed(service): - if not importutils.try_import('neutron_%s' % service): - alembic_util.err(_('Package neutron-%s not installed') % service) +def _get_installed_entrypoint(subproject): + '''Get the entrypoint for the subproject, which must be installed.''' + if subproject not in migration_entrypoints: + alembic_util.err(_('Package %s not installed') % subproject) + return migration_entrypoints[subproject] + + +def _get_subproject_script_location(subproject): + '''Get the script location for the installed subproject.''' + entrypoint = _get_installed_entrypoint(subproject) + return ':'.join([entrypoint.module_name, entrypoint.attrs[0]]) -def get_script_location(neutron_config): - location = '%s.db.migration:alembic_migrations' - return location % _get_neutron_service_base(neutron_config) +def _get_service_script_location(service): + '''Get the script location for the service, which must be installed.''' + return _get_subproject_script_location('neutron-%s' % service) -def get_alembic_config(): - config = alembic_config.Config(os.path.join(os.path.dirname(__file__), - 'alembic.ini')) - config.set_main_option('script_location', get_script_location(CONF)) - _set_version_locations(config) - return config +def _get_subproject_base(subproject): + '''Get the import base name for the installed subproject.''' + entrypoint = _get_installed_entrypoint(subproject) + return entrypoint.module_name.split('.')[0] + + +def get_alembic_configs(): + '''Return a list of alembic configs, one per project. + ''' + + # Get the script locations for the specified or installed projects. + # Which projects to get script locations for is determined by the CLI + # options as follows: + # --service X # only subproject neutron-X + # --subproject Y # only subproject Y + # (none specified) # neutron and all installed subprojects + script_locations = {} + if CONF.service: + script_location = _get_service_script_location(CONF.service) + script_locations['neutron-%s' % CONF.service] = script_location + elif CONF.subproject: + script_location = _get_subproject_script_location(CONF.subproject) + script_locations[CONF.subproject] = script_location + else: + for subproject, ep in migration_entrypoints.items(): + script_locations[subproject] = _get_subproject_script_location( + subproject) + + # Return a list of alembic configs from the projects in the + # script_locations dict. If neutron is in the list it is first. + configs = [] + project_seq = sorted(script_locations.keys()) + # Core neutron must be the first project if there is more than one + if len(project_seq) > 1 and 'neutron' in project_seq: + project_seq.insert(0, project_seq.pop(project_seq.index('neutron'))) + for project in project_seq: + config = alembic_config.Config(neutron_alembic_ini) + config.set_main_option('neutron_project', project) + script_location = script_locations[project] + config.set_main_option('script_location', script_location) + _set_version_locations(config) + config.neutron_config = CONF + configs.append(config) + + return configs + + +def get_neutron_config(): + # Neutron's alembic config is always the first one + return get_alembic_configs()[0] def run_sanity_checks(config, revision): @@ -357,10 +450,14 @@ def run_sanity_checks(config, revision): script_dir.run_env() +def validate_cli_options(): + if CONF.subproject and CONF.service: + alembic_util.err(_("Cannot specify both --service and --subproject.")) + + def main(): CONF(project='neutron') - config = get_alembic_config() - config.neutron_config = CONF - - #TODO(gongysh) enable logging - CONF.command.func(config, CONF.command.name) + validate_cli_options() + for config in get_alembic_configs(): + #TODO(gongysh) enable logging + CONF.command.func(config, CONF.command.name) diff --git a/neutron/tests/functional/db/test_migrations.py b/neutron/tests/functional/db/test_migrations.py index eabe9da2e..4e6ac1481 100644 --- a/neutron/tests/functional/db/test_migrations.py +++ b/neutron/tests/functional/db/test_migrations.py @@ -112,7 +112,7 @@ class _TestModelsMigrations(test_migrations.ModelsMigrationsSync): super(_TestModelsMigrations, self).setUp() self.cfg = self.useFixture(config_fixture.Config()) self.cfg.config(core_plugin=CORE_PLUGIN) - self.alembic_config = migration.get_alembic_config() + self.alembic_config = migration.get_neutron_config() self.alembic_config.neutron_config = cfg.CONF def db_sync(self, engine): @@ -218,7 +218,7 @@ class TestSanityCheck(test_base.DbTestCase): def setUp(self): super(TestSanityCheck, self).setUp() - self.alembic_config = migration.get_alembic_config() + self.alembic_config = migration.get_neutron_config() self.alembic_config.neutron_config = cfg.CONF def test_check_sanity_14be42f3d0a5(self): @@ -246,7 +246,7 @@ class TestWalkMigrations(test_base.DbTestCase): def setUp(self): super(TestWalkMigrations, self).setUp() - self.alembic_config = migration.get_alembic_config() + self.alembic_config = migration.get_neutron_config() self.alembic_config.neutron_config = cfg.CONF def test_no_downgrade(self): diff --git a/neutron/tests/unit/db/test_migration.py b/neutron/tests/unit/db/test_migration.py index 955605aad..87f57f7e1 100644 --- a/neutron/tests/unit/db/test_migration.py +++ b/neutron/tests/unit/db/test_migration.py @@ -13,9 +13,14 @@ # License for the specific language governing permissions and limitations # under the License. +import copy +import os import sys +from alembic import config as alembic_config +import fixtures import mock +import pkg_resources from neutron.db import migration from neutron.db.migration import cli @@ -26,6 +31,21 @@ class FakeConfig(object): service = '' +class MigrationEntrypointsMemento(fixtures.Fixture): + '''Create a copy of the migration entrypoints map so it can be restored + during test cleanup. + ''' + + def _setUp(self): + self.ep_backup = {} + for proj, ep in cli.migration_entrypoints.items(): + self.ep_backup[proj] = copy.copy(ep) + self.addCleanup(self.restore) + + def restore(self): + cli.migration_entrypoints = self.ep_backup + + class TestDbMigration(base.BaseTestCase): def setUp(self): @@ -79,6 +99,32 @@ class TestCli(base.BaseTestCase): self.mock_alembic_err = mock.patch('alembic.util.err').start() self.mock_alembic_err.side_effect = SystemExit + def mocked_root_dir(cfg): + return os.path.join('/fake/dir', cli._get_project_base(cfg)) + mock_root = mock.patch.object(cli, '_get_package_root_dir').start() + mock_root.side_effect = mocked_root_dir + # Avoid creating fake directories + mock.patch('neutron.common.utils.ensure_dir').start() + + # Set up some configs and entrypoints for tests to chew on + self.configs = [] + self.projects = ('neutron', 'networking-foo', 'neutron-fwaas') + ini = os.path.join(os.path.dirname(cli.__file__), 'alembic.ini') + self.useFixture(MigrationEntrypointsMemento()) + cli.migration_entrypoints = {} + for project in self.projects: + config = alembic_config.Config(ini) + config.set_main_option('neutron_project', project) + module_name = project.replace('-', '_') + '.db.migration' + attrs = ('alembic_migrations',) + script_location = ':'.join([module_name, attrs[0]]) + config.set_main_option('script_location', script_location) + self.configs.append(config) + entrypoint = pkg_resources.EntryPoint(project, + module_name, + attrs=attrs) + cli.migration_entrypoints[project] = entrypoint + def _main_test_helper(self, argv, func_name, exp_args=(), exp_kwargs=[{}]): with mock.patch.object(sys, 'argv', argv), mock.patch.object( cli, 'run_sanity_checks'): @@ -112,17 +158,20 @@ class TestCli(base.BaseTestCase): def test_check_migration(self): with mock.patch.object(cli, 'validate_heads_file') as validate: self._main_test_helper(['prog', 'check_migration'], 'branches') - validate.assert_called_once_with(mock.ANY) + self.assertEqual(len(self.projects), validate.call_count) def _test_database_sync_revision(self, separate_branches=True): - with mock.patch.object(cli, 'update_heads_file') as update: - fake_config = FakeConfig() + with mock.patch.object(cli, 'update_heads_file') as update,\ + mock.patch.object(cli, '_use_separate_migration_branches', + return_value=separate_branches): if separate_branches: + mock.patch('os.path.exists').start() expected_kwargs = [ {'message': 'message', 'sql': False, 'autogenerate': True, 'version_path': - cli._get_version_branch_path(fake_config, branch), + cli._get_version_branch_path(config, branch), 'head': cli._get_branch_head(branch)} + for config in self.configs for branch in cli.MIGRATION_BRANCHES] else: expected_kwargs = [{ @@ -133,7 +182,7 @@ class TestCli(base.BaseTestCase): 'revision', (), expected_kwargs ) - update.assert_called_once_with(mock.ANY) + self.assertEqual(len(self.projects), update.call_count) update.reset_mock() for kwarg in expected_kwargs: @@ -145,14 +194,12 @@ class TestCli(base.BaseTestCase): 'revision', (), expected_kwargs ) - update.assert_called_once_with(mock.ANY) + self.assertEqual(len(self.projects), update.call_count) def test_database_sync_revision(self): self._test_database_sync_revision() - @mock.patch.object(cli, '_use_separate_migration_branches', - return_value=False) - def test_database_sync_revision_no_branches(self, *args): + def test_database_sync_revision_no_branches(self): # Test that old branchless approach is still supported self._test_database_sync_revision(separate_branches=False) @@ -201,8 +248,10 @@ class TestCli(base.BaseTestCase): branchless=False): if file_heads is None: file_heads = [] - fake_config = FakeConfig() - with mock.patch('alembic.script.ScriptDirectory.from_config') as fc: + fake_config = self.configs[0] + with mock.patch('alembic.script.ScriptDirectory.from_config') as fc,\ + mock.patch.object(cli, '_use_separate_migration_branches', + return_value=not branchless): fc.return_value.get_heads.return_value = heads with mock.patch('six.moves.builtins.open') as mock_open: mock_open.return_value.__enter__ = lambda s: s @@ -260,7 +309,7 @@ class TestCli(base.BaseTestCase): mock_open.return_value.__enter__ = lambda s: s mock_open.return_value.__exit__ = mock.Mock() - cli.update_heads_file(mock.sentinel.config) + cli.update_heads_file(self.configs[0]) mock_open.return_value.write.assert_called_once_with( '\n'.join(sorted(heads))) @@ -283,6 +332,40 @@ class TestCli(base.BaseTestCase): mock_open.return_value.__enter__ = lambda s: s mock_open.return_value.__exit__ = mock.Mock() - cli.update_heads_file(mock.sentinel.config) + cli.update_heads_file(self.configs[0]) mock_open.return_value.write.assert_called_once_with( '\n'.join(heads)) + + def test_get_project_base(self): + config = alembic_config.Config() + config.set_main_option('script_location', 'a.b.c:d') + proj_base = cli._get_project_base(config) + self.assertEqual('a', proj_base) + + def test_get_root_versions_dir(self): + config = alembic_config.Config() + config.set_main_option('script_location', 'a.b.c:d') + versions_dir = cli._get_root_versions_dir(config) + self.assertEqual('/fake/dir/a/a/b/c/d/versions', versions_dir) + + def test_get_subproject_script_location(self): + foo_ep = cli._get_subproject_script_location('networking-foo') + expected = 'networking_foo.db.migration:alembic_migrations' + self.assertEqual(expected, foo_ep) + + def test_get_subproject_script_location_not_installed(self): + self.assertRaises( + SystemExit, cli._get_subproject_script_location, 'not-installed') + + def test_get_service_script_location(self): + fwaas_ep = cli._get_service_script_location('fwaas') + expected = 'neutron_fwaas.db.migration:alembic_migrations' + self.assertEqual(expected, fwaas_ep) + + def test_get_service_script_location_not_installed(self): + self.assertRaises( + SystemExit, cli._get_service_script_location, 'myaas') + + def test_get_subproject_base_not_installed(self): + self.assertRaises( + SystemExit, cli._get_subproject_base, 'not-installed') diff --git a/setup.cfg b/setup.cfg index 9cda3e8d2..5708c9a31 100644 --- a/setup.cfg +++ b/setup.cfg @@ -199,6 +199,8 @@ oslo.messaging.notify.drivers = neutron.openstack.common.notifier.rpc_notifier2 = oslo_messaging.notify._impl_messaging:MessagingV2Driver neutron.openstack.common.notifier.rpc_notifier = oslo_messaging.notify._impl_messaging:MessagingDriver neutron.openstack.common.notifier.test_notifier = oslo_messaging.notify._impl_test:TestDriver +neutron.db.alembic_migrations = + neutron = neutron.db.migration:alembic_migrations [build_sphinx] all_files = 1 -- 2.45.2