From 468de2d767b6cc984fa4fc3b150721368f889a77 Mon Sep 17 00:00:00 2001 From: Henry Gessau Date: Thu, 5 Nov 2015 11:47:13 -0500 Subject: [PATCH] More instructions for neutron-db-manage revision --autogenerate More detailed instructions for --autogenerate, including how to rely less on devstack. Also some tweaks to scripts and requirements to simplify the process. Closes-Bug: #1197570 Change-Id: I97043b30a8df7408a73af38c6a015fc0dcf82571 --- doc/source/devref/alembic_migrations.rst | 107 +++++++++++++++++++++- neutron/tests/functional/requirements.txt | 1 - test-requirements.txt | 2 + tools/configure_for_func_testing.sh | 30 ++++-- 4 files changed, 129 insertions(+), 11 deletions(-) diff --git a/doc/source/devref/alembic_migrations.rst b/doc/source/devref/alembic_migrations.rst index 8762b689e..fbdba620c 100644 --- a/doc/source/devref/alembic_migrations.rst +++ b/doc/source/devref/alembic_migrations.rst @@ -161,10 +161,109 @@ complete. The operations in the template are those supported by the Alembic migration library. +.. _neutron-db-manage-without-devstack: + +Running neutron-db-manage without devstack +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +When, as a developer, you want to work with the Neutron DB schema and alembic +migrations only, it can be rather tedious to rely on devstack just to get an +up-to-date neutron-db-manage installed. This section describes how to work on +the schema and migration scripts with just the unit test virtualenv and +mysql. You can also operate on a separate test database so you don't mess up +the installed Neutron database. + +Setting up the environment +++++++++++++++++++++++++++ + +Install mysql service +''''''''''''''''''''' + +This only needs to be done once since it is a system install. If you have run +devstack on your system before, then the mysql service is already installed and +you can skip this step. + +Mysql must be configured as installed by devstack, and the following script +accomplishes this without actually running devstack:: + + INSTALL_MYSQL_ONLY=True ./tools/configure_for_func_testing.sh ../devstack + +Run this from the root of the neutron repo. It assumes an up-to-date clone of +the devstack repo is in ``../devstack``. + +Note that you must know the mysql root password. It is derived from (in order +of precedence): + +- ``$MYSQL_PASSWORD`` in your environment +- ``$MYSQL_PASSWORD`` in ``../devstack/local.conf`` +- ``$MYSQL_PASSWORD`` in ``../devstack/localrc`` +- default of 'secretmysql' from ``tools/configure_for_func_testing.sh`` + +Work on a test database +''''''''''''''''''''''' + +Rather than using the neutron database when working on schema and alembic +migration script changes, we can work on a test database. In the examples +below, we use a database named ``testdb``. + +To create the database:: + + mysql -e "create database testdb;" + +You will often need to clear it to re-run operations from a blank database:: + + mysql -e "drop database testdb; create database testdb;" + +To work on the test database instead of the neutron database, point to it with +the ``--database-connection`` option:: + + neutron-db-manage --database-connection mysql+pymysql://root:secretmysql@127.0.0.1/testdb?charset=utf8 + +You may find it convenient to set up an alias (in your .bashrc) for this:: + + alias test-db-manage='neutron-db-manage --database-connection mysql+pymysql://root:secretmysql@127.0.0.1/testdb?charset=utf8' + +Create and activate the virtualenv +'''''''''''''''''''''''''''''''''' + +From the root of the neutron (or sub-project) repo directory, run:: + + tox --notest -r -e py27 + source .tox/py27/bin/activate + +Now you can use the ``test-db-manage`` alias in place of ``neutron-db-manage`` +in the script auto-generation instructions below. + +When you are done, exit the virtualenv:: + + deactivate + + Script Auto-generation ~~~~~~~~~~~~~~~~~~~~~~ -:: +This section describes how to auto-generate an alembic migration script for a +model change. You may either use the system installed devstack environment, or +a virtualenv + testdb environment as described in +:ref:`neutron-db-manage-without-devstack`. + +Stop the neutron service. Work from the base directory of the neutron (or +sub-project) repo. Check out the master branch and and do ``git pull`` to +ensure it is fully up to date. Check out your development branch and rebase to +master. + +**NOTE:** Make sure you have not updated the ``CONTRACT_HEAD`` or +``EXPAND_HEAD`` yet at this point. + +Start with an empty database and upgrade to heads:: + + mysql -e "drop database neutron; create database neutron;" + neutron-db-manage upgrade heads + +The database schema is now created without your model changes. The alembic +``revision --autogenerate`` command will look for differences between the +schema generated by the upgrade command and the schema defined by the models, +including your model updates:: neutron-db-manage revision -m "description of revision" --autogenerate @@ -195,6 +294,12 @@ blank file for a branch via:: in a directory that is named as current release. If not, please raise the issue with the development team (IRC, mailing list, launchpad bug). +**NOTE:** The "description of revision" text should be a simple English +sentence. The first 30 characters of the description will be used in the file +name for the script, with underscores substituted for spaces. If the truncation +occurs at an awkward point in the description, you can modify the script file +name manually before committing. + The timeline on each alembic branch should remain linear and not interleave with other branches, so that there is a clear path when upgrading. To verify that alembic branches maintain linear timelines, you can run this command:: diff --git a/neutron/tests/functional/requirements.txt b/neutron/tests/functional/requirements.txt index 2f9131d0b..62285e9e9 100644 --- a/neutron/tests/functional/requirements.txt +++ b/neutron/tests/functional/requirements.txt @@ -6,4 +6,3 @@ psutil>=1.1.1,<2.0.0 psycopg2 -PyMySQL>=0.6.2 # MIT License diff --git a/test-requirements.txt b/test-requirements.txt index 513242623..d3b06369c 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -22,3 +22,5 @@ tempest-lib>=0.10.0 ddt>=0.7.0 pylint==1.4.4 # GNU GPL v2 reno>=0.1.1 # Apache2 +# Needed to run DB commands in virtualenvs +PyMySQL>=0.6.2 # MIT License diff --git a/tools/configure_for_func_testing.sh b/tools/configure_for_func_testing.sh index 1a0842dd6..5235dc3be 100755 --- a/tools/configure_for_func_testing.sh +++ b/tools/configure_for_func_testing.sh @@ -53,6 +53,7 @@ VENV=${VENV:-dsvm-functional} DEVSTACK_PATH=${DEVSTACK_PATH:-$1} PROJECT_NAME=${PROJECT_NAME:-neutron} REPO_BASE=${GATE_DEST:-$(cd $(dirname "$0")/../.. && pwd)} +INSTALL_MYSQL_ONLY=${INSTALL_MYSQL_ONLY:-False} # The gate should automatically install dependencies. INSTALL_BASE_DEPENDENCIES=${INSTALL_BASE_DEPENDENCIES:-$IS_GATE} @@ -109,7 +110,10 @@ function _install_rpc_backend { } +# _install_databases [install_pg] function _install_databases { + local install_pg=${1:-True} + echo_summary "Installing databases" # Avoid attempting to configure the db if it appears to already @@ -129,10 +133,12 @@ function _install_databases { install_database configure_database_mysql - enable_service postgresql - initialize_database_backends - install_database - configure_database_postgresql + if [[ "$install_pg" == "True" ]]; then + enable_service postgresql + initialize_database_backends + install_database + configure_database_postgresql + fi # Set up the 'openstack_citest' user and database in each backend tmp_dir=$(mktemp -d) @@ -148,14 +154,16 @@ FLUSH PRIVILEGES; EOF /usr/bin/mysql -u root < $tmp_dir/mysql.sql - cat << EOF > $tmp_dir/postgresql.sql + if [[ "$install_pg" == "True" ]]; then + cat << EOF > $tmp_dir/postgresql.sql CREATE USER openstack_citest WITH CREATEDB LOGIN PASSWORD 'openstack_citest'; CREATE DATABASE openstack_citest WITH OWNER openstack_citest; EOF - # User/group postgres needs to be given access to tmp_dir - setfacl -m g:postgres:rwx $tmp_dir - sudo -u postgres /usr/bin/psql --file=$tmp_dir/postgresql.sql + # User/group postgres needs to be given access to tmp_dir + setfacl -m g:postgres:rwx $tmp_dir + sudo -u postgres /usr/bin/psql --file=$tmp_dir/postgresql.sql + fi } @@ -250,5 +258,9 @@ _init if [[ "$IS_GATE" != "True" ]]; then - configure_host_for_func_testing + if [[ "$INSTALL_MYSQL_ONLY" == "True" ]]; then + _install_databases nopg + else + configure_host_for_func_testing + fi fi -- 2.45.2