From 0e5cb31afd4d05e2ab9f63cc27f0389ceafc67ae Mon Sep 17 00:00:00 2001 From: Mate Lakat Date: Mon, 24 Jun 2013 14:37:08 +0100 Subject: [PATCH] xenapi: implement xenserver image to volume Amend image_utils, so that it could handle xenserver type images (vhd chains inside a tgz archive). This requires the vhd-util binary, if the image contains a vhd chain (e.g.: snapshot) DocImpact - Copy a XenServer type OpenStack image (tgz -ed vhd chain) is now supported. - To use the above functionality, the cinder box must have access to the vhd-util binary. In ubuntu, this is done by apt-get installing blktap-utils. Change-Id: I99b58868439c4d5187c2fe3666110bfda0924181 --- cinder/image/image_utils.py | 113 ++++++++++++++- cinder/tests/test_image_utils.py | 221 ++++++++++++++++++++++++++++- cinder/tests/test_xenapi_sm.py | 16 +-- cinder/volume/drivers/xenapi/sm.py | 16 +-- 4 files changed, 339 insertions(+), 27 deletions(-) diff --git a/cinder/image/image_utils.py b/cinder/image/image_utils.py index d5179af8a..b5f67ffda 100644 --- a/cinder/image/image_utils.py +++ b/cinder/image/image_utils.py @@ -26,6 +26,7 @@ we should look at maybe pushign this up to OSLO """ +import contextlib import os import re import tempfile @@ -221,11 +222,12 @@ def fetch_to_raw(context, image_service, # large and cause disk full errors which would confuse users. # Unfortunately it seems that you can't pipe to 'qemu-img convert' because # it seeks. Maybe we can think of something for a future version. - fd, tmp = tempfile.mkstemp(dir=CONF.image_conversion_dir) - os.close(fd) - with fileutils.remove_path_on_error(tmp): + with temporary_file() as tmp: fetch(context, image_service, image_id, tmp, user_id, project_id) + if is_xenserver_image(context, image_service, image_id): + replace_xenserver_image_with_coalesced_vhd(tmp) + data = qemu_img_info(tmp) fmt = data.file_format if fmt is None: @@ -259,7 +261,6 @@ def fetch_to_raw(context, image_service, image_id=image_id, reason=_("Converted to raw, but format is now %s") % data.file_format) - os.unlink(tmp) def upload_volume(context, image_service, image_meta, volume_path): @@ -293,3 +294,107 @@ def upload_volume(context, image_service, image_meta, volume_path): with fileutils.file_open(tmp) as image_file: image_service.update(context, image_id, {}, image_file) os.unlink(tmp) + + +def is_xenserver_image(context, image_service, image_id): + image_meta = image_service.show(context, image_id) + return is_xenserver_format(image_meta) + + +def is_xenserver_format(image_meta): + return ( + image_meta['disk_format'] == 'vhd' + and image_meta['container_format'] == 'ovf' + ) + + +def file_exist(fpath): + return os.path.exists(fpath) + + +def set_vhd_parent(vhd_path, parentpath): + utils.execute('vhd-util', 'modify', '-n', vhd_path, '-p', parentpath) + + +def extract_targz(archive_name, target): + utils.execute('tar', '-xzf', archive_name, '-C', target) + + +def fix_vhd_chain(vhd_chain): + for child, parent in zip(vhd_chain[:-1], vhd_chain[1:]): + set_vhd_parent(child, parent) + + +def get_vhd_size(vhd_path): + out, err = utils.execute('vhd-util', 'query', '-n', vhd_path, '-v') + return int(out) + + +def resize_vhd(vhd_path, size, journal): + utils.execute( + 'vhd-util', 'resize', '-n', vhd_path, '-s', '%d' % size, '-j', journal) + + +def coalesce_vhd(vhd_path): + utils.execute( + 'vhd-util', 'coalesce', '-n', vhd_path) + + +def create_temporary_file(): + fd, tmp = tempfile.mkstemp(dir=CONF.image_conversion_dir) + os.close(fd) + return tmp + + +def rename_file(src, dst): + os.rename(src, dst) + + +@contextlib.contextmanager +def temporary_file(): + try: + tmp = create_temporary_file() + yield tmp + finally: + os.unlink(tmp) + + +def temporary_dir(): + return utils.tempdir(dir=CONF.image_conversion_dir) + + +def coalesce_chain(vhd_chain): + for child, parent in zip(vhd_chain[:-1], vhd_chain[1:]): + with temporary_dir() as directory_for_journal: + size = get_vhd_size(child) + journal_file = os.path.join( + directory_for_journal, 'vhd-util-resize-journal') + resize_vhd(parent, size, journal_file) + coalesce_vhd(child) + + return vhd_chain[-1] + + +def discover_vhd_chain(directory): + counter = 0 + chain = [] + + while True: + fpath = os.path.join(directory, '%d.vhd' % counter) + if file_exist(fpath): + chain.append(fpath) + else: + break + counter += 1 + + return chain + + +def replace_xenserver_image_with_coalesced_vhd(image_file): + with temporary_dir() as tempdir: + extract_targz(image_file, tempdir) + chain = discover_vhd_chain(tempdir) + fix_vhd_chain(chain) + coalesced = coalesce_chain(chain) + os.unlink(image_file) + rename_file(coalesced, image_file) diff --git a/cinder/tests/test_image_utils.py b/cinder/tests/test_image_utils.py index f7d19fa66..86e690d65 100644 --- a/cinder/tests/test_image_utils.py +++ b/cinder/tests/test_image_utils.py @@ -16,10 +16,13 @@ # under the License. """Unit tests for image utils.""" +import contextlib +import mox +import textwrap + from cinder.image import image_utils from cinder import test from cinder import utils -import mox class TestUtils(test.TestCase): @@ -43,3 +46,219 @@ class TestUtils(test.TestCase): image_utils.resize_image(TEST_IMG_SOURCE, TEST_IMG_SIZE_IN_GB) mox.VerifyAll() + + +class TestExtractTo(test.TestCase): + def test_extract_to_calls_tar(self): + mox = self.mox + mox.StubOutWithMock(utils, 'execute') + + utils.execute( + 'tar', '-xzf', 'archive.tgz', '-C', 'targetpath').AndReturn( + ('ignored', 'ignored') + ) + + mox.ReplayAll() + + image_utils.extract_targz('archive.tgz', 'targetpath') + mox.VerifyAll() + + +class TestSetVhdParent(test.TestCase): + def test_vhd_util_call(self): + mox = self.mox + mox.StubOutWithMock(utils, 'execute') + + utils.execute( + 'vhd-util', 'modify', '-n', 'child', '-p', 'parent').AndReturn( + ('ignored', 'ignored') + ) + + mox.ReplayAll() + + image_utils.set_vhd_parent('child', 'parent') + mox.VerifyAll() + + +class TestFixVhdChain(test.TestCase): + def test_empty_chain(self): + mox = self.mox + mox.StubOutWithMock(image_utils, 'set_vhd_parent') + + mox.ReplayAll() + image_utils.fix_vhd_chain([]) + + def test_single_vhd_file_chain(self): + mox = self.mox + mox.StubOutWithMock(image_utils, 'set_vhd_parent') + + mox.ReplayAll() + image_utils.fix_vhd_chain(['0.vhd']) + + def test_chain_with_two_elements(self): + mox = self.mox + mox.StubOutWithMock(image_utils, 'set_vhd_parent') + + image_utils.set_vhd_parent('0.vhd', '1.vhd') + + mox.ReplayAll() + image_utils.fix_vhd_chain(['0.vhd', '1.vhd']) + + +class TestGetSize(test.TestCase): + def test_vhd_util_call(self): + mox = self.mox + mox.StubOutWithMock(utils, 'execute') + + utils.execute( + 'vhd-util', 'query', '-n', 'vhdfile', '-v').AndReturn( + ('1024', 'ignored') + ) + + mox.ReplayAll() + + result = image_utils.get_vhd_size('vhdfile') + mox.VerifyAll() + + self.assertEquals(1024, result) + + +class TestResize(test.TestCase): + def test_vhd_util_call(self): + mox = self.mox + mox.StubOutWithMock(utils, 'execute') + + utils.execute( + 'vhd-util', 'resize', '-n', 'vhdfile', '-s', '1024', + '-j', 'journal').AndReturn(('ignored', 'ignored')) + + mox.ReplayAll() + + image_utils.resize_vhd('vhdfile', 1024, 'journal') + mox.VerifyAll() + + +class TestCoalesce(test.TestCase): + def test_vhd_util_call(self): + mox = self.mox + mox.StubOutWithMock(utils, 'execute') + + utils.execute( + 'vhd-util', 'coalesce', '-n', 'vhdfile' + ).AndReturn(('ignored', 'ignored')) + + mox.ReplayAll() + + image_utils.coalesce_vhd('vhdfile') + mox.VerifyAll() + + +@contextlib.contextmanager +def fake_context(return_value): + yield return_value + + +class TestTemporaryFile(test.TestCase): + def test_file_unlinked(self): + mox = self.mox + mox.StubOutWithMock(image_utils, 'create_temporary_file') + mox.StubOutWithMock(image_utils.os, 'unlink') + + image_utils.create_temporary_file().AndReturn('somefile') + image_utils.os.unlink('somefile') + + mox.ReplayAll() + + with image_utils.temporary_file(): + pass + + def test_file_unlinked_on_error(self): + mox = self.mox + mox.StubOutWithMock(image_utils, 'create_temporary_file') + mox.StubOutWithMock(image_utils.os, 'unlink') + + image_utils.create_temporary_file().AndReturn('somefile') + image_utils.os.unlink('somefile') + + mox.ReplayAll() + + def sut(): + with image_utils.temporary_file(): + raise Exception() + + self.assertRaises(Exception, sut) + + +class TestCoalesceChain(test.TestCase): + def test_single_vhd(self): + mox = self.mox + mox.StubOutWithMock(image_utils, 'get_vhd_size') + mox.StubOutWithMock(image_utils, 'resize_vhd') + mox.StubOutWithMock(image_utils, 'coalesce_vhd') + + mox.ReplayAll() + + result = image_utils.coalesce_chain(['0.vhd']) + mox.VerifyAll() + + self.assertEquals('0.vhd', result) + + def test_chain_of_two_vhds(self): + self.mox.StubOutWithMock(image_utils, 'get_vhd_size') + self.mox.StubOutWithMock(image_utils, 'temporary_dir') + self.mox.StubOutWithMock(image_utils, 'resize_vhd') + self.mox.StubOutWithMock(image_utils, 'coalesce_vhd') + self.mox.StubOutWithMock(image_utils, 'temporary_file') + + image_utils.get_vhd_size('0.vhd').AndReturn(1024) + image_utils.temporary_dir().AndReturn(fake_context('tdir')) + image_utils.resize_vhd('1.vhd', 1024, 'tdir/vhd-util-resize-journal') + image_utils.coalesce_vhd('0.vhd') + self.mox.ReplayAll() + + result = image_utils.coalesce_chain(['0.vhd', '1.vhd']) + self.mox.VerifyAll() + self.assertEquals('1.vhd', result) + + +class TestDiscoverChain(test.TestCase): + def test_discovery_calls(self): + mox = self.mox + mox.StubOutWithMock(image_utils, 'file_exist') + + image_utils.file_exist('some/path/0.vhd').AndReturn(True) + image_utils.file_exist('some/path/1.vhd').AndReturn(True) + image_utils.file_exist('some/path/2.vhd').AndReturn(False) + + mox.ReplayAll() + result = image_utils.discover_vhd_chain('some/path') + mox.VerifyAll() + + self.assertEquals( + ['some/path/0.vhd', 'some/path/1.vhd'], result) + + +class TestXenServerImageToCoalescedVhd(test.TestCase): + def test_calls(self): + mox = self.mox + mox.StubOutWithMock(image_utils, 'temporary_dir') + mox.StubOutWithMock(image_utils, 'extract_targz') + mox.StubOutWithMock(image_utils, 'discover_vhd_chain') + mox.StubOutWithMock(image_utils, 'fix_vhd_chain') + mox.StubOutWithMock(image_utils, 'coalesce_chain') + mox.StubOutWithMock(image_utils.os, 'unlink') + mox.StubOutWithMock(image_utils, 'rename_file') + + image_utils.temporary_dir().AndReturn(fake_context('somedir')) + image_utils.extract_targz('image', 'somedir') + image_utils.discover_vhd_chain('somedir').AndReturn( + ['somedir/0.vhd', 'somedir/1.vhd']) + image_utils.fix_vhd_chain(['somedir/0.vhd', 'somedir/1.vhd']) + image_utils.coalesce_chain( + ['somedir/0.vhd', 'somedir/1.vhd']).AndReturn('somedir/1.vhd') + image_utils.os.unlink('image') + image_utils.rename_file('somedir/1.vhd', 'image') + + mox.ReplayAll() + image_utils.replace_xenserver_image_with_coalesced_vhd('image') + mox.VerifyAll() diff --git a/cinder/tests/test_xenapi_sm.py b/cinder/tests/test_xenapi_sm.py index 4253cf0b0..ba5cc1adb 100644 --- a/cinder/tests/test_xenapi_sm.py +++ b/cinder/tests/test_xenapi_sm.py @@ -275,10 +275,10 @@ class DriverTestCase(test.TestCase): 'server', 'serverpath', '/var/run/sr-mount') mock.StubOutWithMock(drv, '_use_glance_plugin_to_upload_volume') - mock.StubOutWithMock(driver, 'is_xenserver_format') + mock.StubOutWithMock(driver.image_utils, 'is_xenserver_format') context = MockContext('token') - driver.is_xenserver_format('image_meta').AndReturn(True) + driver.image_utils.is_xenserver_format('image_meta').AndReturn(True) drv._use_glance_plugin_to_upload_volume( context, 'volume', 'image_service', 'image_meta').AndReturn( @@ -296,10 +296,10 @@ class DriverTestCase(test.TestCase): 'server', 'serverpath', '/var/run/sr-mount') mock.StubOutWithMock(drv, '_use_image_utils_to_upload_volume') - mock.StubOutWithMock(driver, 'is_xenserver_format') + mock.StubOutWithMock(driver.image_utils, 'is_xenserver_format') context = MockContext('token') - driver.is_xenserver_format('image_meta').AndReturn(False) + driver.image_utils.is_xenserver_format('image_meta').AndReturn(False) drv._use_image_utils_to_upload_volume( context, 'volume', 'image_service', 'image_meta').AndReturn( @@ -358,10 +358,10 @@ class DriverTestCase(test.TestCase): 'server', 'serverpath', '/var/run/sr-mount') mock.StubOutWithMock(drv, '_use_glance_plugin_to_copy_image_to_volume') - mock.StubOutWithMock(driver, 'is_xenserver_image') + mock.StubOutWithMock(driver.image_utils, 'is_xenserver_image') context = MockContext('token') - driver.is_xenserver_image( + driver.image_utils.is_xenserver_image( context, 'image_service', 'image_id').AndReturn(True) drv._use_glance_plugin_to_copy_image_to_volume( context, 'volume', 'image_service', 'image_id').AndReturn('result') @@ -376,10 +376,10 @@ class DriverTestCase(test.TestCase): 'server', 'serverpath', '/var/run/sr-mount') mock.StubOutWithMock(drv, '_use_image_utils_to_pipe_bytes_to_volume') - mock.StubOutWithMock(driver, 'is_xenserver_image') + mock.StubOutWithMock(driver.image_utils, 'is_xenserver_image') context = MockContext('token') - driver.is_xenserver_image( + driver.image_utils.is_xenserver_image( context, 'image_service', 'image_id').AndReturn(False) drv._use_image_utils_to_pipe_bytes_to_volume( context, 'volume', 'image_service', 'image_id').AndReturn(True) diff --git a/cinder/volume/drivers/xenapi/sm.py b/cinder/volume/drivers/xenapi/sm.py index 2b11c89b4..92cd583e7 100644 --- a/cinder/volume/drivers/xenapi/sm.py +++ b/cinder/volume/drivers/xenapi/sm.py @@ -155,7 +155,7 @@ class XenAPINFSDriver(driver.VolumeDriver): pass def copy_image_to_volume(self, context, volume, image_service, image_id): - if is_xenserver_image(context, image_service, image_id): + if image_utils.is_xenserver_image(context, image_service, image_id): return self._use_glance_plugin_to_copy_image_to_volume( context, volume, image_service, image_id) @@ -204,7 +204,7 @@ class XenAPINFSDriver(driver.VolumeDriver): volume['size']) def copy_volume_to_image(self, context, volume, image_service, image_meta): - if is_xenserver_format(image_meta): + if image_utils.is_xenserver_format(image_meta): return self._use_glance_plugin_to_upload_volume( context, volume, image_service, image_meta) @@ -258,15 +258,3 @@ class XenAPINFSDriver(driver.VolumeDriver): self._stats = data return self._stats - - -def is_xenserver_image(context, image_service, image_id): - image_meta = image_service.show(context, image_id) - return is_xenserver_format(image_meta) - - -def is_xenserver_format(image_meta): - return ( - image_meta['disk_format'] == 'vhd' - and image_meta['container_format'] == 'ovf' - ) -- 2.45.2