Merge "xenapi: through-dev raw-tgz image upload to glance"

openstack/nova

Code Issues Proposed changes

Browse Source

This commit is contained in:

Jenkins

2013年09月05日 15:17:58 +00:00

committed by Gerrit Code Review

parent 63a8880c61 ef8017ff79

commit 31783bb30b

3 changed files with 327 additions and 5 deletions

Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL

Download Patch File Download Diff File Expand all files Collapse all files

202
nova/tests/virt/xenapi/image/test_vdi_through_dev.py Normal file

View File

@@ -0,0 +1,202 @@

# vim: tabstop=4 shiftwidth=4 softtabstop=4

#

# Licensed under the Apache License, Version 2.0 (the "License"); you may

# not use this file except in compliance with the License. You may obtain

# a copy of the License at

#

# http://www.apache.org/licenses/LICENSE-2.0

#

# Unless required by applicable law or agreed to in writing, software

# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT

# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the

# License for the specific language governing permissions and limitations

# under the License.

import contextlib

import eventlet

import tarfile

from nova.image import glance

from nova import test

from nova.virt.xenapi import driver

from nova.virt.xenapi.image import vdi_through_dev

@contextlib.contextmanager

def fake_context(result=None):

yield result

class TestDelegatingToCommand(test.TestCase):

def test_upload_image_is_delegated_to_command(self):

command = self.mox.CreateMock(vdi_through_dev.UploadToGlanceAsRawTgz)

self.mox.StubOutWithMock(vdi_through_dev, 'UploadToGlanceAsRawTgz')

vdi_through_dev.UploadToGlanceAsRawTgz(

'ctx', 'session', 'instance', 'vdis', 'image_id').AndReturn(

command)

command.upload_image().AndReturn('result')

self.mox.ReplayAll()

store = vdi_through_dev.VdiThroughDevStore()

result = store.upload_image(

'ctx', 'session', 'instance', 'vdis', 'image_id')

self.assertEquals('result', result)

class TestUploadToGlanceAsRawTgz(test.TestCase):

def test_upload_image(self):

store = vdi_through_dev.UploadToGlanceAsRawTgz(

'context', 'session', 'instance', ['vdi0', 'vdi1'], 'id')

image_service = self.mox.CreateMock(glance.GlanceImageService)

self.mox.StubOutWithMock(store, '_perform_upload')

self.mox.StubOutWithMock(store, '_get_vdi_ref')

self.mox.StubOutWithMock(vdi_through_dev, 'glance')

self.mox.StubOutWithMock(vdi_through_dev, 'vm_utils')

self.mox.StubOutWithMock(vdi_through_dev, 'utils')

store._get_vdi_ref().AndReturn('vdi_ref')

vdi_through_dev.vm_utils.vdi_attached_here(

'session', 'vdi_ref', read_only=True).AndReturn(

fake_context('dev'))

vdi_through_dev.utils.make_dev_path('dev').AndReturn('devpath')

vdi_through_dev.utils.temporary_chown('devpath').AndReturn(

fake_context())

store._perform_upload('devpath')

self.mox.ReplayAll()

store.upload_image()

def test__perform_upload(self):

producer = self.mox.CreateMock(vdi_through_dev.TarGzProducer)

consumer = self.mox.CreateMock(vdi_through_dev.UpdateGlanceImage)

pool = self.mox.CreateMock(eventlet.GreenPool)

store = vdi_through_dev.UploadToGlanceAsRawTgz(

'context', 'session', 'instance', ['vdi0', 'vdi1'], 'id')

self.mox.StubOutWithMock(store, '_create_pipe')

self.mox.StubOutWithMock(store, '_get_virtual_size')

self.mox.StubOutWithMock(producer, 'get_metadata')

self.mox.StubOutWithMock(vdi_through_dev, 'TarGzProducer')

self.mox.StubOutWithMock(vdi_through_dev, 'UpdateGlanceImage')

self.mox.StubOutWithMock(vdi_through_dev, 'eventlet')

producer.get_metadata().AndReturn('metadata')

store._get_virtual_size().AndReturn('324')

store._create_pipe().AndReturn(('readfile', 'writefile'))

vdi_through_dev.TarGzProducer(

'devpath', 'writefile', '324', 'disk.raw').AndReturn(

producer)

vdi_through_dev.UpdateGlanceImage('context', 'id', 'metadata',

'readfile').AndReturn(consumer)

vdi_through_dev.eventlet.GreenPool().AndReturn(pool)

pool.spawn(producer.start)

pool.spawn(consumer.start)

pool.waitall()

self.mox.ReplayAll()

store._perform_upload('devpath')

def test__get_vdi_ref(self):

session = self.mox.CreateMock(driver.XenAPISession)

store = vdi_through_dev.UploadToGlanceAsRawTgz(

'context', session, 'instance', ['vdi0', 'vdi1'], 'id')

session.call_xenapi('VDI.get_by_uuid', 'vdi0').AndReturn('vdi_ref')

self.mox.ReplayAll()

self.assertEquals('vdi_ref', store._get_vdi_ref())

def test__get_virtual_size(self):

session = self.mox.CreateMock(driver.XenAPISession)

store = vdi_through_dev.UploadToGlanceAsRawTgz(

'context', session, 'instance', ['vdi0', 'vdi1'], 'id')

self.mox.StubOutWithMock(store, '_get_vdi_ref')

store._get_vdi_ref().AndReturn('vdi_ref')

session.call_xenapi('VDI.get_virtual_size', 'vdi_ref')

self.mox.ReplayAll()

result = store._get_virtual_size()

def test__create_pipe(self):

store = vdi_through_dev.UploadToGlanceAsRawTgz(

'context', 'session', 'instance', ['vdi0', 'vdi1'], 'id')

self.mox.StubOutWithMock(vdi_through_dev, 'os')

self.mox.StubOutWithMock(vdi_through_dev, 'greenio')

vdi_through_dev.os.pipe().AndReturn(('rpipe', 'wpipe'))

vdi_through_dev.greenio.GreenPipe('rpipe', 'rb', 0).AndReturn('rfile')

vdi_through_dev.greenio.GreenPipe('wpipe', 'wb', 0).AndReturn('wfile')

self.mox.ReplayAll()

result = store._create_pipe()

self.assertEqual(('rfile', 'wfile'), result)

class TestTarGzProducer(test.TestCase):

def test_constructor(self):

producer = vdi_through_dev.TarGzProducer('devpath', 'writefile',

'100', 'fname')

self.assertEqual('devpath', producer.fpath)

self.assertEqual('writefile', producer.output)

self.assertEqual('100', producer.size)

self.assertEqual('writefile', producer.output)

def test_start(self):

outf = self.mox.CreateMock(file)

producer = vdi_through_dev.TarGzProducer('fpath', outf,

'100', 'fname')

tfile = self.mox.CreateMock(tarfile.TarFile)

tinfo = self.mox.CreateMock(tarfile.TarInfo)

inf = self.mox.CreateMock(file)

self.mox.StubOutWithMock(vdi_through_dev, 'tarfile')

self.mox.StubOutWithMock(producer, '_open_file')

vdi_through_dev.tarfile.TarInfo(name='fname').AndReturn(tinfo)

vdi_through_dev.tarfile.open(fileobj=outf, mode='w|gz').AndReturn(

fake_context(tfile))

producer._open_file('fpath', 'rb').AndReturn(fake_context(inf))

tfile.addfile(tinfo, fileobj=inf)

outf.close()

self.mox.ReplayAll()

producer.start()

self.assertEquals(100, tinfo.size)

def test_get_metadata(self):

producer = vdi_through_dev.TarGzProducer('devpath', 'writefile',

'100', 'fname')

self.assertEquals({

'disk_format': 'raw',

'container_format': 'tgz'},

producer.get_metadata())

class TestUpdateGlanceImage(test.TestCase):

def test_start(self):

consumer = vdi_through_dev.UpdateGlanceImage(

'context', 'id', 'metadata', 'stream')

image_service = self.mox.CreateMock(glance.GlanceImageService)

self.mox.StubOutWithMock(vdi_through_dev, 'glance')

vdi_through_dev.glance.get_remote_image_service(

'context', 'id').AndReturn((image_service, 'image_id'))

image_service.update(

'context', 'image_id', 'metadata', 'stream', purge_props=False)

self.mox.ReplayAll()

consumer.start()

123
nova/virt/xenapi/image/vdi_through_dev.py Normal file

View File

@@ -0,0 +1,123 @@

# vim: tabstop=4 shiftwidth=4 softtabstop=4

#

# Licensed under the Apache License, Version 2.0 (the "License"); you may

# not use this file except in compliance with the License. You may obtain

# a copy of the License at

#

# http://www.apache.org/licenses/LICENSE-2.0

#

# Unless required by applicable law or agreed to in writing, software

# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT

# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the

# License for the specific language governing permissions and limitations

# under the License.

import contextlib

import eventlet

from eventlet import greenio

import os

import tarfile

from nova.image import glance

from nova import utils

from nova.virt.xenapi import vm_utils

class VdiThroughDevStore(object):

"""Deal with virtual disks by attaching them to the OS domU.

At the moment it supports upload to Glance, and the upload format is a raw

disk inside a tgz.

"""

def upload_image(self, context, session, instance, vdi_uuids, image_id):

command = UploadToGlanceAsRawTgz(

context, session, instance, vdi_uuids, image_id)

return command.upload_image()

def download_image(self, context, session, instance, image_id):

# TODO(matelakat) Move through-dev image download functionality to this

# method.

raise NotImplementedError()

class UploadToGlanceAsRawTgz(object):

def __init__(self, context, session, instance, vdi_uuids, image_id):

self.context = context

self.image_id = image_id

self.session = session

self.vdi_uuids = vdi_uuids

def _get_virtual_size(self):

return self.session.call_xenapi(

'VDI.get_virtual_size', self._get_vdi_ref())

def _get_vdi_ref(self):

return self.session.call_xenapi('VDI.get_by_uuid', self.vdi_uuids[0])

def _perform_upload(self, devpath):

readfile, writefile = self._create_pipe()

size = self._get_virtual_size()

producer = TarGzProducer(devpath, writefile, size, 'disk.raw')

consumer = UpdateGlanceImage(

self.context, self.image_id, producer.get_metadata(), readfile)

pool = eventlet.GreenPool()

pool.spawn(producer.start)

pool.spawn(consumer.start)

pool.waitall()

def _create_pipe(self):

rpipe, wpipe = os.pipe()

rfile = greenio.GreenPipe(rpipe, 'rb', 0)

wfile = greenio.GreenPipe(wpipe, 'wb', 0)

return rfile, wfile

def upload_image(self):

vdi_ref = self._get_vdi_ref()

with vm_utils.vdi_attached_here(self.session, vdi_ref,

read_only=True) as dev:

devpath = utils.make_dev_path(dev)

with utils.temporary_chown(devpath):

self._perform_upload(devpath)

class TarGzProducer(object):

def __init__(self, devpath, writefile, size, fname):

self.fpath = devpath

self.output = writefile

self.size = size

self.fname = fname

def get_metadata(self):

return {

'disk_format': 'raw',

'container_format': 'tgz'

}

def start(self):

with contextlib.closing(self.output):

tinfo = tarfile.TarInfo(name=self.fname)

tinfo.size = int(self.size)

with tarfile.open(fileobj=self.output, mode='w|gz') as tfile:

with self._open_file(self.fpath, 'rb') as input_file:

tfile.addfile(tinfo, fileobj=input_file)

def _open_file(self, *args):

return open(*args)

class UpdateGlanceImage(object):

def __init__(self, context, image_id, metadata, stream):

self.context = context

self.image_id = image_id

self.metadata = metadata

self.image_stream = stream

def start(self):

image_service, image_id = (

glance.get_remote_image_service(self.context, self.image_id))

image_service.update(self.context, image_id, self.metadata,

self.image_stream, purge_props=False)

7
nova/virt/xenapi/vmops.py

View File

@@ -757,11 +757,8 @@ class VMOps(object):

coalesce together, so, we must wait for this coalescing to occur to

get a stable representation of the data on disk.

3. Push-to-data-store: Once coalesced, we call a plugin on the

XenServer that will bundle the VHDs together and then push the

bundle. Depending on the configured value of

'xenapi_image_upload_handler', image data may be pushed to

Glance or the specified data store.

3. Push-to-data-store: Once coalesced, we call

'xenapi_image_upload_handler' to upload the images.

"""

vm_ref = self._get_vm_opaque_ref(instance)