Merge "Safe db.api.compute_node_get_all() performance improvement"

openstack/nova

Code Issues Proposed changes

Browse Source

This commit is contained in:

Jenkins

2013年09月05日 05:56:41 +00:00

committed by Gerrit Code Review

parent 2fff7add7b 1e90eb66ae

commit 3e46707774

3 changed files with 94 additions and 17 deletions

Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL

Download Patch File Download Diff File Expand all files Collapse all files

16
nova/db/api.py

View File

@@ -183,9 +183,19 @@ def compute_node_get_by_service_id(context, service_id):

return IMPL.compute_node_get_by_service_id(context, service_id)

def compute_node_get_all(context):

"""Get all computeNodes."""

return IMPL.compute_node_get_all(context)

def compute_node_get_all(context, no_date_fields=False):

"""Get all computeNodes.

:param context: The security context

:param no_date_fields: If set to True, excludes 'created_at', 'updated_at',

'deteled_at' and 'deleted' fields from the output,

thus significantly reducing its size.

Set to False by default

:returns: List of dictionaries each containing compute node properties,

including corresponding service and stats

"""

return IMPL.compute_node_get_all(context, no_date_fields)

def compute_node_search_by_hypervisor(context, hypervisor_match):

68
nova/db/sqlalchemy/api.py

View File

@@ -23,6 +23,7 @@ import collections

import copy

import datetime

import functools

import itertools

import sys

import time

import uuid

@@ -516,11 +517,68 @@ def compute_node_get_by_service_id(context, service_id):

@require_admin_context

def compute_node_get_all(context):

return model_query(context, models.ComputeNode).\

options(joinedload('service')).\

options(joinedload('stats')).\

all()

def compute_node_get_all(context, no_date_fields):

# NOTE(msdubov): Using lower-level 'select' queries and joining the tables

# manually here allows to gain 3x speed-up and to have 5x

# less network load / memory usage compared to the sqla ORM.

engine = get_engine()

# Retrieve ComputeNode, Service, Stat.

compute_node = models.ComputeNode.__table__

service = models.Service.__table__

stat = models.ComputeNodeStat.__table__

with engine.begin() as conn:

redundant_columns = set(['deleted_at', 'created_at', 'updated_at',

'deleted']) if no_date_fields else set([])

def filter_columns(table):

return [c for c in table.c if c.name not in redundant_columns]

compute_node_query = select(filter_columns(compute_node)).\

where(compute_node.c.deleted == 0).\

order_by(compute_node.c.service_id)

compute_node_rows = conn.execute(compute_node_query).fetchall()

service_query = select(filter_columns(service)).\

where((service.c.deleted == 0) &

(service.c.binary == 'nova-compute')).\

order_by(service.c.id)

service_rows = conn.execute(service_query).fetchall()

stat_query = select(filter_columns(stat)).\

where(stat.c.deleted == 0).\

order_by(stat.c.compute_node_id)

stat_rows = conn.execute(stat_query).fetchall()

# NOTE(msdubov): Transferring sqla.RowProxy objects to dicts.

compute_nodes = [dict(proxy.items()) for proxy in compute_node_rows]

services = [dict(proxy.items()) for proxy in service_rows]

stats = [dict(proxy.items()) for proxy in stat_rows]

# Join ComputeNode & Service manually.

# NOTE(msdubov): ComputeNodes and Services map 1-to-1.

for node, service in itertools.izip(compute_nodes, services):

node['service'] = service

# Join ComputeNode & ComputeNodeStat manually.

# NOTE(msdubov): ComputeNode and ComputeNodeStat map 1-to-Many.

# Running time is (asymptotically) optimal due to the use

# of iterators (itertools.groupby() for ComputeNodeStat and

# iter() for ComputeNode) - we handle each record only once.

compute_nodes.sort(key=lambda node: node['id'])

compute_nodes_iter = iter(compute_nodes)

for nid, nsts in itertools.groupby(stats, lambda s: s['compute_node_id']):

for node in compute_nodes_iter:

if node['id'] == nid:

node['stats'] = list(nsts)

break

else:

node['stats'] = []

return compute_nodes

@require_admin_context

27
nova/tests/db/test_db_api.py

View File

@@ -5166,8 +5166,8 @@ class ComputeNodeTestCase(test.TestCase, ModelsObjectComparatorMixin):

def setUp(self):

super(ComputeNodeTestCase, self).setUp()

self.ctxt = context.get_admin_context()

self.service_dict = dict(host='host1', binary='binary1',

topic='compute', report_count=1,

self.service_dict = dict(host='host1', binary='nova-compute',

topic=CONF.compute_topic, report_count=1,

disabled=False)

self.service = db.service_create(self.ctxt, self.service_dict)

self.compute_node_dict = dict(vcpus=2, memory_mb=1024, local_gb=2048,

@@ -5208,13 +5208,22 @@ class ComputeNodeTestCase(test.TestCase, ModelsObjectComparatorMixin):

self._stats_equal(self.stats, new_stats)

def test_compute_node_get_all(self):

nodes = db.compute_node_get_all(self.ctxt)

self.assertEqual(1, len(nodes))

node = nodes[0]

self._assertEqualObjects(self.compute_node_dict, node,

ignored_keys=self._ignored_keys + ['stats', 'service'])

new_stats = self._stats_as_dict(node['stats'])

self._stats_equal(self.stats, new_stats)

date_fields = set(['created_at', 'updated_at',

'deleted_at', 'deleted'])

for no_date_fields in [False, True]:

nodes = db.compute_node_get_all(self.ctxt, no_date_fields)

self.assertEqual(1, len(nodes))

node = nodes[0]

self._assertEqualObjects(self.compute_node_dict, node,

ignored_keys=self._ignored_keys +

['stats', 'service'])

node_fields = set(node.keys())

if no_date_fields:

self.assertFalse(date_fields & node_fields)

else:

self.assertTrue(date_fields <= node_fields)

new_stats = self._stats_as_dict(node['stats'])

self._stats_equal(self.stats, new_stats)

def test_compute_node_get(self):

compute_node_id = self.item['id']