Can run swift-bench across multiple cores/servers.

Code Issues Proposed changes

You run one or more swift-bench-client processes like this:
 $ swift-bench-client 127.0.0.1 20001
 $ swift-bench-client 127.0.0.1 20002
Then you run swift-bench with a new option, --bench-clients (-b), which is
specified once for each swift-bench-client:
 $ swift-bench -b 127.0.0.1:20001 -b 127.0.0.1:20002
You get log lines from each client (interleaved) along with a final report
for all clients:
 127.0.0.1:20002 swift-bench-server 2012年08月25日 22:44:06,148 INFO Auth version: 1.0
 127.0.0.1:20001 swift-bench-server 2012年08月25日 22:44:06,148 INFO Auth version: 1.0
 127.0.0.1:20001 swift-bench-server 2012年08月25日 22:44:12,249 INFO 83 PUTS [0 failures], 41.5/s
 127.0.0.1:20002 swift-bench-server 2012年08月25日 22:44:14,430 INFO 74 PUTS [0 failures], 34.3/s
 ...
 127.0.0.1:20002 swift-bench-server 2012年08月25日 22:45:18,942 INFO Auth version: 1.0
 127.0.0.1:20002 swift-bench-server 2012年08月25日 22:45:20,946 INFO 238 DEL [2 failures], 118.9/s
 swift-bench 2012年08月25日 22:45:27,549 INFO 2000 PUTS **FINAL** [0 failures], 56.8/s
 swift-bench 2012年08月25日 22:45:27,550 INFO 30000 GETS **FINAL** [50 failures], 974.6/s
 swift-bench 2012年08月25日 22:45:27,550 INFO 2000 DEL **FINAL** [20 failures], 237.1/s
The concurrency, PUT count, and GET count config settings are divided by
the number of bench_clients. In other words, the same volume of work is
attempted (vs. not specifying --bench-clients), but it can now span
servers and CPU cores.
Benchmark containers are created (if use_proxy = yes) and deleted (if
delete = yes), with appropriate concurrency, in the initiating
swift-bench process, not any of the swift-bench-client processes.
Change-Id: Idbf31a23093244ab357a9bf77e6031257774f24a

This commit is contained in:

Darrell Bishop

2012年08月25日 23:02:53 -07:00

committed by Gerrit Code Review

parent c509ac2371

commit ed3b12d05c

4 changed files with 262 additions and 20 deletions

Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL

Download Patch File Download Diff File Expand all files Collapse all files

24
bin/swift-bench

View File

@@ -21,7 +21,8 @@ import signal

import uuid

from optparse import OptionParser

from swift.common.bench import BenchController

from swift.common.bench import (BenchController, DistributedBenchController,

create_containers, delete_containers)

from swift.common.utils import readconf, LogAdapter

# The defaults should be sufficient to run swift-bench on a SAIO

@@ -49,6 +50,8 @@ CONF_DEFAULTS = {

'devices': 'sdb1', # space-sep list

'log_level': 'INFO',

'timeout': '10',

'auth_version': '1.0',

'bench_clients': [],

}

SAIO_DEFAULTS = {

@@ -81,6 +84,13 @@ if __name__ == '__main__':

help='User name for obtaining an auth token')

parser.add_option('-K', '--key', dest='key',

help='Key for obtaining an auth token')

parser.add_option('-b', '--bench-clients', action='append',

metavar='<ip>:<port>',

help=('A string of the form "<ip>:<port>" which matches '

'the arguments supplied to a swift-bench-client '

'process. This argument must be specified '

'once per swift-bench-client you want to '

'utilize.'))

parser.add_option('-u', '--url', dest='url',

help='Storage URL')

parser.add_option('-c', '--concurrency', dest='concurrency',

@@ -125,6 +135,8 @@ if __name__ == '__main__':

options.put_concurrency = options.concurrency

options.get_concurrency = options.concurrency

options.del_concurrency = options.concurrency

options.containers = ['%s_%d' % (options.container_name, i)

for i in xrange(int(options.num_containers))]

def sigterm(signum, frame):

sys.exit('Termination signal received.')

@@ -145,5 +157,13 @@ if __name__ == '__main__':

'%(message)s')

loghandler.setFormatter(logformat)

controller = BenchController(logger, options)

if options.use_proxy:

create_containers(logger, options)

controller_class = DistributedBenchController if options.bench_clients \

else BenchController

controller = controller_class(logger, options)

controller.run()

if options.delete:

delete_containers(logger, options)

59
bin/swift-bench-client Executable file

View File

@@ -0,0 +1,59 @@

#!/usr/bin/env python

#

# Licensed under the Apache License, Version 2.0 (the "License");

# you may not use this file except in compliance with the License.

# You may obtain a copy of the License at

#

# http://www.apache.org/licenses/LICENSE-2.0

#

# Unless required by applicable law or agreed to in writing, software

# distributed under the License is distributed on an "AS IS" BASIS,

# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or

# implied.

# See the License for the specific language governing permissions and

# limitations under the License.

import logging

import sys

import signal

from optparse import OptionParser

from swift.common.bench import BenchServer

from swift.common.utils import LogAdapter

if __name__ == '__main__':

usage = "usage: %prog <ip> <port>"

usage += "\n\nRun a client for distributed swift-bench runs."

parser = OptionParser(usage=usage)

parser.add_option('-o', '--log-level', dest='log_level',

default='info',

help='Logging level (debug, info, etc)')

if len(sys.argv) != 3:

parser.print_help()

sys.exit(1)

options, args = parser.parse_args()

logger = logging.getLogger()

logger.setLevel({

'debug': logging.DEBUG,

'info': logging.INFO,

'warning': logging.WARNING,

'error': logging.ERROR,

'critical': logging.CRITICAL}.get(

options.log_level.lower(), logging.INFO))

loghandler = logging.StreamHandler()

logger.addHandler(loghandler)

logger = LogAdapter(logger, 'swift-bench-client')

logformat = logging.Formatter('%(server)s %(asctime)s %(levelname)s '

'%(message)s')

loghandler.setFormatter(logformat)

def sigterm(signum, frame):

sys.exit('Termination signal received.')

signal.signal(signal.SIGTERM, sigterm)

signal.signal(signal.SIGINT, sigterm)

server = BenchServer(logger, args[0], args[1])

server.run()