Commit 54e2fb4

committed

Merge: [resnet/mxnet] Apply horovod patch for hvd init

2 parents 2a7c251 + 810bcf3 commit 54e2fb4Copy full SHA for 54e2fb4

File tree

-6

lines changed

-6

lines changed

Lines changed: 1 addition & 1 deletion

Original file line number	Diff line number	Diff line change
`@@ -31,7 +31,7 @@ def add_dali_args(parser):`
`31`	`31`	`group.add_argument('--dali-validation-threads', type=int, default=10, help="number of threads" +\`
`32`	`32`	`"per GPU for DALI for validation")`
`33`	`33`	`group.add_argument('--dali-prefetch-queue', type=int, default=5, help="DALI prefetch queue depth")`
`34`		`- group.add_argument('--dali-nvjpeg-memory-padding', type=int, default=256, help="Memory padding value for nvJPEG (in MB)")`
	`34`	`+ group.add_argument('--dali-nvjpeg-memory-padding', type=int, default=64, help="Memory padding value for nvJPEG (in MB)")`
`35`	`35`	`group.add_argument('--dali-fuse-decoder', type=int, default=1, help="0 or 1 whether to fuse decoder or not")`
`36`	`36`
`37`	`37`	`group.add_argument('--dali-nvjpeg-width-hint', type=int, default=5980, help="Width hint value for nvJPEG (in pixels)")`

Lines changed: 5 additions & 5 deletions

Original file line number	Diff line number	Diff line change
`@@ -483,11 +483,6 @@ def fit(args, model, data_loader):`
`483`	`483`	`# select gpu for horovod process`
`484`	`484`	`if 'horovod' in args.kv_store:`
`485`	`485`	`args.gpus = [args.gpus[hvd.local_rank()]]`
`486`		`- ctx = mx.gpu(hvd.local_rank())`
`487`		`-`
`488`		`- tensor1 = mx.nd.zeros(shape=(1,), dtype='float32', ctx=ctx)`
`489`		`- tensor2 = mx.nd.zeros(shape=(1,), dtype='float32', ctx=ctx)`
`490`		`- tensor1, tensor2 = hvd.grouped_allreduce([tensor1,tensor2])`
`491`	`486`
`492`	`487`	`if args.amp:`
`493`	`488`	`amp.init()`
`@@ -579,6 +574,11 @@ def fit(args, model, data_loader):`
`579`	`574`	`params = model.collect_params()`
`580`	`575`	`if params is not None:`
`581`	`576`	`hvd.broadcast_parameters(params, root_rank=0)`
	`577`	`+ ctx = mx.gpu(hvd.local_rank())`
	`578`	`+ tensor1 = mx.nd.zeros(shape=(1,), dtype='float32', ctx=ctx)`
	`579`	`+ tensor2 = mx.nd.zeros(shape=(1,), dtype='float32', ctx=ctx)`
	`580`	`+ tensor1, tensor2 = hvd.grouped_allreduce([tensor1,tensor2])`
	`581`	`+`
`582`	`582`	`global_metrics = CompositeMeter()`
`583`	`583`	`if args.mode in ['train_val', 'train']:`
`584`	`584`	`global_metrics.register_metric('train.loss', MinMeter())`

Comments

(0)