Commit ae7af2d

authored

Merge pull request #1942 from oesteban/enh/ResourceInitialization

[ENH] Migrating resource handler to Node level

2 parents d82a18f + b812031 commit ae7af2dCopy full SHA for ae7af2d

File tree

+14

-7

lines changed

+14

-7

lines changed

Lines changed: 1 addition & 1 deletion

Original file line number	Diff line number	Diff line change
`@@ -774,7 +774,7 @@ def __init__(self, from_file=None, **inputs):`
`774`	`774`	`self.__class__.__name__)`
`775`	`775`
`776`	`776`	`self.inputs = self.input_spec(**inputs)`
`777`		`- self.estimated_memory_gb = 1`
	`777`	`+ self.estimated_memory_gb = 0.25`
`778`	`778`	`self.num_threads = 1`
`779`	`779`
`780`	`780`	`if from_file is not None:`

Lines changed: 7 additions & 1 deletion

Original file line number	Diff line number	Diff line change
`@@ -79,7 +79,8 @@ class Node(EngineBase):`
`79`	`79`
`80`	`80`	`def __init__(self, interface, name, iterables=None, itersource=None,`
`81`	`81`	`synchronize=False, overwrite=None, needed_outputs=None,`
`82`		`- run_without_submitting=False, **kwargs):`
	`82`	`+ run_without_submitting=False, n_procs=1, mem_gb=None,`
	`83`	`+ **kwargs):`
`83`	`84`	`"""`
`84`	`85`	`Parameters`
`85`	`86`	`----------`
`@@ -168,6 +169,11 @@ def __init__(self, interface, name, iterables=None, itersource=None,`
`168`	`169`	`self.input_source = {}`
`169`	`170`	`self.needed_outputs = []`
`170`	`171`	`self.plugin_args = {}`
	`172`	`+`
	`173`	`+ self._interface.num_threads = n_procs`
	`174`	`+ if mem_gb is not None:`
	`175`	`+ self._interface.estimated_memory_gb = mem_gb`
	`176`	`+`
`171`	`177`	`if needed_outputs:`
`172`	`178`	`self.needed_outputs = sorted(needed_outputs)`
`173`	`179`	`self._got_inputs = False`

Lines changed: 6 additions & 5 deletions

Original file line number	Diff line number	Diff line change
`@@ -231,11 +231,12 @@ def _send_procs_to_workers(self, updatehash=False, graph=None):`
`231`	`231`	`busy_processors += self.procs[jobid]._interface.num_threads`
`232`	`232`
`233`	`233`	`else:`
`234`		`- raise ValueError("Resources required by jobid %d (%f GB, %d threads)"`
`235`		`- "exceed what is available on the system (%f GB, %d threads)"%(jobid,`
`236`		`- self.procs[jobid].__interface.estimated_memory_gb,`
`237`		`- self.procs[jobid].__interface.num_threads,`
`238`		`- self.memory_gb,self.processors))`
	`234`	`+ raise ValueError(`
	`235`	`+ "Resources required by jobid {0} ({3}GB, {4} threads) exceed what is "`
	`236`	`+ "available on the system ({1}GB, {2} threads)".format(`
	`237`	`+ jobid, self.memory_gb, self.processors,`
	`238`	`+ self.procs[jobid]._interface.estimated_memory_gb,`
	`239`	`+ self.procs[jobid]._interface.num_threads))`
`239`	`240`
`240`	`241`	`free_memory_gb = self.memory_gb - busy_memory_gb`
`241`	`242`	`free_processors = self.processors - busy_processors`

Comments

(0)