Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Sign up
Appearance settings

Commit 01a2772

Browse files
Merge pull request #2985 from oesteban/fix/loading-results
FIX: Use ``load_resultfile`` when loading a results pickle
2 parents e788059 + 88b6fc0 commit 01a2772

File tree

4 files changed

+44
-51
lines changed

4 files changed

+44
-51
lines changed

‎nipype/pipeline/engine/nodes.py‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -194,7 +194,8 @@ def interface(self):
194194
@property
195195
def result(self):
196196
"""Get result from result file (do not hold it in memory)"""
197-
return _load_resultfile(self.output_dir(), self.name)[0]
197+
return _load_resultfile(
198+
op.join(self.output_dir(), 'result_%s.pklz' % self.name))[0]
198199

199200
@property
200201
def inputs(self):
@@ -517,7 +518,7 @@ def _get_inputs(self):
517518
logger.debug('input: %s', key)
518519
results_file = info[0]
519520
logger.debug('results file: %s', results_file)
520-
outputs = loadpkl(results_file).outputs
521+
outputs = _load_resultfile(results_file)[0].outputs
521522
if outputs is None:
522523
raise RuntimeError("""\
523524
Error populating the input "%s" of node "%s": the results file of the source node \
@@ -564,7 +565,8 @@ def _run_interface(self, execute=True, updatehash=False):
564565

565566
def _load_results(self):
566567
cwd = self.output_dir()
567-
result, aggregate, attribute_error = _load_resultfile(cwd, self.name)
568+
result, aggregate, attribute_error = _load_resultfile(
569+
op.join(cwd, 'result_%s.pklz' % self.name))
568570
# try aggregating first
569571
if aggregate:
570572
logger.debug('aggregating results')

‎nipype/pipeline/engine/utils.py‎

Lines changed: 6 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,6 @@
1616

1717
from traceback import format_exception
1818
from hashlib import sha1
19-
import gzip
2019

2120
from functools import reduce
2221

@@ -25,6 +24,7 @@
2524

2625
from ... import logging, config, LooseVersion
2726
from ...utils.filemanip import (
27+
Path,
2828
relpath,
2929
makedirs,
3030
fname_presuffix,
@@ -34,6 +34,7 @@
3434
FileNotFoundError,
3535
save_json,
3636
savepkl,
37+
loadpkl,
3738
write_rst_header,
3839
write_rst_dict,
3940
write_rst_list,
@@ -233,7 +234,7 @@ def save_resultfile(result, cwd, name):
233234
logger.debug('saved results in %s', resultsfile)
234235

235236

236-
def load_resultfile(path, name):
237+
def load_resultfile(results_file):
237238
"""
238239
Load InterfaceResult file from path
239240
@@ -253,17 +254,12 @@ def load_resultfile(path, name):
253254
rerun
254255
"""
255256
aggregate = True
256-
resultsoutputfile = os.path.join(path, 'result_%s.pklz'%name)
257+
results_file = Path(results_file)
257258
result = None
258259
attribute_error = False
259-
if os.path.exists(resultsoutputfile):
260-
pkl_file = gzip.open(resultsoutputfile, 'rb')
260+
if results_file.exists():
261261
try:
262-
result = pickle.load(pkl_file)
263-
except UnicodeDecodeError:
264-
# Was this pickle created with Python 2.x?
265-
pickle.load(pkl_file, fix_imports=True, encoding='utf-8')
266-
logger.warning('Successfully loaded pkl in compatibility mode')
262+
result = loadpkl(results_file)
267263
except (traits.TraitError, AttributeError, ImportError,
268264
EOFError) as err:
269265
if isinstance(err, (AttributeError, ImportError)):
@@ -275,8 +271,6 @@ def load_resultfile(path, name):
275271
'some file does not exist. hence trait cannot be set')
276272
else:
277273
aggregate = False
278-
finally:
279-
pkl_file.close()
280274

281275
logger.debug('Aggregate: %s', aggregate)
282276
return result, aggregate, attribute_error

‎nipype/pipeline/plugins/base.py‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -18,9 +18,8 @@
1818
import numpy as np
1919

2020
from ... import logging
21-
from ...utils.filemanip import loadpkl
2221
from ...utils.misc import str2bool
23-
from ..engine.utils import topological_sort
22+
from ..engine.utils import topological_sort, load_resultfile
2423
from ..engine import MapNode
2524
from .tools import report_crash, report_nodes_not_run, create_pyscript
2625

@@ -504,7 +503,7 @@ def _get_result(self, taskid):
504503
result_data['traceback'] = '\n'.join(format_exception(*sys.exc_info()))
505504
else:
506505
results_file = glob(os.path.join(node_dir, 'result_*.pklz'))[0]
507-
result_data = loadpkl(results_file)
506+
result_data = load_resultfile(results_file)
508507
result_out = dict(result=None, traceback=None)
509508
if isinstance(result_data, dict):
510509
result_out['result'] = result_data['result']
@@ -602,7 +601,7 @@ def _get_result(self, taskid):
602601
glob(os.path.join(node_dir, 'result_*.pklz')).pop()
603602

604603
results_file = glob(os.path.join(node_dir, 'result_*.pklz'))[0]
605-
result_data = loadpkl(results_file)
604+
result_data = load_resultfile(results_file)
606605
result_out = dict(result=None, traceback=None)
607606

608607
if isinstance(result_data, dict):

‎nipype/utils/filemanip.py‎

Lines changed: 30 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -676,52 +676,50 @@ def loadcrash(infile, *args):
676676

677677

678678
def loadpkl(infile, versioning=False):
679-
"""Load a zipped or plain cPickled file
680-
"""
679+
"""Load a zipped or plain cPickled file."""
680+
infile=Path(infile)
681681
fmlogger.debug('Loading pkl: %s', infile)
682-
if infile.endswith('pklz'):
683-
pkl_file = gzip.open(infile, 'rb')
684-
else:
685-
pkl_file = open(infile, 'rb')
682+
pklopen = gzip.open if infile.suffix == '.pklz' else open
683+
pkl_metadata = None
686684

687-
ifversioning:
688-
pkl_metadata = {}
685+
withindirectory(str(infile.parent)):
686+
pkl_file = pklopen(infile.name, 'rb')
689687

690-
# Look if pkl file contains version file
691-
try:
688+
try: # Look if pkl file contains version file
692689
pkl_metadata_line = pkl_file.readline()
693690
pkl_metadata = json.loads(pkl_metadata_line)
694-
except:
691+
except (UnicodeDecodeError, json.JSONDecodeError):
692+
pass
693+
finally:
695694
# Could not get version info
696695
pkl_file.seek(0)
697696

698-
try:
699697
try:
700698
unpkl = pickle.load(pkl_file)
701699
except UnicodeDecodeError:
700+
# Was this pickle created with Python 2.x?
702701
unpkl = pickle.load(pkl_file, fix_imports=True, encoding='utf-8')
703-
704-
return unpkl
705-
706-
# Unpickling problems
707-
except Exception as e:
708-
if not versioning:
702+
fmlogger.info('Successfully loaded pkl in compatibility mode.')
703+
# Unpickling problems
704+
except Exception as e:
705+
if not versioning:
706+
raise e
707+
708+
if pkl_metadata and 'version' in pkl_metadata:
709+
from nipype import __version__ as version
710+
if pkl_metadata['version'] != version:
711+
fmlogger.error("""\
712+
Attempted to open a results file generated by Nipype version %s, \
713+
with an incompatible Nipype version (%s)""", pkl_metadata['version'], version)
714+
raise e
715+
fmlogger.error("""\
716+
No metadata was found in the pkl file. Make sure you are currently using \
717+
the same Nipype version from the generated pkl.""")
709718
raise e
710-
711-
from nipype import __version__ as version
712-
713-
if 'version' in pkl_metadata:
714-
if pkl_metadata['version'] != version:
715-
fmlogger.error('Your Nipype version is: %s',
716-
version)
717-
fmlogger.error('Nipype version of the pkl is: %s',
718-
pkl_metadata['version'])
719719
else:
720-
fmlogger.error('No metadata was found in the pkl file.')
721-
fmlogger.error('Make sure that you are using the same Nipype'
722-
'version from the generated pkl.')
723-
724-
raise e
720+
return unpkl
721+
finally:
722+
pkl_file.close()
725723

726724

727725
def crash2txt(filename, record):

0 commit comments

Comments
(0)

AltStyle によって変換されたページ (->オリジナル) /