Commit 2dafb5d

committed

retry loop for file writes

1 parent 45ad14f commit 2dafb5dCopy full SHA for 2dafb5d

File tree

+15

-2

lines changed

+15

-2

lines changed

Lines changed: 1 addition & 0 deletions

Original file line number	Diff line number	Diff line change
`@@ -15,6 +15,7 @@ class Config:`
`15`	`15`	`output_append = True # if the output file exists, append to it`
`16`	`16`	`output_dir = '.'`
`17`	`17`	`output_file_name_template = '{{name}}.md' # a mustache template for the output file name`
	`18`	`+ write_attempts = 3`
`18`	`19`
`19`	`20`	`# Language and style`
`20`	`21`	`language_name = 'python'`

Lines changed: 14 additions & 2 deletions

Original file line number	Diff line number	Diff line change
`@@ -2,6 +2,8 @@`
`2`	`2`	`import glob`
`3`	`3`
`4`	`4`	`import pystache`
	`5`	`+import time`
	`6`	`+import random`
`5`	`7`
`6`	`8`	`from snippet.config import Config`
`7`	`9`	`from snippet.logs import logger`
`@@ -28,8 +30,18 @@ def write_example(config: Config, example_name, example_block):`
`28`	`30`	`os.makedirs(config.output_dir)`
`29`	`31`	`output_file = os.path.join(config.output_dir, output_file_name)`
`30`	`32`	`logger.info('writing %r to %s', example_name, output_file)`
`31`		`- with open(output_file, 'a' if config.output_append else 'w') as fh:`
`32`		`- fh.write(output)`
	`33`	`+ for i in range(1, config.write_attempts + 1):`
	`34`	`+ # we run a retry loop as there may be contention on the output file in`
	`35`	`+ # a multi-process environment`
	`36`	`+ try:`
	`37`	`+ with open(output_file, 'a' if config.output_append else 'w') as fh:`
	`38`	`+ fh.write(output)`
	`39`	`+ break`
	`40`	`+ except IOError as err:`
	`41`	`+ time.sleep(i * 0.5 + 0.1 * random.randint(0, 5))`
	`42`	`+ logger.info('write failed (%s) retrying attempt: %s', err, i)`
	`43`	`+ else:`
	`44`	`+ raise IOError('could not write output file after %s attempts' % config.write_attempts)`
`33`	`45`
`34`	`46`
`35`	`47`	`def load_file_lines(path):`

Comments

(0)