Commit cfcd2ba

committed

update parallel scripts

1 parent e791a95 commit cfcd2baCopy full SHA for cfcd2ba

File tree

-5

lines changed

-5

lines changed

Lines changed: 3 additions & 3 deletions

Original file line number	Diff line number	Diff line change
`@@ -7,7 +7,7 @@`
`7`	`7`	`from scrapers.scraper import connect_to_base, get_driver, parse_html, write_to_file`
`8`	`8`
`9`	`9`
`10`		`-def run_process(filename, headless):`
	`10`	`+def run_process(number, filename, headless):`
`11`	`11`
`12`	`12`	`# init browser`
`13`	`13`	`browser = get_driver(headless)`
`@@ -39,9 +39,9 @@ def run_process(filename, headless):`
`39`	`39`	`output_timestamp = datetime.datetime.now().strftime("%Y%m%d%H%M%S")`
`40`	`40`	`output_filename = f"output_{output_timestamp}.csv"`
`41`	`41`
`42`		`- # scrape and craw`
	`42`	`+ # scrape and crawl`
`43`	`43`	`with Pool(cpu_count() - 1) as p:`
`44`		`- p.starmap(run_process, repeat(output_filename), repeat(headless)))`
	`44`	`+ p.starmap(run_process, zip(range(1, 21), repeat(output_filename), repeat(headless)))`
`45`	`45`	`p.close()`
`46`	`46`	`p.join()`
`47`	`47`

Lines changed: 2 additions & 2 deletions

Original file line number	Diff line number	Diff line change
`@@ -6,7 +6,7 @@`
`6`	`6`	`from scrapers.scraper import connect_to_base, get_driver, parse_html, write_to_file`
`7`	`7`
`8`	`8`
`9`		`-def run_process(filename, headless):`
	`9`	`+def run_process(number, filename, headless):`
`10`	`10`
`11`	`11`	`# init browser`
`12`	`12`	`browser = get_driver(headless)`
`@@ -43,7 +43,7 @@ def run_process(filename, headless):`
`43`	`43`	`with ProcessPoolExecutor() as executor:`
`44`	`44`	`for number in range(1, 21):`
`45`	`45`	`futures.append(`
`46`		`- executor.submit(run_process, output_filename, headless)`
	`46`	`+ executor.submit(run_process, number, output_filename, headless)`
`47`	`47`	`)`
`48`	`48`
`49`	`49`	`wait(futures)`

Comments

(0)