Commit 619b94c

authored

Merge pull request PaddlePaddle#2026 from luyaojie/develop

[DuUIE] encoding fixing for windows

2 parents fcd3d1a + 0e4b2ca commit 619b94cCopy full SHA for 619b94c

7 files changed

Lines changed: 87 additions & 62 deletions

File tree

examples/information_extraction/DuUIE
- README.md
- config
  - multi-task-duuie.yaml
- inference.py
- process_data.py
- run_seq2struct.py
- uie
  - evaluation
    - sel2record.py
  - seq2struct
    - utils.py

`‎examples/information_extraction/DuUIE/README.md‎`

Lines changed: 2 additions & 2 deletions

Original file line number	Diff line number	Diff line change
`@@ -153,7 +153,7 @@ python3 run_seq2struct.py \`
`153`	`153`	`--learning_rate=5e-4 \`
`154`	`154`	`--seed=42 \`
`155`	`155`	`--overwrite_output_dir \`
`156`		`- --gradient_accumulation_steps 1`
	`156`	`+ --gradient_accumulation_steps 2`
`157`	`157`	```
`158`	`158`
`159`	`159`	训练完成后,将生成对应的文件夹 `output/duuie_multi_task_b32_lr5e-4`
`@@ -165,7 +165,7 @@ python3 run_seq2struct.py \`
`165`	`165`
`166`	`166`	``` bash
`167`	`167`	`python process_data.py split-test`
`168`		`-python inference.py --data data/duuie_test_a/* --model output/duuie_multi_task_b32_lr5e-4`
	`168`	`+python inference.py --data data/duuie_test_a --model output/duuie_multi_task_b32_lr5e-4`
`169`	`169`	```
`170`	`170`
`171`	`171`	`### 快速基线第四步:后处理提交结果`

`‎examples/information_extraction/DuUIE/config/multi-task-duuie.yaml‎`

Lines changed: 21 additions & 21 deletions

Original file line number	Diff line number	Diff line change
`@@ -4,61 +4,61 @@ T1:`
`4`	`4`	`sel2record: longer_first_zh`
`5`	`5`	`eval_match_mode: set`
`6`	`6`	`metrics:`
`7`		`- - string-rel-strict-F1`
	`7`	`+ - string-rel-boundary-F1`
`8`	`8`
`9`	`9`	`T2:`
`10`	`10`	`name: DUIE_ORG_SPO`
`11`	`11`	`path: data/duuie_pre/DUIE_ORG_SPO`
`12`	`12`	`sel2record: longer_first_zh`
`13`	`13`	`eval_match_mode: set`
`14`	`14`	`metrics:`
`15`		`- - string-rel-strict-F1`
	`15`	`+ - string-rel-boundary-F1`
`16`	`16`
`17`	`17`	`T3:`
`18`	`18`	`name: 体育竞赛`
`19`	`19`	`path: data/duuie_pre/体育竞赛`
`20`	`20`	`sel2record: longer_first_zh`
`21`		`- eval_match_mode: normal`
	`21`	`+ eval_match_mode: set`
`22`	`22`	`metrics:`
`23`	`23`	`- string-evt-role-F1`
`24`	`24`
`25`	`25`	`T4:`
`26`	`26`	`name: 灾害意外`
`27`	`27`	`path: data/duuie_pre/灾害意外`
`28`	`28`	`sel2record: longer_first_zh`
`29`		`- eval_match_mode: normal`
	`29`	`+ eval_match_mode: set`
`30`	`30`	`metrics:`
`31`	`31`	`- string-evt-role-F1`
`32`	`32`
`33`	`33`	`T5:`
`34`	`34`	`name: CONV-ASA`
`35`	`35`	`path: data/duuie_pre/CONV-ASA`
`36`	`36`	`sel2record: longer_first_zh`
`37`		`- eval_match_mode: normal`
	`37`	`+ eval_match_mode: set`
`38`	`38`	`metrics:`
`39`	`39`	`- string-rel-strict-F1`
`40`	`40`
`41`	`41`	`T6:`
`42`	`42`	`name: MSRA`
`43`	`43`	`path: data/duuie_pre/MSRA`
`44`	`44`	`sel2record: longer_first_zh`
`45`		`- eval_match_mode: normal`
	`45`	`+ eval_match_mode: set`
`46`	`46`	`metrics:`
`47`	`47`	`- string-ent-F1`
`48`	`48`
`49`	`49`	`T7:`
`50`	`50`	`name: PEOPLE_DAILY`
`51`	`51`	`path: data/duuie_pre/PEOPLE_DAILY`
`52`	`52`	`sel2record: longer_first_zh`
`53`		`- eval_match_mode: normal`
	`53`	`+ eval_match_mode: set`
`54`	`54`	`metrics:`
`55`	`55`	`- string-ent-F1`
`56`	`56`
`57`	`57`	`T8:`
`58`	`58`	`name: 金融信息_中标`
`59`	`59`	`path: data/duuie_pre/金融信息_中标`
`60`	`60`	`sel2record: longer_first_zh`
`61`		`- eval_match_mode: normal`
	`61`	`+ eval_match_mode: set`
`62`	`62`	`metrics:`
`63`	`63`	`- string-evt-role-F1`
`64`	`64`
`@@ -67,7 +67,7 @@ T9:`
`67`	`67`	`name: 金融信息_企业融资`
`68`	`68`	`path: data/duuie_pre/金融信息_企业融资`
`69`	`69`	`sel2record: longer_first_zh`
`70`		`- eval_match_mode: normal`
	`70`	`+ eval_match_mode: set`
`71`	`71`	`metrics:`
`72`	`72`	`- string-evt-role-F1`
`73`	`73`
`@@ -76,7 +76,7 @@ T10:`
`76`	`76`	`name: 金融信息_股份回购`
`77`	`77`	`path: data/duuie_pre/金融信息_股份回购`
`78`	`78`	`sel2record: longer_first_zh`
`79`		`- eval_match_mode: normal`
	`79`	`+ eval_match_mode: set`
`80`	`80`	`metrics:`
`81`	`81`	`- string-evt-role-F1`
`82`	`82`
`@@ -85,7 +85,7 @@ T11:`
`85`	`85`	`name: 金融信息_中标`
`86`	`86`	`path: data/duuie_pre/金融信息_中标`
`87`	`87`	`sel2record: longer_first_zh`
`88`		`- eval_match_mode: normal`
	`88`	`+ eval_match_mode: set`
`89`	`89`	`metrics:`
`90`	`90`	`- string-evt-role-F1`
`91`	`91`
`@@ -94,7 +94,7 @@ T12:`
`94`	`94`	`name: 金融信息_高管变动`
`95`	`95`	`path: data/duuie_pre/金融信息_高管变动`
`96`	`96`	`sel2record: longer_first_zh`
`97`		`- eval_match_mode: normal`
	`97`	`+ eval_match_mode: set`
`98`	`98`	`metrics:`
`99`	`99`	`- string-evt-role-F1`
`100`	`100`
`@@ -103,70 +103,70 @@ T13:`
`103`	`103`	`name: 金融信息_亏损`
`104`	`104`	`path: data/duuie_pre/金融信息_亏损`
`105`	`105`	`sel2record: longer_first_zh`
`106`		`- eval_match_mode: normal`
	`106`	`+ eval_match_mode: set`
`107`	`107`	`metrics:`
`108`	`108`	`- string-evt-role-F1`
`109`	`109`
`110`	`110`	`T14:`
`111`	`111`	`name: 金融信息_公司上市`
`112`	`112`	`path: data/duuie_pre/金融信息_公司上市`
`113`	`113`	`sel2record: longer_first_zh`
`114`		`- eval_match_mode: normal`
	`114`	`+ eval_match_mode: set`
`115`	`115`	`metrics:`
`116`	`116`	`- string-evt-role-F1`
`117`	`117`
`118`	`118`	`T15:`
`119`	`119`	`name: 金融信息_被约谈`
`120`	`120`	`path: data/duuie_pre/金融信息_被约谈`
`121`	`121`	`sel2record: longer_first_zh`
`122`		`- eval_match_mode: normal`
	`122`	`+ eval_match_mode: set`
`123`	`123`	`metrics:`
`124`	`124`	`- string-evt-role-F1`
`125`	`125`
`126`	`126`	`T16:`
`127`	`127`	`name: 金融信息_企业收购`
`128`	`128`	`path: data/duuie_pre/金融信息_企业收购`
`129`	`129`	`sel2record: longer_first_zh`
`130`		`- eval_match_mode: normal`
	`130`	`+ eval_match_mode: set`
`131`	`131`	`metrics:`
`132`	`132`	`- string-evt-role-F1`
`133`	`133`
`134`	`134`	`T17:`
`135`	`135`	`name: 金融信息_股东减持`
`136`	`136`	`path: data/duuie_pre/金融信息_股东减持`
`137`	`137`	`sel2record: longer_first_zh`
`138`		`- eval_match_mode: normal`
	`138`	`+ eval_match_mode: set`
`139`	`139`	`metrics:`
`140`	`140`	`- string-evt-role-F1`
`141`	`141`
`142`	`142`	`T18:`
`143`	`143`	`name: 金融信息_解除质押`
`144`	`144`	`path: data/duuie_pre/金融信息_解除质押`
`145`	`145`	`sel2record: longer_first_zh`
`146`		`- eval_match_mode: normal`
	`146`	`+ eval_match_mode: set`
`147`	`147`	`metrics:`
`148`	`148`	`- string-evt-role-F1`
`149`	`149`
`150`	`150`	`T19:`
`151`	`151`	`name: 金融信息_企业破产`
`152`	`152`	`path: data/duuie_pre/金融信息_企业破产`
`153`	`153`	`sel2record: longer_first_zh`
`154`		`- eval_match_mode: normal`
	`154`	`+ eval_match_mode: set`
`155`	`155`	`metrics:`
`156`	`156`	`- string-evt-role-F1`
`157`	`157`
`158`	`158`	`T20:`
`159`	`159`	`name: 金融信息_股东增持`
`160`	`160`	`path: data/duuie_pre/金融信息_股东增持`
`161`	`161`	`sel2record: longer_first_zh`
`162`		`- eval_match_mode: normal`
	`162`	`+ eval_match_mode: set`
`163`	`163`	`metrics:`
`164`	`164`	`- string-evt-role-F1`
`165`	`165`
`166`	`166`	`T21:`
`167`	`167`	`name: 金融信息_质押`
`168`	`168`	`path: data/duuie_pre/金融信息_质押`
`169`	`169`	`sel2record: longer_first_zh`
`170`		`- eval_match_mode: normal`
	`170`	`+ eval_match_mode: set`
`171`	`171`	`metrics:`
`172`	`172`	`- string-evt-role-F1`

`‎examples/information_extraction/DuUIE/inference.py‎`

Lines changed: 13 additions & 8 deletions

Original file line number	Diff line number	Diff line change
`@@ -16,7 +16,7 @@`
`16`	`16`
`17`	`17`
`18`	`18`	`def read_json_file(file_name):`
`19`		`- return [json.loads(line) for line in open(file_name)]`
	`19`	`+ return [json.loads(line) for line in open(file_name, encoding='utf8')]`
`20`	`20`
`21`	`21`
`22`	`22`	`def schema_to_ssi(schema: RecordSchema):`
`@@ -75,15 +75,19 @@ def to_tensor(x):`
`75`	`75`	`return [post_processing(x) for x in pred]`
`76`	`76`
`77`	`77`
	`78`	`+def find_to_predict_folder(folder_name):`
	`79`	`+ for root, dirs, _ in os.walk(folder_name):`
	`80`	`+ for dirname in dirs:`
	`81`	`+ data_name = os.path.join(root, dirname)`
	`82`	`+ if os.path.exists(os.path.join(data_name, 'record.schema')):`
	`83`	`+ yield data_name`
	`84`	`+`
	`85`	`+`
`78`	`86`	`def main():`
`79`	`87`	`import argparse`
`80`	`88`	`parser = argparse.ArgumentParser()`
`81`	`89`	`parser.add_argument(`
`82`		`- '--data',`
`83`		`- '-d',`
`84`		`- required=True,`
`85`		`- nargs='+',`
`86`		`- help='Folder need to been predicted.')`
	`90`	`+ '--data', '-d', required=True, help='Folder need to been predicted.')`
`87`	`91`	`parser.add_argument(`
`88`	`92`	`'--model', '-m', required=True, help='Trained model for inference')`
`89`	`93`	`parser.add_argument(`
`@@ -102,7 +106,8 @@ def main():`
`102`	`106`	`parser.add_argument('--verbose', action='store_true')`
`103`	`107`	`options = parser.parse_args()`
`104`	`108`
`105`		`- data_folder = options.data`
	`109`	+ # Find the folder need to be predicted with `record.schema`
	`110`	`+ data_folder = find_to_predict_folder(options.data)`
`106`	`111`	`model_path = options.model`
`107`	`112`
`108`	`113`	`predictor = Predictor(`
`@@ -142,7 +147,7 @@ def main():`
`142`	`147`	`records += [sel2record.sel2record(pred=p, text=text, tokens=tokens)]`
`143`	`148`
`144`	`149`	`pred_filename = os.path.join(f"{task_folder}", "pred.json")`
`145`		`- with open(pred_filename, 'w') as output:`
	`150`	`+ with open(pred_filename, 'w', encoding='utf8') as output:`
`146`	`151`	`for record in records:`
`147`	`152`	`output.write(json.dumps(record, ensure_ascii=False) + '\n')`
`148`	`153`

0 commit comments

Comments

(0)

Navigation Menu

Search code, repositories, users, issues, pull requests...

Provide feedback

Saved searches

Use saved searches to filter your results more quickly