Commit ad3bd59

committed

Finalized trainer.

1 parent d105ad8 commit ad3bd59Copy full SHA for ad3bd59

File tree

+15

-8

lines changed

+15

-8

lines changed

Lines changed: 5 additions & 1 deletion

Lines changed: 9 additions & 6 deletions

Original file line number	Diff line number	Diff line change
`@@ -6,11 +6,13 @@`
`6`	`6`	`from transformers import Trainer, TrainingArguments`
`7`	`7`	`import os`
`8`	`8`
	`9`	`+os.environ["WANDB_DISABLED"] = "true"`
	`10`	`+`
`9`	`11`	`def encode(lines):`
`10`	`12`	`return tokenizer(lines['text'], add_special_tokens=True, truncation=True, max_length=512)`
`11`	`13`
`12`	`14`	`TRAIN_BASE = False`
`13`		`-TOKENIZER_DIR = "tokenizer"`
	`15`	`+TOKENIZER_DIR = "../tokenizer"`
`14`	`16`
`15`	`17`	`paths = ["../data.txt"]`
`16`	`18`
`@@ -44,11 +46,11 @@ def encode(lines):`
`44`	`46`
`45`	`47`	`config = GPT2Config(`
`46`	`48`	`vocab_size = tokenizer.vocab_size,`
`47`		`- bos_token_id = tokenizer.bos_token_id,`
`48`		`- eos_token_id = tokenizer.eos_token_id`
	`49`	`+ bos_token = tokenizer.bos_token_id,`
	`50`	`+ eos_token = tokenizer.eos_token_id`
`49`	`51`	`)`
`50`	`52`
`51`		`-model = GPT2LMHeadModel()`
	`53`	`+model = GPT2LMHeadModel(config)`
`52`	`54`
`53`	`55`	`dataset = load_dataset("text", data_files=paths)`
`54`	`56`
`@@ -62,12 +64,13 @@ def encode(lines):`
`62`	`64`
`63`	`65`	`training_args = TrainingArguments(`
`64`	`66`	`output_dir="../GPyT",`
`65`		`- per_device_train_batch_size=10,`
`66`	`67`	`overwrite_output_dir=True,`
`67`	`68`	`num_train_epochs=1,`
	`69`	`+ per_device_train_batch_size=10,`
`68`	`70`	`save_steps=100,`
`69`	`71`	`save_total_limit=2,`
`70`		`- prediction_loss_only=True`
	`72`	`+ prediction_loss_only=True,`
	`73`	`+ remove_unused_columns=False`
`71`	`74`	`)`
`72`	`75`
`73`	`76`	`trainer = Trainer(`

Lines changed: 1 addition & 1 deletion

Original file line number	Diff line number	Diff line change
`@@ -3,7 +3,7 @@`
`3`	`3`	`from transformers import GPT2Config, GPT2LMHeadModel, GPT2Tokenizer`
`4`	`4`
`5`	`5`	`TRAIN_BASE = False`
`6`		`-TOKENIZER_DIR = "tokenizer"`
	`6`	`+TOKENIZER_DIR = "../tokenizer"`
`7`	`7`
`8`	`8`	`paths = ["../data.txt"]`
`9`	`9`

Comments

(0)