fix max sequence length for xlmr transform (#1495)

parmeet · web-flow · commit 776a15daed49 · 2022-01-07T22:15:33.000-05:00
diff --git a/torchtext/models/roberta/bundler.py b/torchtext/models/roberta/bundler.py
@@ -158,7 +158,7 @@ def encoderConf(self) -> RobertaEncoderConf:
     transform=lambda: T.Sequential(
         T.SentencePieceTokenizer(urljoin(_TEXT_BUCKET, "xlmr.sentencepiece.bpe.model")),
         T.VocabTransform(load_state_dict_from_url(urljoin(_TEXT_BUCKET, "xlmr.vocab.pt"))),
-        T.Truncate(510),
+        T.Truncate(254),
         T.AddToken(token=0, begin=True),
         T.AddToken(token=2, begin=False),
     )

Original file line number	Diff line number	Diff line change
`@@ -158,7 +158,7 @@ def encoderConf(self) -> RobertaEncoderConf:`
`158`	`158`	`transform=lambda: T.Sequential(`
`159`	`159`	`T.SentencePieceTokenizer(urljoin(_TEXT_BUCKET, "xlmr.sentencepiece.bpe.model")),`
`160`	`160`	`T.VocabTransform(load_state_dict_from_url(urljoin(_TEXT_BUCKET, "xlmr.vocab.pt"))),`
`161`		`- T.Truncate(510),`
	`161`	`+ T.Truncate(254),`
`162`	`162`	`T.AddToken(token=0, begin=True),`
`163`	`163`	`T.AddToken(token=2, begin=False),`
`164`	`164`	`)`