[fbsync] Fix regression on Detection training script (#5985)

Reviewed By: datumbox

Differential Revision: D36413361

fbshipit-source-id: 8e61cae70c97255ebc143991eaad5a07633d317c

Author: NicolasHug

references/detection/train.py

@@ -158,10 +158,7 @@ def main(args):
     device = torch.device(args.device)
 
     if args.use_deterministic_algorithms:
-        torch.backends.cudnn.benchmark = False
         torch.use_deterministic_algorithms(True)
-    else:
-        torch.backends.cudnn.benchmark = True
 
     # Data loading code
     print("Loading data")
@@ -253,8 +250,6 @@ def main(args):
             scaler.load_state_dict(checkpoint["scaler"])
 
     if args.test_only:
-        # We disable the cudnn benchmarking because it can noticeably affect the accuracy
-        torch.backends.cudnn.benchmark = False
         torch.backends.cudnn.deterministic = True
         evaluate(model, data_loader_test, device=device)
         return