Skip to content

Commit 6f016dd

Browse files
datumboxdatumbox
authored
Restructuring metrics meta-data (#5859)
* Restructuring metrics meta-data for detection, segmentation and optical flow. * Renaming acc to pixel_acc for segmentation * Restructure video meta-data. * Restructure classification and quantization meta-data. * Fix tests. * Fix documentation
1 parent c82b86d commit 6f016dd

35 files changed

+599
-298
lines changed

‎docs/source/conf.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -347,8 +347,8 @@ def generate_classification_table():
347347
content = [
348348
(
349349
f":class:`{w} <{type(w).__name__}>`",
350-
w.meta["acc@1"],
351-
w.meta["acc@5"],
350+
w.meta["metrics"]["acc@1"],
351+
w.meta["metrics"]["acc@5"],
352352
f"{w.meta['num_params']/1e6:.1f}M",
353353
f"`link <{w.meta['recipe']}>`__",
354354
)

‎test/test_extended_models.py

Lines changed: 24 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -79,20 +79,32 @@ def test_naming_conventions(model_fn):
7979
)
8080
@run_if_test_with_extended
8181
def test_schema_meta_validation(model_fn):
82-
# TODO: add list of permitted fields
83-
classification_fields = ["categories", "acc@1", "acc@5"]
82+
# list of all possible supported high-level fields for weights meta-data
83+
permitted_fields = {
84+
"backend",
85+
"categories",
86+
"keypoint_names",
87+
"license",
88+
"metrics",
89+
"min_size",
90+
"num_params",
91+
"recipe",
92+
"unquantized",
93+
}
94+
# mandatory fields for each computer vision task
95+
classification_fields = {"categories", ("metrics", "acc@1"), ("metrics", "acc@5")}
8496
defaults = {
85-
"all": ["recipe", "num_params", "min_size"],
97+
"all": {"metrics", "min_size", "num_params", "recipe"},
8698
"models": classification_fields,
87-
"detection": ["categories", "map"],
88-
"quantization": classification_fields + ["backend", "unquantized"],
89-
"segmentation": ["categories", "mIoU", "acc"],
99+
"detection": {"categories", ("metrics", "box_map")},
100+
"quantization": classification_fields | {"backend", "unquantized"},
101+
"segmentation": {"categories", ("metrics", "miou"), ("metrics", "pixel_acc")},
90102
"video": classification_fields,
91-
"optical_flow": [],
103+
"optical_flow": set(),
92104
}
93105
model_name = model_fn.__name__
94106
module_name = model_fn.__module__.split(".")[-2]
95-
fields = set(defaults["all"] + defaults[module_name])
107+
fields = defaults["all"] | defaults[module_name]
96108

97109
weights_enum = _get_model_weights(model_fn)
98110
if len(weights_enum) == 0:
@@ -102,9 +114,10 @@ def test_schema_meta_validation(model_fn):
102114
incorrect_params = []
103115
bad_names = []
104116
for w in weights_enum:
105-
missing_fields = fields - set(w.meta.keys())
106-
if missing_fields:
107-
problematic_weights[w] = missing_fields
117+
missing_fields = fields - (set(w.meta.keys()) | set(("metrics", x) for x in w.meta.get("metrics", {}).keys()))
118+
unsupported_fields = set(w.meta.keys()) - permitted_fields
119+
if missing_fields or unsupported_fields:
120+
problematic_weights[w] = {"missing": missing_fields, "unsupported": unsupported_fields}
108121
if w == weights_enum.DEFAULT:
109122
if module_name == "quantization":
110123
# parameters() count doesn't work well with quantization, so we check against the non-quantized

‎torchvision/models/alexnet.py

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -61,8 +61,10 @@ class AlexNet_Weights(WeightsEnum):
6161
"min_size": (63, 63),
6262
"categories": _IMAGENET_CATEGORIES,
6363
"recipe": "https://github.com/pytorch/vision/tree/main/references/classification#alexnet-and-vgg",
64-
"acc@1": 56.522,
65-
"acc@5": 79.066,
64+
"metrics": {
65+
"acc@1": 56.522,
66+
"acc@5": 79.066,
67+
},
6668
},
6769
)
6870
DEFAULT = IMAGENET1K_V1

‎torchvision/models/convnext.py

Lines changed: 16 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -217,8 +217,10 @@ class ConvNeXt_Tiny_Weights(WeightsEnum):
217217
meta={
218218
**_COMMON_META,
219219
"num_params": 28589128,
220-
"acc@1": 82.520,
221-
"acc@5": 96.146,
220+
"metrics": {
221+
"acc@1": 82.520,
222+
"acc@5": 96.146,
223+
},
222224
},
223225
)
224226
DEFAULT = IMAGENET1K_V1
@@ -231,8 +233,10 @@ class ConvNeXt_Small_Weights(WeightsEnum):
231233
meta={
232234
**_COMMON_META,
233235
"num_params": 50223688,
234-
"acc@1": 83.616,
235-
"acc@5": 96.650,
236+
"metrics": {
237+
"acc@1": 83.616,
238+
"acc@5": 96.650,
239+
},
236240
},
237241
)
238242
DEFAULT = IMAGENET1K_V1
@@ -245,8 +249,10 @@ class ConvNeXt_Base_Weights(WeightsEnum):
245249
meta={
246250
**_COMMON_META,
247251
"num_params": 88591464,
248-
"acc@1": 84.062,
249-
"acc@5": 96.870,
252+
"metrics": {
253+
"acc@1": 84.062,
254+
"acc@5": 96.870,
255+
},
250256
},
251257
)
252258
DEFAULT = IMAGENET1K_V1
@@ -259,8 +265,10 @@ class ConvNeXt_Large_Weights(WeightsEnum):
259265
meta={
260266
**_COMMON_META,
261267
"num_params": 197767336,
262-
"acc@1": 84.414,
263-
"acc@5": 96.976,
268+
"metrics": {
269+
"acc@1": 84.414,
270+
"acc@5": 96.976,
271+
},
264272
},
265273
)
266274
DEFAULT = IMAGENET1K_V1

‎torchvision/models/densenet.py

Lines changed: 16 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -279,8 +279,10 @@ class DenseNet121_Weights(WeightsEnum):
279279
meta={
280280
**_COMMON_META,
281281
"num_params": 7978856,
282-
"acc@1": 74.434,
283-
"acc@5": 91.972,
282+
"metrics": {
283+
"acc@1": 74.434,
284+
"acc@5": 91.972,
285+
},
284286
},
285287
)
286288
DEFAULT = IMAGENET1K_V1
@@ -293,8 +295,10 @@ class DenseNet161_Weights(WeightsEnum):
293295
meta={
294296
**_COMMON_META,
295297
"num_params": 28681000,
296-
"acc@1": 77.138,
297-
"acc@5": 93.560,
298+
"metrics": {
299+
"acc@1": 77.138,
300+
"acc@5": 93.560,
301+
},
298302
},
299303
)
300304
DEFAULT = IMAGENET1K_V1
@@ -307,8 +311,10 @@ class DenseNet169_Weights(WeightsEnum):
307311
meta={
308312
**_COMMON_META,
309313
"num_params": 14149480,
310-
"acc@1": 75.600,
311-
"acc@5": 92.806,
314+
"metrics": {
315+
"acc@1": 75.600,
316+
"acc@5": 92.806,
317+
},
312318
},
313319
)
314320
DEFAULT = IMAGENET1K_V1
@@ -321,8 +327,10 @@ class DenseNet201_Weights(WeightsEnum):
321327
meta={
322328
**_COMMON_META,
323329
"num_params": 20013928,
324-
"acc@1": 76.896,
325-
"acc@5": 93.370,
330+
"metrics": {
331+
"acc@1": 76.896,
332+
"acc@5": 93.370,
333+
},
326334
},
327335
)
328336
DEFAULT = IMAGENET1K_V1

‎torchvision/models/detection/faster_rcnn.py

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -383,7 +383,9 @@ class FasterRCNN_ResNet50_FPN_Weights(WeightsEnum):
383383
**_COMMON_META,
384384
"num_params": 41755286,
385385
"recipe": "https://github.com/pytorch/vision/tree/main/references/detection#faster-r-cnn-resnet-50-fpn",
386-
"map": 37.0,
386+
"metrics": {
387+
"box_map": 37.0,
388+
},
387389
},
388390
)
389391
DEFAULT = COCO_V1
@@ -397,7 +399,9 @@ class FasterRCNN_ResNet50_FPN_V2_Weights(WeightsEnum):
397399
**_COMMON_META,
398400
"num_params": 43712278,
399401
"recipe": "https://github.com/pytorch/vision/pull/5763",
400-
"map": 46.7,
402+
"metrics": {
403+
"box_map": 46.7,
404+
},
401405
},
402406
)
403407
DEFAULT = COCO_V1
@@ -411,7 +415,9 @@ class FasterRCNN_MobileNet_V3_Large_FPN_Weights(WeightsEnum):
411415
**_COMMON_META,
412416
"num_params": 19386354,
413417
"recipe": "https://github.com/pytorch/vision/tree/main/references/detection#faster-r-cnn-mobilenetv3-large-fpn",
414-
"map": 32.8,
418+
"metrics": {
419+
"box_map": 32.8,
420+
},
415421
},
416422
)
417423
DEFAULT = COCO_V1
@@ -425,7 +431,9 @@ class FasterRCNN_MobileNet_V3_Large_320_FPN_Weights(WeightsEnum):
425431
**_COMMON_META,
426432
"num_params": 19386354,
427433
"recipe": "https://github.com/pytorch/vision/tree/main/references/detection#faster-r-cnn-mobilenetv3-large-320-fpn",
428-
"map": 22.8,
434+
"metrics": {
435+
"box_map": 22.8,
436+
},
429437
},
430438
)
431439
DEFAULT = COCO_V1

‎torchvision/models/detection/fcos.py

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -655,7 +655,9 @@ class FCOS_ResNet50_FPN_Weights(WeightsEnum):
655655
"categories": _COCO_CATEGORIES,
656656
"min_size": (1, 1),
657657
"recipe": "https://github.com/pytorch/vision/tree/main/references/detection#fcos-resnet-50-fpn",
658-
"map": 39.2,
658+
"metrics": {
659+
"box_map": 39.2,
660+
},
659661
},
660662
)
661663
DEFAULT = COCO_V1

‎torchvision/models/detection/keypoint_rcnn.py

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -322,8 +322,10 @@ class KeypointRCNN_ResNet50_FPN_Weights(WeightsEnum):
322322
**_COMMON_META,
323323
"num_params": 59137258,
324324
"recipe": "https://github.com/pytorch/vision/issues/1606",
325-
"map": 50.6,
326-
"map_kp": 61.1,
325+
"metrics": {
326+
"box_map": 50.6,
327+
"kp_map": 61.1,
328+
},
327329
},
328330
)
329331
COCO_V1 = Weights(
@@ -333,8 +335,10 @@ class KeypointRCNN_ResNet50_FPN_Weights(WeightsEnum):
333335
**_COMMON_META,
334336
"num_params": 59137258,
335337
"recipe": "https://github.com/pytorch/vision/tree/main/references/detection#keypoint-r-cnn",
336-
"map": 54.6,
337-
"map_kp": 65.0,
338+
"metrics": {
339+
"box_map": 54.6,
340+
"kp_map": 65.0,
341+
},
338342
},
339343
)
340344
DEFAULT = COCO_V1

‎torchvision/models/detection/mask_rcnn.py

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -364,8 +364,10 @@ class MaskRCNN_ResNet50_FPN_Weights(WeightsEnum):
364364
**_COMMON_META,
365365
"num_params": 44401393,
366366
"recipe": "https://github.com/pytorch/vision/tree/main/references/detection#mask-r-cnn",
367-
"map": 37.9,
368-
"map_mask": 34.6,
367+
"metrics": {
368+
"box_map": 37.9,
369+
"mask_map": 34.6,
370+
},
369371
},
370372
)
371373
DEFAULT = COCO_V1
@@ -379,8 +381,10 @@ class MaskRCNN_ResNet50_FPN_V2_Weights(WeightsEnum):
379381
**_COMMON_META,
380382
"num_params": 46359409,
381383
"recipe": "https://github.com/pytorch/vision/pull/5773",
382-
"map": 47.4,
383-
"map_mask": 41.8,
384+
"metrics": {
385+
"box_map": 47.4,
386+
"mask_map": 41.8,
387+
},
384388
},
385389
)
386390
DEFAULT = COCO_V1

‎torchvision/models/detection/retinanet.py

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -687,7 +687,9 @@ class RetinaNet_ResNet50_FPN_Weights(WeightsEnum):
687687
**_COMMON_META,
688688
"num_params": 34014999,
689689
"recipe": "https://github.com/pytorch/vision/tree/main/references/detection#retinanet",
690-
"map": 36.4,
690+
"metrics": {
691+
"box_map": 36.4,
692+
},
691693
},
692694
)
693695
DEFAULT = COCO_V1
@@ -701,7 +703,9 @@ class RetinaNet_ResNet50_FPN_V2_Weights(WeightsEnum):
701703
**_COMMON_META,
702704
"num_params": 38198935,
703705
"recipe": "https://github.com/pytorch/vision/pull/5756",
704-
"map": 41.5,
706+
"metrics": {
707+
"box_map": 41.5,
708+
},
705709
},
706710
)
707711
DEFAULT = COCO_V1

‎torchvision/models/detection/ssd.py

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,9 @@ class SSD300_VGG16_Weights(WeightsEnum):
3434
"categories": _COCO_CATEGORIES,
3535
"min_size": (1, 1),
3636
"recipe": "https://github.com/pytorch/vision/tree/main/references/detection#ssd300-vgg16",
37-
"map": 25.1,
37+
"metrics": {
38+
"box_map": 25.1,
39+
},
3840
},
3941
)
4042
DEFAULT = COCO_V1

‎torchvision/models/detection/ssdlite.py

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -193,7 +193,9 @@ class SSDLite320_MobileNet_V3_Large_Weights(WeightsEnum):
193193
"categories": _COCO_CATEGORIES,
194194
"min_size": (1, 1),
195195
"recipe": "https://github.com/pytorch/vision/tree/main/references/detection#ssdlite320-mobilenetv3-large",
196-
"map": 21.3,
196+
"metrics": {
197+
"box_map": 21.3,
198+
},
197199
},
198200
)
199201
DEFAULT = COCO_V1

0 commit comments

Comments
 (0)