Skip to content

Commit fcf88ae

Browse files
JulianJulian
committed
Add a way to run the test suite with xfail instead of skip.
Makes it easier to see when something's fixed. As part of this, also 'explode' which tests we skip, so now it's just those that fail, even though that can be lots more verbose.
1 parent 86a5fff commit fcf88ae

File tree

2 files changed

+189
-21
lines changed

2 files changed

+189
-21
lines changed

‎jsonschema/tests/_suite.py

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -194,7 +194,12 @@ def fn(this):
194194

195195
fn.__name__ = self.method_name
196196
reason = skip(self)
197-
return unittest.skipIf(reason is not None, reason)(fn)
197+
if reason is None:
198+
return fn
199+
elif os.environ.get("JSON_SCHEMA_EXPECTED_FAILURES", "0") != "0":
200+
return unittest.expectedFailure(fn)
201+
else:
202+
return unittest.skip(reason)(fn)
198203

199204
def validate(self, Validator, **kwargs):
200205
Validator.check_schema(self.schema)

‎jsonschema/tests/test_jsonschema_test_suite.py

Lines changed: 183 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88

99
import sys
1010

11-
from jsonschema.tests._helpers import bug
11+
from jsonschema.tests._helpers import bug, test_suite_bug
1212
from jsonschema.tests._suite import Suite
1313
import jsonschema
1414

@@ -160,6 +160,7 @@ def leap_second(test):
160160
or skip(
161161
message=bug(371),
162162
subject="ref",
163+
valid=False,
163164
case_description=(
164165
"$ref prevents a sibling id from changing the base uri"
165166
),
@@ -244,16 +245,6 @@ def leap_second(test):
244245
or leap_second(test)
245246
or missing_format(jsonschema.draft6_format_checker)(test)
246247
or complex_email_validation(test)
247-
or skip(
248-
message="id is incorrectly finding non-ids",
249-
subject="id",
250-
description="const at const_not_anchor does not match",
251-
)(test)
252-
or skip(
253-
message="id is incorrectly finding non-ids",
254-
subject="id",
255-
description="const at const_not_id does not match",
256-
)(test)
257248
or skip(
258249
message=bug(),
259250
subject="refRemote",
@@ -342,31 +333,152 @@ def leap_second(test):
342333
skip(
343334
message="unevaluatedItems is different in 2019-09 (needs work).",
344335
subject="unevaluatedItems",
336+
description="uncle keyword evaluation is not significant",
337+
)(test)
338+
or skip(
339+
message="unevaluatedItems is different in 2019-09 (needs work).",
340+
subject="unevaluatedItems",
341+
description="when one schema matches and has unevaluated items",
342+
)(test)
343+
or skip(
344+
message="unevaluatedItems is different in 2019-09 (needs work).",
345+
subject="unevaluatedItems",
346+
description="when two schemas match and has unevaluated items",
345347
)(test)
346348
or skip(
347-
message="dynamicRef support isn't working yet.",
349+
message="unevaluatedItems is different in 2019-09 (needs work).",
350+
subject="unevaluatedItems",
351+
description="when if matches and it has unevaluated items",
352+
)(test)
353+
or skip(
354+
message="unevaluatedItems is different in 2019-09 (needs work).",
355+
subject="unevaluatedItems",
356+
case_description="unevaluatedItems with nested tuple",
357+
description="with unevaluated items",
358+
)(test)
359+
or skip(
360+
message="unevaluatedItems is different in 2019-09 (needs work).",
361+
subject="unevaluatedItems",
362+
case_description="unevaluatedItems with not",
363+
description="with unevaluated items",
364+
)(test)
365+
or skip(
366+
message="unevaluatedItems is different in 2019-09 (needs work).",
367+
subject="unevaluatedItems",
368+
case_description="unevaluatedItems with oneOf",
369+
description="with unevaluated items",
370+
)(test)
371+
or skip(
372+
message="unevaluatedItems is different in 2019-09 (needs work).",
373+
subject="unevaluatedItems",
374+
case_description="unevaluatedItems with $ref",
375+
description="with unevaluated items",
376+
)(test)
377+
or skip(
378+
message="unevaluatedItems is different in 2019-09 (needs work).",
379+
subject="unevaluatedItems",
380+
case_description="unevaluatedItems with tuple",
381+
description="with unevaluated items",
382+
)(test)
383+
or skip(
384+
message="unevaluatedItems is different in 2019-09 (needs work).",
385+
subject="unevaluatedItems",
386+
description="when if doesn't match and it has unevaluated items",
387+
)(test)
388+
or skip(
389+
message="recursiveRef support isn't working yet.",
348390
subject="recursiveRef",
391+
case_description=(
392+
"$recursiveRef with no $recursiveAnchor in "
393+
"the initial target schema resource"
394+
),
395+
description=(
396+
"leaf node does not match: recursion uses the inner schema"
397+
),
398+
)(test)
399+
or skip(
400+
message="recursiveRef support isn't working yet.",
401+
subject="recursiveRef",
402+
description="leaf node matches: recursion uses the inner schema",
403+
)(test)
404+
or skip(
405+
message="recursiveRef support isn't working yet.",
406+
subject="recursiveRef",
407+
case_description=(
408+
"dynamic $recursiveRef destination (not predictable "
409+
"at schema compile time)"
410+
),
411+
description="integer node",
349412
)(test)
350413
or skip(
351-
message="These tests depends on dynamicRef working.",
414+
message="recursiveRef support isn't working yet.",
415+
subject="recursiveRef",
416+
case_description=(
417+
"multiple dynamic paths to the $recursiveRef keyword"
418+
),
419+
description="recurse to integerNode - floats are not allowed",
420+
)(test)
421+
or skip(
422+
message="recursiveRef support isn't working yet.",
423+
subject="recursiveRef",
424+
description="integer does not match as a property value",
425+
)(test)
426+
or skip(
427+
message="recursiveRef support isn't working yet.",
428+
subject="recursiveRef",
429+
description=(
430+
"leaf node does not match: "
431+
"recursion only uses inner schema"
432+
),
433+
)(test)
434+
or skip(
435+
message="recursiveRef support isn't working yet.",
436+
subject="recursiveRef",
437+
description=(
438+
"leaf node matches: "
439+
"recursion only uses inner schema"
440+
),
441+
)(test)
442+
or skip(
443+
message="recursiveRef support isn't working yet.",
444+
subject="recursiveRef",
445+
description=(
446+
"two levels, integer does not match as a property value"
447+
),
448+
)(test)
449+
or skip(
450+
message="recursiveRef support isn't working yet.",
451+
subject="recursiveRef",
452+
description="recursive mismatch",
453+
)(test)
454+
or skip(
455+
message="recursiveRef support isn't working yet.",
456+
subject="recursiveRef",
457+
description="two levels, no match",
458+
)(test)
459+
or skip(
460+
message="recursiveRef support isn't working yet.",
352461
subject="id",
353462
case_description=(
354463
"Invalid use of fragments in location-independent $id"
355464
),
356465
)(test)
357466
or skip(
358-
message="These tests depends on dynamicRef working.",
467+
message="dynamicRef support isn't fully working yet.",
359468
subject="defs",
360469
description="invalid definition schema",
361470
)(test)
362471
or skip(
363-
message="These tests depends on dynamicRef working.",
472+
message="dynamicRef support isn't fully working yet.",
364473
subject="anchor",
365474
case_description="same $anchor with different base uri",
366475
)(test)
367476
or skip(
368-
message="Vocabulary support is not yet present.",
477+
message=test_suite_bug(574),
369478
subject="vocabulary",
479+
description=(
480+
"no validation: invalid number, but it still validates"
481+
),
370482
)(test)
371483
or skip(
372484
message=bug(),
@@ -404,21 +516,72 @@ def leap_second(test):
404516
skip=lambda test: (
405517
narrow_unicode_build(test)
406518
or skip(
407-
message="dynamicRef support isn't working yet.",
519+
message="dynamicRef support isn't fully working yet.",
520+
subject="dynamicRef",
521+
description="The recursive part is not valid against the root",
522+
)(test)
523+
or skip(
524+
message="dynamicRef support isn't fully working yet.",
525+
subject="dynamicRef",
526+
description="correct extended schema",
527+
case_description=(
528+
"Tests for implementation dynamic anchor and reference link. "
529+
"Reference should be independent of any possible ordering."
530+
),
531+
)(test)
532+
or skip(
533+
message="dynamicRef support isn't fully working yet.",
408534
subject="dynamicRef",
535+
description="incorrect extended schema",
536+
case_description=(
537+
"Tests for implementation dynamic anchor and reference link. "
538+
"Reference should be independent of any possible ordering."
539+
),
409540
)(test)
410541
or skip(
411-
message="These tests depends on dynamicRef working.",
542+
message="dynamicRef support isn't fully working yet.",
543+
subject="dynamicRef",
544+
description=(
545+
"/then/$defs/thingy is the final stop for the $dynamicRef"
546+
),
547+
)(test)
548+
or skip(
549+
message="dynamicRef support isn't fully working yet.",
550+
subject="dynamicRef",
551+
description=(
552+
"string matches /$defs/thingy, but the $dynamicRef "
553+
"does not stop here"
554+
),
555+
)(test)
556+
or skip(
557+
message="dynamicRef support isn't fully working yet.",
558+
subject="dynamicRef",
559+
description=(
560+
"string matches /$defs/thingy, but the $dynamicRef "
561+
"does not stop here"
562+
),
563+
)(test)
564+
or skip(
565+
message="dynamicRef support isn't fully working yet.",
566+
subject="dynamicRef",
567+
description="recurse to integerNode - floats are not allowed",
568+
)(test)
569+
or skip(
570+
message="dynamicRef support isn't fully working yet.",
412571
subject="defs",
572+
description="invalid definition schema",
413573
)(test)
414574
or skip(
415-
message="These tests depends on dynamicRef working.",
575+
message="dynamicRef support isn't fully working yet.",
416576
subject="anchor",
417577
case_description="same $anchor with different base uri",
418578
)(test)
419579
or skip(
420-
message="Vocabulary support is not yet present.",
580+
message=test_suite_bug(574),
421581
subject="vocabulary",
582+
description=(
583+
"no validation: invalid number, but it still validates"
584+
),
422585
)(test)
423586
or skip(
424587
message=bug(),

0 commit comments

Comments
 (0)