Skip to content

Commit e994ef4

Browse files
mpkorstanjempkorstanje
committed
[Core] Improve undefined step reporting
The various ways to execute Cucumber report undefined steps inconsistently. Partially because different tools have different reporting needs. Nevertheless: When reporting per scenario the suggestion should include snippets for all undefined steps in that scenario. This applies to: - JUnit4 - TestNG - JUnit5 - Teamcity Plugin When reporting per test run the suggestion should include snippets for all undefined steps in the execution. This applies to: - CLI When printing snippets they should be copy-pasted into an IDE without further editing. This means individual snippets or groups of snippets should not be separated by spacers, text or anything else. Fixes: #2024
1 parent e0ad566 commit e994ef4

File tree

18 files changed

+502
-240
lines changed

18 files changed

+502
-240
lines changed

‎core/src/main/java/io/cucumber/core/plugin/DefaultSummaryPrinter.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ public void setEventPublisher(EventPublisher publisher) {
3737
}
3838

3939
private void handleSnippetsSuggestedEvent(SnippetsSuggestedEvent event) {
40-
this.snippets.addAll(event.getSnippets());
40+
this.snippets.addAll(event.getSuggestion().getSnippets());
4141
}
4242

4343
private void print() {

‎core/src/main/java/io/cucumber/core/plugin/TeamCityPlugin.java‎

Lines changed: 32 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
import io.cucumber.plugin.event.PickleStepTestStep;
1212
import io.cucumber.plugin.event.Result;
1313
import io.cucumber.plugin.event.SnippetsSuggestedEvent;
14+
import io.cucumber.plugin.event.SnippetsSuggestedEvent.Suggestion;
1415
import io.cucumber.plugin.event.Status;
1516
import io.cucumber.plugin.event.TestCase;
1617
import io.cucumber.plugin.event.TestCaseFinished;
@@ -69,7 +70,7 @@ public class TeamCityPlugin implements EventListener {
6970
private static final String TEMPLATE_TEST_FAILED = TEAMCITY_PREFIX
7071
+ "[testFailed timestamp = '%s' duration = '%s' message = '%s' details = '%s' name = '%s']";
7172
private static final String TEMPLATE_TEST_IGNORED = TEAMCITY_PREFIX
72-
+ "[testIgnored timestamp = '%s' duration = '%s' message = '%s' name = '%s']";
73+
+ "[testIgnored timestamp = '%s' duration = '%s' message = '%s' details = '%s' name = '%s']";
7374

7475
private static final String TEMPLATE_PROGRESS_COUNTING_STARTED = TEAMCITY_PREFIX
7576
+ "[customProgressStatus testsCategory = 'Scenarios' count = '0' timestamp = '%s']";
@@ -86,7 +87,7 @@ public class TeamCityPlugin implements EventListener {
8687
private static final Pattern LAMBDA_GLUE_CODE_LOCATION_PATTERN = Pattern.compile("^(.*)\\.(.*)\\(.*:.*\\)");
8788

8889
private final PrintStream out;
89-
private final List<SnippetsSuggestedEvent> snippets = new ArrayList<>();
90+
private final List<SnippetsSuggestedEvent> suggestions = new ArrayList<>();
9091
private final Map<URI, Collection<Node>> parsedTestSources = new HashMap<>();
9192
private List<Node> currentStack = new ArrayList<>();
9293

@@ -244,18 +245,18 @@ private void printTestStepFinished(TestStepFinished event) {
244245

245246
Throwable error = event.getResult().getError();
246247
Status status = event.getResult().getStatus();
248+
247249
switch (status) {
248250
case SKIPPED:
249251
print(TEMPLATE_TEST_IGNORED, timeStamp, duration, error == null ? "Step skipped" : error.getMessage(),
250-
name);
252+
getSnippets(event), name);
251253
break;
252254
case PENDING:
253255
print(TEMPLATE_TEST_IGNORED, timeStamp, duration, error == null ? "Step pending" : error.getMessage(),
254-
name);
256+
"", name);
255257
break;
256258
case UNDEFINED:
257-
PickleStepTestStep testStep = (PickleStepTestStep) event.getTestStep();
258-
print(TEMPLATE_TEST_FAILED, timeStamp, duration, "Step undefined", getSnippet(testStep), name);
259+
print(TEMPLATE_TEST_FAILED, timeStamp, duration, "Step undefined", getSnippets(event), name);
259260
break;
260261
case AMBIGUOUS:
261262
case FAILED:
@@ -268,6 +269,14 @@ private void printTestStepFinished(TestStepFinished event) {
268269
print(TEMPLATE_TEST_FINISHED, timeStamp, duration, name);
269270
}
270271

272+
private String getSnippets(TestStepFinished event) {
273+
if (event.getTestStep() instanceof PickleStepTestStep) {
274+
PickleStepTestStep testStep = (PickleStepTestStep) event.getTestStep();
275+
return getSnippet(testStep);
276+
}
277+
return "";
278+
}
279+
271280
private String extractStackTrace(Throwable error) {
272281
ByteArrayOutputStream s = new ByteArrayOutputStream();
273282
PrintStream printStream = new PrintStream(s);
@@ -300,23 +309,23 @@ private String extractName(TestStep step) {
300309
}
301310

302311
private String getSnippet(PickleStepTestStep testStep) {
303-
StringBuilder builder = new StringBuilder();
304-
305-
if (snippets.isEmpty()) {
306-
return builder.toString();
307-
}
308-
309-
snippets.stream()
310-
.filter(snippet -> snippet.getStepLocation().equals(testStep.getStep().getLocation()) &&
311-
snippet.getUri().equals(testStep.getUri()))
312+
return suggestions.stream()
313+
.filter(suggestions -> suggestions.getUri().equals(testStep.getUri()) &&
314+
suggestions.getStepLocation().equals(testStep.getStep().getLocation()))
312315
.findFirst()
313-
.ifPresent(event -> {
314-
builder.append("You can implement missing steps with the snippets below:\n");
315-
event.getSnippets().forEach(snippet -> {
316-
builder.append(snippet);
317-
builder.append("\n");
318-
});
319-
});
316+
.map(SnippetsSuggestedEvent::getSuggestion)
317+
.map(Suggestion::getSnippets)
318+
.map(this::printSnippets)
319+
.orElse("");
320+
}
321+
322+
private String printSnippets(List<String> snippets) {
323+
StringBuilder builder = new StringBuilder();
324+
builder.append("You can implement this step using the snippet(s) below:\n\n");
325+
snippets.forEach(snippet -> {
326+
builder.append(snippet);
327+
builder.append("\n");
328+
});
320329
return builder.toString();
321330
}
322331

@@ -342,7 +351,7 @@ private void printTestRunFinished(TestRunFinished event) {
342351
}
343352

344353
private void handleSnippetSuggested(SnippetsSuggestedEvent event) {
345-
snippets.add(event);
354+
suggestions.add(event);
346355
}
347356

348357
private void handleEmbedEvent(EmbedEvent event) {

‎core/src/main/java/io/cucumber/core/runner/Runner.java‎

Lines changed: 16 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,9 @@
1111
import io.cucumber.core.snippets.SnippetGenerator;
1212
import io.cucumber.core.stepexpression.StepTypeRegistry;
1313
import io.cucumber.plugin.event.HookType;
14+
import io.cucumber.plugin.event.Location;
1415
import io.cucumber.plugin.event.SnippetsSuggestedEvent;
16+
import io.cucumber.plugin.event.SnippetsSuggestedEvent.Suggestion;
1517

1618
import java.net.URI;
1719
import java.util.ArrayList;
@@ -147,17 +149,26 @@ private PickleStepDefinitionMatch matchStepToStepDefinition(Pickle pickle, Step
147149
if (match != null) {
148150
return match;
149151
}
150-
List<String> snippets = generateSnippetsForStep(step);
151-
if (!snippets.isEmpty()) {
152-
bus.send(new SnippetsSuggestedEvent(bus.getInstant(), pickle.getUri(), pickle.getScenarioLocation(),
153-
step.getLocation(), snippets));
154-
}
152+
emitSnippetSuggestedEvent(pickle, step);
155153
return new UndefinedPickleStepDefinitionMatch(pickle.getUri(), step);
156154
} catch (AmbiguousStepDefinitionsException e) {
157155
return new AmbiguousPickleStepDefinitionsMatch(pickle.getUri(), step, e);
158156
}
159157
}
160158

159+
private void emitSnippetSuggestedEvent(Pickle pickle, Step step) {
160+
List<String> snippets = generateSnippetsForStep(step);
161+
if (snippets.isEmpty()) {
162+
return;
163+
}
164+
Suggestion suggestion = new Suggestion(step.getText(), snippets);
165+
Location scenarioLocation = pickle.getScenarioLocation();
166+
Location stepLocation = step.getLocation();
167+
SnippetsSuggestedEvent event = new SnippetsSuggestedEvent(bus.getInstant(), pickle.getUri(), scenarioLocation,
168+
stepLocation, suggestion);
169+
bus.send(event);
170+
}
171+
161172
private List<HookTestStep> createAfterStepHooks(List<String> tags) {
162173
return createTestStepsForHooks(tags, glue.getAfterStepHooks(), HookType.AFTER_STEP);
163174
}

‎core/src/main/java/io/cucumber/core/runtime/TestCaseResultObserver.java‎

Lines changed: 5 additions & 56 deletions
Original file line numberDiff line numberDiff line change
@@ -2,79 +2,46 @@
22

33
import io.cucumber.plugin.event.EventHandler;
44
import io.cucumber.plugin.event.EventPublisher;
5-
import io.cucumber.plugin.event.PickleStepTestStep;
65
import io.cucumber.plugin.event.Result;
76
import io.cucumber.plugin.event.SnippetsSuggestedEvent;
87
import io.cucumber.plugin.event.Status;
98
import io.cucumber.plugin.event.TestCaseFinished;
10-
import io.cucumber.plugin.event.TestStep;
11-
import io.cucumber.plugin.event.TestStepFinished;
129

13-
import java.net.URI;
1410
import java.util.ArrayList;
1511
import java.util.List;
16-
import java.util.Map;
17-
import java.util.TreeMap;
1812
import java.util.function.Function;
1913
import java.util.function.Supplier;
2014

2115
import static io.cucumber.plugin.event.Status.PASSED;
2216
import static io.cucumber.plugin.event.Status.PENDING;
2317
import static io.cucumber.plugin.event.Status.SKIPPED;
2418
import static io.cucumber.plugin.event.Status.UNDEFINED;
19+
import static java.util.Collections.unmodifiableList;
2520
import static java.util.Objects.requireNonNull;
2621

2722
public final class TestCaseResultObserver implements AutoCloseable {
2823

2924
private final EventPublisher bus;
30-
private final Map<StepLocation, List<String>> snippetsPerStep = new TreeMap<>();
3125
private final List<Suggestion> suggestions = new ArrayList<>();
3226
private final EventHandler<SnippetsSuggestedEvent> snippetsSuggested = this::handleSnippetSuggestedEvent;
33-
private final EventHandler<TestStepFinished> testStepFinished = this::handleTestStepFinished;
3427
private Result result;
3528
private final EventHandler<TestCaseFinished> testCaseFinished = this::handleTestCaseFinished;
3629

3730
public TestCaseResultObserver(EventPublisher bus) {
3831
this.bus = bus;
3932
bus.registerHandlerFor(SnippetsSuggestedEvent.class, snippetsSuggested);
40-
bus.registerHandlerFor(TestStepFinished.class, testStepFinished);
4133
bus.registerHandlerFor(TestCaseFinished.class, testCaseFinished);
4234
}
4335

4436
@Override
4537
public void close() {
4638
bus.removeHandlerFor(SnippetsSuggestedEvent.class, snippetsSuggested);
47-
bus.removeHandlerFor(TestStepFinished.class, testStepFinished);
4839
bus.removeHandlerFor(TestCaseFinished.class, testCaseFinished);
4940
}
5041

5142
private void handleSnippetSuggestedEvent(SnippetsSuggestedEvent event) {
52-
snippetsPerStep.putIfAbsent(new StepLocation(
53-
event.getUri(),
54-
event.getStepLine()),
55-
event.getSnippets());
56-
}
57-
58-
private void handleTestStepFinished(TestStepFinished event) {
59-
Result result = event.getResult();
60-
Status status = result.getStatus();
61-
if (!status.is(UNDEFINED)) {
62-
return;
63-
}
64-
65-
TestStep testStep = event.getTestStep();
66-
if (!(testStep instanceof PickleStepTestStep)) {
67-
return;
68-
}
69-
70-
PickleStepTestStep pickleStepTestStep = (PickleStepTestStep) testStep;
71-
String stepText = pickleStepTestStep.getStepText();
72-
73-
List<String> snippets = snippetsPerStep.get(
74-
new StepLocation(
75-
pickleStepTestStep.getUri(),
76-
pickleStepTestStep.getStepLine()));
77-
suggestions.add(new Suggestion(stepText, snippets));
43+
SnippetsSuggestedEvent.Suggestion s = event.getSuggestion();
44+
suggestions.add(new Suggestion(s.getStep(), s.getSnippets()));
7845
}
7946

8047
private void handleTestCaseFinished(TestCaseFinished event) {
@@ -117,32 +84,14 @@ public TestCaseFailed(Throwable throwable) {
11784

11885
}
11986

120-
private static final class StepLocation implements Comparable<StepLocation> {
121-
122-
private final URI uri;
123-
private final int line;
124-
125-
private StepLocation(URI uri, int line) {
126-
this.uri = uri;
127-
this.line = line;
128-
}
129-
130-
@Override
131-
public int compareTo(StepLocation o) {
132-
int order = uri.compareTo(o.uri);
133-
return order != 0 ? order : Integer.compare(line, o.line);
134-
}
135-
136-
}
137-
13887
public static final class Suggestion {
13988

14089
final String step;
14190
final List<String> snippets;
14291

14392
public Suggestion(String step, List<String> snippets) {
144-
this.step = step;
145-
this.snippets = snippets;
93+
this.step = requireNonNull(step);
94+
this.snippets = unmodifiableList(requireNonNull(snippets));
14695
}
14796

14897
public String getStep() {

‎core/src/test/java/io/cucumber/core/plugin/CanonicalEventOrderTest.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
import io.cucumber.plugin.event.Location;
55
import io.cucumber.plugin.event.Result;
66
import io.cucumber.plugin.event.SnippetsSuggestedEvent;
7+
import io.cucumber.plugin.event.SnippetsSuggestedEvent.Suggestion;
78
import io.cucumber.plugin.event.Status;
89
import io.cucumber.plugin.event.TestCase;
910
import io.cucumber.plugin.event.TestCaseStarted;
@@ -48,13 +49,13 @@ class CanonicalEventOrderTest {
4849
URI.create("file:path/to/1.feature"),
4950
new Location(0, -1),
5051
new Location(0, -1),
51-
Collections.emptyList());
52+
new Suggestion("", Collections.emptyList()));
5253
private final Event suggested2 = new SnippetsSuggestedEvent(
5354
ofEpochMilli(5),
5455
URI.create("file:path/to/1.feature"),
5556
new Location(0, -1),
5657
new Location(0, -1),
57-
Collections.emptyList());
58+
new Suggestion("", Collections.emptyList()));
5859
private final Event feature1Case1Started = createTestCaseEvent(
5960
ofEpochMilli(5),
6061
URI.create("file:path/to/1.feature"),

‎core/src/test/java/io/cucumber/core/plugin/DefaultSummaryPrinterTest.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
import io.cucumber.plugin.event.Location;
66
import io.cucumber.plugin.event.Result;
77
import io.cucumber.plugin.event.SnippetsSuggestedEvent;
8+
import io.cucumber.plugin.event.SnippetsSuggestedEvent.Suggestion;
89
import io.cucumber.plugin.event.Status;
910
import io.cucumber.plugin.event.TestRunFinished;
1011
import org.junit.jupiter.api.BeforeEach;
@@ -44,14 +45,14 @@ void does_not_print_duplicate_snippets() {
4445
URI.create("classpath:com/example.feature"),
4546
new Location(12, -1),
4647
new Location(13, -1),
47-
singletonList("snippet")));
48+
new Suggestion("", singletonList("snippet"))));
4849

4950
bus.send(new SnippetsSuggestedEvent(
5051
bus.getInstant(),
5152
URI.create("classpath:com/example.feature"),
5253
new Location(12, -1),
5354
new Location(14, -1),
54-
singletonList("snippet")));
55+
new Suggestion("", singletonList("snippet"))));
5556

5657
bus.send(new TestRunFinished(bus.getInstant(), new Result(Status.PASSED, Duration.ZERO, null)));
5758

‎core/src/test/java/io/cucumber/core/plugin/TeamCityPluginTest.java‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -199,7 +199,8 @@ void should_print_error_message_for_undefined_steps() {
199199
Feature feature = TestFeatureParser.parse("path/test.feature", "" +
200200
"Feature: feature name\n" +
201201
" Scenario: scenario name\n" +
202-
" Given first step\n");
202+
" Given first step\n" +
203+
" Given second step\n");
203204

204205
ByteArrayOutputStream out = new ByteArrayOutputStream();
205206
Runtime.builder()
@@ -211,7 +212,9 @@ void should_print_error_message_for_undefined_steps() {
211212
.run();
212213

213214
assertThat(out, bytesContainsString("" +
214-
"##teamcity[testFailed timestamp = '1970-01-01T12:00:00.000+0000' duration = '0' message = 'Step undefined' details = 'You can implement missing steps with the snippets below:|n|n' name = 'first step']\n"));
215+
"##teamcity[testFailed timestamp = '1970-01-01T12:00:00.000+0000' duration = '0' message = 'Step undefined' details = 'You can implement this step using the snippet(s) below:|n|n|n' name = 'first step']\n"));
216+
assertThat(out, bytesContainsString("" +
217+
"##teamcity[testIgnored timestamp = '1970-01-01T12:00:00.000+0000' duration = '0' message = 'Step skipped' details = 'You can implement this step using the snippet(s) below:|n|n|n' name = 'second step']\n"));
215218
}
216219

217220
@Test

0 commit comments

Comments
 (0)