diff --git a/src/test/java/scala/tools/refactoring/tests/util/ScalaVersionTestRule.java b/src/test/java/scala/tools/refactoring/tests/util/ScalaVersionTestRule.java
index 0877c13c..b52853cd 100644
--- a/src/test/java/scala/tools/refactoring/tests/util/ScalaVersionTestRule.java
+++ b/src/test/java/scala/tools/refactoring/tests/util/ScalaVersionTestRule.java
@@ -18,12 +18,12 @@ public void evaluate() throws Throwable {
 
   public Statement apply(Statement stmt, FrameworkMethod meth, Object arg2) {
     ScalaVersion onlyOn = meth.getAnnotation(ScalaVersion.class);
-    String versionString = Properties.versionString();
+    String versionNumberString = Properties.versionNumberString();
 
     if (onlyOn != null) {
-      if (!onlyOn.doesNotMatch().isEmpty() && versionString.contains(onlyOn.doesNotMatch())) {
+      if (!onlyOn.doesNotMatch().isEmpty() && versionNumberString.startsWith(onlyOn.doesNotMatch())) {
         return new EmptyStatement();
-      } else if (versionString.contains(onlyOn.matches())) {
+      } else if (versionNumberString.startsWith(onlyOn.matches())) {
         return stmt;
       } else {
         return new EmptyStatement();