Fix getIntegerTypeOrder() to properly handle enums by first unwrapping their underlying integer type. This is a precondition for calling getIntegerRank().

Fixes an assertion failure in a test case involving vectors.

Fixes <rdar://problem/15091442>

Please somebody check this.

llvm-svn: 192334

Author: tkremenek

clang/lib/AST/ASTContext.cpp

@@ -4417,12 +4417,27 @@ Qualifiers::ObjCLifetime ASTContext::getInnerObjCOwnership(QualType T) const {
   return Qualifiers::OCL_None;
 }
 
+static const Type *getIntegerTypeForEnum(const EnumType *ET) {
+  // Incomplete enum types are not treated as integer types.
+  // FIXME: In C++, enum types are never integer types.
+  if (ET->getDecl()->isComplete() && !ET->getDecl()->isScoped())
+    return ET->getDecl()->getIntegerType().getTypePtr();
+  return NULL;
+}
+
 /// getIntegerTypeOrder - Returns the highest ranked integer type:
 /// C99 6.3.1.8p1.  If LHS > RHS, return 1.  If LHS == RHS, return 0. If
 /// LHS < RHS, return -1.
 int ASTContext::getIntegerTypeOrder(QualType LHS, QualType RHS) const {
   const Type *LHSC = getCanonicalType(LHS).getTypePtr();
   const Type *RHSC = getCanonicalType(RHS).getTypePtr();
+
+  // Unwrap enums to their underlying type.
+  if (const EnumType *ET = dyn_cast<EnumType>(LHSC))
+    LHSC = getIntegerTypeForEnum(ET);
+  if (const EnumType *ET = dyn_cast<EnumType>(RHSC))
+    RHSC = getIntegerTypeForEnum(ET);
+
   if (LHSC == RHSC) return 0;
 
   bool LHSUnsigned = LHSC->isUnsignedIntegerType();

clang/test/Sema/ext_vector_casts.c

@@ -5,6 +5,9 @@ typedef __attribute__(( ext_vector_type(4) )) int int4;
 typedef __attribute__(( ext_vector_type(8) )) short short8;
 typedef __attribute__(( ext_vector_type(4) )) float float4;
 typedef float t3 __attribute__ ((vector_size (16)));
+typedef __typeof__(sizeof(int)) size_t;
+typedef unsigned long ulong2 __attribute__ ((ext_vector_type(2)));
+typedef size_t stride4 __attribute__((ext_vector_type(4)));
 
 static void test() {
     float2 vec2;
@@ -50,3 +53,24 @@ void inc(float2 f2) {
   f2++; // expected-error{{cannot increment value of type 'float2'}}
   __real f2; // expected-error{{invalid type 'float2' to __real operator}}
 }
+
+typedef enum
+{
+    uchar_stride = 1,
+    uchar4_stride = 4,
+    ushort4_stride = 8,
+    short4_stride = 8,
+    uint4_stride = 16,
+    int4_stride = 16,
+    float4_stride = 16,
+} PixelByteStride;
+
+stride4 RDar15091442_get_stride4(int4 x, PixelByteStride pixelByteStride);
+stride4 RDar15091442_get_stride4(int4 x, PixelByteStride pixelByteStride)
+{
+    stride4 stride;
+    // This previously caused an assertion failure.
+    stride.lo = ((ulong2) x) * pixelByteStride; // no-warning
+    return stride;
+}
+