[ownership] Fix the eager specializer for ownership

docs/SIL.rst

@@ -5148,6 +5148,21 @@ unchecked_bitwise_cast
 Bitwise copies an object of type ``A`` into a new object of type ``B``
 of the same size or smaller.
 
+unchecked_value_cast
+````````````````````
+::
+
+   sil-instruction ::= 'unchecked_value_cast' sil-operand 'to' sil-type
+
+   %1 = unchecked_value_cast %0 : $A to $B
+
+Bitwise copies an object of type ``A`` into a new layout-compatible object of
+type ``B`` of the same size.
+
+This instruction is assumed to forward a fixed ownership (set upon its
+construction) and lowers to 'unchecked_bitwise_cast' in non-ossa code. This
+causes the cast to lose its guarantee of layout-compatibility.
+
 ref_to_raw_pointer
 ``````````````````
 ::

include/swift/SIL/SILBuilder.h

@@ -1102,6 +1102,12 @@ class SILBuilder {
         getSILDebugLocation(Loc), Op, Ty, getFunction(), C.OpenedArchetypes));
   }
 
+  UncheckedValueCastInst *createUncheckedValueCast(SILLocation Loc, SILValue Op,
+                                                   SILType Ty) {
+    return insert(UncheckedValueCastInst::create(
+        getSILDebugLocation(Loc), Op, Ty, getFunction(), C.OpenedArchetypes));
+  }
+
   RefToBridgeObjectInst *createRefToBridgeObject(SILLocation Loc, SILValue Ref,
                                                  SILValue Bits) {
     auto Ty = SILType::getBridgeObjectType(getASTContext());
@@ -1847,7 +1853,18 @@ class SILBuilder {
   // Unchecked cast helpers
   //===--------------------------------------------------------------------===//
 
-  // Create the appropriate cast instruction based on result type.
+  /// Create the appropriate cast instruction based on result type.
+  ///
+  /// NOTE: We allow for non-layout compatible casts that shrink the underlying
+  /// type we are bit casting!
+  SingleValueInstruction *
+  createUncheckedReinterpretCast(SILLocation Loc, SILValue Op, SILType Ty);
+
+  /// Create an appropriate cast instruction based on result type.
+  ///
+  /// NOTE: This assumes that the input and the result cast are layout
+  /// compatible. Reduces to createUncheckedReinterpretCast when ownership is
+  /// disabled.
   SingleValueInstruction *createUncheckedBitCast(SILLocation Loc, SILValue Op,
                                                  SILType Ty);
 

include/swift/SIL/SILCloner.h

@@ -1538,6 +1538,16 @@ visitUncheckedBitwiseCastInst(UncheckedBitwiseCastInst *Inst) {
                                     getOpType(Inst->getType())));
 }
 
+template <typename ImplClass>
+void SILCloner<ImplClass>::visitUncheckedValueCastInst(
+    UncheckedValueCastInst *Inst) {
+  getBuilder().setCurrentDebugScope(getOpScope(Inst->getDebugScope()));
+  recordClonedInstruction(Inst, getBuilder().createUncheckedValueCast(
+                                    getOpLocation(Inst->getLoc()),
+                                    getOpValue(Inst->getOperand()),
+                                    getOpType(Inst->getType())));
+}
+
 template<typename ImplClass>
 void
 SILCloner<ImplClass>::

include/swift/SIL/SILInstruction.h

@@ -4576,6 +4576,23 @@ class UncheckedBitwiseCastInst final
          SILFunction &F, SILOpenedArchetypesState &OpenedArchetypes);
 };
 
+/// Bitwise copy a value into another value of the same size.
+class UncheckedValueCastInst final
+    : public UnaryInstructionWithTypeDependentOperandsBase<
+          SILInstructionKind::UncheckedValueCastInst, UncheckedValueCastInst,
+          OwnershipForwardingConversionInst> {
+  friend SILBuilder;
+
+  UncheckedValueCastInst(SILDebugLocation DebugLoc, SILValue Operand,
+                         ArrayRef<SILValue> TypeDependentOperands, SILType Ty)
+      : UnaryInstructionWithTypeDependentOperandsBase(
+            DebugLoc, Operand, TypeDependentOperands, Ty,
+            Operand.getOwnershipKind()) {}
+  static UncheckedValueCastInst *
+  create(SILDebugLocation DebugLoc, SILValue Operand, SILType Ty,
+         SILFunction &F, SILOpenedArchetypesState &OpenedArchetypes);
+};
+
 /// Build a Builtin.BridgeObject from a heap object reference by bitwise-or-ing
 /// in bits from a word.
 class RefToBridgeObjectInst

include/swift/SIL/SILNodes.def

@@ -500,6 +500,8 @@ ABSTRACT_VALUE_AND_INST(SingleValueInstruction, ValueBase, SILInstruction)
                       ConversionInst, None, DoesNotRelease)
     SINGLE_VALUE_INST(UncheckedBitwiseCastInst, unchecked_bitwise_cast,
                       ConversionInst, None, DoesNotRelease)
+    SINGLE_VALUE_INST(UncheckedValueCastInst, unchecked_value_cast,
+                      ConversionInst, None, DoesNotRelease)
     SINGLE_VALUE_INST(RefToRawPointerInst, ref_to_raw_pointer,
                       ConversionInst, None, DoesNotRelease)
     SINGLE_VALUE_INST(RawPointerToRefInst, raw_pointer_to_ref,

include/swift/SILOptimizer/Utils/CFGOptUtils.h

@@ -107,6 +107,10 @@ bool splitCriticalEdgesFrom(SILBasicBlock *fromBB,
                             DominanceInfo *domInfo = nullptr,
                             SILLoopInfo *loopInfo = nullptr);
 
+/// Splits all critical edges that have `toBB` as a destination.
+bool splitCriticalEdgesTo(SILBasicBlock *toBB, DominanceInfo *domInfo = nullptr,
+                          SILLoopInfo *loopInfo = nullptr);
+
 /// Splits the edges between two basic blocks.
 ///
 /// Updates dominance information and loop information if not null.

lib/IRGen/IRGenSIL.cpp

@@ -1026,6 +1026,9 @@ class IRGenSILFunction :
   void visitUncheckedAddrCastInst(UncheckedAddrCastInst *i);
   void visitUncheckedTrivialBitCastInst(UncheckedTrivialBitCastInst *i);
   void visitUncheckedBitwiseCastInst(UncheckedBitwiseCastInst *i);
+  void visitUncheckedValueCastInst(UncheckedValueCastInst *i) {
+    llvm_unreachable("Should never be seen in Lowered SIL");
+  }
   void visitRefToRawPointerInst(RefToRawPointerInst *i);
   void visitRawPointerToRefInst(RawPointerToRefInst *i);
   void visitThinToThickFunctionInst(ThinToThickFunctionInst *i);

lib/IRGen/LoadableByAddress.cpp

@@ -1544,7 +1544,7 @@ void LoadableStorageAllocation::
                                             storageType);
     if (pass.containsDifferentFunctionSignature(pass.F->getLoweredFunctionType(),
                                                 storageType)) {
-      auto *castInstr = argBuilder.createUncheckedBitCast(
+      auto *castInstr = argBuilder.createUncheckedReinterpretCast(
           RegularLocation(const_cast<ValueDecl *>(arg->getDecl())), arg,
           newSILType);
       arg->replaceAllUsesWith(castInstr);
@@ -1912,8 +1912,8 @@ static void castTupleInstr(SingleValueInstruction *instr, IRGenModule &Mod,
   switch (instr->getKind()) {
   // Add cast to the new sil function type:
   case SILInstructionKind::TupleExtractInst: {
-    castInstr = castBuilder.createUncheckedBitCast(instr->getLoc(), instr,
-                                                   newSILType.getObjectType());
+    castInstr = castBuilder.createUncheckedReinterpretCast(
+        instr->getLoc(), instr, newSILType.getObjectType());
     break;
   }
   case SILInstructionKind::TupleElementAddrInst: {
@@ -2471,8 +2471,8 @@ getOperandTypeWithCastIfNecessary(SILInstruction *containingInstr, SILValue op,
     }
     assert(currSILType.isObject() && "Expected an object type");
     if (newSILType != currSILType) {
-      auto castInstr = builder.createUncheckedBitCast(containingInstr->getLoc(),
-                                                      op, newSILType);
+      auto castInstr = builder.createUncheckedReinterpretCast(
+          containingInstr->getLoc(), op, newSILType);
       return castInstr;
     }
   }
@@ -2653,8 +2653,8 @@ bool LoadableByAddress::recreateUncheckedEnumDataInstr(
     auto *takeEnum = enumBuilder.createUncheckedEnumData(
         enumInstr->getLoc(), enumInstr->getOperand(), enumInstr->getElement(),
         caseTy);
-    newInstr = enumBuilder.createUncheckedBitCast(enumInstr->getLoc(), takeEnum,
-                                                  newType);
+    newInstr = enumBuilder.createUncheckedReinterpretCast(enumInstr->getLoc(),
+                                                          takeEnum, newType);
   } else {
     newInstr = enumBuilder.createUncheckedEnumData(
         enumInstr->getLoc(), enumInstr->getOperand(), enumInstr->getElement(),
@@ -2708,7 +2708,7 @@ bool LoadableByAddress::fixStoreToBlockStorageInstr(
   if (destType.getObjectType() != srcType) {
     // Add cast to destType
     SILBuilderWithScope castBuilder(instr);
-    auto *castInstr = castBuilder.createUncheckedBitCast(
+    auto *castInstr = castBuilder.createUncheckedReinterpretCast(
         instr->getLoc(), src, destType.getObjectType());
     instr->setOperand(StoreInst::Src, castInstr);
   }

lib/SIL/IR/OperandOwnership.cpp

@@ -350,6 +350,7 @@ FORWARD_ANY_OWNERSHIP_INST(DestructureTuple)
 FORWARD_ANY_OWNERSHIP_INST(InitExistentialRef)
 FORWARD_ANY_OWNERSHIP_INST(DifferentiableFunction)
 FORWARD_ANY_OWNERSHIP_INST(LinearFunction)
+FORWARD_ANY_OWNERSHIP_INST(UncheckedValueCast)
 #undef FORWARD_ANY_OWNERSHIP_INST
 
 // An instruction that forwards a constant ownership or trivial ownership.

lib/SIL/IR/SILBuilder.cpp

@@ -141,7 +141,8 @@ SILBuilder::createClassifyBridgeObject(SILLocation Loc, SILValue value) {
 
 // Create the appropriate cast instruction based on result type.
 SingleValueInstruction *
-SILBuilder::createUncheckedBitCast(SILLocation Loc, SILValue Op, SILType Ty) {
+SILBuilder::createUncheckedReinterpretCast(SILLocation Loc, SILValue Op,
+                                           SILType Ty) {
   assert(isLoadableOrOpaque(Ty));
   if (Ty.isTrivial(getFunction()))
     return insert(UncheckedTrivialBitCastInst::create(
@@ -156,6 +157,26 @@ SILBuilder::createUncheckedBitCast(SILLocation Loc, SILValue Op, SILType Ty) {
       getSILDebugLocation(Loc), Op, Ty, getFunction(), C.OpenedArchetypes));
 }
 
+// Create the appropriate cast instruction based on result type.
+SingleValueInstruction *
+SILBuilder::createUncheckedBitCast(SILLocation Loc, SILValue Op, SILType Ty) {
+  // Without ownership, delegate to unchecked reinterpret cast.
+  if (!hasOwnership())
+    return createUncheckedReinterpretCast(Loc, Op, Ty);
+
+  assert(isLoadableOrOpaque(Ty));
+  if (Ty.isTrivial(getFunction()))
+    return insert(UncheckedTrivialBitCastInst::create(
+        getSILDebugLocation(Loc), Op, Ty, getFunction(), C.OpenedArchetypes));
+
+  if (SILType::canRefCast(Op->getType(), Ty, getModule()))
+    return createUncheckedRefCast(Loc, Op, Ty);
+
+  // The destination type is nontrivial, and may be smaller than the source
+  // type, so RC identity cannot be assumed.
+  return createUncheckedValueCast(Loc, Op, Ty);
+}
+
 BranchInst *SILBuilder::createBranch(SILLocation Loc,
                                      SILBasicBlock *TargetBlock,
                                      OperandValueArrayRef Args) {

lib/SIL/IR/SILInstructions.cpp

@@ -2217,6 +2217,21 @@ UncheckedRefCastInst::create(SILDebugLocation DebugLoc, SILValue Operand,
                                              TypeDependentOperands, Ty);
 }
 
+UncheckedValueCastInst *
+UncheckedValueCastInst::create(SILDebugLocation DebugLoc, SILValue Operand,
+                               SILType Ty, SILFunction &F,
+                               SILOpenedArchetypesState &OpenedArchetypes) {
+  SILModule &Mod = F.getModule();
+  SmallVector<SILValue, 8> TypeDependentOperands;
+  collectTypeDependentOperands(TypeDependentOperands, OpenedArchetypes, F,
+                               Ty.getASTType());
+  unsigned size =
+      totalSizeToAlloc<swift::Operand>(1 + TypeDependentOperands.size());
+  void *Buffer = Mod.allocateInst(size, alignof(UncheckedValueCastInst));
+  return ::new (Buffer)
+      UncheckedValueCastInst(DebugLoc, Operand, TypeDependentOperands, Ty);
+}
+
 UncheckedAddrCastInst *
 UncheckedAddrCastInst::create(SILDebugLocation DebugLoc, SILValue Operand,
                               SILType Ty, SILFunction &F,

lib/SIL/IR/SILPrinter.cpp

@@ -1620,6 +1620,9 @@ class SILPrinter : public SILInstructionVisitor<SILPrinter> {
   void visitUncheckedBitwiseCastInst(UncheckedBitwiseCastInst *CI) {
     printUncheckedConversionInst(CI, CI->getOperand());
   }
+  void visitUncheckedValueCastInst(UncheckedValueCastInst *CI) {
+    printUncheckedConversionInst(CI, CI->getOperand());
+  }
   void visitRefToRawPointerInst(RefToRawPointerInst *CI) {
     printUncheckedConversionInst(CI, CI->getOperand());
   }

lib/SIL/IR/ValueOwnership.cpp

@@ -254,6 +254,7 @@ FORWARDING_OWNERSHIP_INST(Tuple)
 FORWARDING_OWNERSHIP_INST(UncheckedRefCast)
 FORWARDING_OWNERSHIP_INST(UnconditionalCheckedCast)
 FORWARDING_OWNERSHIP_INST(Upcast)
+FORWARDING_OWNERSHIP_INST(UncheckedValueCast)
 FORWARDING_OWNERSHIP_INST(UncheckedEnumData)
 FORWARDING_OWNERSHIP_INST(SelectEnum)
 FORWARDING_OWNERSHIP_INST(Enum)

lib/SIL/Parser/ParseSIL.cpp

@@ -3238,6 +3238,7 @@ bool SILParser::parseSpecificSILInstruction(SILBuilder &B,
   case SILInstructionKind::UncheckedAddrCastInst:
   case SILInstructionKind::UncheckedTrivialBitCastInst:
   case SILInstructionKind::UncheckedBitwiseCastInst:
+  case SILInstructionKind::UncheckedValueCastInst:
   case SILInstructionKind::UpcastInst:
   case SILInstructionKind::AddressToPointerInst:
   case SILInstructionKind::BridgeObjectToRefInst:
@@ -3307,6 +3308,9 @@ bool SILParser::parseSpecificSILInstruction(SILBuilder &B,
     case SILInstructionKind::UncheckedBitwiseCastInst:
       ResultVal = B.createUncheckedBitwiseCast(InstLoc, Val, Ty);
       break;
+    case SILInstructionKind::UncheckedValueCastInst:
+      ResultVal = B.createUncheckedValueCast(InstLoc, Val, Ty);
+      break;
     case SILInstructionKind::UpcastInst:
       ResultVal = B.createUpcast(InstLoc, Val, Ty);
       break;

lib/SIL/Utils/OwnershipUtils.cpp

@@ -34,6 +34,7 @@ bool swift::isOwnershipForwardingValueKind(SILNodeKind kind) {
   case SILNodeKind::LinearFunctionInst:
   case SILNodeKind::OpenExistentialRefInst:
   case SILNodeKind::UpcastInst:
+  case SILNodeKind::UncheckedValueCastInst:
   case SILNodeKind::UncheckedRefCastInst:
   case SILNodeKind::ConvertFunctionInst:
   case SILNodeKind::RefToBridgeObjectInst:

lib/SIL/Verifier/SILVerifier.cpp

@@ -1904,6 +1904,12 @@ class SILVerifier : public SILVerifierBase<SILVerifier> {
         "Inst with qualified ownership in a function that is not qualified");
   }
 
+  void checkUncheckedValueCastInst(UncheckedValueCastInst *) {
+    require(
+        F.hasOwnership(),
+        "Inst with qualified ownership in a function that is not qualified");
+  }
+
   template <class AI>
   void checkAccessEnforcement(AI *AccessInst) {
     if (AccessInst->getModule().getStage() != SILStage::Raw) {

lib/SILGen/SILGenApply.cpp

@@ -1340,8 +1340,8 @@ class SILGenApply : public Lowering::ExprVisitor<SILGenApply> {
           && loweredResultTy.getASTType()->hasDynamicSelfType()) {
         assert(selfMetaTy);
         selfValue = SGF.emitManagedRValueWithCleanup(
-          SGF.B.createUncheckedBitCast(loc, selfValue.forward(SGF),
-                                       loweredResultTy));
+            SGF.B.createUncheckedReinterpretCast(loc, selfValue.forward(SGF),
+                                                 loweredResultTy));
       } else {
         selfValue = SGF.emitManagedRValueWithCleanup(
             SGF.B.createUpcast(loc, selfValue.forward(SGF), loweredResultTy));

lib/SILGen/SILGenBuilder.cpp

@@ -610,7 +610,7 @@ ManagedValue SILGenBuilder::createUncheckedBitCast(SILLocation loc,
                                                    ManagedValue value,
                                                    SILType type) {
   CleanupCloner cloner(*this, value);
-  SILValue cast = createUncheckedBitCast(loc, value.getValue(), type);
+  SILValue cast = createUncheckedReinterpretCast(loc, value.getValue(), type);
 
   // Currently createUncheckedBitCast only produces these
   // instructions. We assert here to make sure if this changes, this code is

lib/SILGen/SILGenBuilder.h

@@ -279,7 +279,7 @@ class SILGenBuilder : public SILBuilder {
   ManagedValue createUncheckedAddrCast(SILLocation loc, ManagedValue op,
                                        SILType resultTy);
 
-  using SILBuilder::createUncheckedBitCast;
+  using SILBuilder::createUncheckedReinterpretCast;
   ManagedValue createUncheckedBitCast(SILLocation loc, ManagedValue original,
                                       SILType type);
 

lib/SILGen/SILGenExpr.cpp

@@ -2066,8 +2066,8 @@ RValue RValueEmitter::visitUnderlyingToOpaqueExpr(UnderlyingToOpaqueExpr *E,
   if (value.getType() == opaqueTL.getLoweredType())
     return RValue(SGF, E, value);
 
-  auto cast = SGF.B.createUncheckedBitCast(E, value.forward(SGF),
-                                           opaqueTL.getLoweredType());
+  auto cast = SGF.B.createUncheckedReinterpretCast(E, value.forward(SGF),
+                                                   opaqueTL.getLoweredType());
   value = SGF.emitManagedRValueWithCleanup(cast);
 
   return RValue(SGF, E, value);