From 4ae64aa44a6186c3cd663e33cdc3a7f89209afd2 Mon Sep 17 00:00:00 2001 From: Anton Korobeynikov Date: Wed, 17 Jul 2024 18:51:04 -0700 Subject: [PATCH 1/2] Initial support for differentiation of throwing functions --- .../SILOptimizer/Differentiation/Common.h | 6 + lib/AST/Builtins.cpp | 2 + lib/SILGen/SILGenBuiltin.cpp | 24 +- .../Differentiation/LinearMapInfo.cpp | 19 +- .../Differentiation/PullbackCloner.cpp | 208 ++++++++++++-- .../Differentiation/VJPCloner.cpp | 262 +++++++++++++++++- lib/Sema/TypeCheckAttr.cpp | 11 +- .../OptionalDifferentiation.swift | 13 + ...erentiation_control_flow_diagnostics.swift | 16 +- .../differentiation_diagnostics.swift | 19 +- .../SILOptimizer/optional_error.swift | 12 +- .../issue-70819-try-apply-adjoint.swift | 24 ++ test/AutoDiff/validation-test/throw.swift | 81 ++++++ 13 files changed, 635 insertions(+), 62 deletions(-) create mode 100644 test/AutoDiff/compiler_crashers_fixed/issue-70819-try-apply-adjoint.swift create mode 100644 test/AutoDiff/validation-test/throw.swift diff --git a/include/swift/SILOptimizer/Differentiation/Common.h b/include/swift/SILOptimizer/Differentiation/Common.h index d43bf3897cd6c..2fb2614cf70c9 100644 --- a/include/swift/SILOptimizer/Differentiation/Common.h +++ b/include/swift/SILOptimizer/Differentiation/Common.h @@ -277,6 +277,12 @@ inline void createEntryArguments(SILFunction *f) { indResTy = indResTy.mapTypeOutOfContext(); createFunctionArgument(f->mapTypeIntoContext(indResTy).getAddressType()); } + if (auto indErrorResTy = conv.getIndirectErrorResultType(f->getTypeExpansionContext())) { + if (indErrorResTy.hasArchetype()) + indErrorResTy = indErrorResTy.mapTypeOutOfContext(); + createFunctionArgument(f->mapTypeIntoContext(indErrorResTy).getAddressType()); + } + for (auto paramTy : conv.getParameterSILTypes(f->getTypeExpansionContext())) { if (paramTy.hasArchetype()) paramTy = paramTy.mapTypeOutOfContext(); diff --git a/lib/AST/Builtins.cpp b/lib/AST/Builtins.cpp index 1cc678274ed4a..8b701180a2bd2 100644 --- a/lib/AST/Builtins.cpp +++ b/lib/AST/Builtins.cpp @@ -2892,6 +2892,7 @@ ValueDecl *swift::getBuiltinValueDecl(ASTContext &Context, Identifier Id) { if (!autodiff::getBuiltinApplyDerivativeConfig( OperationName, kind, arity, throws)) return nullptr; + // TODO: Support somehow typed throws return getAutoDiffApplyDerivativeFunction(Context, Id, kind, arity, throws, /*thrownType=*/Type()); } @@ -2901,6 +2902,7 @@ ValueDecl *swift::getBuiltinValueDecl(ASTContext &Context, Identifier Id) { if (!autodiff::getBuiltinApplyTransposeConfig( OperationName, arity, throws)) return nullptr; + // TODO: Support somehow typed throws return getAutoDiffApplyTransposeFunction(Context, Id, arity, throws, /*thrownType=*/Type()); } diff --git a/lib/SILGen/SILGenBuiltin.cpp b/lib/SILGen/SILGenBuiltin.cpp index 2b54dcf6bce3c..ee5df5957535d 100644 --- a/lib/SILGen/SILGenBuiltin.cpp +++ b/lib/SILGen/SILGenBuiltin.cpp @@ -1191,11 +1191,8 @@ static ManagedValue emitBuiltinTypeTrait(SILGenFunction &SGF, static ManagedValue emitBuiltinAutoDiffApplyDerivativeFunction( AutoDiffDerivativeFunctionKind kind, unsigned arity, - bool throws, SILGenFunction &SGF, SILLocation loc, - SubstitutionMap substitutions, ArrayRef args, SGFContext C) { - // FIXME(https://github.com/apple/swift/issues/54259): Support throwing functions. - assert(!throws && "Throwing functions are not yet supported"); - + SILGenFunction &SGF, SILLocation loc, SubstitutionMap substitutions, + ArrayRef args, SGFContext C) { auto origFnVal = args[0]; SmallVector origFnArgVals; for (auto& arg : args.drop_front(1)) @@ -1213,7 +1210,8 @@ static ManagedValue emitBuiltinAutoDiffApplyDerivativeFunction( origFnVal = SGF.B.createBeginBorrow(loc, origFnVal); SILValue derivativeFn = SGF.B.createDifferentiableFunctionExtract( loc, kind, origFnVal.getValue()); - auto derivativeFnType = derivativeFn->getType().castTo(); + SILType derivativeType = derivativeFn->getType(); + auto derivativeFnType = derivativeType.castTo(); assert(derivativeFnType->getNumResults() == 2); assert(derivativeFnType->getNumParameters() == origFnArgVals.size()); @@ -1240,8 +1238,10 @@ static ManagedValue emitBuiltinAutoDiffApplyDerivativeFunction( applyArgs.push_back(SGF.B.createTupleElementAddr(loc, indResBuffer, 0)); for (auto origFnArgVal : origFnArgVals) applyArgs.push_back(origFnArgVal); - auto differential = SGF.B.createApply(loc, derivativeFn, SubstitutionMap(), - applyArgs); + auto differential = + SGF.emitApplyWithRethrow(loc, + derivativeFn, derivativeType, + SubstitutionMap(), applyArgs); derivativeFn = SILValue(); @@ -1253,8 +1253,10 @@ static ManagedValue emitBuiltinAutoDiffApplyDerivativeFunction( } // Do the apply for the direct result case. - auto resultTuple = SGF.B.createApply( - loc, derivativeFn, SubstitutionMap(), origFnArgVals); + auto resultTuple = + SGF.emitApplyWithRethrow(loc, + derivativeFn, derivativeType, + SubstitutionMap(), origFnArgVals); derivativeFn = SILValue(); @@ -1324,7 +1326,7 @@ static ManagedValue emitBuiltinApplyDerivative( builtinName, kind, arity, throws); assert(successfullyParsed); return emitBuiltinAutoDiffApplyDerivativeFunction( - kind, arity, throws, SGF, loc, substitutions, args, C); + kind, arity, SGF, loc, substitutions, args, C); } static ManagedValue emitBuiltinApplyTranspose( diff --git a/lib/SILOptimizer/Differentiation/LinearMapInfo.cpp b/lib/SILOptimizer/Differentiation/LinearMapInfo.cpp index d1a4141f22a4b..13d9c2443e8cb 100644 --- a/lib/SILOptimizer/Differentiation/LinearMapInfo.cpp +++ b/lib/SILOptimizer/Differentiation/LinearMapInfo.cpp @@ -144,14 +144,15 @@ void LinearMapInfo::populateBranchingTraceDecl(SILBasicBlock *originalBB, heapAllocatedContext = true; decl->setInterfaceType(astCtx.TheRawPointerType); } else { // Otherwise the payload is the linear map tuple. - auto *linearMapStructTy = getLinearMapTupleType(predBB); + auto *linearMapTupleTy = getLinearMapTupleType(predBB); // Do not create entries for unreachable predecessors - if (!linearMapStructTy) + if (!linearMapTupleTy) continue; - auto canLinearMapStructTy = linearMapStructTy->getCanonicalType(); + + auto canLinearMapTupleTy = linearMapTupleTy->getCanonicalType(); decl->setInterfaceType( - canLinearMapStructTy->hasArchetype() - ? canLinearMapStructTy->mapTypeOutOfContext() : canLinearMapStructTy); + canLinearMapTupleTy->hasArchetype() + ? canLinearMapTupleTy->mapTypeOutOfContext() : canLinearMapTupleTy); } // Create enum element and enum case declarations. auto *paramList = ParameterList::create(astCtx, {decl}); @@ -183,6 +184,7 @@ Type LinearMapInfo::getLinearMapType(ADContext &context, FullApplySite fai) { auto hasActiveResults = llvm::any_of(allResults, [&](SILValue res) { return activityInfo.isActive(res, config); }); + bool hasActiveSemanticResultArgument = false; bool hasActiveArguments = false; auto numIndirectResults = fai.getNumIndirectSILResults(); @@ -311,10 +313,11 @@ Type LinearMapInfo::getLinearMapType(ADContext &context, FullApplySite fai) { params, silFnTy->getAllResultsInterfaceType().getASTType(), info); } - if (astFnTy->hasArchetype()) - return astFnTy->mapTypeOutOfContext(); + Type resultType = astFnTy->hasArchetype() ? astFnTy->mapTypeOutOfContext() : astFnTy; + if (fai.getKind() == FullApplySiteKind::TryApplyInst) + resultType = resultType->wrapInOptionalType(); - return astFnTy; + return resultType; } void LinearMapInfo::generateDifferentiationDataStructures( diff --git a/lib/SILOptimizer/Differentiation/PullbackCloner.cpp b/lib/SILOptimizer/Differentiation/PullbackCloner.cpp index 31af1f8691379..cee5ed02b22a1 100644 --- a/lib/SILOptimizer/Differentiation/PullbackCloner.cpp +++ b/lib/SILOptimizer/Differentiation/PullbackCloner.cpp @@ -81,7 +81,9 @@ class PullbackCloner::Implementation final /// Mapping from original basic blocks to corresponding pullback basic blocks. /// Pullback basic blocks always have the predecessor as the single argument. - llvm::DenseMap pullbackBBMap; + /// Pullback block might be split into SESE region. Here we record entry and + /// exit blocks of this region + llvm::DenseMap> pullbackBBMap; /// Mapping from original basic blocks and original values to corresponding /// adjoint values. @@ -117,6 +119,11 @@ class PullbackCloner::Implementation final llvm::DenseMap> blockTemporaries; + /// Adjoints for result values of `try_apply` instruction. These are only + /// available in normal destination basic block. Therefore we keep them + /// on the side. + llvm::DenseMap tryApplyAdjoints; + /// The scope cloner. ScopeCloner scopeCloner; @@ -616,7 +623,7 @@ class PullbackCloner::Implementation final auto pullbackBBArg = activeValuePullbackBBArgumentMap[{origBB, activeValue}]; assert(pullbackBBArg); - assert(pullbackBBArg->getParent() == getPullbackBlock(origBB)); + assert(pullbackBBArg->getParent() == getPullbackBlock(origBB).first); return pullbackBBArg; } @@ -807,7 +814,8 @@ class PullbackCloner::Implementation final // CFG mapping //--------------------------------------------------------------------------// - SILBasicBlock *getPullbackBlock(SILBasicBlock *originalBlock) { + std::pair + getPullbackBlock(const SILBasicBlock *originalBlock) { return pullbackBBMap.lookup(originalBlock); } @@ -948,8 +956,11 @@ class PullbackCloner::Implementation final auto &s = llvm::dbgs() << "[ADJ] Emitted in pullback (pb bb" << builder.getInsertionBB()->getDebugID() << "):\n"; auto afterInsertion = builder.getInsertionPoint(); - for (auto it = ++beforeInsertion; it != afterInsertion; ++it) - s << *it; + if (beforeInsertion->getParent() == afterInsertion->getParent()) + for (auto it = ++beforeInsertion; it != afterInsertion; ++it) + s << *it; + else + s << "insertion spread over multiple BBs\n"; }); } @@ -989,26 +1000,136 @@ class PullbackCloner::Implementation final return; } - buildPullbackCall(ai); + auto &nestedApplyInfo = getContext().getNestedApplyInfo(); + auto applyInfoLookup = nestedApplyInfo.find(ai); + // If no `NestedApplyInfo` was found, then this task doesn't need to be + // differentiated. + if (applyInfoLookup == nestedApplyInfo.end()) { + // Must not be active. + assert(!getActivityInfo().isActive(ai, getConfig())); + return; + } + + // Replace a call to a function with a call to its pullback. + buildPullbackCall(ai, applyInfoLookup->getSecond()); } - void buildPullbackCall(FullApplySite fai) { - auto loc = fai->getLoc(); - auto *bb = fai->getParent(); + void visitTryApplyInst(TryApplyInst *tai) { + assert(getPullbackInfo().shouldDifferentiateApplySite(tai)); // Replace a call to a function with a call to its pullback. auto &nestedApplyInfo = getContext().getNestedApplyInfo(); - auto applyInfoLookup = nestedApplyInfo.find(fai); + auto applyInfoLookup = nestedApplyInfo.find(tai); // If no `NestedApplyInfo` was found, then this task doesn't need to be // differentiated. if (applyInfoLookup == nestedApplyInfo.end()) { + auto *normalBlock = tai->getNormalBB(); + assert(normalBlock->getNumArguments() == 1 && + "Expected try apply to have a single result"); // Must not be active. - // TODO: Do we need to check token result for begin_apply? - SILValue result = fai.getResult(); - assert(!result || !getActivityInfo().isActive(result, getConfig())); + assert(!getActivityInfo().isActive(normalBlock->getArgument(0), getConfig())); return; } + + // try_apply pullback produces value only on non-error path, therefore + // its pullback might be not available. Therefore we wrap pullback into Optional + // in the linear map tuple. Build a diamond-shaped CFG fragment here, emitting + // switch_enum to unwrap an Optional. If pullback is present we + // emit a pullback call and propagate adjoints of arguments to successor block. + // Otherwise, there is no pullback call, so we only propagate adjoints of would-be + // arguments. So, the code looks like as follows: + // bb1(%35 : $Float, %36 : @owned $(_: Optional<@callee_guaranteed (Float) -> Float>)): + // %37 = destructure_tuple %36 + // switch_enum %37, case #Optional.some!enumelt: bb8, case #Optional.none!enumelt: bb9 + // bb8(%39 : @owned $@callee_guaranteed (Float) -> Float) + // %40 = apply %39(%0) : $@callee_guaranteed (Float) -> Float + // accumulate adjoints (using %35)... + // %54 = load [trivial] %42 + // br bb10(%54) + // bb9: + // br bb10(%35) + // bb10(%60 : $Float): + // ... + auto loc = tai->getLoc(); + auto optPullback = getPullbackTupleElement(tai); + auto *pbBB = builder.getInsertionBB(); + auto *normalPbBB = pbBB->split(builder.getInsertionPoint()); + auto *errorPbBB = getPullback().createBasicBlockAfter(normalPbBB); + auto *afterTryApplyPbBB = getPullback().createBasicBlockAfter(errorPbBB); + + // Note that we cannot simply assign map value as DenseMap entry references + // could be invalidated on insertion, so map[a] = map[b] may trigger + // use-after-free + auto currentBT = blockTemporaries[pbBB]; + blockTemporaries.insert({afterTryApplyPbBB, currentBT}); + blockTemporaries[pbBB].clear(); + + auto pullback = + builder.createSwitchOptional(loc, optPullback, + normalPbBB, errorPbBB, + optPullback->getOwnershipKind()); + + SmallVector adjArgs; + FullApplySite fai(tai); + auto *originalBB = tai->getParent(); auto &applyInfo = applyInfoLookup->getSecond(); + for (unsigned i : applyInfo.config.parameterIndices->getIndices()) { + unsigned argIdx = fai.getNumIndirectSILResults() + fai.getNumIndirectSILErrorResults() + i; + auto origArg = fai.getArgument(argIdx); + auto paramInfo = fai.getSubstCalleeConv().getParamInfoForSILArg(argIdx); + if (paramInfo.isAutoDiffSemanticResult()) + continue; + if (getTangentValueCategory(origArg) != SILValueCategory::Object) + continue; + + adjArgs.push_back(origArg); + afterTryApplyPbBB->createPhiArgument(origArg->getType(), + OwnershipKind::Owned); + } + + { + builder.setInsertionPoint(errorPbBB); + + SmallVector outAdjArgs; + for (auto arg : adjArgs) { + auto argAdj = getAdjointValue(originalBB, arg); + outAdjArgs.push_back(materializeAdjointDirect(argAdj, loc)); + } + + cleanUpTemporariesForBlock(errorPbBB, loc); + builder.createBranch(loc, afterTryApplyPbBB, outAdjArgs); + } + + { + builder.setInsertionPoint(normalPbBB); + buildPullbackCall(tai, applyInfo, pullback); + + SmallVector outAdjArgs; + for (auto arg : adjArgs) { + auto argAdj = getAdjointValue(originalBB, arg); + outAdjArgs.push_back(materializeAdjointDirect(argAdj, loc)); + } + + cleanUpTemporariesForBlock(normalPbBB, loc); + builder.createBranch(loc, afterTryApplyPbBB, outAdjArgs); + } + + builder.setInsertionPoint(afterTryApplyPbBB); + for (auto argAndIdx : llvm::enumerate(adjArgs)) { + auto forwardedArgAdj = + makeConcreteAdjointValue(afterTryApplyPbBB->getArgument(argAndIdx.index())); + setAdjointValue(originalBB, argAndIdx.value(), forwardedArgAdj); + } + + pullbackBBMap[tai->getParent()].second = afterTryApplyPbBB; + } + + void buildPullbackCall(FullApplySite fai, NestedApplyInfo &applyInfo, + SILValue pullback = SILValue()) { + auto loc = fai->getLoc(); + auto *bb = fai->getParent(); + if (!pullback) + pullback = getPullbackTupleElement(fai); // Get the original result of the `apply` instruction. const auto &conv = fai.getSubstCalleeConv(); @@ -1020,7 +1141,8 @@ class PullbackCloner::Implementation final collectAllActualResultsInTypeOrder(fai, origDirectResults, origAllResults); // Append semantic result arguments after original results. for (auto paramIdx : applyInfo.config.parameterIndices->getIndices()) { - unsigned argIdx = fai.getNumIndirectSILResults() + paramIdx; + unsigned argIdx = fai.getNumIndirectSILResults() + + fai.getNumIndirectSILErrorResults() + paramIdx; auto paramInfo = conv.getParamInfoForSILArg(argIdx); if (!paramInfo.isAutoDiffSemanticResult()) continue; @@ -1033,7 +1155,6 @@ class PullbackCloner::Implementation final // Handle callee pullback indirect results. // Create local allocations for these and destroy them after the call. - auto pullback = getPullbackTupleElement(fai); auto pullbackType = remapType(pullback->getType()).castTo(); @@ -1053,16 +1174,22 @@ class PullbackCloner::Implementation final unsigned firstSemanticParamResultIdx = conv.getResults().size(); unsigned firstYieldResultIndex = firstSemanticParamResultIdx + conv.getNumAutoDiffSemanticResultParameters(); + for (auto resultIndex : applyInfo.config.resultIndices->getIndices()) { if (resultIndex >= firstYieldResultIndex) continue; assert(resultIndex < origAllResults.size()); auto origResult = origAllResults[resultIndex]; + // Get the seed (i.e. adjoint value of the original result). SILValue seed; switch (getTangentValueCategory(origResult)) { case SILValueCategory::Object: - seed = materializeAdjointDirect(getAdjointValue(bb, origResult), loc); + // Adjoint for normal try_apply result is available in the normal destination BB. + // Get it from there. + seed = (fai.getKind() == FullApplySiteKind::TryApplyInst ? + tryApplyAdjoints.at(cast(fai.getInstruction())) : + materializeAdjointDirect(getAdjointValue(bb, origResult), loc)); break; case SILValueCategory::Address: seed = getAdjointBuffer(bb, origResult); @@ -1120,7 +1247,7 @@ class PullbackCloner::Implementation final // Accumulate adjoints for original differentiation parameters. auto allResultsIt = allResults.begin(); for (unsigned i : applyInfo.config.parameterIndices->getIndices()) { - unsigned argIdx = fai.getNumIndirectSILResults() + i; + unsigned argIdx = fai.getNumIndirectSILResults() + fai.getNumIndirectSILErrorResults() + i; auto origArg = fai.getArgument(argIdx); // Skip adjoint accumulation for semantic results arguments. auto paramInfo = fai.getSubstCalleeConv().getParamInfoForSILArg(argIdx); @@ -1203,7 +1330,7 @@ class PullbackCloner::Implementation final return; } - buildPullbackCall(bai); + buildPullbackCall(bai, applyInfoLookup->second); } void visitBeginApplyInst(BeginApplyInst *bai) { @@ -2278,7 +2405,7 @@ bool PullbackCloner::Implementation::run() { for (auto *origBB : originalBlocks) { auto *pullbackBB = pullback.createBasicBlock(); - pullbackBBMap.insert({origBB, pullbackBB}); + pullbackBBMap.insert({origBB, { pullbackBB, pullbackBB } }); auto pbTupleLoweredType = remapType(getPullbackInfo().getLinearMapTupleLoweredType(origBB)); // If the BB is the original exit, then the pullback block that we just @@ -2469,7 +2596,7 @@ bool PullbackCloner::Implementation::run() { if (loop == nullptr) continue; SILBasicBlock *loopHeader = loop->getHeader(); - SILBasicBlock *pbLoopHeader = getPullbackBlock(loopHeader); + SILBasicBlock *pbLoopHeader = getPullbackBlock(loopHeader).first; LLVM_DEBUG(getADDebugStream() << "Original bb" << bb->getDebugID() << " belongs to a loop, original header bb" @@ -2597,7 +2724,7 @@ bool PullbackCloner::Implementation::run() { // Prepare and emit a `return` in the pullback exit block. auto *origEntry = getOriginal().getEntryBlock(); - auto *pbExit = getPullbackBlock(origEntry); + auto *pbExit = getPullbackBlock(origEntry).second; builder.setCurrentDebugScope(pbExit->back().getDebugScope()); builder.setInsertionPoint(pbExit); @@ -2895,7 +3022,7 @@ SILBasicBlock *PullbackCloner::Implementation::buildPullbackSuccessor( SmallDenseMap &pullbackTrampolineBlockMap) { // Get the pullback block and optional pullback trampoline block of the // predecessor block. - auto *pullbackBB = getPullbackBlock(origPredBB); + auto *pullbackBB = getPullbackBlock(origPredBB).first; auto *pullbackTrampolineBB = getPullbackTrampolineBlock(origPredBB, origBB); // If the predecessor block does not have a corresponding pullback // trampoline block, then the pullback successor is the pullback block. @@ -2994,7 +3121,7 @@ SILBasicBlock *PullbackCloner::Implementation::buildPullbackSuccessor( void PullbackCloner::Implementation::visitSILBasicBlock(SILBasicBlock *bb) { auto pbLoc = getPullback().getLocation(); // Get the corresponding pullback basic block. - auto *pbBB = getPullbackBlock(bb); + auto *pbBB = getPullbackBlock(bb).first; builder.setInsertionPoint(pbBB); LLVM_DEBUG({ @@ -3019,6 +3146,9 @@ void PullbackCloner::Implementation::visitSILBasicBlock(SILBasicBlock *bb) { return; } + // If visitor changed current BB, update it here as well. + pbBB = builder.getInsertionBB(); + // Emit a branching terminator for the block. // If the original block is the original entry, then the pullback block is // the pullback exit. This is handled specially in @@ -3100,7 +3230,7 @@ void PullbackCloner::Implementation::visitSILBasicBlock(SILBasicBlock *bb) { if (isSwitchEnumInstOnOptional(termInst)) { accumulateAdjointValueForOptional(bb, incomingValue, concreteBBArgAdjCopy); } else { - blockTemporaries[getPullbackBlock(predBB)].insert( + blockTemporaries[getPullbackBlock(predBB).first].insert( concreteBBArgAdjCopy); addAdjointValue(predBB, incomingValue, makeConcreteAdjointValue(concreteBBArgAdjCopy), pbLoc); @@ -3125,6 +3255,36 @@ void PullbackCloner::Implementation::visitSILBasicBlock(SILBasicBlock *bb) { break; } } + } else if (auto *tai = dyn_cast(bbArg->getSingleTerminator())) { + // try_apply does not provide result, so there is no value to propagate + // adjoint to. Prepare adjoint and associate it "on side", so it will be + // used as seed during pullback call emission + if (!getPullbackInfo().shouldDifferentiateApplySite(tai)) + continue; + + LLVM_DEBUG(getADDebugStream() << "Creating adjoint value to active try_apply " + << *tai << " in " << bb->getDebugID() << " destination "); + + auto *predBB = tai->getParentBlock(); + switch (getTangentValueCategory(bbArg)) { + case SILValueCategory::Object: { + auto bbArgAdj = getAdjointValue(bb, bbArg); + auto concreteBBArgAdj = materializeAdjointDirect(bbArgAdj, pbLoc); + auto concreteBBArgAdjCopy = + builder.emitCopyValueOperation(pbLoc, concreteBBArgAdj); + + blockTemporaries[getPullbackBlock(predBB).first].insert(concreteBBArgAdjCopy); + auto [_, inserted] = tryApplyAdjoints.try_emplace(tai, concreteBBArgAdjCopy); + assert(inserted && "should have unique adjoint for try_apply"); + break; + } + case SILValueCategory::Address: { + auto bbArgAdjBuf = getAdjointBuffer(bb, bbArg); + auto [_, inserted] = tryApplyAdjoints.try_emplace(tai, bbArgAdjBuf); + assert(inserted && "should have unique adjoint for try_apply"); + break; + } + } } else { LLVM_DEBUG(getADDebugStream() << "do not know how to handle this incoming bb argument"); diff --git a/lib/SILOptimizer/Differentiation/VJPCloner.cpp b/lib/SILOptimizer/Differentiation/VJPCloner.cpp index 7a11087553685..a5f2bcfb6cf98 100644 --- a/lib/SILOptimizer/Differentiation/VJPCloner.cpp +++ b/lib/SILOptimizer/Differentiation/VJPCloner.cpp @@ -989,16 +989,268 @@ class VJPCloner::Implementation final void visitTryApplyInst(TryApplyInst *tai) { Builder.setCurrentDebugScope(getOpScope(tai->getDebugScope())); - // Build pullback struct value for original block. - auto *pbTupleVal = buildPullbackValueTupleValue(tai); - // Create a new `try_apply` instruction. - auto args = getOpValueArray<8>(tai->getArguments()); - getBuilder().createTryApply( + + // If callee should not be differentiated, do standard cloning. + if (!pullbackInfo.shouldDifferentiateApplySite(tai)) { + LLVM_DEBUG(getADDebugStream() << "No active results:\n" << *tai << '\n'); + + // Build pullback struct value for original block. + auto *pbTupleVal = buildPullbackValueTupleValue(tai); + // Create a new `try_apply` instruction. + auto args = getOpValueArray<8>(tai->getArguments()); + getBuilder().createTryApply( tai->getLoc(), getOpValue(tai->getCallee()), getOpSubstitutionMap(tai->getSubstitutionMap()), args, createTrampolineBasicBlock(tai, pbTupleVal, tai->getNormalBB()), createTrampolineBasicBlock(tai, pbTupleVal, tai->getErrorBB()), tai->getApplyOptions()); + + return; + } + + auto loc = tai->getLoc(); + auto &builder = getBuilder(); + auto origCallee = getOpValue(tai->getCallee()); + auto originalFnTy = origCallee->getType().castTo(); + auto *origBB = tai->getParent(); + + LLVM_DEBUG(getADDebugStream() << "VJP-transforming:\n" << *tai << '\n'); + + // Get the minimal parameter and result indices required for differentiating + // this `apply`. + SmallVector allResults; + SmallVector activeParamIndices; + SmallVector activeResultIndices; + collectMinimalIndicesForFunctionCall(tai, getConfig(), activityInfo, + allResults, activeParamIndices, + activeResultIndices); + assert(!activeParamIndices.empty() && "Parameter indices cannot be empty"); + assert(!activeResultIndices.empty() && "Result indices cannot be empty"); + LLVM_DEBUG(auto &s = getADDebugStream() << "Active indices: params=("; + llvm::interleave( + activeParamIndices.begin(), activeParamIndices.end(), + [&s](unsigned i) { s << i; }, [&s] { s << ", "; }); + s << "), results=("; llvm::interleave( + activeResultIndices.begin(), activeResultIndices.end(), + [&s](unsigned i) { s << i; }, [&s] { s << ", "; }); + s << ")\n";); + + // Form expected indices. + AutoDiffConfig config( + IndexSubset::get(getASTContext(), + tai->getArgumentsWithoutIndirectResults().size(), + activeParamIndices), + IndexSubset::get(getASTContext(), + tai->getSubstCalleeType()->getNumAutoDiffSemanticResults(), + activeResultIndices)); + + // Emit the VJP. + SILValue vjpValue; + + if (diagnoseNondifferentiableOriginalFunctionType(originalFnTy, + tai, origCallee, config)) { + errorOccurred = true; + return; + } + + // If the original `apply` instruction has a substitution map, then the + // applied function is specialized. + // In the VJP, specialization is also necessary for parity. The original + // function operand is specialized with a remapped version of same + // substitution map using an argument-less `partial_apply`. + if (tai->getSubstitutionMap().empty()) { + origCallee = builder.emitCopyValueOperation(loc, origCallee); + } else { + auto substMap = getOpSubstitutionMap(tai->getSubstitutionMap()); + auto vjpPartialApply = builder.createPartialApply( + tai->getLoc(), origCallee, substMap, {}, + ParameterConvention::Direct_Guaranteed); + origCallee = vjpPartialApply; + originalFnTy = origCallee->getType().castTo(); + + // Diagnose if new original function type is non-differentiable. + if (diagnoseNondifferentiableOriginalFunctionType(originalFnTy, + tai, origCallee, config)) { + errorOccurred = true; + return; + } + } + + auto *diffFuncInst = context.createDifferentiableFunction( + builder, loc, config.parameterIndices, config.resultIndices, + origCallee); + + // Record the `differentiable_function` instruction. + context.getDifferentiableFunctionInstWorklist().push_back(diffFuncInst); + + builder.emitScopedBorrowOperation( + loc, diffFuncInst, [&](SILValue borrowedADFunc) { + auto extractedVJP = + builder.createDifferentiableFunctionExtract( + loc, NormalDifferentiableFunctionTypeComponent::VJP, + borrowedADFunc); + vjpValue = builder.emitCopyValueOperation(loc, extractedVJP); + }); + builder.emitDestroyValueOperation(loc, diffFuncInst); + + // Record desired/actual VJP indices. + // Temporarily set original pullback type to `None`. + NestedApplyInfo info{config, /*originalPullbackType*/ std::nullopt}; + auto insertion = context.getNestedApplyInfo().try_emplace(tai, info); + auto &nestedApplyInfo = insertion.first->getSecond(); + nestedApplyInfo = info; + + // Call the VJP using the original parameters. + SmallVector vjpArgs; + auto vjpFnTy = getOpType(vjpValue->getType()).castTo(); + auto numVJPArgs = + vjpFnTy->getNumParameters() + vjpFnTy->getNumIndirectFormalResults() + + (vjpFnTy->hasIndirectErrorResult() ? 1 : 0); + vjpArgs.reserve(numVJPArgs); + // Collect substituted arguments. + for (auto origArg : tai->getArguments()) + vjpArgs.push_back(getOpValue(origArg)); + assert(vjpArgs.size() == numVJPArgs); + + // Create placeholder trampoline BB for error destination + auto *errorBB = + vjp->createBasicBlockBefore(getOpBasicBlock(tai->getErrorBB())); + for (auto *arg : getOpBasicBlock(tai->getErrorBB())->getArguments().drop_back()) + errorBB->createPhiArgument(arg->getType(), arg->getOwnershipKind()); + + // Create placeholder trampoline BB for normal destination + auto *normalBB = + vjp->createBasicBlockBefore(getOpBasicBlock(tai->getNormalBB())); + normalBB->createPhiArgument( + vjpFnTy->getDirectFormalResultsType(getModule(), + TypeExpansionContext::minimal()), + tai->getNormalBB()->getArgument(0)->getOwnershipKind()); + + // Apply the VJP. + // The VJP should be specialized, so no substitution map is necessary. + auto args = getOpValueArray<8>(tai->getArguments()); + auto *vjpCall = builder.createTryApply(loc, vjpValue, SubstitutionMap(), + vjpArgs, + normalBB, errorBB, + tai->getApplyOptions()); + + LLVM_DEBUG(getADDebugStream() << "Applied vjp function\n" << *vjpCall); + + // Perform necessary cleanup on error path and forward the error result. + // There is no pullback here. + { + SILBuilder trampolineBuilder(errorBB); + trampolineBuilder.setCurrentDebugScope(getOpScope(tai->getDebugScope())); + + trampolineBuilder.emitDestroyValueOperation(loc, vjpValue); + + auto pullbackType = pullbackInfo.lookUpLinearMapType(tai); + auto pullbackFnType = pullbackType->getOptionalObjectType(); + auto loweredPullbackType = getOpType(getLoweredType(pullbackFnType)); + auto tupleLoweredTy = remapType(pullbackInfo.getLinearMapTupleLoweredType(origBB)); + + // Find `Optional.none` EnumElementDecl. + auto noneEltDecl = getASTContext().getOptionalNoneDecl(); + // %enum = enum $Optional, #Optional.none!enumelt + auto bbPullbackValues = getPullbackValues(origBB); + bbPullbackValues.push_back( + trampolineBuilder.createEnum(loc, SILValue(), noneEltDecl, + SILType::getOptionalType(loweredPullbackType))); + + auto pbTupleVal = trampolineBuilder.createTuple(loc, + tupleLoweredTy, bbPullbackValues); + + auto *succEnumVal = + buildPredecessorEnumValue(trampolineBuilder, origBB, tai->getErrorBB(), pbTupleVal); + SmallVector forwardedArguments( + errorBB->getArguments().begin(), errorBB->getArguments().end()); + forwardedArguments.push_back(succEnumVal); + trampolineBuilder.createBranch(loc, getOpBasicBlock(tai->getErrorBB()), + forwardedArguments); + } + + // Capture the pullback on normal path and forward result + { + SILBuilder trampolineBuilder(normalBB); + trampolineBuilder.setCurrentDebugScope(getOpScope(tai->getDebugScope())); + + trampolineBuilder.emitDestroyValueOperation(loc, vjpValue); + + // Get the VJP results (original results and pullback). + SmallVector vjpDirectResults; + extractAllElements(normalBB->getArgument(0), trampolineBuilder, vjpDirectResults); + ArrayRef originalDirectResults = + ArrayRef(vjpDirectResults).drop_back(1); + SILValue originalDirectResult = + joinElements(originalDirectResults, trampolineBuilder, vjpCall->getLoc()); + SILValue pullback = vjpDirectResults.back(); + { + auto pullbackFnType = pullback->getType().castTo(); + auto pullbackUnsubstFnType = + pullbackFnType->getUnsubstitutedType(getModule()); + if (pullbackFnType != pullbackUnsubstFnType) { + pullback = trampolineBuilder.createConvertFunction( + loc, pullback, + SILType::getPrimitiveObjectType(pullbackUnsubstFnType), + /*withoutActuallyEscaping*/ false); + } + } + + // Checkpoint the pullback. + auto pullbackType = pullbackInfo.lookUpLinearMapType(tai); + auto pullbackFnType = pullbackType->getOptionalObjectType(); + + // If actual pullback type does not match lowered pullback type, reabstract + // the pullback using a thunk. + auto actualPullbackType = + getOpType(pullback->getType()).getAs(); + auto loweredPullbackType = + getOpType(getLoweredType(pullbackFnType)).castTo(); + if (!loweredPullbackType->isEqual(actualPullbackType)) { + // Set non-reabstracted original pullback type in nested apply info. + nestedApplyInfo.originalPullbackType = actualPullbackType; + SILOptFunctionBuilder fb(context.getTransform()); + pullback = reabstractFunction( + trampolineBuilder, fb, loc, pullback, loweredPullbackType, + [this](SubstitutionMap subs) -> SubstitutionMap { + return this->getOpSubstitutionMap(subs); + }); + } + + // Technically, the pullback value is not available in originalBB, + // however, we record it for the try_apply's BB. This is safe as try_apply + // is a terminator and we are emitting the pullback manually + nestedApplyInfo.pullbackIdx = pullbackValues[origBB].size(); + + // Find `Optional.some` EnumElementDecl. + auto someEltDecl = getASTContext().getOptionalSomeDecl(); + // %enum = enum $Optional, #Optional.some!enumelt, + // %pullback : $PullbackType + pullback = trampolineBuilder.createEnum(loc, pullback, someEltDecl, + SILType::getOptionalType(pullback->getType())); + pullbackValues[origBB].push_back(pullback); + + auto tupleLoweredTy = remapType(pullbackInfo.getLinearMapTupleLoweredType(origBB)); + auto pbTupleVal = trampolineBuilder.createTuple(loc, + tupleLoweredTy, getPullbackValues(origBB)); + + auto *succEnumVal = + buildPredecessorEnumValue(trampolineBuilder, tai->getParent(), tai->getNormalBB(), + pbTupleVal); + SmallVector forwardedArguments{originalDirectResult, + succEnumVal}; + trampolineBuilder.createBranch(loc, getOpBasicBlock(tai->getNormalBB()), + forwardedArguments); + } + + // Some instructions that produce the callee may have been cloned. + // If the original callee did not have any users beyond this `apply`, + // recursively kill the cloned callee. + if (auto *origCallee = cast_or_null( + tai->getCallee()->getDefiningInstruction())) + if (origCallee->hasOneUse()) + recursivelyDeleteTriviallyDeadInstructions( + getOpValue(origCallee)->getDefiningInstruction()); } void visitDifferentiableFunctionInst(DifferentiableFunctionInst *dfi) { diff --git a/lib/Sema/TypeCheckAttr.cpp b/lib/Sema/TypeCheckAttr.cpp index 3e3eb031ca389..c22dada3e1e7e 100644 --- a/lib/Sema/TypeCheckAttr.cpp +++ b/lib/Sema/TypeCheckAttr.cpp @@ -6121,14 +6121,17 @@ static bool checkFunctionSignature( /// generic signature. static AnyFunctionType * makeFunctionType(ArrayRef parameters, Type resultType, + bool throws, Type thrownError, GenericSignature genericSignature) { // FIXME: Verify ExtInfo state is correct, not working by accident. if (genericSignature) { GenericFunctionType::ExtInfo info; + info = info.withThrows(throws, thrownError); return GenericFunctionType::get(genericSignature, parameters, resultType, info); } FunctionType::ExtInfo info; + info = info.withThrows(throws, thrownError); return FunctionType::get(parameters, resultType, info); } @@ -6154,6 +6157,7 @@ getDerivativeOriginalFunctionType(AnyFunctionType *derivativeFnTy) { auto originalResult = derivativeResult->getElement(0).getType(); auto *originalType = makeFunctionType( curryLevels.back()->getParams(), originalResult, + curryLevels.back()->isThrowing(), curryLevels.back()->getThrownError(), curryLevels.size() == 1 ? derivativeFnTy->getOptGenericSignature() : nullptr); @@ -6165,6 +6169,7 @@ getDerivativeOriginalFunctionType(AnyFunctionType *derivativeFnTy) { AnyFunctionType *curryLevel = pair.value(); originalType = makeFunctionType(curryLevel->getParams(), originalType, + curryLevel->isThrowing(), curryLevel->getThrownError(), i == curryLevelsWithoutLast.size() - 1 ? derivativeFnTy->getOptGenericSignature() : nullptr); @@ -6255,15 +6260,19 @@ getTransposeOriginalFunctionType(AnyFunctionType *transposeFnType, // `(Self) -> () -> `. if (isCurried) { assert(selfType && "`Self` type should be resolved"); - originalType = makeFunctionType(originalParams, originalResult, nullptr); + originalType = makeFunctionType(originalParams, originalResult, + transposeFnType->isThrowing(), transposeFnType->getThrownError(), + /*genericSignature=*/nullptr); originalType = makeFunctionType(AnyFunctionType::Param(selfType), originalType, + /*throws=*/false, Type(), transposeFnType->getOptGenericSignature()); } // Otherwise, the original function type is simply: // `() -> `. else { originalType = makeFunctionType(originalParams, originalResult, + transposeFnType->isThrowing(), transposeFnType->getThrownError(), transposeFnType->getOptGenericSignature()); } return originalType; diff --git a/stdlib/public/Differentiation/OptionalDifferentiation.swift b/stdlib/public/Differentiation/OptionalDifferentiation.swift index 1a239a68400d2..6e188d1626871 100644 --- a/stdlib/public/Differentiation/OptionalDifferentiation.swift +++ b/stdlib/public/Differentiation/OptionalDifferentiation.swift @@ -70,3 +70,16 @@ extension Optional.TangentVector: CustomReflectable { return value.customMirror } } + +@derivative(of: ??) +@_transparent +@_alwaysEmitIntoClient +func _vjpNilCoalescing(optional: T?, defaultValue: @autoclosure () throws -> T) + rethrows -> (value: T, pullback: (T.TangentVector) -> Optional.TangentVector) { + let hasValue = optional != nil + let value = try optional ?? defaultValue() + func pullback(_ v: T.TangentVector) -> Optional.TangentVector { + return hasValue ? .init(v) : .zero + } + return (value, pullback) +} diff --git a/test/AutoDiff/SILOptimizer/differentiation_control_flow_diagnostics.swift b/test/AutoDiff/SILOptimizer/differentiation_control_flow_diagnostics.swift index 6c6ba960de7c0..1bb778717039f 100644 --- a/test/AutoDiff/SILOptimizer/differentiation_control_flow_diagnostics.swift +++ b/test/AutoDiff/SILOptimizer/differentiation_control_flow_diagnostics.swift @@ -84,25 +84,29 @@ func testTryApply(_ x: Float) -> Float { return x } -// expected-error @+1 {{function is not differentiable}} @differentiable(reverse) -// expected-note @+1 {{when differentiating this function definition}} func withoutDerivative( at x: T, in body: (T) throws -> R ) rethrows -> R { - // expected-note @+1 {{expression is not differentiable}} + // expected-error @+2 {{expression is not differentiable}} + // expected-note @+1 {{opaque non-'@differentiable' function is not differentiable}} try body(x) } // Tests active `try_apply`. -// expected-error @+1 {{function is not differentiable}} @differentiable(reverse) -// expected-note @+1 {{when differentiating this function definition}} func testNilCoalescing(_ maybeX: Float?) -> Float { - // expected-note @+1 {{expression is not differentiable}} return maybeX ?? 10 } +// expected-error @+1 {{function is not differentiable}} +@differentiable(reverse) +// expected-note @+1 {{when differentiating this function definition}} +func testNilCoalescingActive(_ maybeX: Float?, y: Float) -> Float { + // expected-note @+1 {{cannot differentiate through a non-differentiable argument; do you want to use 'withoutDerivative(at:)'?}} + return maybeX ?? y +} + // Test unsupported differentiation of active enum values. // expected-error @+1 {{function is not differentiable}} diff --git a/test/AutoDiff/SILOptimizer/differentiation_diagnostics.swift b/test/AutoDiff/SILOptimizer/differentiation_diagnostics.swift index 6df2c5c5300ac..ddf7269ab09b6 100644 --- a/test/AutoDiff/SILOptimizer/differentiation_diagnostics.swift +++ b/test/AutoDiff/SILOptimizer/differentiation_diagnostics.swift @@ -46,14 +46,21 @@ func try_apply_rethrows(_ x: Float) -> Float { return x } -// This generates `try_apply` which we do not know to handle yet, therefore -// one should use a.differentialMap here. If / when differentiation of throwing -// functions will be supported, we'd need to remove this diagnostics. -// expected-error @+2 {{function is not differentiable}} -// expected-note @+2 {{when differentiating this function definition}} +func throwing_result(_ x: Float) throws -> Float { return x; } + +@differentiable(reverse) +func active_try_apply(_ x: Float) -> Float { + do { + return try throwing_result(x); + } catch { + return x + } +} + @differentiable(reverse) func map_nondiff(_ a: [Float]) -> [Float] { - // expected-note @+1 {{expression is not differentiable}} + // expected-error @+2 {{expression is not differentiable}} + // expected-note @+1 {{cannot differentiate functions that have not been marked '@differentiable' and that are defined in other files}} return a.map { $0 } } diff --git a/test/AutoDiff/SILOptimizer/optional_error.swift b/test/AutoDiff/SILOptimizer/optional_error.swift index 23104118acae0..3ca8a5fa85a44 100644 --- a/test/AutoDiff/SILOptimizer/optional_error.swift +++ b/test/AutoDiff/SILOptimizer/optional_error.swift @@ -8,12 +8,22 @@ import _Differentiation func o(ff: F) -> Double { var y = ff.i?.first { $0 >= 0.0 } ?? 0.0 while 0.0 < y { - // expected-note @+1 {{expression is not differentiable}} + // This one is not differentiable since rhs of ?? is an autoclosure that we cannot differentiate wrt. + // The variant below has rhs as non-active and therefore everything works. + // expected-note @+1 {{cannot differentiate through a non-differentiable argument; do you want to use 'withoutDerivative(at:)'}} y = ff.g() ?? y } return y } +func o2(ff: F) -> Double { + var y = ff.i?.first { $0 >= 0.0 } ?? 0.0 + while 0.0 < y { + y = ff.g() ?? 42 + } + return y +} + public struct F: Differentiable { @noDerivative var i: [Double]? {return nil} func g() -> Double? {return nil} diff --git a/test/AutoDiff/compiler_crashers_fixed/issue-70819-try-apply-adjoint.swift b/test/AutoDiff/compiler_crashers_fixed/issue-70819-try-apply-adjoint.swift new file mode 100644 index 0000000000000..d0e6b421a2bd4 --- /dev/null +++ b/test/AutoDiff/compiler_crashers_fixed/issue-70819-try-apply-adjoint.swift @@ -0,0 +1,24 @@ +// RUN: %target-build-swift %s + +// https://github.com/swiftlang/swift/issues/70819 +// Fixes "note: do not know how to handle this incoming bb argument" assertion +// Here try_apply is produced so the result value is available in the successor BB + +import _Differentiation + +@differentiable(reverse) +func test1(input: Float) throws -> Float { + return input +} + +@differentiable(reverse) +func test2(input: Float) -> Float { + do { + return try test1(input: input) + } catch { + return 0.0 + } +} + +let (value, gradient) = valueWithGradient(at: 1.0, of: test2) +print("Value: \(value), gradient: \(gradient)") diff --git a/test/AutoDiff/validation-test/throw.swift b/test/AutoDiff/validation-test/throw.swift new file mode 100644 index 0000000000000..589b7df376752 --- /dev/null +++ b/test/AutoDiff/validation-test/throw.swift @@ -0,0 +1,81 @@ +// RUN: %target-run-simple-swift +// REQUIRES: executable_test + +import StdlibUnittest +import DifferentiationUnittest + +var ThrowingTests = TestSuite("Throwing") + +enum E: Error { + case error +} + +ThrowingTests.testWithLeakChecking("SimpleTry") { + @differentiable(reverse) + func f(x: Double) throws -> Double { + if x < 0 { + throw E.error + } else { + return x * x + } + } + + expectEqual(4.0, gradient(at: 2.0, of: {x in try! f(x: x)})) +} + +ThrowingTests.testWithLeakChecking("ActiveTry") { + @differentiable(reverse) + func f(x: Double) throws -> Double { + if x < 0 { + throw E.error + } else { + return x * x + } + } + + @differentiable(reverse) + func g(x: Double) -> Double { + do { + return try f(x: x) + } catch { + return 2*x + } + } + + @differentiable(reverse) + func h(x: Double) -> Double { + let y = 5*x; + do { + let z = -x; + return try f(x: z) + } catch { + return 2*y + } + } + + expectEqual(4.0, gradient(at: 2.0, of: g)) + expectEqual(2.0, gradient(at: -2.0, of: g)) + expectEqual(10.0, gradient(at: 2.0, of: h)) + expectEqual(-4.0, gradient(at: -2.0, of: h)) +} + +ThrowingTests.testWithLeakChecking("ActiveGenericTry") { + @differentiable(reverse where T : Differentiable) + func f(x: T) throws -> T { + return x + } + + @differentiable(reverse where T : Differentiable) + func g(x: T) -> T { + do { + return try f(x: x) + } catch { + return x + } + } + + expectEqual(1.0, gradient(at: 2.0, of: g)) +} + + +runAllTests() From c942e9a4fe29f9dc00733b63f8bf1d1a7051cba4 Mon Sep 17 00:00:00 2001 From: Anton Korobeynikov Date: Mon, 30 Jun 2025 22:18:51 -0700 Subject: [PATCH 2/2] Enforce ownership for pullback --- lib/SILOptimizer/Differentiation/VJPCloner.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/SILOptimizer/Differentiation/VJPCloner.cpp b/lib/SILOptimizer/Differentiation/VJPCloner.cpp index a5f2bcfb6cf98..bf43cf098d996 100644 --- a/lib/SILOptimizer/Differentiation/VJPCloner.cpp +++ b/lib/SILOptimizer/Differentiation/VJPCloner.cpp @@ -1227,7 +1227,8 @@ class VJPCloner::Implementation final // %enum = enum $Optional, #Optional.some!enumelt, // %pullback : $PullbackType pullback = trampolineBuilder.createEnum(loc, pullback, someEltDecl, - SILType::getOptionalType(pullback->getType())); + SILType::getOptionalType(pullback->getType()), + OwnershipKind::Owned); pullbackValues[origBB].push_back(pullback); auto tupleLoweredTy = remapType(pullbackInfo.getLinearMapTupleLoweredType(origBB));