[OpenMP] Split EmitOMPWorksharingLoop into loop and body Claude assisted with this patch.
diff --git a/clang/lib/CodeGen/CGStmtOpenMP.cpp b/clang/lib/CodeGen/CGStmtOpenMP.cpp index a761ac6..72ec3e3 100644 --- a/clang/lib/CodeGen/CGStmtOpenMP.cpp +++ b/clang/lib/CodeGen/CGStmtOpenMP.cpp
@@ -3782,6 +3782,62 @@ }; } // namespace +bool CodeGenFunction::EmitOMPWorksharingLoopBody( + const OMPLoopDirective &S, LValue IL, bool HasLinears, + const CodeGenLoopStructureTy &CodeGenLoopStructure) { + OpenMPDirectiveKind EKind = getEffectiveDirectiveKind(S); + OMPPrivateScope LoopScope(*this); + if (EmitOMPFirstprivateClause(S, LoopScope) || HasLinears) { + // Emit implicit barrier to synchronize threads and avoid data races on + // initialization of firstprivate variables and post-update of + // lastprivate variables. + CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(), OMPD_unknown, + /*EmitChecks=*/false, + /*ForceSimpleCall=*/true); + } + EmitOMPPrivateClause(S, LoopScope); + CGOpenMPRuntime::LastprivateConditionalRAII LPCRegion( + *this, S, EmitLValue(S.getIterationVariable())); + bool HasLastprivateClause = EmitOMPLastprivateClauseInit(S, LoopScope); + EmitOMPReductionClauseInit(S, LoopScope); + EmitOMPPrivateLoopCounters(S, LoopScope); + EmitOMPLinearClause(S, LoopScope); + (void)LoopScope.Privatize(); + if (isOpenMPTargetExecutionDirective(EKind)) + CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(*this, S); + + // Emit the loop structure around the body of the construct. For "no-loop" + // codegen this emits the body a single time instead. + CodeGenLoopStructure(*this, LoopScope); + + if (isOpenMPSimdDirective(EKind)) { + EmitOMPSimdFinal(S, [IL, &S](CodeGenFunction &CGF) { + return CGF.Builder.CreateIsNotNull( + CGF.EmitLoadOfScalar(IL, S.getBeginLoc())); + }); + } + EmitOMPReductionClauseFinal(S, + /*ReductionKind=*/isOpenMPSimdDirective(EKind) + ? /*Parallel and Simd*/ OMPD_parallel_for_simd + : /*Parallel only*/ OMPD_parallel); + // Emit post-update of the reduction variables if IsLastIter != 0. + emitPostUpdateForReductionClause(*this, S, [IL, &S](CodeGenFunction &CGF) { + return CGF.Builder.CreateIsNotNull( + CGF.EmitLoadOfScalar(IL, S.getBeginLoc())); + }); + // Emit final copy of the lastprivate variables if IsLastIter != 0. + if (HasLastprivateClause) + EmitOMPLastprivateClauseFinal( + S, isOpenMPSimdDirective(EKind), + Builder.CreateIsNotNull(EmitLoadOfScalar(IL, S.getBeginLoc()))); + LoopScope.restoreMap(); + EmitOMPLinearClauseFinal(S, [IL, &S](CodeGenFunction &CGF) { + return CGF.Builder.CreateIsNotNull( + CGF.EmitLoadOfScalar(IL, S.getBeginLoc())); + }); + return HasLastprivateClause; +} + bool CodeGenFunction::EmitOMPWorksharingLoop( const OMPLoopDirective &S, Expr *EUB, const CodeGenLoopBoundsTy &CodeGenLoopBounds, @@ -3844,28 +3900,13 @@ LValue IL = EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getIsLastIterVariable())); - // Emit 'then' code. - { + // Emit 'then' code: the schedule and the loop that the body of the + // construct is wrapped in. + auto &&CodeGenLoopStructure = [&S, EUB, &CGDispatchBounds, IVExpr, Ordered, + &RT, LB, UB, ST, + IL](CodeGenFunction &CGF, + OMPPrivateScope &LoopScope) { OpenMPDirectiveKind EKind = getEffectiveDirectiveKind(S); - OMPPrivateScope LoopScope(*this); - if (EmitOMPFirstprivateClause(S, LoopScope) || HasLinears) { - // Emit implicit barrier to synchronize threads and avoid data races on - // initialization of firstprivate variables and post-update of - // lastprivate variables. - CGM.getOpenMPRuntime().emitBarrierCall( - *this, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false, - /*ForceSimpleCall=*/true); - } - EmitOMPPrivateClause(S, LoopScope); - CGOpenMPRuntime::LastprivateConditionalRAII LPCRegion( - *this, S, EmitLValue(S.getIterationVariable())); - HasLastprivateClause = EmitOMPLastprivateClauseInit(S, LoopScope); - EmitOMPReductionClauseInit(S, LoopScope); - EmitOMPPrivateLoopCounters(S, LoopScope); - EmitOMPLinearClause(S, LoopScope); - (void)LoopScope.Privatize(); - if (isOpenMPTargetExecutionDirective(EKind)) - CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(*this, S); // Detect the loop schedule kind and chunk. const Expr *ChunkExpr = nullptr; @@ -3877,23 +3918,23 @@ ChunkExpr = C->getChunkSize(); } else { // Default behaviour for schedule clause. - CGM.getOpenMPRuntime().getDefaultScheduleAndChunk( - *this, S, ScheduleKind.Schedule, ChunkExpr); + CGF.CGM.getOpenMPRuntime().getDefaultScheduleAndChunk( + CGF, S, ScheduleKind.Schedule, ChunkExpr); } bool HasChunkSizeOne = false; llvm::Value *Chunk = nullptr; if (ChunkExpr) { - Chunk = EmitScalarExpr(ChunkExpr); - Chunk = EmitScalarConversion(Chunk, ChunkExpr->getType(), - S.getIterationVariable()->getType(), - S.getBeginLoc()); + Chunk = CGF.EmitScalarExpr(ChunkExpr); + Chunk = CGF.EmitScalarConversion(Chunk, ChunkExpr->getType(), + S.getIterationVariable()->getType(), + S.getBeginLoc()); Expr::EvalResult Result; - if (ChunkExpr->EvaluateAsInt(Result, getContext())) { + if (ChunkExpr->EvaluateAsInt(Result, CGF.getContext())) { llvm::APSInt EvaluatedChunk = Result.Val.getInt(); HasChunkSizeOne = (EvaluatedChunk.getLimitedValue() == 1); } } - const unsigned IVSize = getContext().getTypeSize(IVExpr->getType()); + const unsigned IVSize = CGF.getContext().getTypeSize(IVExpr->getType()); const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation(); // OpenMP 4.5, 2.7.1 Loop Construct, Description. // If the static schedule kind is specified or if the ordered clause is @@ -3911,7 +3952,7 @@ // disagree; the assert guards the invariant that makes this safe today, // aka that the implicit GPU default schedule is always static chunk-one. ScheduleKind.UseFusedDistChunkSchedule = - canEmitGPUFusedDistSchedule(CGM, S, EKind); + canEmitGPUFusedDistSchedule(CGF.CGM, S, EKind); assert((!ScheduleKind.UseFusedDistChunkSchedule || StaticChunkedOne) && "fused distribute schedule requires a static chunk-one schedule"); bool IsMonotonic = @@ -3925,10 +3966,10 @@ /* Chunked */ Chunk != nullptr) || StaticChunkedOne) && !Ordered) { - JumpDest LoopExit = - getJumpDestInCurrentScope(createBasicBlock("omp.loop.exit")); + JumpDest LoopExit = CGF.getJumpDestInCurrentScope( + CGF.createBasicBlock("omp.loop.exit")); emitCommonSimdLoop( - *this, S, + CGF, S, [&S, EKind](CodeGenFunction &CGF, PrePostActionTy &) { if (isOpenMPSimdDirective(EKind)) { CGF.EmitOMPSimdInit(S); @@ -3979,13 +4020,13 @@ }, [](CodeGenFunction &) {}); }); - EmitBlock(LoopExit.getBlock()); + CGF.EmitBlock(LoopExit.getBlock()); // Tell the runtime we are done. auto &&CodeGen = [&S](CodeGenFunction &CGF) { CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(), OMPD_for); }; - OMPCancelStack.emitExit(*this, EKind, CodeGen); + CGF.OMPCancelStack.emitExit(CGF, EKind, CodeGen); } else { // Emit the outer loop, which requests its work chunk [LB..UB] from // runtime and runs the inner loop to process it. @@ -3993,36 +4034,12 @@ ST.getAddress(), IL.getAddress(), Chunk, EUB); LoopArguments.DKind = OMPD_for; - EmitOMPForOuterLoop(ScheduleKind, IsMonotonic, S, LoopScope, Ordered, - LoopArguments, CGDispatchBounds); + CGF.EmitOMPForOuterLoop(ScheduleKind, IsMonotonic, S, LoopScope, + Ordered, LoopArguments, CGDispatchBounds); } - if (isOpenMPSimdDirective(EKind)) { - EmitOMPSimdFinal(S, [IL, &S](CodeGenFunction &CGF) { - return CGF.Builder.CreateIsNotNull( - CGF.EmitLoadOfScalar(IL, S.getBeginLoc())); - }); - } - EmitOMPReductionClauseFinal( - S, /*ReductionKind=*/isOpenMPSimdDirective(EKind) - ? /*Parallel and Simd*/ OMPD_parallel_for_simd - : /*Parallel only*/ OMPD_parallel); - // Emit post-update of the reduction variables if IsLastIter != 0. - emitPostUpdateForReductionClause( - *this, S, [IL, &S](CodeGenFunction &CGF) { - return CGF.Builder.CreateIsNotNull( - CGF.EmitLoadOfScalar(IL, S.getBeginLoc())); - }); - // Emit final copy of the lastprivate variables if IsLastIter != 0. - if (HasLastprivateClause) - EmitOMPLastprivateClauseFinal( - S, isOpenMPSimdDirective(EKind), - Builder.CreateIsNotNull(EmitLoadOfScalar(IL, S.getBeginLoc()))); - LoopScope.restoreMap(); - EmitOMPLinearClauseFinal(S, [IL, &S](CodeGenFunction &CGF) { - return CGF.Builder.CreateIsNotNull( - CGF.EmitLoadOfScalar(IL, S.getBeginLoc())); - }); - } + }; + HasLastprivateClause = + EmitOMPWorksharingLoopBody(S, IL, HasLinears, CodeGenLoopStructure); DoacrossCleanupScope.ForceCleanup(); // We're now done with the loop, so jump to the continuation block. if (ContBlock) {
diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h index 7bdc79d..dae9315 100644 --- a/clang/lib/CodeGen/CodeGenFunction.h +++ b/clang/lib/CodeGen/CodeGenFunction.h
@@ -4136,7 +4136,36 @@ /// Helper for the OpenMP loop directives. void EmitOMPLoopBody(const OMPLoopDirective &D, JumpDest LoopExit); - /// Emit code for the worksharing loop-based directive. + /// Codegen lambda emitting the loop structure a worksharing loop body is + /// wrapped in: the schedule initialization and the loop itself. It is passed + /// the privatization scope set up by EmitOMPWorksharingLoopBody(). + typedef llvm::function_ref<void(CodeGenFunction &, OMPPrivateScope &)> + CodeGenLoopStructureTy; + + /// Emit the body of a worksharing loop-based directive: the data-sharing + /// clauses (firstprivate, private, lastprivate, reduction, linear), the + /// privatized loop counters, the work of the construct itself - emitted by + /// \p CodeGenLoopStructure - and the finalization of those clauses. + /// + /// This is factored out of EmitOMPWorksharingLoop() so that it can also be + /// used on its own by "no-loop" codegen, i.e. when iterations can be mapped + /// 1:1 onto threads and no loop has to be emitted at all. In that case + /// \p CodeGenLoopStructure emits the loop body a single time instead of a + /// loop around it. + /// + /// \param IL LValue of the "is last iteration" helper variable, used to + /// guard the finalization of the clauses above. + /// \param HasLinears Whether EmitOMPLinearClauseInit() emitted any linear + /// clause initialization for \p S. + /// \return true, if this construct has any lastprivate clause, false - + /// otherwise. + bool EmitOMPWorksharingLoopBody( + const OMPLoopDirective &S, LValue IL, bool HasLinears, + const CodeGenLoopStructureTy &CodeGenLoopStructure); + + /// Emit code for the worksharing loop-based directive: the loop structure + /// (precondition, helper variables, schedule and the loop itself) around the + /// body emitted by EmitOMPWorksharingLoopBody(). /// \return true, if this construct has any lastprivate clause, false - /// otherwise. bool EmitOMPWorksharingLoop(const OMPLoopDirective &S, Expr *EUB,