Skip to content

Commit

Permalink
Revert D50471345: Multisect successfully blamed "D50471345: [Kineto] …
Browse files Browse the repository at this point in the history 
…Make CUPTI lazy init" for test or build failures (#832)

Summary:
Pull Request resolved: #832

This diff is reverting D50471345
D50471345: [Kineto] Make CUPTI lazy init by aaronenyeshi has been identified to be causing the following test or build failures:

Reviewed By: davidberard98

Differential Revision: D51382925

fbshipit-source-id: 4bea1e4f62e967f84fa453be9df4ea4d4d5c91a5
  • Loading branch information
@facebook-github-bot
generatedunixname89002005232357 authored and facebook-github-bot committed Nov 16, 2023
1 parent 2cb8872 commit d1b7159
Show file tree
Hide file tree
Showing 4 changed files with 73 additions and 78 deletions.
37 changes: 23 additions & 14 deletions libkineto/src/CuptiActivityApi.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,13 +41,19 @@ inline bool cuptiLazyInit_() {

inline void reenableCuptiCallbacks_(std::shared_ptr<CuptiCallbackApi>& cbapi_) {
// Re-enable callbacks from the past if they exist.
VLOG(1) << "Re-enable previous CUPTI callbacks - Starting";
bool status = cbapi_->reenableCallbacks();
LOG(INFO) << " CUPTI subscriber after enable:" << cbapi_->getCuptiSubscriber();
if (!status) {
LOG(WARNING) << "Re-enable previous CUPTI callbacks - Failed to reenableCallbacks";
LOG(INFO) << "Re-enable previous CUPTI callbacks - Starting";
VLOG(1) << " CUPTI subscriber before reinit:" << cbapi_->getCuptiSubscriber();
cbapi_->initCallbackApi();
if (cbapi_->initSuccess()) {
VLOG(1) << " CUPTI subscriber after reinit:" << cbapi_->getCuptiSubscriber();
bool status = cbapi_->reenableCallbacks();
if (!status) {
LOG(WARNING) << "Re-enable previous CUPTI callbacks - Failed to reenableCallbacks";
} else {
LOG(INFO) << "Re-enable previous CUPTI callbacks - Successful";
}
} else {
VLOG(1) << "Re-enable previous CUPTI callbacks - Successful";
LOG(WARNING) << "Re-enable previous CUPTI callbacks - Failed to initCallbackApi";
}
}
#endif
Expand Down Expand Up @@ -300,13 +306,10 @@ void CuptiActivityApi::bufferCompleted(
void CuptiActivityApi::enableCuptiActivities(
const std::set<ActivityType>& selected_activities) {
#ifdef HAS_CUPTI
// Lazily support re-init of CUPTI Callbacks, if they were finalized before.
auto cbapi_ = CuptiCallbackApi::singleton();
if (!tracingEnabled_ && !cbapi_->initStatus()) {
cbapi_->initCallbackApi();
// Lazily support init of CUPTI Callbacks.
if (cuptiLazyInit_()) {
reenableCuptiCallbacks_(cbapi_);
}
if (!tracingEnabled_ && !cbapi_->initSuccess() && cuptiLazyInit_()) {
reenableCuptiCallbacks_(cbapi_);
}
cbapi_.reset();

Expand Down Expand Up @@ -393,6 +396,13 @@ void CuptiActivityApi::teardownContext() {
// PyTorch Profiler is synchronous, so teardown needs to be run async in this thread.
std::thread teardownThread([&] {
auto cbapi_ = CuptiCallbackApi::singleton();
if (!cbapi_->initSuccess()) {
cbapi_->initCallbackApi();
if (!cbapi_->initSuccess()) {
LOG(WARNING) << "CUPTI Callback failed to init, skipping teardown";
return;
}
}
// Subscribe callbacks to call cuptiFinalize in the exit callback of these APIs
bool status = cbapi_->enableCallbackDomain(CUPTI_CB_DOMAIN_RUNTIME_API);
status = status && cbapi_->enableCallbackDomain(CUPTI_CB_DOMAIN_DRIVER_API);
Expand All @@ -404,7 +414,7 @@ void CuptiActivityApi::teardownContext() {
// Force Flush before finalize
CUPTI_CALL(cuptiActivityFlushAll(CUPTI_ACTIVITY_FLAG_FLUSH_FORCED));

VLOG(1) << " CUPTI subscriber before finalize:" << cbapi_->getCuptiSubscriber();
LOG(INFO) << " CUPTI subscriber before finalize:" << cbapi_->getCuptiSubscriber();
teardownCupti_ = 1;
std::unique_lock<std::mutex> lck(finalizeMutex_);
finalizeCond_.wait(lck, [&]{return teardownCupti_ == 0;});
Expand All @@ -417,7 +427,6 @@ void CuptiActivityApi::teardownContext() {
// Remove the callbacks used specifically for cuptiFinalize
cbapi_->disableCallbackDomain(CUPTI_CB_DOMAIN_RUNTIME_API);
cbapi_->disableCallbackDomain(CUPTI_CB_DOMAIN_DRIVER_API);
cbapi_->deinitCallbackApi();

// Re-init CUPTI Callbacks if Lazy Re-init is not enabled.
if (!cuptiLazyInit_()) {
Expand Down
29 changes: 1 addition & 28 deletions libkineto/src/CuptiCallbackApi.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -165,43 +165,19 @@ std::shared_ptr<CuptiCallbackApi> CuptiCallbackApi::singleton() {

void CuptiCallbackApi::initCallbackApi() {
#ifdef HAS_CUPTI
if (initSuccess_) {
return;
}

lastCuptiStatus_ = CUPTI_ERROR_UNKNOWN;
lastCuptiStatus_ = CUPTI_CALL_NOWARN(
cuptiSubscribe(&subscriber_,
(CUpti_CallbackFunc)callback_switchboard,
nullptr));
if (lastCuptiStatus_ != CUPTI_SUCCESS) {
LOG(WARNING) << "Failed cuptiSubscribe, status: " << lastCuptiStatus_;
LOG(WARNING) << "CUPTI initialization failed - "
<< "CUDA profiler activities will be missing";
if (lastCuptiStatus_ == CUPTI_ERROR_INSUFFICIENT_PRIVILEGES) {
LOG(INFO) << "For CUPTI_ERROR_INSUFFICIENT_PRIVILEGES, refer to "
<< "https://developer.nvidia.com/nvidia-development-tools-solutions-err-nvgpuctrperm-cupti";
}
VLOG(1) << "Failed cuptiSubscribe, status: " << lastCuptiStatus_;
}

initSuccess_ = (lastCuptiStatus_ == CUPTI_SUCCESS);
#endif
}

void CuptiCallbackApi::deinitCallbackApi() {
#ifdef HAS_CUPTI
if (!initSuccess_) {
return;
}
lastCuptiStatus_ = CUPTI_CALL_NOWARN(
cuptiUnsubscribe(subscriber_));
if (lastCuptiStatus_ != CUPTI_SUCCESS) {
LOG(WARNING) << "Failed cuptiUnsubscribe, status: " << lastCuptiStatus_;
}
initSuccess_ = false;
#endif
}

CuptiCallbackApi::CallbackList* CuptiCallbackApi::CallbackTable::lookup(
CUpti_CallbackDomain domain, CuptiCallBackID cbid) {
size_t idx;
Expand Down Expand Up @@ -286,7 +262,6 @@ bool CuptiCallbackApi::deleteCallback(
bool CuptiCallbackApi::enableCallback(
CUpti_CallbackDomain domain, CUpti_CallbackId cbid) {
#ifdef HAS_CUPTI
initCallbackApi();
if (initSuccess_) {
lastCuptiStatus_ = CUPTI_CALL_NOWARN(
cuptiEnableCallback(1, subscriber_, domain, cbid));
Expand All @@ -313,7 +288,6 @@ bool CuptiCallbackApi::disableCallback(
bool CuptiCallbackApi::enableCallbackDomain(
CUpti_CallbackDomain domain) {
#ifdef HAS_CUPTI
initCallbackApi();
if (initSuccess_) {
lastCuptiStatus_ = CUPTI_CALL_NOWARN(
cuptiEnableDomain(1, subscriber_, domain));
Expand All @@ -339,7 +313,6 @@ bool CuptiCallbackApi::disableCallbackDomain(

bool CuptiCallbackApi::reenableCallbacks() {
#ifdef HAS_CUPTI
initCallbackApi();
if (initSuccess_) {
for (auto& cbpair : enabledCallbacks_) {
if ((uint32_t)cbpair.second == MAX_CUPTI_CALLBACK_ID_ALL) {
Expand Down
3 changes: 1 addition & 2 deletions libkineto/src/CuptiCallbackApi.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,9 +72,8 @@ class CuptiCallbackApi {
static std::shared_ptr<CuptiCallbackApi> singleton();

void initCallbackApi();
void deinitCallbackApi();

bool initStatus() const {
bool initSuccess() const {
return initSuccess_;
}

Expand Down
82 changes: 48 additions & 34 deletions libkineto/src/init.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
namespace KINETO_NAMESPACE {

#ifdef HAS_CUPTI
static bool initialized = false;
static std::mutex initMutex;

bool enableEventProfiler() {
Expand All @@ -45,14 +46,25 @@ static void initProfilers(
VLOG(0) << "CUDA Context created";
std::lock_guard<std::mutex> lock(initMutex);

CUpti_ResourceData* d = (CUpti_ResourceData*)cbInfo;
CUcontext ctx = d->context;
ConfigLoader& config_loader = libkineto::api().configLoader();
config_loader.initBaseConfig();
auto config = config_loader.getConfigCopy();
if (config->eventProfilerEnabled()) {
EventProfilerController::start(ctx, config_loader);
LOG(INFO) << "Kineto EventProfiler started";
if (!initialized) {
libkineto::api().initProfilerIfRegistered();
initialized = true;
VLOG(0) << "libkineto profilers activated";
}

if (!enableEventProfiler()) {
VLOG(0) << "Kineto EventProfiler disabled, skipping start";
return;
} else {
CUpti_ResourceData* d = (CUpti_ResourceData*)cbInfo;
CUcontext ctx = d->context;
ConfigLoader& config_loader = libkineto::api().configLoader();
config_loader.initBaseConfig();
auto config = config_loader.getConfigCopy();
if (config->eventProfilerEnabled()) {
EventProfilerController::start(ctx, config_loader);
LOG(INFO) << "Kineto EventProfiler started";
}
}
}

Expand All @@ -76,10 +88,12 @@ static void stopProfiler(
VLOG(0) << "CUDA Context destroyed";
std::lock_guard<std::mutex> lock(initMutex);

CUpti_ResourceData* d = (CUpti_ResourceData*)cbInfo;
CUcontext ctx = d->context;
EventProfilerController::stopIfEnabled(ctx);
LOG(INFO) << "Kineto EventProfiler stopped";
if (enableEventProfiler()) {
CUpti_ResourceData* d = (CUpti_ResourceData*)cbInfo;
CUcontext ctx = d->context;
EventProfilerController::stopIfEnabled(ctx);
LOG(INFO) << "Kineto EventProfiler stopped";
}
}

static std::unique_ptr<CuptiRangeProfilerInit> rangeProfilerInit;
Expand Down Expand Up @@ -110,29 +124,31 @@ void libkineto_init(bool cpuOnly, bool logOnError) {
#endif

#ifdef HAS_CUPTI
bool initRangeProfiler = false;
if (!cpuOnly && enableEventProfiler() ) {
if (!cpuOnly) {
// libcupti will be lazily loaded on this call.
// If it is not available (e.g. CUDA is not installed),
// then this call will return an error and we just abort init.
auto cbapi = CuptiCallbackApi::singleton();
cbapi->initCallbackApi();
bool status = false;
initRangeProfiler = true;

const CUpti_CallbackDomain domain = CUPTI_CB_DOMAIN_RESOURCE;
status = cbapi->registerCallback(
domain, CuptiCallbackApi::RESOURCE_CONTEXT_CREATED, initProfilers);
status = status && cbapi->registerCallback(
domain, CuptiCallbackApi::RESOURCE_CONTEXT_DESTROYED, stopProfiler);

if (status) {
status = cbapi->enableCallback(
domain, CuptiCallbackApi::RESOURCE_CONTEXT_CREATED);
status = status && cbapi->enableCallback(
domain, CuptiCallbackApi::RESOURCE_CONTEXT_DESTROYED);
bool initRangeProfiler = true;

if (cbapi->initSuccess()){
const CUpti_CallbackDomain domain = CUPTI_CB_DOMAIN_RESOURCE;
status = cbapi->registerCallback(
domain, CuptiCallbackApi::RESOURCE_CONTEXT_CREATED, initProfilers);
status = status && cbapi->registerCallback(
domain, CuptiCallbackApi::RESOURCE_CONTEXT_DESTROYED, stopProfiler);

if (status) {
status = cbapi->enableCallback(
domain, CuptiCallbackApi::RESOURCE_CONTEXT_CREATED);
status = status && cbapi->enableCallback(
domain, CuptiCallbackApi::RESOURCE_CONTEXT_DESTROYED);
}
}

if (!cbapi->initStatus() || !status) {
if (!cbapi->initSuccess() || !status) {
initRangeProfiler = false;
cpuOnly = true;
if (logOnError) {
Expand All @@ -143,13 +159,11 @@ void libkineto_init(bool cpuOnly, bool logOnError) {
<< "https://developer.nvidia.com/nvidia-development-tools-solutions-err-nvgpuctrperm-cupti";
}
}
} else {
VLOG(0) << "Kineto EventProfiler disabled, skipping it";
}

// initialize CUPTI Range Profiler API
if (initRangeProfiler) {
rangeProfilerInit = std::make_unique<CuptiRangeProfilerInit>();
// initialize CUPTI Range Profiler API
if (initRangeProfiler) {
rangeProfilerInit = std::make_unique<CuptiRangeProfilerInit>();
}
}

if (shouldPreloadCuptiInstrumentation()) {
Expand Down

0 comments on commit d1b7159

Please sign in to comment.