Skip to content

Commit

Permalink
Improve shards file exception handling
Browse files Browse the repository at this point in the history
  • Loading branch information
@miguelportilla
miguelportilla committed Aug 7, 2018
1 parent cb7322c commit 384c6b8
Show file tree
Hide file tree
Showing 4 changed files with 218 additions and 154 deletions.
169 changes: 91 additions & 78 deletions src/ripple/nodestore/impl/DatabaseShardImp.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,67 +92,77 @@ DatabaseShardImp::init()
return true;
}

// Find shards
for (auto const& d : directory_iterator(dir_))
try
{
if (!is_directory(d))
continue;
// Find shards
for (auto const& d : directory_iterator(dir_))
{
if (!is_directory(d))
continue;

// Validate shard directory name is numeric
auto dirName = d.path().stem().string();
if (!std::all_of(
dirName.begin(),
dirName.end(),
[](auto c){
// Validate shard directory name is numeric
auto dirName = d.path().stem().string();
if (!std::all_of(
dirName.begin(),
dirName.end(),
[](auto c) {
return ::isdigit(static_cast<unsigned char>(c));
}))
{
continue;
}

auto const shardIndex {std::stoul(dirName)};
if (shardIndex < earliestShardIndex())
{
JLOG(j_.fatal()) <<
"Invalid shard index " << shardIndex <<
". Earliest shard index " << earliestShardIndex();
return false;
}
{
continue;
}

// Check if a previous import failed
if (is_regular_file(dir_ / std::to_string(shardIndex) /
importMarker_))
{
JLOG(j_.warn()) <<
"shard " << shardIndex <<
" previously failed import, removing";
if (!this->remove(dir_ / std::to_string(shardIndex)))
auto const shardIndex {std::stoul(dirName)};
if (shardIndex < earliestShardIndex())
{
JLOG(j_.fatal()) <<
"Invalid shard index " << shardIndex <<
". Earliest shard index " << earliestShardIndex();
return false;
continue;
}
}

auto shard = std::make_unique<Shard>(
*this, shardIndex, cacheSz_, cacheAge_, j_);
if (!shard->open(config_, scheduler_))
return false;
usedDiskSpace_ += shard->fileSize();
if (shard->complete())
complete_.emplace(shard->index(), std::move(shard));
else
{
if (incomplete_)
// Check if a previous import failed
if (is_regular_file(
dir_ / std::to_string(shardIndex) / importMarker_))
{
JLOG(j_.fatal()) <<
"More than one control file found";
JLOG(j_.warn()) <<
"shard " << shardIndex <<
" previously failed import, removing";
remove_all(dir_ / std::to_string(shardIndex));
continue;
}

auto shard {std::make_unique<Shard>(
*this, shardIndex, cacheSz_, cacheAge_, j_)};
if (!shard->open(config_, scheduler_))
return false;

usedDiskSpace_ += shard->fileSize();
if (shard->complete())
complete_.emplace(shard->index(), std::move(shard));
else
{
if (incomplete_)
{
JLOG(j_.fatal()) <<
"More than one control file found";
return false;
}
incomplete_ = std::move(shard);
}
incomplete_ = std::move(shard);
}
}
catch (std::exception const& e)
{
JLOG(j_.error()) <<
"exception: " << e.what();
return false;
}

if (!incomplete_ && complete_.empty())
{
// New Shard Store, calculate file descriptor requirements
if (maxDiskSpace_ > space(dir_).free)
if (maxDiskSpace_ > available())
{
JLOG(j_.error()) <<
"Insufficient disk space";
Expand Down Expand Up @@ -185,7 +195,7 @@ DatabaseShardImp::prepareLedger(std::uint32_t validLedgerSeq)
canAdd_ = false;
return boost::none;
}
if (avgShardSz_ > boost::filesystem::space(dir_).free)
if (avgShardSz_ > available())
{
JLOG(j_.error()) <<
"Insufficient disk space";
Expand All @@ -211,9 +221,9 @@ DatabaseShardImp::prepareLedger(std::uint32_t validLedgerSeq)
if (!incomplete_->open(config_, scheduler_))
{
incomplete_.reset();
this->remove(dir_ / std::to_string(*shardIndex));
return boost::none;
}

return incomplete_->prepare();
}

Expand Down Expand Up @@ -254,6 +264,7 @@ DatabaseShardImp::prepareShard(std::uint32_t shardIndex)
{
return false;
}

if (complete_.find(shardIndex) != complete_.end())
{
JLOG(j_.debug()) <<
Expand Down Expand Up @@ -287,7 +298,7 @@ DatabaseShardImp::prepareShard(std::uint32_t shardIndex)
"Exceeds maximum size";
return false;
}
if (sz > space(dir_).free)
if (sz > available())
{
JLOG(j_.error()) <<
"Insufficient disk space";
Expand Down Expand Up @@ -321,10 +332,19 @@ DatabaseShardImp::importShard(std::uint32_t shardIndex,
boost::filesystem::path const& srcDir, bool validate)
{
using namespace boost::filesystem;
if (!is_directory(srcDir) || is_empty(srcDir))
try
{
if (!is_directory(srcDir) || is_empty(srcDir))
{
JLOG(j_.error()) <<
"Invalid source directory " << srcDir.string();
return false;
}
}
catch (std::exception const& e)
{
JLOG(j_.error()) <<
"Invalid source directory " << srcDir.string();
"exception: " << e.what();
return false;
}

Expand All @@ -334,12 +354,10 @@ DatabaseShardImp::importShard(std::uint32_t shardIndex,
{
rename(src, dst);
}
catch (const filesystem_error& e)
catch (std::exception const& e)
{
JLOG(j_.error()) <<
"rename " << src.string() <<
" to " << dst.string() <<
": Exception, " << e.code().message();
"exception: " << e.what();
return false;
}
return true;
Expand Down Expand Up @@ -367,27 +385,25 @@ DatabaseShardImp::importShard(std::uint32_t shardIndex,
*this, shardIndex, cacheSz_, cacheAge_, j_)};
auto fail = [&](std::string msg)
{
if (!msg.empty())
{
JLOG(j_.error()) << msg;
}
JLOG(j_.error()) << msg;
shard.release();
move(dstDir, srcDir);
return false;
};

if (!shard->open(config_, scheduler_))
return fail({});
return fail("Failure");
if (!shard->complete())
return fail("Incomplete shard");

// Verify database integrity
try
{
// Verify database integrity
shard->getBackend()->verify();
}
catch (std::exception const& e)
{
return fail(std::string("Verify: Exception, ") + e.what());
return fail(std::string("exception: ") + e.what());
}

// Validate shard ledgers
Expand All @@ -397,14 +413,14 @@ DatabaseShardImp::importShard(std::uint32_t shardIndex,
// so the database can fetch data from it
it->second = shard.get();
l.unlock();
auto valid {shard->validate(app_)};
auto const valid {shard->validate(app_)};
l.lock();
if (!valid)
{
it = preShards_.find(shardIndex);
if(it != preShards_.end())
it->second = nullptr;
return fail({});
return fail("failed validation");
}
}

Expand Down Expand Up @@ -634,7 +650,7 @@ DatabaseShardImp::import(Database& source)
canAdd_ = false;
break;
}
if (avgShardSz_ > boost::filesystem::space(dir_).free)
if (avgShardSz_ > available())
{
JLOG(j_.error()) <<
"Insufficient disk space";
Expand Down Expand Up @@ -686,7 +702,6 @@ DatabaseShardImp::import(Database& source)
if (!shard->open(config_, scheduler_))
{
shard.reset();
this->remove(shardDir);
continue;
}

Expand All @@ -699,7 +714,7 @@ DatabaseShardImp::import(Database& source)
"shard " << shardIndex <<
" unable to create temp marker file";
shard.reset();
this->remove(shardDir);
removeAll(shardDir, j_);
continue;
}
ofs.close();
Expand Down Expand Up @@ -727,7 +742,7 @@ DatabaseShardImp::import(Database& source)
JLOG(j_.debug()) <<
"shard " << shardIndex <<
" successfully imported";
this->remove(markerFile);
removeAll(markerFile, j_);
break;
}
}
Expand All @@ -738,7 +753,7 @@ DatabaseShardImp::import(Database& source)
"shard " << shardIndex <<
" failed to import";
shard.reset();
this->remove(shardDir);
removeAll(shardDir, j_);
}
}

Expand Down Expand Up @@ -1070,7 +1085,7 @@ DatabaseShardImp::updateStats(std::lock_guard<std::mutex>&)
else
{
auto const sz = maxDiskSpace_ - usedDiskSpace_;
if (sz > space(dir_).free)
if (sz > available())
{
JLOG(j_.warn()) <<
"Max Shard Store size exceeds "
Expand Down Expand Up @@ -1110,21 +1125,19 @@ DatabaseShardImp::selectCache(std::uint32_t seq)
return {};
}

bool
DatabaseShardImp::remove(boost::filesystem::path const& path)
std::uint64_t
DatabaseShardImp::available() const
{
try
{
boost::filesystem::remove_all(path);
return boost::filesystem::space(dir_).available;
}
catch (const boost::filesystem::filesystem_error& e)
catch (std::exception const& e)
{
JLOG(j_.error()) <<
"remove_all " << path.string() <<
": Exception, " << e.code().message();
return false;
"exception: " << e.what();
return 0;
}
return true;
}

} // NodeStore
Expand Down
5 changes: 3 additions & 2 deletions src/ripple/nodestore/impl/DatabaseShardImp.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -241,8 +241,9 @@ class DatabaseShardImp : public DatabaseShard
1, static_cast<int>(complete_.size() + (incomplete_ ? 1 : 0))));
}

bool
remove(boost::filesystem::path const& path);
// Returns available storage space
std::uint64_t
available() const;
};

} // NodeStore
Expand Down
Loading

0 comments on commit 384c6b8

Please sign in to comment.