From 3e3f686b011789635d06bba2de89c48a1d7e2a42 Mon Sep 17 00:00:00 2001 From: jiangph Date: Mon, 2 Dec 2019 11:06:49 +0800 Subject: [PATCH] Only close LRU in IndexService and call rename in IndexCleanupService --- .../clouseau/IndexCleanupService.scala | 38 ++++++++++++++++--- .../clouseau/IndexManagerService.scala | 6 +-- .../com/cloudant/clouseau/IndexService.scala | 21 +++------- .../scala/com/cloudant/clouseau/Utils.scala | 32 ---------------- .../clouseau/IndexCleanupServiceSpec.scala | 11 ++++-- 5 files changed, 49 insertions(+), 59 deletions(-) diff --git a/src/main/scala/com/cloudant/clouseau/IndexCleanupService.scala b/src/main/scala/com/cloudant/clouseau/IndexCleanupService.scala index d428836c..60a4d46e 100644 --- a/src/main/scala/com/cloudant/clouseau/IndexCleanupService.scala +++ b/src/main/scala/com/cloudant/clouseau/IndexCleanupService.scala @@ -17,6 +17,9 @@ import java.io.File import java.util.regex.Pattern import org.apache.log4j.Logger import scalang._ +import java.util.Calendar +import java.util.TimeZone +import java.text.SimpleDateFormat class IndexCleanupService(ctx: ServiceContext[ConfigurationArgs]) extends Service(ctx) with Instrumented { @@ -74,17 +77,42 @@ class IndexCleanupService(ctx: ServiceContext[ConfigurationArgs]) extends Servic softDelete(file, path, includePattern) } - logger.info(fileOrDir.getAbsolutePath) val m = includePattern.matcher(fileOrDir.getAbsolutePath) if (m.find) { logger.info("Soft-deleting index " + m.group) - call('main, ('soft_delete, m.group)) match { + call('main, ('close_lru, m.group)) match { case 'ok => - 'ok - case ('error, 'not_found) => - Utils.rename(rootDir, path, fileOrDir.getName) + rename(path, fileOrDir.getName) } } } + def rename(dbName: String, sig: String) { + val srcParentDir = new File(rootDir, dbName) + val sdf = new SimpleDateFormat("yyyyMMdd'.'HHmmss") + sdf.setTimeZone(TimeZone.getTimeZone("UTC")) + val sdfNow = sdf.format(Calendar.getInstance().getTime()) + // move timestamp information in dbName to end of destination path + // for example, from foo.1234567890 to foo.20170912.092828.deleted.1234567890 + val destParentPath = dbName.dropRight(10) + sdfNow + ".deleted." + dbName.takeRight(10) + val destParentDir = new File(rootDir, destParentPath) + logger.info("Renaming '%s' to '%s'".format( + srcParentDir.getAbsolutePath, destParentDir.getAbsolutePath) + ) + if (!srcParentDir.isDirectory) { + return + } + if (!destParentDir.exists) { + destParentDir.mkdirs + } + + val srcDir = new File(srcParentDir, sig) + val destDir = new File(destParentDir, sig) + + if (!srcDir.renameTo(destDir)) { + logger.error("Failed to rename directory from '%s' to '%s'".format( + srcDir.getAbsolutePath, destDir.getAbsolutePath)) + } + } + } diff --git a/src/main/scala/com/cloudant/clouseau/IndexManagerService.scala b/src/main/scala/com/cloudant/clouseau/IndexManagerService.scala index 6e46c8d3..f6c75855 100644 --- a/src/main/scala/com/cloudant/clouseau/IndexManagerService.scala +++ b/src/main/scala/com/cloudant/clouseau/IndexManagerService.scala @@ -119,12 +119,12 @@ class IndexManagerService(ctx: ServiceContext[ConfigurationArgs]) extends Servic pid ! 'delete 'ok } - case ('soft_delete, path: String) => + case ('close_lru, path: String) => lru.get(path) match { case null => - ('error, 'not_found) + 'ok case pid: Pid => - pid ! 'soft_delete + pid ! ('sync_close, 'closing) 'ok } case DiskSizeMsg(path: String) => diff --git a/src/main/scala/com/cloudant/clouseau/IndexService.scala b/src/main/scala/com/cloudant/clouseau/IndexService.scala index 86e2a152..0d997f71 100644 --- a/src/main/scala/com/cloudant/clouseau/IndexService.scala +++ b/src/main/scala/com/cloudant/clouseau/IndexService.scala @@ -144,6 +144,9 @@ class IndexService(ctx: ServiceContext[IndexServiceArgs]) extends Service(ctx) w 'ok case 'info => ('ok, getInfo) + case 'sync_close => + exit(msg) + 'ok } override def handleCast(msg: Any) = msg match { @@ -178,13 +181,6 @@ class IndexService(ctx: ServiceContext[IndexServiceArgs]) extends Service(ctx) w dir.deleteFile(name) } exit('deleted) - case 'soft_delete => - ctx.args.writer.close() - val rootDir = new File(ctx.args.config.getString("clouseau.dir", "target/indexes")) - val path = ctx.args.name.substring(0, ctx.args.name.lastIndexOf('/')) - val sig = ctx.args.name.substring(ctx.args.name.lastIndexOf('/') + 1) - Utils.rename(rootDir, path, sig) - exit('deleted) case 'maybe_commit => commit(pendingSeq, pendingPurgeSeq) case ('committed, newUpdateSeq: Long, newPurgeSeq: Long) => @@ -217,7 +213,6 @@ class IndexService(ctx: ServiceContext[IndexServiceArgs]) extends Service(ctx) w } override def exit(msg: Any) { - debug("Closed with reason: %.1000s".format(msg)) try { reader.close() } catch { @@ -228,13 +223,9 @@ class IndexService(ctx: ServiceContext[IndexServiceArgs]) extends Service(ctx) w } catch { case e: AlreadyClosedException => 'ignored case e: IOException => - try { - ctx.args.writer.close(); - } finally { - val dir = ctx.args.writer.getDirectory - if (IndexWriter.isLocked(dir)) { - IndexWriter.unlock(dir); - } + val dir = ctx.args.writer.getDirectory + if (IndexWriter.isLocked(dir)) { + IndexWriter.unlock(dir); } warn("Error while closing writer", e) } finally { diff --git a/src/main/scala/com/cloudant/clouseau/Utils.scala b/src/main/scala/com/cloudant/clouseau/Utils.scala index a58c2057..8791e197 100644 --- a/src/main/scala/com/cloudant/clouseau/Utils.scala +++ b/src/main/scala/com/cloudant/clouseau/Utils.scala @@ -18,9 +18,6 @@ import org.apache.lucene.util.NumericUtils import org.apache.log4j.Logger import java.io.File import java.io.IOException -import java.util.Calendar -import java.util.TimeZone -import java.text.SimpleDateFormat object Utils { @@ -35,33 +32,4 @@ object Utils { new BytesRef(string) } - def rename(rootDir: File, dbName: String, sig: String) { - val logger = Logger.getLogger("clouseau.utils") - val srcParentDir = new File(rootDir, dbName) - val sdf = new SimpleDateFormat("yyyyMMdd'.'HHmmss") - sdf.setTimeZone(TimeZone.getTimeZone("UTC")) - val sdfNow = sdf.format(Calendar.getInstance().getTime()) - // move timestamp information in dbName to end of destination path - // for example, from foo.1234567890 to foo.20170912.092828.deleted.1234567890 - val destParentPath = dbName.dropRight(10) + sdfNow + ".deleted." + dbName.takeRight(10) - val destParentDir = new File(rootDir, destParentPath) - logger.info("Renaming '%s' to '%s'".format( - srcParentDir.getAbsolutePath, destParentDir.getAbsolutePath) - ) - if (!srcParentDir.isDirectory) { - return - } - if (!destParentDir.exists) { - destParentDir.mkdirs - } - - val srcDir = new File(srcParentDir, sig) - val destDir = new File(destParentDir, sig) - - if (!srcDir.renameTo(destDir)) { - logger.error("Failed to rename directory from '%s' to '%s'".format( - srcDir.getAbsolutePath, destDir.getAbsolutePath)) - } - } - } diff --git a/src/test/scala/com/cloudant/clouseau/IndexCleanupServiceSpec.scala b/src/test/scala/com/cloudant/clouseau/IndexCleanupServiceSpec.scala index ea9c5f08..3361ca7d 100644 --- a/src/test/scala/com/cloudant/clouseau/IndexCleanupServiceSpec.scala +++ b/src/test/scala/com/cloudant/clouseau/IndexCleanupServiceSpec.scala @@ -23,9 +23,9 @@ class IndexCleanupServiceSpec extends SpecificationWithJUnit { "the index clean-up service" should { "soft-delete index when database is deleted" in new cleanup_service { - node.cast(service, SoftDeletePathMsg("foo.1234567890")) must be equalTo 'ok + node.cast(cleanup, SoftDeletePathMsg("shards/00000000-ffffffff/foo.1234567890")) must be equalTo 'ok Thread.sleep(1000) - val indexdir = new File("target", "indexes") + val indexdir = new File(new File(new File("target", "indexes"), "shards"), "00000000-ffffffff") var subdirlist = List[String]() for (file <- indexdir.listFiles if file.getName contains ".deleted") { @@ -41,7 +41,9 @@ class IndexCleanupServiceSpec extends SpecificationWithJUnit { trait cleanup_service extends RunningNode { val config = new SystemConfiguration() val args = new ConfigurationArgs(config) - val service = node.spawnService[IndexCleanupService, ConfigurationArgs](args) + val cleanup = node.spawnService[IndexCleanupService, ConfigurationArgs](args) + var manager = node.spawnService[IndexManagerService, ConfigurationArgs]('main, args) + val mbox = node.spawnMbox val dir = new File("target", "indexes") @@ -51,7 +53,8 @@ trait cleanup_service extends RunningNode { } } - val foodir = new File(new File("target", "indexes"), "foo.1234567890") + val foodir = new File(new File(new File(new File(new File("target", "indexes"), "shards"), + "00000000-ffffffff"), "foo.1234567890"), "5838a59330e52227a58019dc1b9edd6e") if (!foodir.exists) { foodir.mkdirs }