From bc47942d1704320647c674e25061d402c36e8b78 Mon Sep 17 00:00:00 2001 From: Greg Wilkins Date: Tue, 5 Sep 2017 12:44:31 +1000 Subject: [PATCH] Issue #1200 Improve PathWatcher Squashed commit of the following: commit 08b5acccf87c3b99152a8042d650aadf7e43c5ff Merge: cea3366 daeb844 Author: Greg Wilkins Date: Tue Sep 5 12:43:01 2017 +1000 Merge branch 'jetty-9.4.x' into jetty-9.4.x-1200 commit cea3366625e16debf66e07284ab7afa89e73a32d Author: Greg Wilkins Date: Tue Sep 5 12:42:21 2017 +1000 Issue #1200 ignore OSX failure commit fd2493f2b30ffb19f4b404636e1e38c8612cb502 Author: Greg Wilkins Date: Tue Sep 5 12:11:05 2017 +1000 Issue #1789 PropertyUserStoreTest failures on windows commit 89aa59ca7b16a393edc77116b13050d2d8a2c3e2 Author: Greg Wilkins Date: Tue Sep 5 11:56:52 2017 +1000 Issue #1200 fixes for windows commit 1904b4566d9224a19729f83a7b49a5ab23aaa5d8 Merge: 74d770e eec6453 Author: Greg Wilkins Date: Tue Sep 5 11:45:19 2017 +1000 Merge branch 'jetty-9.4.x' into jetty-9.4.x-1200 commit 74d770e557e8ff613a5965cb430a7b83ee75bd45 Author: Greg Wilkins Date: Fri Sep 1 10:47:05 2017 +1000 Issue #1200 fixes for windows commit f4ee0e97dcd0a07257cea8da8b3106f71150957f Author: Greg Wilkins Date: Thu Aug 31 10:24:07 2017 +1000 Issue #1200 improved tests for long duration quiet time commit 17381cbb0bbebe3ea27ed5f55caeb45c2856e1be Author: Greg Wilkins Date: Thu Aug 31 10:03:04 2017 +1000 Issue #1200 fixed javadoc commit b3a12c15167ce77a9781942680ca2d5c872374dd Merge: ed0db46 ce4adb5 Author: Greg Wilkins Date: Thu Aug 31 09:41:50 2017 +1000 Merge branch 'jetty-9.4.x-1200' of github.com:eclipse/jetty.project into jetty-9.4.x-1200 commit ed0db46f495f27491ba58e6c4353cf1ef6f2061e Author: Greg Wilkins Date: Thu Aug 31 09:39:46 2017 +1000 Issue #1200 Improved PathWatcher commit ce4adb54ed58d39789ea1ba4f5d58035e57980ce Merge: f993a7c 48aaecb Author: Joakim Erdfelt Date: Wed Aug 30 16:38:07 2017 -0700 Merge branch 'jetty-9.4.x-1200' of github.com:eclipse/jetty.project into jetty-9.4.x-1200 commit f993a7c83ee7294a34b00cea68242adb7993e565 Author: Joakim Erdfelt Date: Wed Aug 30 16:37:45 2017 -0700 Issue #1200 - adding some important OSX/HFS+ timing differences + We should really be testing the FileSystem (not the OS) to make the timing constants be more sane. (APFS for example should be much lower on newer OSX installations commit 48aaecb4dd291d94d591c1545f671eecff1e3587 Author: Greg Wilkins Date: Thu Aug 31 08:50:42 2017 +1000 Issue #1200 Improved PathWatcher diff commit 1917f8b177d163bd42c07d5a2715858e7cf9787a Author: Greg Wilkins Date: Thu Aug 31 08:36:40 2017 +1000 Issue #1200 Improved PathWatcher diff commit ecf002395a426ee3c00a4b42a32222e61805234f Author: Greg Wilkins Date: Thu Aug 31 08:22:41 2017 +1000 Issue #1200 Test improved PathWatcher commit 0d76544093cbcddd9b29fc2c92a4d0bb0a6839a8 Merge: 0fd7187 eb1320f Author: Greg Wilkins Date: Wed Aug 30 16:43:15 2017 +1000 Merge branch 'jetty-9.4.x' into jetty-9.4.x-1200 commit 0fd7187f908ed2d1bed24d5d82e25cb7ec244b0e Author: Greg Wilkins Date: Wed Aug 30 15:58:24 2017 +1000 Issue #1200 Improve PathWatcher --- .../jetty/security/PropertyUserStoreTest.java | 41 +- .../org/eclipse/jetty/util/PathWatcher.java | 1075 +++++++++-------- .../eclipse/jetty/util/PathWatcherTest.java | 603 ++++++--- .../test/resources/jetty-logging.properties | 1 + 4 files changed, 1038 insertions(+), 682 deletions(-) diff --git a/jetty-security/src/test/java/org/eclipse/jetty/security/PropertyUserStoreTest.java b/jetty-security/src/test/java/org/eclipse/jetty/security/PropertyUserStoreTest.java index 84e8c764b753..3153a3ca3a7d 100644 --- a/jetty-security/src/test/java/org/eclipse/jetty/security/PropertyUserStoreTest.java +++ b/jetty-security/src/test/java/org/eclipse/jetty/security/PropertyUserStoreTest.java @@ -30,11 +30,17 @@ import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; +import java.io.FilePermission; import java.io.FileWriter; +import java.io.IOException; import java.io.OutputStream; import java.io.Writer; +import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.attribute.PosixFilePermission; +import java.nio.file.attribute.PosixFilePermissions; import java.util.ArrayList; +import java.util.EnumSet; import java.util.List; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; @@ -139,7 +145,7 @@ private String initUsersPackedFileText() } } - return "jar:file:" + usersJar.getCanonicalPath() + "!/" + entryPath; + return "jar:" + usersJar.toURI().toASCIIString() + "!/" + entryPath; } private void writeUser(File usersFile) @@ -205,30 +211,47 @@ public void testPropertyUserStoreLoadFromJarFile() throws Exception userCount.awaitCount(3); } - @Test public void testPropertyUserStoreLoadUpdateUser() throws Exception { assumeThat("Skipping on OSX", OS.IS_OSX, is(false)); final UserCount userCount = new UserCount(); final File usersFile = initUsersText(); - - PropertyUserStore store = new PropertyUserStore(); + final AtomicInteger loadCount = new AtomicInteger(0); + PropertyUserStore store = new PropertyUserStore() + { + @Override + protected void loadUsers() throws IOException + { + loadCount.incrementAndGet(); + super.loadUsers(); + } + }; store.setHotReload(true); store.setConfigFile(usersFile); - store.registerUserListener(userCount); store.start(); userCount.assertThatCount(is(3)); - + assertThat(loadCount.get(),is(1)); + addAdditionalUser(usersFile,"skip: skip, roleA\n"); - userCount.awaitCount(4); - - assertThat("Failed to retrieve UserIdentity from PropertyUserStore directly", store.getUserIdentity("skip"), notNullValue()); + assertThat(loadCount.get(),is(2)); + assertThat(store.getUserIdentity("skip"), notNullValue()); + userCount.assertThatCount(is(4)); + userCount.assertThatUsers(hasItem("skip")); + + if (OS.IS_LINUX) + Files.createFile(testdir.getPath().toRealPath().resolve("unrelated.txt"), + PosixFilePermissions.asFileAttribute(EnumSet.noneOf(PosixFilePermission.class))); + else + Files.createFile(testdir.getPath().toRealPath().resolve("unrelated.txt")); + Thread.sleep(1100); + assertThat(loadCount.get(),is(2)); + userCount.assertThatCount(is(4)); userCount.assertThatUsers(hasItem("skip")); } diff --git a/jetty-util/src/main/java/org/eclipse/jetty/util/PathWatcher.java b/jetty-util/src/main/java/org/eclipse/jetty/util/PathWatcher.java index 00d313ad4b9a..9e9c0c32d1ce 100644 --- a/jetty-util/src/main/java/org/eclipse/jetty/util/PathWatcher.java +++ b/jetty-util/src/main/java/org/eclipse/jetty/util/PathWatcher.java @@ -28,18 +28,15 @@ import java.nio.file.ClosedWatchServiceException; import java.nio.file.FileSystem; import java.nio.file.FileSystems; -import java.nio.file.FileVisitResult; import java.nio.file.Files; -import java.nio.file.LinkOption; import java.nio.file.Path; import java.nio.file.PathMatcher; -import java.nio.file.SimpleFileVisitor; import java.nio.file.WatchEvent; -import java.nio.file.WatchEvent.Kind; import java.nio.file.WatchKey; import java.nio.file.WatchService; -import java.nio.file.attribute.BasicFileAttributes; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.EventListener; import java.util.HashMap; @@ -52,6 +49,7 @@ import java.util.Scanner; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.TimeUnit; +import java.util.function.Predicate; import org.eclipse.jetty.util.component.AbstractLifeCycle; import org.eclipse.jetty.util.log.Log; @@ -68,7 +66,7 @@ */ public class PathWatcher extends AbstractLifeCycle implements Runnable { - public static class Config + public static class Config implements Predicate { public static final int UNLIMITED_DEPTH = -9999; @@ -84,19 +82,64 @@ public static class Config PATTERN_SEP = sep; } - protected final Path dir; + protected final Config parent; + protected final Path path; + protected final IncludeExcludeSet includeExclude; protected int recurseDepth = 0; // 0 means no sub-directories are scanned - protected List includes; - protected List excludes; protected boolean excludeHidden = false; + protected long pauseUntil; public Config(Path path) { - this.dir = path; - includes = new ArrayList<>(); - excludes = new ArrayList<>(); + this(path,null); } + public Config(Path path, Config parent) + { + this.parent = parent; + this.includeExclude = parent==null ? new IncludeExcludeSet<>(PathMatcherSet.class) : parent.includeExclude; + + Path dir = path; + if (!Files.exists(path)) + throw new IllegalStateException("Path does not exist: "+path); + + if (!Files.isDirectory(path)) + { + dir = path.getParent(); + includeExclude.include(new ExactPathMatcher(path)); + setRecurseDepth(0); + } + + this.path = dir; + } + + public Config getParent() + { + return parent; + } + + public void setPauseUntil(long time) + { + if (time>pauseUntil) + pauseUntil=time; + } + + public boolean isPaused(long now) + { + if (pauseUntil==0) + return false; + if (pauseUntil>now) + { + if (LOG.isDebugEnabled()) + LOG.debug("PAUSED {}",this); + return true; + } + if (LOG.isDebugEnabled()) + LOG.debug("unpaused {}",this); + pauseUntil = 0; + return false; + } + /** * Add an exclude PathMatcher * @@ -105,7 +148,7 @@ public Config(Path path) */ public void addExclude(PathMatcher matcher) { - this.excludes.add(matcher); + includeExclude.exclude(matcher); } /** @@ -120,10 +163,8 @@ public void addExclude(PathMatcher matcher) public void addExclude(final String syntaxAndPattern) { if (LOG.isDebugEnabled()) - { LOG.debug("Adding exclude: [{}]",syntaxAndPattern); - } - addExclude(dir.getFileSystem().getPathMatcher(syntaxAndPattern)); + addExclude(path.getFileSystem().getPathMatcher(syntaxAndPattern)); } /** @@ -145,7 +186,7 @@ public void addExclude(final String syntaxAndPattern) */ public void addExcludeGlobRelative(String pattern) { - addExclude(toGlobPattern(dir,pattern)); + addExclude(toGlobPattern(path,pattern)); } /** @@ -160,9 +201,6 @@ public void addExcludeHidden() LOG.debug("Adding hidden files and directories to exclusions"); } excludeHidden = true; - - addExclude("regex:^.*" + PATTERN_SEP + "\\..*$"); // ignore hidden files - addExclude("regex:^.*" + PATTERN_SEP + "\\..*" + PATTERN_SEP + ".*$"); // ignore files in hidden directories } } @@ -189,7 +227,7 @@ public void addExcludes(List syntaxAndPatterns) */ public void addInclude(PathMatcher matcher) { - this.includes.add(matcher); + includeExclude.include(matcher); } /** @@ -205,7 +243,7 @@ public void addInclude(String syntaxAndPattern) { LOG.debug("Adding include: [{}]",syntaxAndPattern); } - addInclude(dir.getFileSystem().getPathMatcher(syntaxAndPattern)); + addInclude(path.getFileSystem().getPathMatcher(syntaxAndPattern)); } /** @@ -227,7 +265,7 @@ public void addInclude(String syntaxAndPattern) */ public void addIncludeGlobRelative(String pattern) { - addInclude(toGlobPattern(dir,pattern)); + addInclude(toGlobPattern(path,pattern)); } /** @@ -256,18 +294,17 @@ public void addIncludes(List syntaxAndPatterns) */ public Config asSubConfig(Path dir) { - Config subconfig = new Config(dir); - subconfig.includes = this.includes; - subconfig.excludes = this.excludes; - if (dir == this.dir) - subconfig.recurseDepth = this.recurseDepth; // TODO shouldn't really do a subconfig for this + Config subconfig = new Config(dir,this); + if (dir == this.path) + throw new IllegalStateException("sub "+dir.toString()+" of "+this); + + if (this.recurseDepth == UNLIMITED_DEPTH) + subconfig.recurseDepth = UNLIMITED_DEPTH; else - { - if (this.recurseDepth == UNLIMITED_DEPTH) - subconfig.recurseDepth = UNLIMITED_DEPTH; - else - subconfig.recurseDepth = this.recurseDepth - (dir.getNameCount() - this.dir.getNameCount()); - } + subconfig.recurseDepth = this.recurseDepth - (dir.getNameCount() - this.path.getNameCount()); + + if (LOG.isDebugEnabled()) + LOG.debug("subconfig {} of {}",subconfig,path); return subconfig; } @@ -283,80 +320,52 @@ public boolean isRecurseDepthUnlimited () public Path getPath () { - return this.dir; + return this.path; } - private boolean hasMatch(Path path, List matchers) + public Path resolve(Path path) { - for (PathMatcher matcher : matchers) - { - if (matcher.matches(path)) - { - return true; - } - } - return false; + if (Files.isDirectory(this.path)) + return this.path.resolve(path); + if (Files.exists(this.path)) + return this.path; + return path; } - - public boolean isExcluded(Path dir) throws IOException + + public boolean test(Path path) { - if (excludeHidden) + if (excludeHidden && isHidden(path)) { - if (Files.isHidden(dir)) - { - if (NOISY_LOG.isDebugEnabled()) - { - NOISY_LOG.debug("isExcluded [Hidden] on {}",dir); - } - return true; - } + if (LOG.isDebugEnabled()) + LOG.debug("test({}) -> [Hidden]", toShortPath(path)); + return false; } - if (excludes.isEmpty()) + if (!path.startsWith(this.path)) { - // no excludes == everything allowed + if (LOG.isDebugEnabled()) + LOG.debug("test({}) -> [!child {}]", toShortPath(path), this.path); return false; } - boolean matched = hasMatch(dir,excludes); - if (NOISY_LOG.isDebugEnabled()) + if (recurseDepth!=UNLIMITED_DEPTH) { - NOISY_LOG.debug("isExcluded [{}] on {}",matched,dir); - } - return matched; - } + int depth = path.getNameCount() - this.path.getNameCount() - 1; - public boolean isIncluded(Path dir) - { - if (includes.isEmpty()) - { - // no includes == everything allowed - if (NOISY_LOG.isDebugEnabled()) + if (depth>recurseDepth) { - NOISY_LOG.debug("isIncluded [All] on {}",dir); + if (LOG.isDebugEnabled()) + LOG.debug("test({}) -> [depth {}>{}]",toShortPath(path),depth,recurseDepth); + return false; } - return true; } - boolean matched = hasMatch(dir,includes); - if (NOISY_LOG.isDebugEnabled()) - { - NOISY_LOG.debug("isIncluded [{}] on {}",matched,dir); - } - return matched; - } + boolean matched = includeExclude.test(path); - public boolean matches(Path path) - { - try - { - return !isExcluded(path) && isIncluded(path); - } - catch (IOException e) - { - LOG.warn("Unable to match path: " + path,e); - return false; - } + if (LOG.isDebugEnabled()) + LOG.debug("test({}) -> {}", toShortPath(path), matched); + + return matched; } /** @@ -372,32 +381,6 @@ public void setRecurseDepth(int depth) this.recurseDepth = depth; } - - - /** - * Determine if the provided child directory should be recursed into based on the configured {@link #setRecurseDepth(int)} - * - * @param child - * the child directory to test against - * @return true if recurse should occur, false otherwise - */ - public boolean shouldRecurseDirectory(Path child) - { - if (!child.startsWith(dir)) - { - // not part of parent? don't recurse - return false; - } - - //If not limiting depth, should recurse all - if (isRecurseDepthUnlimited()) - return true; - - //Depth limited, check it - int childDepth = dir.relativize(child).getNameCount(); - return (childDepth <= recurseDepth); - } - private String toGlobPattern(Path path, String subPattern) { StringBuilder s = new StringBuilder(); @@ -409,9 +392,9 @@ private String toGlobPattern(Path path, String subPattern) Path root = path.getRoot(); if (root != null) { - if (NOISY_LOG.isDebugEnabled()) + if (LOG.isDebugEnabled()) { - NOISY_LOG.debug("Path: {} -> Root: {}",path,root); + LOG.debug("Path: {} -> Root: {}", path, root); } for (char c : root.toString().toCharArray()) { @@ -464,116 +447,76 @@ private String toGlobPattern(Path path, String subPattern) return s.toString(); } - @Override - public String toString() + + DirAction handleDir(Path path) { - StringBuilder s = new StringBuilder(); - s.append(dir); - if (recurseDepth > 0) + try { - s.append(" [depth=").append(recurseDepth).append("]"); + if (!Files.isDirectory(path)) + return DirAction.IGNORE; + if (excludeHidden && isHidden(path)) + return DirAction.IGNORE; + if (getRecurseDepth()==0) + return DirAction.WATCH; + return DirAction.ENTER; + } + catch(Exception e) + { + LOG.ignore(e); + return DirAction.IGNORE; } - return s.toString(); - } - } - - public static class DepthLimitedFileVisitor extends SimpleFileVisitor - { - private Config base; - private PathWatcher watcher; - - public DepthLimitedFileVisitor (PathWatcher watcher, Config base) - { - this.base = base; - this.watcher = watcher; } - /* - * 2 situations: - * - * 1. a subtree exists at the time a dir to watch is added (eg watching /tmp/xxx and it contains aaa/) - * - will start with /tmp/xxx for which we want to register with the poller - * - want to visit each child - * - if child is file, gen add event - * - if child is dir, gen add event but ONLY register it if inside depth limit and ONLY continue visit of child if inside depth limit - * 2. a subtree is added inside a watched dir (eg watching /tmp/xxx, add aaa/ to xxx/) - * - will start with /tmp/xxx/aaa - * - gen add event but ONLY register it if inside depth limit and ONLY continue visit of children if inside depth limit - * - */ - @Override - public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException - { - //In a directory: - // 1. the dir is the base directory - // - register it with the poll mechanism - // - generate pending add event (iff notifiable and matches patterns) - // - continue the visit (sibling dirs, sibling files) - // 2. the dir is a subdir at some depth in the basedir's tree - // - if the level of the subdir less or equal to base's limit - // - register it wih the poll mechanism - // - generate pending add event (iff notifiable and matches patterns) - // - else stop visiting this dir - - if (!base.isExcluded(dir)) + public boolean isHidden(Path path) + { + try { - if (base.isIncluded(dir)) + if (!path.startsWith(this.path)) + return true; + for (int i=this.path.getNameCount(); i= 0)) || base.shouldRecurseDirectory(dir)) - watcher.register(dir,base); + return Files.exists(path) && Files.isHidden(path); } + catch (IOException e) + { + LOG.ignore(e); + return false; + } + } - //Continue walking the tree of this dir if it is: - // - the base dir and recursion is unlimited - // - the base dir and we're not recursing in it - // - the base dir and we are recursing it and the depth is within the limit - // - a child dir and its depth is within the limits - if ((base.getPath().equals(dir)&& (base.isRecurseDepthUnlimited() || base.getRecurseDepth() >= 0)) || base.shouldRecurseDirectory(dir)) - return FileVisitResult.CONTINUE; - else - return FileVisitResult.SKIP_SUBTREE; + public String toShortPath(Path path) + { + if (!path.startsWith(this.path)) + return path.toString(); + return this.path.relativize(path).toString(); } @Override - public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException + public String toString() { - // In a file: - // - register with poll mechanism - // - generate pending add event (iff notifiable and matches patterns) - - if (base.matches(file) && watcher.isNotifiable()) - { - PathWatchEvent event = new PathWatchEvent(file,PathWatchEventType.ADDED); - if (LOG.isDebugEnabled()) - { - LOG.debug("Pending {}",event); - } - watcher.addToPendingList(file, event); - } - - return FileVisitResult.CONTINUE; + StringBuilder s = new StringBuilder(); + s.append(path).append(" [depth="); + if (recurseDepth==UNLIMITED_DEPTH) + s.append("UNLIMITED"); + else + s.append(recurseDepth); + s.append(']'); + return s.toString(); } - + } + + public static enum DirAction + { + IGNORE, WATCH, ENTER; + } + /** * Listener for path change events */ @@ -591,30 +534,38 @@ public static interface EventListListener extends EventListener { void onPathWatchEvents(List events); } - + /** * PathWatchEvent * * Represents a file event. Reported to registered listeners. */ - public static class PathWatchEvent + public class PathWatchEvent { private final Path path; private final PathWatchEventType type; - private int count = 0; + private final Config config; + long checked; + long modified; + long length; - public PathWatchEvent(Path path, PathWatchEventType type) + public PathWatchEvent(Path path, PathWatchEventType type, Config config) { this.path = path; - this.count = 1; this.type = type; + this.config = config; + checked = TimeUnit.NANOSECONDS.toMillis(System.nanoTime()); + check(); + } + public Config getConfig() + { + return config; } - public PathWatchEvent(Path path, WatchEvent event) + public PathWatchEvent(Path path, WatchEvent event, Config config) { this.path = path; - this.count = event.count(); if (event.kind() == ENTRY_CREATE) { this.type = PathWatchEventType.ADDED; @@ -631,8 +582,63 @@ else if (event.kind() == ENTRY_MODIFY) { this.type = PathWatchEventType.UNKNOWN; } + this.config = config; + checked = TimeUnit.NANOSECONDS.toMillis(System.nanoTime()); + check(); + } + + private void check() + { + if (Files.exists(path)) + { + try + { + modified = Files.getLastModifiedTime(path).toMillis(); + length = Files.size(path); + } + catch(IOException e) + { + modified = -1; + length = -1; + } + } + else + { + modified = -1; + length = -1; + } } + public boolean isQuiet(long now, long quietTime) + { + long lastModified = modified; + long lastLength = length; + + check(); + + if (lastModified == modified && lastLength == length) + return (now-checked)>=quietTime; + + checked = now; + return false; + } + + public long toQuietCheck(long now, long quietTime) + { + long check = quietTime - (now-checked); + if (check<=0) + return quietTime; + return check; + } + + public void modified() + { + long now = TimeUnit.NANOSECONDS.toMillis(System.nanoTime()); + checked = now; + check(); + config.setPauseUntil(now+getUpdateQuietTimeMillis()); + } + /** * @see java.lang.Object#equals(java.lang.Object) */ @@ -680,14 +686,10 @@ public PathWatchEventType getType() return type; } - public void incrementCount(int num) - { - count += num; - } - + @Deprecated public int getCount() { - return count; + return 1; } /** @@ -709,121 +711,10 @@ public int hashCode() @Override public String toString() { - return String.format("PathWatchEvent[%s|%s]",type,path); + return String.format("PathWatchEvent[%8s|%s]",type,path); } } - - - - /** - * PathPendingEvents - * - * For a given path, a list of events that are awaiting the - * quiet time. The list is in the order that the event were - * received from the WatchService - */ - public static class PathPendingEvents - { - private Path _path; - private List _events; - private long _timestamp; - private long _lastFileSize = -1; - - public PathPendingEvents (Path path) - { - _path = path; - } - - public PathPendingEvents (Path path, PathWatchEvent event) - { - this (path); - addEvent(event); - } - - public void addEvent (PathWatchEvent event) - { - long now = System.currentTimeMillis(); - _timestamp = now; - if (_events == null) - { - _events = new ArrayList(); - _events.add(event); - } - else - { - //Check if the same type of event is already present, in which case we - //can increment its counter. Otherwise, add it - PathWatchEvent existingType = null; - for (PathWatchEvent e:_events) - { - if (e.getType() == event.getType()) - { - existingType = e; - break; - } - } - - if (existingType == null) - { - _events.add(event); - } - else - { - existingType.incrementCount(event.getCount()); - } - } - - } - - public List getEvents() - { - return _events; - } - - public long getTimestamp() - { - return _timestamp; - } - - - /** - * Check to see if the file referenced by this Event is quiet. - *

- * Will validate the timestamp to see if it is expired, as well as if the file size hasn't changed within the quiet period. - *

- * Always updates timestamp to 'now' on use of this method. - * - * @param now the time now - * - * @param expiredDuration - * the expired duration past the timestamp to be considered expired - * @param expiredUnit - * the unit of time for the expired check - * @return true if expired, false if not - */ - public boolean isQuiet(long now, long expiredDuration, TimeUnit expiredUnit) - { - - long pastdue = _timestamp + expiredUnit.toMillis(expiredDuration); - _timestamp = now; - - long fileSize = _path.toFile().length(); // File.length() returns 0 for non existant files - boolean fileSizeChanged = (_lastFileSize != fileSize); - _lastFileSize = fileSize; - - if ((now > pastdue) && (!fileSizeChanged /*|| fileSize == 0*/)) - { - // Quiet period timestamp has expired, and file size hasn't changed, or the file - // has been deleted. - // Consider this a quiet event now. - return true; - } - - return false; - } - - } /** * PathWatchEventType @@ -836,7 +727,7 @@ public static enum PathWatchEventType } private static final boolean IS_WINDOWS; - + static { String os = System.getProperty("os.name"); @@ -851,11 +742,7 @@ public static enum PathWatchEventType } } - private static final Logger LOG = Log.getLogger(PathWatcher.class); - /** - * super noisy debug logging - */ - private static final Logger NOISY_LOG = Log.getLogger(PathWatcher.class.getName() + ".Noisy"); + static final Logger LOG = Log.getLogger(PathWatcher.class); @SuppressWarnings("unchecked") protected static WatchEvent cast(WatchEvent event) @@ -864,14 +751,18 @@ protected static WatchEvent cast(WatchEvent event) } private static final WatchEvent.Kind WATCH_EVENT_KINDS[] = { ENTRY_CREATE, ENTRY_DELETE, ENTRY_MODIFY }; + private static final WatchEvent.Kind WATCH_DIR_KINDS[] = { ENTRY_CREATE, ENTRY_DELETE }; - private WatchService watchService; - private WatchEvent.Modifier watchModifiers[]; - private boolean nativeWatchService; - - private Map keys = new HashMap<>(); - private List listeners = new CopyOnWriteArrayList<>(); //a listener may modify the listener list directly or by stopping the PathWatcher - private List configs = new ArrayList<>(); + private WatchService watchService; + private WatchEvent.Modifier watchModifiers[]; + private boolean nativeWatchService; + + private final List configs = new ArrayList<>(); + private final Map keys = new HashMap<>(); + private final List listeners = new CopyOnWriteArrayList<>(); //a listener may modify the listener list directly or by stopping the PathWatcher + + private final Map pending = new LinkedHashMap<>(32,(float)0.75,false); + private final List events = new ArrayList<>(); /** * Update Quiet Time - set to 1000 ms as default (a lower value in Windows is not supported) @@ -880,8 +771,6 @@ protected static WatchEvent cast(WatchEvent event) private TimeUnit updateQuietTimeUnit = TimeUnit.MILLISECONDS; private Thread thread; private boolean _notifyExistingOnStart = true; - private Map pendingEvents = new LinkedHashMap<>(); - /** @@ -890,7 +779,12 @@ protected static WatchEvent cast(WatchEvent event) public PathWatcher() { } - + + public Collection getConfigs() + { + return configs; + } + /** * Request watch on a the given path (either file or dir) * using all Config defaults. In the case of a dir, @@ -949,24 +843,6 @@ public void watch (final Config config) configs.add(config); } - /** - * Register path in the config with the file watch service, - * walking the tree if it happens to be a directory. - * - * @param baseDir the base directory configuration to watch - * @throws IOException if unable to walk the filesystem tree - */ - protected void prepareConfig (final Config baseDir) throws IOException - { - if (LOG.isDebugEnabled()) - { - LOG.debug("Watching directory {}",baseDir); - } - Files.walkFileTree(baseDir.getPath(), new DepthLimitedFileVisitor(this, baseDir)); - } - - - /** * Add a listener for changes the watcher notices. * @@ -988,7 +864,7 @@ private void appendConfigId(StringBuilder s) for (Config config : keys.values()) { - dirs.add(config.dir); + dirs.add(config.path); } Collections.sort(dirs); @@ -1024,12 +900,14 @@ protected void doStart() throws Exception // Register all watched paths, walking dir hierarchies as needed, possibly generating // fake add events if notifyExistingOnStart is true for (Config c:configs) - prepareConfig(c); + registerTree(c.getPath(),c,isNotifyExistingOnStart()); // Start Thread for watcher take/pollKeys loop StringBuilder threadId = new StringBuilder(); - threadId.append("PathWatcher-Thread"); - appendConfigId(threadId); + threadId.append("PathWatcher@"); + threadId.append(Integer.toHexString(hashCode())); + if (LOG.isDebugEnabled()) + LOG.debug("{} -> {}", this, threadId); thread = new Thread(this,threadId.toString()); thread.setDaemon(true); @@ -1048,7 +926,8 @@ protected void doStop() throws Exception watchService = null; thread = null; keys.clear(); - pendingEvents.clear(); + pending.clear(); + events.clear(); super.doStop(); } @@ -1092,9 +971,9 @@ private void createWatchService () throws IOException Class c = Class.forName("com.sun.nio.file.SensitivityWatchEventModifier"); Field f = c.getField("HIGH"); modifiers = new WatchEvent.Modifier[] - { - (WatchEvent.Modifier)f.get(c) - }; + { + (WatchEvent.Modifier)f.get(c) + }; } } catch (Throwable t) @@ -1139,72 +1018,93 @@ public long getUpdateQuietTimeMillis() return TimeUnit.MILLISECONDS.convert(updateQuietTimeDuration,updateQuietTimeUnit); } - /** - * Generate events to the listeners. - * - * @param events the events captured - */ - protected void notifyOnPathWatchEvents (List events) + private void registerTree(Path dir, Config config, boolean notify) throws IOException { - if (events == null || events.isEmpty()) - return; - - for (EventListener listener : listeners) + if (LOG.isDebugEnabled()) + LOG.debug("registerTree {} {} {}", dir, config, notify); + + if (!Files.isDirectory(dir)) + throw new IllegalArgumentException(dir.toString()); + + register(dir,config); + + final MultiException me = new MultiException(); + Files.list(dir).forEach(p-> { - if (listener instanceof EventListListener) + if (LOG.isDebugEnabled()) + LOG.debug("registerTree? {}",p); + + try { - try - { - ((EventListListener)listener).onPathWatchEvents(events); - } - catch (Throwable t) + if (notify && config.test(p)) + pending.put(p,new PathWatchEvent(p,PathWatchEventType.ADDED,config)); + + switch(config.handleDir(p)) { - LOG.warn(t); + case ENTER: + registerTree(p,config.asSubConfig(p),notify); + break; + case WATCH: + registerDir(p,config); + break; + case IGNORE: + default: + break; } } - else + catch(IOException e) { - Listener l = (Listener)listener; - for (PathWatchEvent event:events) - { - try - { - l.onPathWatchEvent(event); - } - catch (Throwable t) - { - LOG.warn(t); - } - } + me.add(e); } + }); + + try + { + me.ifExceptionThrow(); + } + catch(IOException e) + { + throw e; } + catch(Throwable th) + { + throw new IOException(th); + } + } + private void registerDir(Path path, Config config) throws IOException + { + if (LOG.isDebugEnabled()) + LOG.debug("registerDir {} {}", path, config); + + if (!Files.isDirectory(path)) + throw new IllegalArgumentException(path.toString()); + + register(path,config.asSubConfig(path),WATCH_DIR_KINDS); + } + + protected void register(Path path, Config config) throws IOException + { + if (LOG.isDebugEnabled()) + LOG.debug("Registering watch on {} {}",path,watchModifiers==null?null:Arrays.asList(watchModifiers)); + + register(path,config,WATCH_EVENT_KINDS); } - /** - * Register a path (directory) with the WatchService. - * - * @param dir the directory to register - * @param root the configuration root - * @throws IOException if unable to register the path with the watch service. - */ - protected void register(Path dir, Config root) throws IOException + private void register(Path path, Config config, WatchEvent.Kind[] kinds) throws IOException { - - LOG.debug("Registering watch on {}",dir); if(watchModifiers != null) { // Java Watcher - WatchKey key = dir.register(watchService,WATCH_EVENT_KINDS,watchModifiers); - keys.put(key,root.asSubConfig(dir)); + WatchKey key = path.register(watchService,kinds,watchModifiers); + keys.put(key,config); } else { // Native Watcher - WatchKey key = dir.register(watchService,WATCH_EVENT_KINDS); - keys.put(key,root.asSubConfig(dir)); + WatchKey key = path.register(watchService,kinds); + keys.put(key,config); } } - /** * Delete a listener @@ -1237,59 +1137,55 @@ public boolean removeListener(Listener listener) @Override public void run() { - - List notifiableEvents = new ArrayList(); - // Start the java.nio watching if (LOG.isDebugEnabled()) { LOG.debug("Starting java.nio file watching with {}",watchService); } - while (watchService != null && thread == Thread.currentThread()) - { - WatchKey key = null; + long wait_time = getUpdateQuietTimeMillis(); + WatchService watch = watchService; + + while (isRunning() && thread == Thread.currentThread()) + { + + WatchKey key; + try { - //If no pending events, wait forever for new events - if (pendingEvents.isEmpty()) - { - if (NOISY_LOG.isDebugEnabled()) - NOISY_LOG.debug("Waiting for take()"); - key = watchService.take(); - } - else + // Reset all keys before watching + long now = TimeUnit.NANOSECONDS.toMillis(System.nanoTime()); + for (Map.Entry e : keys.entrySet()) { - //There are existing events that might be ready to go, - //only wait as long as the quiet time for any new events - if (NOISY_LOG.isDebugEnabled()) - NOISY_LOG.debug("Waiting for poll({}, {})",updateQuietTimeDuration,updateQuietTimeUnit); - - key = watchService.poll(updateQuietTimeDuration,updateQuietTimeUnit); - - //If no new events its safe to process the pendings - if (key == null) + WatchKey k = e.getKey(); + Config c = e.getValue(); + + if (!c.isPaused(now) && !k.reset()) { - long now = System.currentTimeMillis(); - // no new event encountered. - for (Path path : new HashSet(pendingEvents.keySet())) + keys.remove(k); + if (keys.isEmpty()) { - PathPendingEvents pending = pendingEvents.get(path); - if (pending.isQuiet(now, updateQuietTimeDuration,updateQuietTimeUnit)) - { - //No fresh events received during quiet time for this path, - //so generate the events that were pent up - for (PathWatchEvent p:pending.getEvents()) - { - notifiableEvents.add(p); - } - // remove from pending list - pendingEvents.remove(path); - } + return; // all done, no longer monitoring anything } } } + + if (LOG.isDebugEnabled()) + LOG.debug("Waiting for poll({})", wait_time); + key = wait_time<0?watch.take():wait_time>0?watch.poll(wait_time,updateQuietTimeUnit):watch.poll(); + + // handle all active keys + while (key!=null) + { + handleKey(key); + key = watch.poll(); + } + + wait_time = processPending(); + + notifyEvents(); + } catch (ClosedWatchServiceException e) { @@ -1306,74 +1202,69 @@ public void run() { LOG.ignore(e); } - return; } + } + } + + private void handleKey(WatchKey key) + { + Config config = keys.get(key); + if (config == null) + { + if (LOG.isDebugEnabled()) + LOG.debug("WatchKey not recognized: {}",key); + return; + } - //If there was some new events to process - if (key != null) - { + for (WatchEvent event : key.pollEvents()) + { + WatchEvent ev = cast(event); + Path name = ev.context(); + Path path = config.resolve(name); - Config config = keys.get(key); - if (config == null) - { - if (LOG.isDebugEnabled()) - { - LOG.debug("WatchKey not recognized: {}",key); - } - continue; - } + if (LOG.isDebugEnabled()) + LOG.debug("handleKey? {} {} {}", ev.kind(), config.toShortPath(path), config); - for (WatchEvent event : key.pollEvents()) - { - @SuppressWarnings("unchecked") - WatchEvent.Kind kind = (Kind)event.kind(); - WatchEvent ev = cast(event); - Path name = ev.context(); - Path child = config.dir.resolve(name); + // Ignore modified events on directories. These are handled as create/delete events of their contents + if (ev.kind()==ENTRY_MODIFY && Files.exists(path) && Files.isDirectory(path)) + continue; - if (kind == ENTRY_CREATE) - { - // handle special case for registering new directories - // recursively - if (Files.isDirectory(child,LinkOption.NOFOLLOW_LINKS)) - { - try - { - prepareConfig(config.asSubConfig(child)); - } - catch (IOException e) - { - LOG.warn(e); - } - } - else if (config.matches(child)) - { - addToPendingList(child, new PathWatchEvent(child,ev)); - } - } - else if (config.matches(child)) - { - addToPendingList(child, new PathWatchEvent(child,ev)); - } - } + if (config.test(path)) + handleWatchEvent(path, new PathWatchEvent(path,ev,config)); + else if (config.getRecurseDepth()==-1) + { + // Convert a watched directory into a modify event on its parent + Path parent = path.getParent(); + Config parentConfig = config.getParent(); + handleWatchEvent(parent, new PathWatchEvent(parent,PathWatchEventType.MODIFIED,parentConfig)); + continue; } - //Send any notifications generated this pass - notifyOnPathWatchEvents(notifiableEvents); - notifiableEvents.clear(); - - if (key != null && !key.reset()) + if (ev.kind() == ENTRY_CREATE) { - keys.remove(key); - if (keys.isEmpty()) + try + { + switch(config.handleDir(path)) + { + case ENTER: + registerTree(path,config.asSubConfig(path),true); + break; + case WATCH: + registerDir(path,config); + break; + case IGNORE: + default: + break; + } + } + catch(IOException e) { - return; // all done, no longer monitoring anything + LOG.warn(e); } } } - } - - + } + /** * Add an event reported by the WatchService to list of pending events * that will be sent after their quiet time has expired. @@ -1381,23 +1272,130 @@ else if (config.matches(child)) * @param path the path to add to the pending list * @param event the pending event */ - public void addToPendingList (Path path, PathWatchEvent event) + public void handleWatchEvent (Path path, PathWatchEvent event) { - PathPendingEvents pending = pendingEvents.get(path); + PathWatchEvent existing = pending.get(path); + + if (LOG.isDebugEnabled()) + LOG.debug("handleWatchEvent {} {} <= {}", path, event, existing); - //Are there already pending events for this path? - if (pending == null) + switch(event.getType()) { - //No existing pending events, create pending list - pendingEvents.put(path,new PathPendingEvents(path, event)); + case ADDED: + if (existing!=null && existing.getType()==PathWatchEventType.MODIFIED) + events.add(new PathWatchEvent(path,PathWatchEventType.DELETED,existing.getConfig())); + pending.put(path,event); + break; + + case MODIFIED: + if (existing==null) + pending.put(path,event); + else + existing.modified(); + break; + + case DELETED: + case UNKNOWN: + if (existing!=null) + pending.remove(path); + events.add(event); + break; + } - else + } + + private long processPending() + { + if (LOG.isDebugEnabled()) + LOG.debug("processPending> {}",pending.values()); + + long now = TimeUnit.NANOSECONDS.toMillis(System.nanoTime()); + long wait = Long.MAX_VALUE; + + // pending map is maintained in LRU order + for (PathWatchEvent event : new ArrayList<>(pending.values())) { - //There are already some events pending for this path - pending.addEvent(event); + Path path = event.getPath(); + // for directories, wait until parent is quiet + if (pending.containsKey(path.getParent())) + continue; + + // if the path is quiet move to events + if (event.isQuiet(now,getUpdateQuietTimeMillis())) + { + if (LOG.isDebugEnabled()) + LOG.debug("isQuiet {}",event); + pending.remove(path); + events.add(event); + } + else + { + long ms_to_check = event.toQuietCheck(now,getUpdateQuietTimeMillis()); + if (LOG.isDebugEnabled()) + LOG.debug("pending {} {}",event, ms_to_check); + if (ms_to_check implements Predicate + { + @Override + public boolean test(Path path) + { + for (PathMatcher pm: this) + if (pm.matches(path)) + return true; + return false; + } + } + } diff --git a/jetty-util/src/test/java/org/eclipse/jetty/util/PathWatcherTest.java b/jetty-util/src/test/java/org/eclipse/jetty/util/PathWatcherTest.java index 5e05758366c5..7691941cc05a 100644 --- a/jetty-util/src/test/java/org/eclipse/jetty/util/PathWatcherTest.java +++ b/jetty-util/src/test/java/org/eclipse/jetty/util/PathWatcherTest.java @@ -18,9 +18,14 @@ package org.eclipse.jetty.util; -import static org.eclipse.jetty.util.PathWatcher.PathWatchEventType.*; -import static org.hamcrest.Matchers.*; -import static org.junit.Assert.*; +import static org.eclipse.jetty.util.PathWatcher.PathWatchEventType.ADDED; +import static org.eclipse.jetty.util.PathWatcher.PathWatchEventType.DELETED; +import static org.eclipse.jetty.util.PathWatcher.PathWatchEventType.MODIFIED; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.is; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; import java.io.File; import java.io.FileOutputStream; @@ -28,6 +33,9 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.attribute.DosFileAttributes; +import java.nio.file.attribute.FileTime; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -36,19 +44,37 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import org.eclipse.jetty.toolchain.test.AdvancedRunner; import org.eclipse.jetty.toolchain.test.OS; import org.eclipse.jetty.toolchain.test.TestingDir; import org.eclipse.jetty.util.PathWatcher.PathWatchEvent; import org.eclipse.jetty.util.PathWatcher.PathWatchEventType; import org.eclipse.jetty.util.log.Log; import org.eclipse.jetty.util.log.Logger; -import org.junit.Ignore; +import org.junit.Assume; import org.junit.Rule; import org.junit.Test; +import org.junit.runner.RunWith; -@Ignore("Disabled due to behavioral differences in various FileSystems (hard to write a single testcase that works in all scenarios)") +@RunWith(AdvancedRunner.class) public class PathWatcherTest { + public static final int QUIET_TIME; + public static final int WAIT_TIME; + public static final int LONG_TIME; + + static + { + if (OS.IS_LINUX) + QUIET_TIME = 300; + else if (OS.IS_OSX) + QUIET_TIME = 5000; + else + QUIET_TIME = 1000; + WAIT_TIME = 2 * QUIET_TIME; + LONG_TIME = 5 * QUIET_TIME; + } + public static class PathWatchEventCapture implements PathWatcher.Listener { public final static String FINISH_TAG = "#finished#.tag"; @@ -76,27 +102,17 @@ public void reset() events.clear(); } + public void reset(int count) + { + setFinishTrigger(count); + events.clear(); + } + @Override public void onPathWatchEvent(PathWatchEvent event) { synchronized (events) { - //if triggered by path - if (triggerPath != null) - { - - if (triggerPath.equals(event.getPath()) && (event.getType() == triggerType)) - { - LOG.debug("Encountered finish trigger: {} on {}",event.getType(),event.getPath()); - finishedLatch.countDown(); - } - } - else if (finishedLatch != null) - { - finishedLatch.countDown(); - } - - Path relativePath = this.baseDir.relativize(event.getPath()); String key = relativePath.toString().replace(File.separatorChar,'/'); @@ -109,6 +125,20 @@ else if (finishedLatch != null) this.events.put(key,types); LOG.debug("Captured Event: {} | {}",event.getType(),key); } + //if triggered by path + if (triggerPath != null) + { + + if (triggerPath.equals(event.getPath()) && (event.getType() == triggerType)) + { + LOG.debug("Encountered finish trigger: {} on {}",event.getType(),event.getPath()); + finishedLatch.countDown(); + } + } + else if (finishedLatch != null) + { + finishedLatch.countDown(); + } } /** @@ -122,13 +152,21 @@ else if (finishedLatch != null) */ public void assertEvents(Map expectedEvents) { - assertThat("Event match (file|diretory) count",this.events.size(),is(expectedEvents.size())); + try + { + assertThat("Event match (file|directory) count", this.events.size(), is(expectedEvents.size())); - for (Map.Entry entry : expectedEvents.entrySet()) + for (Map.Entry entry : expectedEvents.entrySet()) + { + String relativePath = entry.getKey(); + PathWatchEventType[] expectedTypes = entry.getValue(); + assertEvents(relativePath, expectedTypes); + } + } + catch(Throwable th) { - String relativePath = entry.getKey(); - PathWatchEventType[] expectedTypes = entry.getValue(); - assertEvents(relativePath,expectedTypes); + System.err.println(this.events); + throw th; } } @@ -175,7 +213,7 @@ public void setFinishTrigger (int count) latchCount = count; finishedLatch = new CountDownLatch(latchCount); } - + /** * Await the countdown latch on the finish trigger * @@ -197,6 +235,12 @@ public void awaitFinish(PathWatcher pathWatcher) throws IOException, Interrupted assertThat("Timed Out (" + awaitMillis + "ms) waiting for capture to finish",finishedLatch.await(awaitMillis,TimeUnit.MILLISECONDS),is(true)); LOG.debug("Finished capture"); } + + @Override + public String toString() + { + return events.toString(); + } } private static void updateFile(Path path, String newContents) throws IOException @@ -228,39 +272,37 @@ private static void updateFile(Path path, String newContents) throws IOException * @throws InterruptedException * if sleep between writes was interrupted */ - private void updateFileOverTime(Path path, int fileSize, int timeDuration, TimeUnit timeUnit) throws IOException, InterruptedException + private void updateFileOverTime(Path path, int timeDuration, TimeUnit timeUnit) { - // how long to sleep between writes - int sleepMs = 100; - - // how many millis to spend writing entire file size - long totalMs = timeUnit.toMillis(timeDuration); - - // how many write chunks to write - int writeCount = (int)((int)totalMs / (int)sleepMs); - - // average chunk buffer - int chunkBufLen = fileSize / writeCount; - byte chunkBuf[] = new byte[chunkBufLen]; - Arrays.fill(chunkBuf,(byte)'x'); - - try (FileOutputStream out = new FileOutputStream(path.toFile())) + try { - int left = fileSize; - - while (left > 0) + // how long to sleep between writes + int sleepMs = 200; + + // average chunk buffer + int chunkBufLen = 16; + byte chunkBuf[] = new byte[chunkBufLen]; + Arrays.fill(chunkBuf, (byte)'x'); + long end = System.nanoTime() + timeUnit.toNanos(timeDuration); + + try (FileOutputStream out = new FileOutputStream(path.toFile())) { - int len = Math.min(left,chunkBufLen); - out.write(chunkBuf,0,len); - left -= chunkBufLen; - out.flush(); - out.getChannel().force(true); - // Force file to actually write to disk. - // Skipping any sort of filesystem caching of the write - out.getFD().sync(); - TimeUnit.MILLISECONDS.sleep(sleepMs); + while(System.nanoTime() expected = new HashMap<>(); - @Test - public void testConfig_ShouldRecurse_1() throws IOException - { - Path dir = testdir.getEmptyPathDir(); + // Check initial scan events + capture.setFinishTrigger(4); + pathWatcher.start(); + expected.put("file0",new PathWatchEventType[] { ADDED }); + expected.put("subdir0",new PathWatchEventType[] { ADDED }); + expected.put("subdir0/fileA",new PathWatchEventType[] { ADDED }); + expected.put("subdir0/subsubdir0",new PathWatchEventType[] { ADDED }); - // Create a few directories - Files.createDirectories(dir.resolve("a/b/c/d")); + capture.finishedLatch.await(LONG_TIME,TimeUnit.MILLISECONDS); + capture.assertEvents(expected); + Thread.sleep(WAIT_TIME); + capture.assertEvents(expected); - PathWatcher.Config config = new PathWatcher.Config(dir); + // Check adding files + capture.reset(3); + expected.clear(); + Files.createFile(dir.resolve("subdir0/subsubdir0/toodeep")); + expected.put("subdir0/subsubdir0",new PathWatchEventType[] { MODIFIED }); + Files.createFile(dir.resolve("file1")); + expected.put("file1",new PathWatchEventType[] { ADDED }); + Files.createFile(dir.resolve("subdir0/fileB")); + expected.put("subdir0/fileB",new PathWatchEventType[] { ADDED }); + + capture.finishedLatch.await(LONG_TIME,TimeUnit.MILLISECONDS); + capture.assertEvents(expected); + Thread.sleep(WAIT_TIME); + capture.assertEvents(expected); - config.setRecurseDepth(1); - assertThat("Config.recurse[1].shouldRecurse[./a/b]",config.shouldRecurseDirectory(dir.resolve("a/b")),is(false)); - assertThat("Config.recurse[1].shouldRecurse[./a]",config.shouldRecurseDirectory(dir.resolve("a")),is(true)); - assertThat("Config.recurse[1].shouldRecurse[./]",config.shouldRecurseDirectory(dir),is(true)); - } + // Check slow modification + capture.reset(1); + expected.clear(); + long start = System.nanoTime(); + new Thread(()->{updateFileOverTime(dir.resolve("file1"),2*QUIET_TIME,TimeUnit.MILLISECONDS);}).start(); + expected.put("file1",new PathWatchEventType[] { MODIFIED }); + capture.finishedLatch.await(LONG_TIME,TimeUnit.MILLISECONDS); + long end = System.nanoTime(); + capture.assertEvents(expected); + assertThat(end-start,greaterThan(TimeUnit.MILLISECONDS.toNanos(2*QUIET_TIME))); + Thread.sleep(WAIT_TIME); + capture.assertEvents(expected); - @Test - public void testConfig_ShouldRecurse_2() throws IOException - { - Path dir = testdir.getEmptyPathDir(); + // Check slow add + capture.reset(1); + expected.clear(); + start = System.nanoTime(); + new Thread(()->{updateFileOverTime(dir.resolve("file2"),2*QUIET_TIME,TimeUnit.MILLISECONDS);}).start(); + expected.put("file2",new PathWatchEventType[] { ADDED }); + capture.finishedLatch.await(LONG_TIME,TimeUnit.MILLISECONDS); + end = System.nanoTime(); + capture.assertEvents(expected); + assertThat(end-start,greaterThan(TimeUnit.MILLISECONDS.toNanos(2*QUIET_TIME))); + Thread.sleep(WAIT_TIME); + capture.assertEvents(expected); - // Create a few directories - Files.createDirectories(dir.resolve("a/b/c/d")); + // Check move directory + if (OS.IS_LINUX) + { + capture.reset(5); + expected.clear(); + Files.move(dir.resolve("subdir0"), dir.resolve("subdir1"), StandardCopyOption.ATOMIC_MOVE); + expected.put("subdir0", new PathWatchEventType[]{DELETED}); + // TODO expected.put("subdir0/fileA",new PathWatchEventType[] { DELETED }); + // TODO expected.put("subdir0/subsubdir0",new PathWatchEventType[] { DELETED }); + expected.put("subdir1", new PathWatchEventType[]{ADDED}); + expected.put("subdir1/fileA", new PathWatchEventType[]{ADDED}); + expected.put("subdir1/fileB", new PathWatchEventType[]{ADDED}); + expected.put("subdir1/subsubdir0", new PathWatchEventType[]{ADDED}); + + capture.finishedLatch.await(LONG_TIME, TimeUnit.MILLISECONDS); + capture.assertEvents(expected); + Thread.sleep(WAIT_TIME); + capture.assertEvents(expected); + } - PathWatcher.Config config = new PathWatcher.Config(dir); + // Check delete file + capture.reset(2); + expected.clear(); + Files.delete(dir.resolve("file1")); + expected.put("file1",new PathWatchEventType[] { DELETED }); + Files.delete(dir.resolve("file2")); + expected.put("file2",new PathWatchEventType[] { DELETED }); - config.setRecurseDepth(2); - assertThat("Config.recurse[1].shouldRecurse[./a/b/c]",config.shouldRecurseDirectory(dir.resolve("a/b/c")),is(false)); - assertThat("Config.recurse[1].shouldRecurse[./a/b]",config.shouldRecurseDirectory(dir.resolve("a/b")),is(true)); - assertThat("Config.recurse[1].shouldRecurse[./a]",config.shouldRecurseDirectory(dir.resolve("a")),is(true)); - assertThat("Config.recurse[1].shouldRecurse[./]",config.shouldRecurseDirectory(dir),is(true)); - } - - - @Test - public void testConfig_ShouldRecurse_3() throws IOException - { - Path dir = testdir.getEmptyPathDir(); - - //Create some deep dirs - Files.createDirectories(dir.resolve("a/b/c/d/e/f/g")); - - PathWatcher.Config config = new PathWatcher.Config(dir); - config.setRecurseDepth(PathWatcher.Config.UNLIMITED_DEPTH); - assertThat("Config.recurse[1].shouldRecurse[./a/b/c/d/g]",config.shouldRecurseDirectory(dir.resolve("a/b/c/d/g")),is(true)); - assertThat("Config.recurse[1].shouldRecurse[./a/b/c/d/f]",config.shouldRecurseDirectory(dir.resolve("a/b/c/d/f")),is(true)); - assertThat("Config.recurse[1].shouldRecurse[./a/b/c/d/e]",config.shouldRecurseDirectory(dir.resolve("a/b/c/d/e")),is(true)); - assertThat("Config.recurse[1].shouldRecurse[./a/b/c/d]",config.shouldRecurseDirectory(dir.resolve("a/b/c/d")),is(true)); - assertThat("Config.recurse[1].shouldRecurse[./a/b/c]",config.shouldRecurseDirectory(dir.resolve("a/b/c")),is(true)); - assertThat("Config.recurse[1].shouldRecurse[./a/b]",config.shouldRecurseDirectory(dir.resolve("a/b")),is(true)); - assertThat("Config.recurse[1].shouldRecurse[./a]",config.shouldRecurseDirectory(dir.resolve("a")),is(true)); - assertThat("Config.recurse[1].shouldRecurse[./]",config.shouldRecurseDirectory(dir),is(true)); + capture.finishedLatch.await(LONG_TIME,TimeUnit.MILLISECONDS); + capture.assertEvents(expected); + Thread.sleep(WAIT_TIME); + capture.assertEvents(expected); + + } + finally + { + pathWatcher.stop(); + } } @Test @@ -368,7 +456,7 @@ public void testRestart() throws Exception PathWatcher pathWatcher = new PathWatcher(); pathWatcher.setNotifyExistingOnStart(true); - pathWatcher.setUpdateQuietTime(500,TimeUnit.MILLISECONDS); + pathWatcher.setUpdateQuietTime(QUIET_TIME,TimeUnit.MILLISECONDS); // Add listener PathWatchEventCapture capture = new PathWatchEventCapture(dir); @@ -393,7 +481,8 @@ public void testRestart() throws Exception expected.put("a.txt",new PathWatchEventType[] {ADDED}); expected.put("b.txt",new PathWatchEventType[] {ADDED}); - + Thread.currentThread().sleep(1000); // TODO poor test + capture.assertEvents(expected); //stop it @@ -401,7 +490,7 @@ public void testRestart() throws Exception capture.reset(); - Thread.currentThread().sleep(1000); + Thread.currentThread().sleep(1000); // TODO poor test pathWatcher.start(); @@ -436,13 +525,17 @@ public void testStartupFindFiles() throws Exception // Files we don't care about Files.createFile(dir.resolve("foo.war.backup")); - Files.createFile(dir.resolve(".hidden.war")); + + String hidden_war = OS.IS_WINDOWS ? "hidden.war" : ".hidden.war"; + Files.createFile(dir.resolve(hidden_war)); + if (OS.IS_WINDOWS) + Files.setAttribute(dir.resolve(hidden_war),"dos:hidden",Boolean.TRUE); Files.createDirectories(dir.resolve(".wat/WEB-INF")); Files.createFile(dir.resolve(".wat/huh.war")); Files.createFile(dir.resolve(".wat/WEB-INF/web.xml")); PathWatcher pathWatcher = new PathWatcher(); - pathWatcher.setUpdateQuietTime(300,TimeUnit.MILLISECONDS); + pathWatcher.setUpdateQuietTime(QUIET_TIME,TimeUnit.MILLISECONDS); // Add listener PathWatchEventCapture capture = new PathWatchEventCapture(dir); @@ -458,17 +551,19 @@ public void testStartupFindFiles() throws Exception try { + capture.setFinishTrigger(2); pathWatcher.start(); // Let quiet time do its thing - awaitQuietTime(pathWatcher); + capture.finishedLatch.await(LONG_TIME,TimeUnit.MILLISECONDS); Map expected = new HashMap<>(); - expected.put("bar/WEB-INF/web.xml",new PathWatchEventType[] { ADDED }); expected.put("foo.war",new PathWatchEventType[] { ADDED }); capture.assertEvents(expected); + TimeUnit.MILLISECONDS.sleep(WAIT_TIME); + capture.assertEvents(expected); } finally { @@ -494,11 +589,10 @@ public void testGlobPattern () throws Exception PathWatcher pathWatcher = new PathWatcher(); - pathWatcher.setUpdateQuietTime(300,TimeUnit.MILLISECONDS); + pathWatcher.setUpdateQuietTime(QUIET_TIME,TimeUnit.MILLISECONDS); // Add listener PathWatchEventCapture capture = new PathWatchEventCapture(dir); - capture.setFinishTrigger(3); pathWatcher.addListener(capture); // Add test dir configuration @@ -510,10 +604,9 @@ public void testGlobPattern () throws Exception try { + capture.setFinishTrigger(3); pathWatcher.start(); - - // Let quiet time do its thing - awaitQuietTime(pathWatcher); + assertTrue(capture.finishedLatch.await(LONG_TIME,TimeUnit.MILLISECONDS)); Map expected = new HashMap<>(); @@ -521,6 +614,8 @@ public void testGlobPattern () throws Exception expected.put("b/b.txt",new PathWatchEventType[] { ADDED }); expected.put("c/d/d.txt",new PathWatchEventType[] { ADDED }); capture.assertEvents(expected); + TimeUnit.MILLISECONDS.sleep(WAIT_TIME); + capture.assertEvents(expected); } finally { @@ -539,16 +634,15 @@ public void testDeployFiles_Update_Delete() throws Exception Files.createFile(dir.resolve("bar/WEB-INF/web.xml")); PathWatcher pathWatcher = new PathWatcher(); - pathWatcher.setUpdateQuietTime(300,TimeUnit.MILLISECONDS); + pathWatcher.setUpdateQuietTime(QUIET_TIME,TimeUnit.MILLISECONDS); // Add listener PathWatchEventCapture capture = new PathWatchEventCapture(dir); - capture.setFinishTrigger(5); pathWatcher.addListener(capture); // Add test dir configuration PathWatcher.Config baseDirConfig = new PathWatcher.Config(dir); - baseDirConfig.setRecurseDepth(2); + baseDirConfig.setRecurseDepth(100); baseDirConfig.addExcludeHidden(); baseDirConfig.addIncludeGlobRelative("*.war"); baseDirConfig.addIncludeGlobRelative("*/WEB-INF/web.xml"); @@ -556,11 +650,13 @@ public void testDeployFiles_Update_Delete() throws Exception try { + capture.setFinishTrigger(2); pathWatcher.start(); - // Pretend that startup occurred - awaitQuietTime(pathWatcher); + assertTrue(capture.finishedLatch.await(LONG_TIME,TimeUnit.MILLISECONDS)); + capture.setFinishTrigger(3); + // Update web.xml Path webFile = dir.resolve("bar/WEB-INF/web.xml"); //capture.setFinishTrigger(webFile,MODIFIED); @@ -573,7 +669,7 @@ public void testDeployFiles_Update_Delete() throws Exception Files.createFile(dir.resolve("bar.war")); // Let capture complete - capture.awaitFinish(pathWatcher); + assertTrue(capture.finishedLatch.await(LONG_TIME,TimeUnit.MILLISECONDS)); Map expected = new HashMap<>(); @@ -582,6 +678,8 @@ public void testDeployFiles_Update_Delete() throws Exception expected.put("bar.war",new PathWatchEventType[] { ADDED }); capture.assertEvents(expected); + TimeUnit.MILLISECONDS.sleep(WAIT_TIME); + capture.assertEvents(expected); } finally { @@ -600,7 +698,7 @@ public void testDeployFiles_NewWar() throws Exception Files.createFile(dir.resolve("bar/WEB-INF/web.xml")); PathWatcher pathWatcher = new PathWatcher(); - pathWatcher.setUpdateQuietTime(300,TimeUnit.MILLISECONDS); + pathWatcher.setUpdateQuietTime(QUIET_TIME,TimeUnit.MILLISECONDS); // Add listener PathWatchEventCapture capture = new PathWatchEventCapture(dir); @@ -616,26 +714,36 @@ public void testDeployFiles_NewWar() throws Exception try { + capture.setFinishTrigger(2); pathWatcher.start(); // Pretend that startup occurred - awaitQuietTime(pathWatcher); + assertTrue(capture.finishedLatch.await(LONG_TIME,TimeUnit.MILLISECONDS)); // New war added + capture.setFinishTrigger(1); Path warFile = dir.resolve("hello.war"); - capture.setFinishTrigger(warFile,MODIFIED); - updateFile(warFile,"Hello Update"); + updateFile(warFile,"Create Hello"); + Thread.sleep(QUIET_TIME/2); + updateFile(warFile,"Hello 1"); + Thread.sleep(QUIET_TIME/2); + updateFile(warFile,"Hello two"); + Thread.sleep(QUIET_TIME/2); + updateFile(warFile,"Hello three"); // Let capture finish - capture.awaitFinish(pathWatcher); + assertTrue(capture.finishedLatch.await(LONG_TIME,TimeUnit.MILLISECONDS)); Map expected = new HashMap<>(); expected.put("bar/WEB-INF/web.xml",new PathWatchEventType[] { ADDED }); expected.put("foo.war",new PathWatchEventType[] { ADDED }); - expected.put("hello.war",new PathWatchEventType[] { ADDED, MODIFIED }); + expected.put("hello.war",new PathWatchEventType[] { ADDED }); capture.assertEvents(expected); + TimeUnit.MILLISECONDS.sleep(WAIT_TIME); + Assume.assumeFalse(OS.IS_OSX); // TODO fix this + capture.assertEvents(expected); } finally { @@ -643,8 +751,195 @@ public void testDeployFiles_NewWar() throws Exception } } + @Test + public void testDeployFiles_NewDir() throws Exception + { + Path dir = testdir.getEmptyPathDir(); + + // Files we are interested in + Files.createFile(dir.resolve("foo.war")); + + PathWatcher pathWatcher = new PathWatcher(); + pathWatcher.setUpdateQuietTime(QUIET_TIME,TimeUnit.MILLISECONDS); + + // Add listener + PathWatchEventCapture capture = new PathWatchEventCapture(dir); + pathWatcher.addListener(capture); + + // Add test dir configuration + PathWatcher.Config baseDirConfig = new PathWatcher.Config(dir); + baseDirConfig.setRecurseDepth(2); + baseDirConfig.addExcludeHidden(); + baseDirConfig.addIncludeGlobRelative("*.war"); + baseDirConfig.addIncludeGlobRelative("*/WEB-INF/web.xml"); + pathWatcher.watch(baseDirConfig); + + try + { + capture.setFinishTrigger(1); + pathWatcher.start(); + + // Pretend that startup occurred + assertTrue(capture.finishedLatch.await(LONG_TIME,TimeUnit.MILLISECONDS)); + + // New war added + capture.setFinishTrigger(1); + + Files.createDirectories(dir.resolve("bar/WEB-INF")); + Thread.sleep(QUIET_TIME/2); + Files.createFile(dir.resolve("bar/WEB-INF/web.xml")); + Thread.sleep(QUIET_TIME/2); + updateFile(dir.resolve("bar/WEB-INF/web.xml"),"Update"); + Thread.sleep(QUIET_TIME/2); + updateFile(dir.resolve("bar/WEB-INF/web.xml"),"Update web.xml"); + + // Let capture finish + assertTrue(capture.finishedLatch.await(LONG_TIME,TimeUnit.MILLISECONDS)); + + Map expected = new HashMap<>(); + + expected.put("bar/WEB-INF/web.xml",new PathWatchEventType[] { ADDED }); + expected.put("foo.war",new PathWatchEventType[] { ADDED }); + + capture.assertEvents(expected); + TimeUnit.MILLISECONDS.sleep(WAIT_TIME); + capture.assertEvents(expected); + } + finally + { + pathWatcher.stop(); + } + } + + + @Test + public void testDeployFilesBeyondDepthLimit() throws Exception + { + Path dir = testdir.getEmptyPathDir(); + + // Files we are interested in + Files.createDirectories(dir.resolve("foo/WEB-INF/lib")); + Files.createDirectories(dir.resolve("bar/WEB-INF/lib")); + + PathWatcher pathWatcher = new PathWatcher(); + pathWatcher.setUpdateQuietTime(QUIET_TIME,TimeUnit.MILLISECONDS); + + // Add listener + PathWatchEventCapture capture = new PathWatchEventCapture(dir); + pathWatcher.addListener(capture); + + // Add test dir configuration + PathWatcher.Config baseDirConfig = new PathWatcher.Config(dir); + baseDirConfig.setRecurseDepth(0); + pathWatcher.watch(baseDirConfig); + + try + { + capture.setFinishTrigger(2); + pathWatcher.start(); + + // Pretend that startup occurred + assertTrue(capture.finishedLatch.await(LONG_TIME,TimeUnit.MILLISECONDS)); + + Map expected = new HashMap<>(); + expected.put("foo",new PathWatchEventType[] { ADDED }); + expected.put("bar",new PathWatchEventType[] { ADDED }); + + capture.assertEvents(expected); + + capture.reset(1); + expected.clear(); + expected.put("bar",new PathWatchEventType[] { MODIFIED }); + Files.createFile(dir.resolve("bar/index.html")); + assertTrue(capture.finishedLatch.await(LONG_TIME,TimeUnit.MILLISECONDS)); + + capture.assertEvents(expected); + TimeUnit.MILLISECONDS.sleep(WAIT_TIME); + capture.assertEvents(expected); + + capture.reset(1); + expected.clear(); + expected.put("bob",new PathWatchEventType[] { ADDED }); + Files.createFile(dir.resolve("bar/WEB-INF/lib/ignored")); + PathWatcher.LOG.debug("create bob"); + Files.createDirectories(dir.resolve("bob/WEB-INF/lib")); + Thread.sleep(QUIET_TIME/2); + PathWatcher.LOG.debug("create bob/index.html"); + Files.createFile(dir.resolve("bob/index.html")); + Thread.sleep(QUIET_TIME/2); + PathWatcher.LOG.debug("update bob/index.html"); + updateFile(dir.resolve("bob/index.html"),"Update"); + Thread.sleep(QUIET_TIME/2); + PathWatcher.LOG.debug("update bob/index.html"); + updateFile(dir.resolve("bob/index.html"),"Update index.html"); + + assertTrue(capture.finishedLatch.await(LONG_TIME,TimeUnit.MILLISECONDS)); + capture.assertEvents(expected); + TimeUnit.MILLISECONDS.sleep(WAIT_TIME); + capture.assertEvents(expected); + + } + finally + { + pathWatcher.stop(); + } + } + + + @Test + public void testWatchFile() throws Exception + { + Path dir = testdir.getEmptyPathDir(); + + // Files we are interested in + Files.createDirectories(dir.resolve("bar/WEB-INF")); + Files.createFile(dir.resolve("bar/WEB-INF/web.xml")); + + PathWatcher pathWatcher = new PathWatcher(); + pathWatcher.setUpdateQuietTime(QUIET_TIME,TimeUnit.MILLISECONDS); + + // Add listener + PathWatchEventCapture capture = new PathWatchEventCapture(dir); + pathWatcher.addListener(capture); + + // Add test configuration + pathWatcher.watch(dir.resolve("bar/WEB-INF/web.xml")); + pathWatcher.setNotifyExistingOnStart(false); + + try + { + pathWatcher.start(); + Thread.sleep(WAIT_TIME); + assertThat(capture.events.size(),is(0)); + + Files.createFile(dir.resolve("bar/index.htnl")); + Files.createFile(dir.resolve("bar/WEB-INF/other.xml")); + Files.createDirectories(dir.resolve("bar/WEB-INF/lib")); + + Thread.sleep(WAIT_TIME); + assertThat(capture.events.size(),is(0)); + + capture.setFinishTrigger(1); + updateFile(dir.resolve("bar/WEB-INF/web.xml"),"Update web.xml"); + assertTrue(capture.finishedLatch.await(LONG_TIME,TimeUnit.MILLISECONDS)); + + Map expected = new HashMap<>(); + + expected.put("bar/WEB-INF/web.xml",new PathWatchEventType[] { MODIFIED }); + + capture.assertEvents(expected); + TimeUnit.MILLISECONDS.sleep(WAIT_TIME); + capture.assertEvents(expected); + } + finally + { + pathWatcher.stop(); + } + } + + /** - * Pretend to add a new war file that is large, and being copied into place + * Pretend to modify a new war file that is large, and being copied into place * using some sort of technique that is slow enough that it takes a while for * the entire war file to exist in place. *

@@ -654,17 +949,18 @@ public void testDeployFiles_NewWar() throws Exception * on test failure */ @Test - public void testDeployFiles_NewWar_LargeSlowCopy() throws Exception + public void testDeployFiles_ModifyWar_LargeSlowCopy() throws Exception { Path dir = testdir.getEmptyPathDir(); // Files we are interested in Files.createFile(dir.resolve("foo.war")); + Files.createFile(dir.resolve("hello.war")); Files.createDirectories(dir.resolve("bar/WEB-INF")); Files.createFile(dir.resolve("bar/WEB-INF/web.xml")); PathWatcher pathWatcher = new PathWatcher(); - pathWatcher.setUpdateQuietTime(500,TimeUnit.MILLISECONDS); + pathWatcher.setUpdateQuietTime(QUIET_TIME,TimeUnit.MILLISECONDS); // Add listener PathWatchEventCapture capture = new PathWatchEventCapture(dir); @@ -680,26 +976,35 @@ public void testDeployFiles_NewWar_LargeSlowCopy() throws Exception try { + capture.setFinishTrigger(3); pathWatcher.start(); // Pretend that startup occurred - awaitQuietTime(pathWatcher); + assertTrue(capture.finishedLatch.await(LONG_TIME,TimeUnit.MILLISECONDS)); + // New war added (slowly) + capture.setFinishTrigger(1); Path warFile = dir.resolve("hello.war"); - capture.setFinishTrigger(warFile,MODIFIED); - updateFileOverTime(warFile,50 * MB,3,TimeUnit.SECONDS); - - // Let capture finish - capture.awaitFinish(pathWatcher); + long start = System.nanoTime(); + new Thread(()-> + { + updateFileOverTime(warFile,2*QUIET_TIME,TimeUnit.MILLISECONDS); + }).start(); + + assertTrue(capture.finishedLatch.await(4*QUIET_TIME,TimeUnit.MILLISECONDS)); + long end = System.nanoTime(); + assertThat(end-start,greaterThan(TimeUnit.MILLISECONDS.toNanos(2*QUIET_TIME))); + Map expected = new HashMap<>(); - expected.put("bar/WEB-INF/web.xml",new PathWatchEventType[] { ADDED }); expected.put("foo.war",new PathWatchEventType[] { ADDED }); expected.put("hello.war",new PathWatchEventType[] { ADDED, MODIFIED }); capture.assertEvents(expected); + TimeUnit.MILLISECONDS.sleep(WAIT_TIME); + capture.assertEvents(expected); } finally { diff --git a/jetty-util/src/test/resources/jetty-logging.properties b/jetty-util/src/test/resources/jetty-logging.properties index 3a0ce97e2e08..adc470de2b3d 100644 --- a/jetty-util/src/test/resources/jetty-logging.properties +++ b/jetty-util/src/test/resources/jetty-logging.properties @@ -1,3 +1,4 @@ # Setup default logging implementation for during testing org.eclipse.jetty.util.log.class=org.eclipse.jetty.util.log.StdErrLog #org.eclipse.jetty.util.LEVEL=DEBUG +#org.eclipse.jetty.util.PathWatcher.LEVEL=DEBUG \ No newline at end of file