Merge pull request #76 from shashi/compathelper/new_version/2023-07-2…

…4-18-44-10-621-04117316811

CompatHelper: bump compat for Dagger to 0.18, (keep existing compat)

.gitignore

@@ -1 +1,2 @@
 Manifest.toml
+.vscode

Project.toml

@@ -11,7 +11,7 @@ Glob = "c27321d9-0574-5035-807b-f59d2c89b15c"
 
 [compat]
 AbstractTrees = "0.3, 0.4"
-Dagger = "0.9, 0.10, 0.11, 0.12.3, 0.13, 0.14, 0.15, 0.16, 0.17"
+Dagger = "0.9, 0.10, 0.11, 0.12.3, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18"
 FilePathsBase = "0.9"
 Glob = "1.3"
 julia = "1"

appveyor.yml

@@ -1,10 +1,10 @@
 environment:
   matrix:
-  - julia_version: 1
+  - julia_version: 1.9
 #  - julia_version: nightly
 
 platform:
-  - x86 # 32-bit
+  - x64
 
 # # Uncomment the following lines to allow failures on nightly julia
 # # (tests will run but not make your overall status red)

src/values.jl

@@ -108,6 +108,20 @@ has a value associated with it.
 (see `load` and `mapvalues` for associating values with files.)
 """
 function save(f, t::Node; lazy=nothing, exec=true)
+    # Dagger 0.18 does not allow passing Thunks between workers, so we create a copy of t where
+    # they are removed.
+    # Note that doing something like setvalue(x, NoValue()) when calling saver below won't work 
+    # since each x drags along the whole tree through its parent reference
+    # One could disconnect the parent as well, but users might have code which does things like
+    # dirname = path(parent(file)) e.g. when storing more than one file.
+    # We should probably remove all values here so we don't end up passing a huge tree between
+    # workers though...
+    t_nothunks = map(t) do n
+        n[] isa Thunk && return setvalue(n, NoValue())
+        n[] isa Chunk && return setvalue(n, NoValue())
+        n
+    end |> setparent
+
     t′ = mapvalued(t) do x
         isempty(x) && return NoValue()
 
@@ -121,7 +135,8 @@ function save(f, t::Node; lazy=nothing, exec=true)
             f(setvalue(file, val))
         end
 
-        setvalue(x, lazify(lazy, saver)(x, x[]))
+        relativepath = replace(path(x), path(t) => "")
+        setvalue(x, lazify(lazy, saver)(t_nothunks[relativepath], x[]))
     end
 
     # placeholder task that waits and returns nothing

test/basics.jl

@@ -140,100 +140,93 @@ import FileTrees: attach
 end
 
 @testset "values" begin
-    t1 = FileTrees.load(path, t)
-    if isdir("test_dir")
-        rm("test_dir", recursive=true)
-    end
+    mktempdir() do rootpath
+        testdir = joinpath(rootpath, "test_dir")
+        t1 = FileTrees.load(path, t)
 
-    @test get(t1["a/b/a"]) == string(p"." / "a" / "b" / "a")
+        @test get(t1["a/b/a"]) == string(p"." / "a" / "b" / "a")
 
-    @test reducevalues(*, mapvalues(lowercase, t1)) == lowercase(reducevalues(*, t1))
+        @test reducevalues(*, mapvalues(lowercase, t1)) == lowercase(reducevalues(*, t1))
 
-    FileTrees.save(maketree("test_dir" => [t1])) do f
-        @test f isa File
-        open(path(f), "w") do io
-            print(io, get(f))
+        FileTrees.save(maketree(testdir => [t1])) do f
+            @test f isa File
+            @test parent(f) isa FileTree
+            open(path(f), "w") do io
+                print(io, get(f))
+            end
         end
-    end
 
-    t2 = FileTree("test_dir")
-    t3 = FileTrees.load(t2) do f
-        open(path(f), "r") do io
-            String(read(io))
+        t2 = FileTree(testdir)
+        t3 = FileTrees.load(t2) do f
+            open(path(f), "r") do io
+                String(read(io))
+            end
         end
-    end
 
-    t4 = filter(!isempty, t1)
+        t4 = filter(!isempty, t1)
 
-    @test isequal(t3, FileTrees.rename(t4, "test_dir"))
-    if isdir("test_dir")
-        rm("test_dir", recursive=true)
-    end
+        @test isequal(t3, FileTrees.rename(t4, testdir))
 
-    x1 = maketree("a"=>[(name="b", value=1)])
-    x2 = mapvalues(x->NoValue(), x1, lazy=true)
-    @test !isempty(values(x2))
-    @test isempty(values(exec(x2)))
-    x3 = mapvalues(x->rand(), x2)
-    @test !isempty(values(x3))
-    @test isempty(values(exec(x3)))
+        x1 = maketree("a"=>[(name="b", value=1)])
+        x2 = mapvalues(x->NoValue(), x1, lazy=true)
+        @test !isempty(values(x2))
+        @test isempty(values(exec(x2)))
+        x3 = mapvalues(x->rand(), x2)
+        @test !isempty(values(x3))
+        @test isempty(values(exec(x3)))
 
-    # issue 16
-    @test_throws ArgumentError reducevalues(+, maketree("." => []))
-    @test reducevalues(+, maketree("." => []), init=0) === 0
+        # issue 16
+        @test_throws ArgumentError reducevalues(+, maketree("." => []))
+        @test reducevalues(+, maketree("." => []), init=0) === 0
 
-    @test_throws Union{ArgumentError,MethodError} reducevalues(+, maketree("." => []), associative=false)
-    @test reducevalues(+, maketree("." => []), init=0, associative=false) === 0
+        @test_throws Union{ArgumentError,MethodError} reducevalues(+, maketree("." => []), associative=false)
+        @test reducevalues(+, maketree("." => []), init=0, associative=false) === 0
 
-    # issue 23
-    @test FileTrees.save(identity, maketree([])) == nothing
+        # issue 23
+        @test FileTrees.save(identity, maketree([])) === nothing
+    end
 end
 
 @testset "lazy-exec" begin
 
-    if isdir("test_dir_lazy")
-        rm("test_dir_lazy", recursive=true)
-    end
+    mktempdir() do rootpath
+        testdir = joinpath(rootpath, "test_dir_lazy")
 
+        t1 = FileTrees.load(uppercase∘path, t, lazy=true)
 
-    t1 = FileTrees.load(uppercase∘path, t, lazy=true)
+        @test get(t1["a/b/a"]) isa Thunk
+        @test get(exec(t1)["a/b/a"]) == string(p"."/"A"/"B"/"A")
+        # Exec a single File
+        @test get(exec(t1["a/b/a"])) == string(p"."/"A"/"B"/"A")
 
-    @test get(t1["a/b/a"]) isa Thunk
-    @test get(exec(t1)["a/b/a"]) == string(p"."/"A"/"B"/"A")
-    # Exec a single File
-    @test get(exec(t1["a/b/a"])) == string(p"."/"A"/"B"/"A")
+        @test exec(reducevalues(*, mapvalues(lowercase, t1))) == lowercase(exec(reducevalues(*, t1)))
 
-    @test exec(reducevalues(*, mapvalues(lowercase, t1))) == lowercase(exec(reducevalues(*, t1)))
-
-    s = FileTrees.save(maketree("test_dir_lazy" => [t1])) do f
-        open(path(f), "w") do io
-            print(io, get(f))
+        s = FileTrees.save(maketree(testdir => [t1])) do f
+            @test f isa File
+            @test parent(f) isa FileTree
+            open(path(f), "w") do io
+                print(io, get(f))
+            end
         end
-    end
-
-    @test isdir("test_dir_lazy")
-    @test isfile("test_dir_lazy/a/b/a")
 
+        @test isfile(joinpath(testdir, "a/b/a"))
 
-    t2 = FileTree("test_dir_lazy")
-    t3 = FileTrees.load(t2; lazy=true) do f
-        open(path(f), "r") do io
-            (String(read(io)), now())
+        t2 = FileTree(testdir)
+        t3 = FileTrees.load(t2; lazy=true) do f
+            open(path(f), "r") do io
+                (String(read(io)), now())
+            end
         end
-    end
-    toc = now()
-    sleep(0.01)
-    tic = exec(reducevalues((x,y)->x, mapvalues(last, t3)))
-
-    @test tic > toc
+        toc = now()
+        sleep(0.01)
+        tic = exec(reducevalues((x,y)->x, mapvalues(last, t3)))
 
-    t4 = filter(!isempty, t1) |> exec
+        @test tic > toc
 
-    t5 = mapvalues(first, t3) |> exec
-    @test isequal(t5, FileTrees.rename(t4, "test_dir_lazy"))
+        t4 = filter(!isempty, t1) |> exec
 
-    if isdir("test_dir_lazy")
-        rm("test_dir_lazy", recursive=true)
+        t5 = mapvalues(first, t3) |> exec
+        @test isequal(t5, FileTrees.rename(t4, testdir))
     end
 end