Skip to content

Commit

Permalink
Fix issue with Parallel reapply
Browse files Browse the repository at this point in the history
  • Loading branch information
juliohm committed Nov 4, 2021
1 parent 7d6c578 commit 63ec04f
Show file tree
Hide file tree
Showing 3 changed files with 11 additions and 3 deletions.
2 changes: 1 addition & 1 deletion src/transforms.jl
Original file line number Diff line number Diff line change
Expand Up @@ -201,7 +201,7 @@ function reapply(transform::Colwise, table, cache)
end

# parallel map with multiple threads
vals = foldxt(vcat, Map(colfunc), 1:length(names))
vals = tcollect(colfunc(i) for i in 1:length(names))

# new table with transformed columns
newtable = (; vals...) |> Tables.materializer(table)
Expand Down
4 changes: 2 additions & 2 deletions src/transforms/parallel.jl
Original file line number Diff line number Diff line change
Expand Up @@ -78,9 +78,9 @@ function reapply(p::Parallel, table, cache)
caches = cache[2]

# reapply transforms in parallel
f((t,c)) = reapply(t, table, c) |> first
f(t, c) = reapply(t, table, c) |> first
itr = zip(p.transforms, caches)
tables = foldxt(vcat, Map(f), itr)
tables = tcollect(f(t, c) for (t, c) in itr)

# table with concatenated columns
newtable = tablehcat(tables)
Expand Down
8 changes: 8 additions & 0 deletions test/transforms.jl
Original file line number Diff line number Diff line change
Expand Up @@ -406,5 +406,13 @@
@test isapprox(std(n.x), 1.0, atol=1e-8)
@test isapprox(mean(r.x), mean(t.x), atol=1e-8)
@test isapprox(std(r.x), std(t.x), atol=1e-8)

# reapply with Parallel transform
t = Table(x=rand(1000))
T = ZScore() Quantile()
n1, c1 = apply(T, t)
n2, c2 = reapply(T, t, c1)
@test n1 == n2
@test c1 == c2
end
end

0 comments on commit 63ec04f

Please sign in to comment.