-
Notifications
You must be signed in to change notification settings - Fork 11
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #20 from jlmelville/loadindex
Loadindex
- Loading branch information
Showing
5 changed files
with
166 additions
and
154 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,97 @@ | ||
| // Taken from RcppParallel.h and then modified slightly to rename header guards | ||
| // and namespaces to avoid any potential clashes. RcppParallel is licensed under | ||
| // GPLv2 or later: | ||
|
|
||
| // pfor.h a version of parallel for based on RcppParallel | ||
| // Copyright (C) 2020 James Melville | ||
| // | ||
| // This program is free software; you can redistribute it and/or | ||
| // modify it under the terms of the GNU General Public License | ||
| // as published by the Free Software Foundation; either version 2 | ||
| // of the License, or (at your option) any later version. | ||
| // | ||
| // This program is distributed in the hope that it will be useful, | ||
| // but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
| // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
| // GNU General Public License for more details. | ||
| // | ||
| // You should have received a copy of the GNU General Public License | ||
| // along with this program; if not, write to the Free Software | ||
| // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, | ||
| // USA. | ||
|
|
||
| #ifndef PFORR | ||
| #define PFORR | ||
|
|
||
| #include <thread> | ||
| #include <utility> | ||
| #include <vector> | ||
|
|
||
| namespace pforr { | ||
|
|
||
| using IndexRange = std::pair<std::size_t, std::size_t>; | ||
|
|
||
| template <typename Worker> | ||
| auto worker_thread(Worker &worker, const IndexRange &range) -> void { | ||
| try { | ||
| worker(range.first, range.second); | ||
| } catch (...) { | ||
| } | ||
| } | ||
|
|
||
| // Function to calculate the ranges for a given input | ||
| inline auto split_input_range(const IndexRange &range, std::size_t n_threads, | ||
| std::size_t grain_size) | ||
| -> std::vector<IndexRange> { | ||
|
|
||
| // compute grain_size (including enforcing requested minimum) | ||
| std::size_t length = range.second - range.first; | ||
| if (n_threads == 1) | ||
| grain_size = length; | ||
| else if ((length % n_threads) == 0) // perfect division | ||
| grain_size = (std::max)(length / n_threads, grain_size); | ||
| else // imperfect division, divide by threads - 1 | ||
| grain_size = (std::max)(length / (n_threads - 1), grain_size); | ||
|
|
||
| // allocate ranges | ||
| std::vector<IndexRange> ranges; | ||
| std::size_t begin = range.first; | ||
| while (begin < range.second) { | ||
| std::size_t end = (std::min)(begin + grain_size, range.second); | ||
| ranges.emplace_back(begin, end); | ||
| begin = end; | ||
| } | ||
|
|
||
| return ranges; | ||
| } | ||
|
|
||
| // Execute the Worker over the IndexRange in parallel | ||
| template <typename Worker> | ||
| inline void parallel_for(std::size_t begin, std::size_t end, Worker &worker, | ||
| std::size_t n_threads, std::size_t grain_size = 1) { | ||
| if (n_threads == 0) { | ||
| worker(begin, end); | ||
| return; | ||
| } | ||
| // split the work | ||
| IndexRange input_range(begin, end); | ||
| std::vector<IndexRange> ranges = | ||
| split_input_range(input_range, n_threads, grain_size); | ||
|
|
||
| std::vector<std::thread> threads; | ||
| threads.reserve(ranges.size()); | ||
| for (auto &range : ranges) { | ||
| threads.push_back( | ||
| std::thread(&worker_thread<Worker>, std::ref(worker), range)); | ||
| } | ||
|
|
||
| for (auto &thread : threads) { | ||
| thread.join(); | ||
| } | ||
|
|
||
| return; | ||
| } | ||
|
|
||
| } // namespace pforr | ||
|
|
||
| #endif // PFORR |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,47 @@ | ||
| library(RcppHNSW) | ||
| context("Save/load index") | ||
|
|
||
| num_elements <- nrow(uirism) | ||
| dim <- ncol(uirism) | ||
|
|
||
| M <- 16 | ||
| ef_construction <- 10 | ||
| p <- new(HnswL2, dim, num_elements, M, ef_construction) | ||
|
|
||
| for (i in 1:num_elements) { | ||
| p$addItem(uirism[i, ]) | ||
| } | ||
|
|
||
| nn4idx <- matrix(0L, nrow = num_elements, ncol = 4) | ||
| nn4dist <- matrix(0.0, nrow = num_elements, ncol = 4) | ||
|
|
||
| for (i in 1:num_elements) { | ||
| res <- p$getNNsList(uirism[i, ], k = 4, TRUE) | ||
| nn4idx[i, ] <- res$item | ||
| nn4dist[i, ] <- res$distance | ||
| } | ||
|
|
||
| temp_file <- tempfile() | ||
| on.exit(unlink(temp_file), add = TRUE) | ||
| p$save(temp_file) | ||
|
|
||
| nn4idx_aftersave <- matrix(0L, nrow = num_elements, ncol = 4) | ||
| nn4dist_aftersave <- matrix(0.0, nrow = num_elements, ncol = 4) | ||
| for (i in 1:num_elements) { | ||
| res_aftersave <- p$getNNsList(uirism[i, ], k = 4, TRUE) | ||
| nn4idx_aftersave[i, ] <- res_aftersave$item | ||
| nn4dist_aftersave[i, ] <- res_aftersave$distance | ||
| } | ||
| expect_equal(nn4idx, nn4idx_aftersave) | ||
| expect_equal(nn4dist, nn4dist_aftersave) | ||
|
|
||
| pload <- new(HnswL2, dim, temp_file) | ||
| nn4idx_afterload <- matrix(0L, nrow = num_elements, ncol = 4) | ||
| nn4dist_afterload <- matrix(0.0, nrow = num_elements, ncol = 4) | ||
| for (i in 1:num_elements) { | ||
| res_afterload <- pload$getNNsList(uirism[i, ], k = 4, TRUE) | ||
| nn4idx_afterload[i, ] <- res_afterload$item | ||
| nn4dist_afterload[i, ] <- res_afterload$distance | ||
| } | ||
| expect_equal(nn4idx, nn4idx_afterload) | ||
| expect_equal(nn4dist, nn4dist_afterload) |