finch-tensor · willow-ahrens · Jan 11, 2024 · Dec 15, 2023 · Dec 15, 2023 · Dec 15, 2023
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md
@@ -0,0 +1 @@
+src/CONTRIBUTING.md
diff --git a/docs/make.jl b/docs/make.jl
@@ -17,24 +17,55 @@ makedocs(;
     repo="https://github.com/willow-ahrens/Finch.jl/blob/{commit}{path}#{line}",
     sitename="Finch.jl",
     format=Documenter.HTML(;
+        repolink="https://github.com/willow-ahrens/Finch.jl/blob/{commit}{path}#{line}",
         prettyurls=get(ENV, "CI", "false") == "true",
         canonical="https://willow-ahrens.github.io/Finch.jl",
         assets=["assets/favicon.ico"],
     ),
     pages=[
         "Home" => "index.md",
-        "Usage" => "usage.md",
-        "Semantics" => "semantics.md",
-        "Array Formats" => "tensors.md",
-        "Custom Functions" => "algebra.md",
-        "Tensor File I/O" => "fileio.md",
-        "C, C++, ..." => "interop.md",
-        "Performance Tips" => "performance.md",
-        "Internals" => "internals.md",
-        "Directory Structure" => "directory_structure.md",
-        "Contributing" => "CONTRIBUTING.md",
-        "Tutorials" => [
-            "Benchmarking" => "benchmark.md",
+        #"Getting Started" => "getting_started.md",
+        #"Practical Tutorials and Use Cases" => "tutorials_use_cases/tutorials_use_cases.md",
+        "Comprehensive Guides" => [
+            "Calling Finch" => "guides/calling_finch.md",
+            "The Finch Language" => "guides/finch_language.md",
+            "Exploration of Tensor Formats" => "guides/tensor_formats.md",
+            #"Dimensionalization" => "guides/dimensionalization.md",
+            #"Tensor Lifecycles" => "guides/tensor_lifecycles.md",
+            #"Special Tensors" => [
+            #    "Overview" => "guides/special_tensors/overview.md",
+            #    "Wrapper Tensors" => "guides/special_tensors/wrapper_tensors.md",
+            #    "Symbolic Tensors" => "guides/special_tensors/symbolic_tensors.md",
+            #    "Early Break Strategies" => "guides/special_tensors/early_break.md",
+            #],
+            #"Index Sugar" => "guides/index_sugar.md",
+            "Custom Operators" => "guides/custom_operators.md",
+            #"Parallelization and Architectures" => "guides/parallelization.md",
+            "FileIO" => "guides/fileio.md",
+            "Interoperability" => "guides/interoperability.md",
+            "Optimization Tips" => "guides/optimization_tips.md",
+            "Benchmarking Tips" => "guides/benchmarking_tips.md",
+            #"Debugging Tips" => "guides/debugging_tips.md",
+        ],
+        "Technical Reference" => [
+        #    "Finch Core API" => "reference/core_api.md",
+        #    "Function and Method Reference" => "reference/function_method_ref.md",
+            "Advanced Implementation Details" => [
+                "Internals" => "reference/advanced_implementation/internals.md",
+        #        "Looplets and Coiteration" => "reference/advanced_implementation/looplets_coiteration.md",
+        #        "Concordization" => "reference/advanced_implementation/concordization.md",
+        #        "Local Variables and Constant Propagation" => "reference/advanced_implementation/local_variables.md",
+        #        "Tensor Interface" => "reference/advanced_implementation/tensor_interface.md",
+        #        "Looplet Interface" => "reference/advanced_implementation/looplet_interface.md",
+            ],
+        ],
+        "Community and Contributions" => "CONTRIBUTING.md",
+        "Appendices and Additional Resources" => [
+            #"Glossary" => "appendices/glossary.md",
+            #"FAQs" => "appendices/faqs.md",
+            "Directory Structure" => "appendices/directory_structure.md",
+            #"Changelog" => "appendices/changelog.md",
+            #"Publications and Articles" => "appendices/publications_articles.md",
         ],
     ],
     warnonly=[:missing_docs],
@@ -43,4 +74,4 @@ makedocs(;
 deploydocs(;
     repo="github.com/willow-ahrens/Finch.jl",
     devbranch="main",
-)
+)
diff --git a/docs/src/CONTRIBUTING.md b/docs/src/CONTRIBUTING.md
diff --git a/docs/src/CONTRIBUTING.md b/docs/src/CONTRIBUTING.md
@@ -0,0 +1,95 @@
+We welcome contributions to Finch, and follow the [Julia contributing
+guidelines](https://github.com/JuliaLang/julia/blob/master/CONTRIBUTING.md).  If
+you use or want to use Finch and have a question or bug, please do file a
+[Github issue](https://github.com/willow-ahrens/Finch.jl/issues)!  If you want
+to contribute to Finch, please first file an issue to double check that there is
+interest from a contributor in the feature.
+
+## Versions
+
+Finch is currently in a pre-release state. The API is not yet stable, and
+breaking changes may occur between minor versions. We follow [semantic
+versioning](https://semver.org/) and will release 1.0 when the API is stable.
+The main branch of the Finch repo is the most up-to-date development branch.
+While it is not stable, it should always pass tests.
+
+Contributors will develop and test Finch from a local directory. Please see the
+[Package documentation](https://pkgdocs.julialang.org/v1/getting-started/) for more
+info, particularly the section on [developing](https://pkgdocs.julialang.org/v1/managing-packages/#developing).
+
+To determine which version of
+Finch you have, run `Pkg.status("Finch")` in the Julia REPL. If the installed
+version of Finch tracks a local path, the output will include the path like so:
+
+```
+Status `~/.julia/environments/v1.9/Project.toml`
+  [9177782c] Finch v0.5.4 `~/Projects/Finch.jl`
+```
+
+If the installed version of Finch tracks a particular version (probably not what
+you want since it will not reflect local changes), the output will look like this:
+
+```
+Status `~/.julia/environments/v1.8/Project.toml`
+  [9177782c] Finch v0.5.4
+```
+
+## Utilities
+
+Finch include several scripts that can be executed directly, e.g. `runtests.jl`.
+These scripts are all have local [Pkg
+environments](https://pkgdocs.julialang.org/v1/getting-started/#Getting-Started-with-Environments).
+The scripts include convenience headers to automatically use their respective
+environments, so you won't need to worry about `--project=.` flags, etc.
+
+## Testing
+
+All pull requests should pass continuous integration testing before merging.
+The test suite has a few options, which are accessible through running the test
+suite directly as `./test/runtests.jl`.
+
+Finch compares compiler output against reference versions.
+
+If you have the appropriate permissions, you can run the
+[FixBot](https://github.com/willow-ahrens/Finch.jl/actions/workflows/FixBot.yml)
+github action on your PR branch to automatically generate output for both 32-bit
+and 64-bit builds.
+
+If you run the test suite directly you can pass the `--overwrite` flag to tell
+the test suite to overwrite the reference.  Because the reference output depends
+on the system word size, you'll need to generate reference output for 32-bit and
+64-bit builds of Julia to get Finch to pass tests. The easiest way to do this is
+to run each 32-bit or 64-bit build of Julia on a system that supports it. You
+can [Download](https://julialang.org/downloads/) multiple builds yourself or use
+[juliaup](https://github.com/JuliaLang/juliaup) to manage multiple versions.
+Using juliaup, it might look like this:
+
+```
+julia +release~x86 test/runtests.jl --overwrite
+julia +release~x64 test/runtests.jl --overwrite
+```
+
+The test suite takes a while to run. You can filter to only run a selection of
+test suites by specifying them as positional arguments, e.g.
+
+```
+./test/runtests.jl constructors conversions representation
+```
+
+This information is summarized with `./test/runtests.jl --help`
+
+## Benchmarking
+
+The Finch test suite includes a benchmarking script that measures Finch
+performance on a variety of kernels. It also includes some scripts to help
+compare Finch performance on the feature branch to the main branch. To run the
+benchmarking script, run `./benchmarks/runbenchmarks.jl`. To run the comparison
+script, run `./benchmarks/runjudge.jl`. Both scripts take a while to run and
+generate a report at the end.
+
+## Documentation
+
+The `/docs` directory includes Finch documentation in `/src`, and a built
+website in `/build`. You can build the website with `./docs/make.jl`. You can
+run doctests with `./docs/test.jl`, and fix doctests with `./docs/fix.jl`,
+though both are included as part of the test suite.
diff --git a/docs/src/README.md b/docs/src/README.md
diff --git a/docs/src/appendices/changelog.md b/docs/src/appendices/changelog.md
@@ -0,0 +1 @@
+# TODO
diff --git a/docs/src/directory_structure.md → docs/src/appendices/directory_structure.md b/docs/src/directory_structure.md → docs/src/appendices/directory_structure.md
diff --git a/docs/src/appendices/faqs.md b/docs/src/appendices/faqs.md
@@ -0,0 +1 @@
+# TODO
diff --git a/docs/src/appendices/glossary.md b/docs/src/appendices/glossary.md
@@ -0,0 +1 @@
+# TODO
diff --git a/docs/src/appendices/publications_articles.md b/docs/src/appendices/publications_articles.md
@@ -0,0 +1 @@
+# TODO
diff --git a/docs/src/getting_started.md b/docs/src/getting_started.md
@@ -0,0 +1 @@
+# TODO
diff --git a/docs/src/benchmark.md → docs/src/guides/benchmarking_tips.md b/docs/src/benchmark.md → docs/src/guides/benchmarking_tips.md
@@ -2,7 +2,7 @@
 CurrentModule = Finch
 ```
 
-# Benchmarking
+# Benchmarking Tips
 
 Julia code is [nototoriously
 fussy](https://github.com/JuliaCI/BenchmarkTools.jl#why-does-this-package-exist)

diff --git a/docs/src/usage.md → docs/src/guides/calling_finch.md b/docs/src/usage.md → docs/src/guides/calling_finch.md
diff --git a/docs/src/algebra.md → docs/src/guides/custom_operators.md b/docs/src/algebra.md → docs/src/guides/custom_operators.md
@@ -1,8 +1,7 @@
 ```@meta
 CurrentModule = Finch
 ```
-# Custom Functions
-
+# Custom Operators
 
 ## User Functions
 

diff --git a/docs/src/guides/debugging_tips.md b/docs/src/guides/debugging_tips.md
@@ -0,0 +1 @@
+# TODO
diff --git a/docs/src/guides/dimensionalization.md b/docs/src/guides/dimensionalization.md
@@ -0,0 +1 @@
+# TODO
diff --git a/docs/src/fileio.md → docs/src/guides/fileio.md b/docs/src/fileio.md → docs/src/guides/fileio.md
diff --git a/docs/src/semantics.md → docs/src/guides/finch_language.md b/docs/src/semantics.md → docs/src/guides/finch_language.md
diff --git a/docs/src/guides/index_sugar.md b/docs/src/guides/index_sugar.md
@@ -0,0 +1 @@
+# TODO
diff --git a/docs/src/interop.md → docs/src/guides/interoperability.md b/docs/src/interop.md → docs/src/guides/interoperability.md
diff --git a/docs/src/performance.md → docs/src/guides/optimization_tips.md b/docs/src/performance.md → docs/src/guides/optimization_tips.md
@@ -1,7 +1,7 @@
 ```@meta
 CurrentModule = Finch
 ```
-# Performance Tips for Finch
+# Optimization Tips for Finch
 
 It's easy to ask Finch to run the same operation in different ways. However,
 different approaches have different performance. The right approach really
@@ -208,7 +208,7 @@ end
 
 ## Use Known Functions
 
-Unless you declare the properties of your functions using Finch's [Custom Functions](@ref) interface, Finch doesn't know how they work. For example, using a lambda obscures
+Unless you declare the properties of your functions using Finch's [Custom Operators](@ref) interface, Finch doesn't know how they work. For example, using a lambda obscures
 the meaning of `*`.
 
 ```jldoctest example1

diff --git a/docs/src/guides/parallelization.md b/docs/src/guides/parallelization.md
@@ -0,0 +1 @@
+# TODO
diff --git a/docs/src/guides/special_tensors/early_break.md b/docs/src/guides/special_tensors/early_break.md
@@ -0,0 +1 @@
+# TODO
diff --git a/docs/src/guides/special_tensors/overview.md b/docs/src/guides/special_tensors/overview.md
@@ -0,0 +1 @@
+# TODO
diff --git a/docs/src/guides/special_tensors/symbolic_tensors.md b/docs/src/guides/special_tensors/symbolic_tensors.md
@@ -0,0 +1 @@
+# TODO
diff --git a/docs/src/guides/special_tensors/wrapper_tensors.md b/docs/src/guides/special_tensors/wrapper_tensors.md
@@ -0,0 +1 @@
+# TODO
diff --git a/docs/src/tensors.md → docs/src/guides/tensor_formats.md b/docs/src/tensors.md → docs/src/guides/tensor_formats.md
@@ -1,6 +1,24 @@
 ```@meta
 CurrentModule = Finch
 ```
+
+<!--
+# Tensor Formats
+
+Finch stores tensors in a variety of formats, each with its own advantages and
+disadvantages. The following table summarizes the formats supported by Finch,
+and some of their key properties.
+
+# Custom Formats
+
+Finch also supports custom tensor formats. Finch represents tensors
+hierarchically in a tree, where each node in the tree is a vector of subtensors
+and the leaves are the elements.  Thus, a matrix is analogous to a vector of
+vectors, and a 3-tensor is analogous to a vector of vectors of vectors.  The
+vectors at each level of the tensor all have the same structure, which can be
+selected by the user. If the user wishes to 
+-->
+
 # Level Formats
 
 Finch implements a flexible array datastructure called a `Tensor`. Finch tensors represent
@@ -85,7 +103,7 @@ When we print the tree in text, positions are numbered from top to bottom.
 However, if we visualize our tree with the root at the top, positions range from
 left to right:
 
-![Dense Format Index Tree](assets/levels-A-d-d-e.png)
+![Dense Format Index Tree](../assets/levels-A-d-d-e.png)
 
 Because our array is sparse, (mostly zero, or another fill value), it would be
 more efficient to store only the nonzero values. In Finch, each level is
@@ -106,7 +124,7 @@ Dense [:,1:3]
 │ ├─[3]: 5.5
 ```
 
-![CSC Format Index Tree](assets/levels-A-d-sl-e.png)
+![CSC Format Index Tree](../assets/levels-A-d-sl-e.png)
 
 Our `Dense(SparseList(Element(0.0)))` format is also known as
 ["CSC"](https://en.wikipedia.org/wiki/Sparse_matrix#Compressed_sparse_column_.28CSC_or_CCS.29)
@@ -130,7 +148,7 @@ SparseList (0.0) [:,1:3]
 │ ├─[3]: 5.5
 ```
 
-![DCSC Format Index Tree](assets/levels-A-sl-sl-e.png)
+![DCSC Format Index Tree](../assets/levels-A-sl-sl-e.png)
 
 Here we see that the entirely zero column has also been compressed. The
 `SparseList(SparseList(Element(0.0)))` format is also known as
@@ -155,7 +173,7 @@ SparseCOO (0.0) [1:4,1:3]
 ├─├─[3, 3]: 5.5
 ```
 
-![COO Format Index Tree](assets/levels-A-sc2-e.png)
+![COO Format Index Tree](../assets/levels-A-sc2-e.png)
 
 The COO format is compact and straightforward, but doesn't support random
 access. For random access, one should use the `SparseHash` format. A full listing