Website & API Doc site generator using DocFx script

.gitignore

@@ -49,4 +49,9 @@ release/
 .tools/
 
 # NUnit test result file produced by nunit3-console.exe
-[Tt]est[Rr]esult.xml
+[Tt]est[Rr]esult.xml
+apidocs/_site/*
+apidocs/api/*.yml
+apidocs/tools/docfx.zip
+apidocs/tools/docfx/*
+apidocs/api/.manifest

Lucene.Net.sln

@@ -94,6 +94,14 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Lucene.Net.ICU", "src\Lucen
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Lucene.Net.Tests.ICU", "src\Lucene.Net.Tests.ICU\Lucene.Net.Tests.ICU.csproj", "{D5AA1A22-1B28-4DF6-BFDA-02519A189839}"
 EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "apidocs", "apidocs", "{58FD6E39-F30F-4566-90E5-B7C9D6BC0660}"
+	ProjectSection(SolutionItems) = preProject
+		apidocs\docfx.filter.yml = apidocs\docfx.filter.yml
+		apidocs\docfx.json = apidocs\docfx.json
+		apidocs\docs.ps1 = apidocs\docs.ps1
+		apidocs\index.md = apidocs\index.md
+		apidocs\toc.yml = apidocs\toc.yml
+	EndProjectSection
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Lucene.Net.Analysis.SmartCn", "src\Lucene.Net.Analysis.SmartCn\Lucene.Net.Analysis.SmartCn.csproj", "{DBA35EDF-A0FF-4DF7-AE4F-A103B01CD488}"
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Lucene.Net.Tests.Analysis.SmartCn", "src\Lucene.Net.Tests.Analysis.SmartCn\Lucene.Net.Tests.Analysis.SmartCn.csproj", "{8C8D78D3-BFFD-4301-953B-FE5350B2AEEB}"

apidocs/api/index.md

@@ -0,0 +1,153 @@
+<!-- TODO: Create the packages table of contents ... manually? Post processor?  -->
+
+
+Apache Lucene is a high-performance, full-featured text search engine library.
+Here's a simple example how to use Lucene for indexing and searching (using JUnit
+to check if the results are what we expect):
+
+
+<!-- TODO: Convert this to .NET  -->
+```csharp
+    Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_CURRENT);
+
+    // Store the index in memory:
+    Directory directory = new RAMDirectory();
+    // To store an index on disk, use this instead:
+    //Directory directory = FSDirectory.open("/tmp/testindex");
+    IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_CURRENT, analyzer);
+    IndexWriter iwriter = new IndexWriter(directory, config);
+    Document doc = new Document();
+    String text = "This is the text to be indexed.";
+    doc.add(new Field("fieldname", text, TextField.TYPE_STORED));
+    iwriter.addDocument(doc);
+    iwriter.close();
+
+    // Now search the index:
+    DirectoryReader ireader = DirectoryReader.open(directory);
+    IndexSearcher isearcher = new IndexSearcher(ireader);
+    // Parse a simple query that searches for "text":
+    QueryParser parser = new QueryParser(Version.LUCENE_CURRENT, "fieldname", analyzer);
+    Query query = parser.parse("text");
+    ScoreDoc[] hits = isearcher.search(query, null, 1000).scoreDocs;
+    assertEquals(1, hits.length);
+    // Iterate through the results:
+    for (int i = 0; i < hits.length; i++) {
+        Document hitDoc = isearcher.doc(hits[i].doc);
+        assertEquals("This is the text to be indexed.", hitDoc.get("fieldname"));
+    }
+    ireader.close();
+    directory.close();
+```
+
+The Lucene API is divided into several packages:
+
+<!-- TODO: Fix links  -->
+
+<ul>
+<li>
+<b>{@link org.apache.lucene.analysis}</b>
+defines an abstract {@link org.apache.lucene.analysis.Analyzer Analyzer}
+API for converting text from a {@link java.io.Reader}
+into a {@link org.apache.lucene.analysis.TokenStream TokenStream},
+an enumeration of token {@link org.apache.lucene.util.Attribute Attribute}s.&nbsp;
+A TokenStream can be composed by applying {@link org.apache.lucene.analysis.TokenFilter TokenFilter}s
+to the output of a {@link org.apache.lucene.analysis.Tokenizer Tokenizer}.&nbsp;
+Tokenizers and TokenFilters are strung together and applied with an {@link org.apache.lucene.analysis.Analyzer Analyzer}.&nbsp;
+<a href="../analyzers-common/overview-summary.html">analyzers-common</a> provides a number of Analyzer implementations, including 
+<a href="../analyzers-common/org/apache/lucene/analysis/core/StopAnalyzer.html">StopAnalyzer</a>
+and the grammar-based <a href="../analyzers-common/org/apache/lucene/analysis/standard/StandardAnalyzer.html">StandardAnalyzer</a>.</li>
+
+<li>
+<b>{@link org.apache.lucene.codecs}</b>
+provides an abstraction over the encoding and decoding of the inverted index structure,
+as well as different implementations that can be chosen depending upon application needs.
+
+<li>
+<b>{@link org.apache.lucene.document}</b>
+provides a simple {@link org.apache.lucene.document.Document Document}
+class.&nbsp; A Document is simply a set of named {@link org.apache.lucene.document.Field Field}s,
+whose values may be strings or instances of {@link java.io.Reader}.</li>
+
+<li>
+<b>{@link org.apache.lucene.index}</b>
+provides two primary classes: {@link org.apache.lucene.index.IndexWriter IndexWriter},
+which creates and adds documents to indices; and {@link org.apache.lucene.index.IndexReader},
+which accesses the data in the index.</li>
+
+<li>
+<b>{@link org.apache.lucene.search}</b>
+provides data structures to represent queries (ie {@link org.apache.lucene.search.TermQuery TermQuery}
+for individual words, {@link org.apache.lucene.search.PhraseQuery PhraseQuery} 
+for phrases, and {@link org.apache.lucene.search.BooleanQuery BooleanQuery} 
+for boolean combinations of queries) and the {@link org.apache.lucene.search.IndexSearcher IndexSearcher}
+which turns queries into {@link org.apache.lucene.search.TopDocs TopDocs}.
+A number of <a href="../queryparser/overview-summary.html">QueryParser</a>s are provided for producing
+query structures from strings or xml.
+
+<li>
+<b>{@link org.apache.lucene.store}</b>
+defines an abstract class for storing persistent data, the {@link org.apache.lucene.store.Directory Directory},
+which is a collection of named files written by an {@link org.apache.lucene.store.IndexOutput IndexOutput}
+and read by an {@link org.apache.lucene.store.IndexInput IndexInput}.&nbsp;
+Multiple implementations are provided, including {@link org.apache.lucene.store.FSDirectory FSDirectory},
+which uses a file system directory to store files, and {@link org.apache.lucene.store.RAMDirectory RAMDirectory}
+which implements files as memory-resident data structures.</li>
+
+<li>
+<b>{@link org.apache.lucene.util}</b>
+contains a few handy data structures and util classes, ie {@link org.apache.lucene.util.OpenBitSet OpenBitSet}
+and {@link org.apache.lucene.util.PriorityQueue PriorityQueue}.</li>
+</ul>
+To use Lucene, an application should:
+<ol>
+<li>
+Create {@link org.apache.lucene.document.Document Document}s by
+adding
+{@link org.apache.lucene.document.Field Field}s;</li>
+
+<li>
+Create an {@link org.apache.lucene.index.IndexWriter IndexWriter}
+and add documents to it with {@link org.apache.lucene.index.IndexWriter#addDocument(Iterable) addDocument()};</li>
+
+<li>
+Call <a href="../queryparser/org/apache/lucene/queryparser/classic/QueryParserBase.html#parse(java.lang.String)">QueryParser.parse()</a>
+to build a query from a string; and</li>
+
+<li>
+Create an {@link org.apache.lucene.search.IndexSearcher IndexSearcher}
+and pass the query to its {@link org.apache.lucene.search.IndexSearcher#search(org.apache.lucene.search.Query, int) search()}
+method.</li>
+</ol>
+Some simple examples of code which does this are:
+<ul>
+<li>
+&nbsp;<a href="../demo/src-html/org/apache/lucene/demo/IndexFiles.html">IndexFiles.java</a> creates an
+index for all the files contained in a directory.</li>
+
+<li>
+&nbsp;<a href="../demo/src-html/org/apache/lucene/demo/SearchFiles.html">SearchFiles.java</a> prompts for
+queries and searches an index.</li>
+</ul>
+
+<!-- TODO: Fix this  -->
+
+To demonstrate these, try something like:
+<blockquote><tt>> <b>java -cp lucene-core.jar:lucene-demo.jar:lucene-analyzers-common.jar org.apache.lucene.demo.IndexFiles -index index -docs rec.food.recipes/soups</b></tt>
+<br><tt>adding rec.food.recipes/soups/abalone-chowder</tt>
+<br><tt>&nbsp; </tt>[ ... ]
+
+<p><tt>> <b>java -cp lucene-core.jar:lucene-demo.jar:lucene-queryparser.jar:lucene-analyzers-common.jar org.apache.lucene.demo.SearchFiles</b></tt>
+<br><tt>Query: <b>chowder</b></tt>
+<br><tt>Searching for: chowder</tt>
+<br><tt>34 total matching documents</tt>
+<br><tt>1. rec.food.recipes/soups/spam-chowder</tt>
+<br><tt>&nbsp; </tt>[ ... thirty-four documents contain the word "chowder" ... ]
+
+<p><tt>Query: <b>"clam chowder" AND Manhattan</b></tt>
+<br><tt>Searching for: +"clam chowder" +manhattan</tt>
+<br><tt>2 total matching documents</tt>
+<br><tt>1. rec.food.recipes/soups/clam-chowder</tt>
+<br><tt>&nbsp; </tt>[ ... two documents contain the phrase "clam chowder"
+and the word "manhattan" ... ]
+<br>&nbsp;&nbsp;&nbsp; [ Note: "+" and "-" are canonical, but "AND", "OR"
+and "NOT" may be used. ]</blockquote>

apidocs/docfx.filter.yml

@@ -0,0 +1,8 @@
+apiRules:
+  - include:
+      uidRegex: ^Lucene\.Net
+  - exclude:
+      hasAttribute:
+      uid: System.ComponentModel.EditorBrowsableAttribute
+      ctorArguments:
+        - System.ComponentModel.EditorBrowsableState.Never

apidocs/docfx.json

@@ -0,0 +1,79 @@
+{
+  "metadata": [
+    {
+      "src": [
+        {
+          "files": [
+            "**.csproj"
+          ],
+          "exclude": [
+            "**/obj/**",
+            "**/bin/**",
+            "_site/**",
+            "**/Lucene.Net.Test*/**"
+          ],
+          "cwd": "../src"
+        }
+      ],
+      "dest": "api",
+      "filter": "docfx.filter.yml"
+    }
+  ],
+  "build": {
+    "content": [
+      {
+        "files": [
+          "api/**.yml",
+          "api/index.md"
+        ]
+      },
+      {
+        "files": [
+          "docs/**.md",
+          "docs/**/toc.yml",
+          "toc.yml",
+          "*.md"
+        ],
+        "exclude": [
+          "obj/**",
+          "_site/**"
+        ]
+      }
+    ],
+    "resource": [
+      {
+        "files": [
+          "images/**"
+        ],
+        "exclude": [
+          "obj/**",
+          "_site/**"
+        ]
+      }
+    ],
+    "overwrite": [
+      {
+        "files": [
+          "apidoc/**.md"
+        ],
+        "exclude": [
+          "obj/**",
+          "_site/**"
+        ]
+      }
+    ],
+	"globalMetadata": {
+      "_appTitle": "Apache Lucene.NET 4.8.0 Documentation",
+      "_enableSearch": false,
+      "_disableContribution": false
+    },
+    "dest": "_site",
+    "globalMetadataFiles": [],
+    "fileMetadataFiles": [],
+    "template": [
+      "default", "lucenetemplate"
+    ],
+    "postProcessors": [],
+    "noLangKeyword": false
+  }
+}

apidocs/docs.ps1

@@ -0,0 +1,78 @@
+# -----------------------------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the ""License""); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+# 
+# http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an ""AS IS"" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# -----------------------------------------------------------------------------------
+
+param (
+	[Parameter(Mandatory=$false)]
+	[int]
+	$ServeDocs = 0,
+	[Parameter(Mandatory=$false)]
+	[int]
+	$Clean = 1
+)
+
+$PSScriptFilePath = (Get-Item $MyInvocation.MyCommand.Path).FullName
+$RepoRoot = (get-item $PSScriptFilePath).Directory.Parent.FullName;
+$ApiDocsFolder = Join-Path -Path $RepoRoot -ChildPath "apidocs";
+$ToolsFolder = Join-Path -Path $ApiDocsFolder -ChildPath "tools";
+#ensure the /build/tools folder
+New-Item $ToolsFolder -type directory -force
+
+# Go get docfx.exe if we don't have it
+$DocFxExe = "$ToolsFolder\docfx\docfx.exe"
+$FileExists = Test-Path $DocFxExe 
+If ($FileExists -eq $False) {
+	Write-Host "Retrieving docfx..."
+	$DocFxZip = "$ToolsFolder\docfx.zip"
+	$SourceDocFx = "https://github.com/dotnet/docfx/releases/download/v2.17.7/docfx.zip"
+	Invoke-WebRequest $SourceDocFx -OutFile $DocFxZip
+	#unzip
+	Expand-Archive $DocFxZip -DestinationPath (Join-Path -Path $ToolsFolder -ChildPath "docfx")
+}
+
+# delete anything that already exists
+if ($Clean -eq 1) {
+	Remove-Item (Join-Path -Path $ApiDocsFolder "obj\*") -recurse -force -ErrorAction SilentlyContinue
+	Remove-Item (Join-Path -Path $ApiDocsFolder "obj") -force -ErrorAction SilentlyContinue
+	Remove-Item (Join-Path -Path $ApiDocsFolder "api\*") -exclude "*.md" -recurse -force -ErrorAction SilentlyContinue
+	##Remove-Item (Join-Path -Path $ApiDocsFolder "api") -force -ErrorAction SilentlyContinue
+}
+
+# NOTE: There's a ton of Lucene docs that we want to copy and re-format. I'm not sure if we can really automate this 
+# in a great way since the docs seem to be in many places, for example:
+# Home page - 	https://github.com/apache/lucene-solr/blob/branch_4x/lucene/site/xsl/index.xsl
+# Wiki docs - 	https://wiki.apache.org/lucene-java/FrontPage?action=show&redirect=FrontPageEN - not sure where the source is for this
+# Html pages - 	Example: https://github.com/apache/lucene-solr/blob/releases/lucene-solr/4.8.0/lucene/highlighter/src/java/org/apache/lucene/search/highlight/package.html - these seem to be throughout the source
+#				For these ones, could we go fetch them and download all *.html files from Git?
+
+$DocFxJson = Join-Path -Path $RepoRoot "apidocs\docfx.json"
+
+Write-Host "Building metadata..."
+& $DocFxExe metadata $DocFxJson
+if($?) { 
+	if ($ServeDocs -eq 0){
+		# build the output		
+		Write-Host "Building docs..."
+		& $DocFxExe build $DocFxJson	
+	}
+	else {
+		# build + serve (for testing)
+		Write-Host "starting website..."
+		& $DocFxExe $DocFxJson --serve
+	}
+}

apidocs/docs/index.md

@@ -0,0 +1,17 @@
+Apache Lucene.NET 4.8.0 Documentation
+===============
+
+Apache Lucene.NET™ 4.8.0 Documentation
+---------------
+
+Lucene is a Java full-text search engine. Lucene.NET is not a complete application, but rather a code library and API that can easily be used to add search capabilities to applications.
+
+This is the official documentation for Apache Lucene.NET 4.8.0. Additional documentation is available in the Wiki.
+
+## Getting Started
+
+The following section is intended as a "getting started" guide. It has three audiences: first-time users looking to install Apache Lucene in their application; developers looking to modify or base the applications they develop on Lucene; and developers looking to become involved in and contribute to the development of Lucene. The goal is to help you "get started". It does not go into great depth on some of the conceptual or inner details of Lucene:
+
+* Lucene demo, its usage, and sources: Tutorial and walk-through of the command-line Lucene demo.
+* Introduction to Lucene's APIs: High-level summary of the different Lucene packages.
+* Analysis overview: Introduction to Lucene's analysis API. See also the TokenStream consumer workflow.