paralellism goes brrr

To quantify brrr (tested with the same file 200 times:

Benchmark 1: ./before
  Time (mean ± σ):     216.9 ms ±  22.9 ms    [User: 75.8 ms, System: 28.7 ms]
  Range (min … max):   177.9 ms … 256.4 ms    15 runs

Benchmark 2: ./after
  Time (mean ± σ):     139.4 ms ±  15.3 ms    [User: 92.5 ms, System: 53.0 ms]
  Range (min … max):   112.0 ms … 165.2 ms    23 runs

Summary
  './after' ran
    1.56 ± 0.24 times faster than './before'

Signed-off-by: Moritz Poldrack <[email protected]>

markdownRender.go

@@ -2,12 +2,14 @@ package main
 
 import (
 	"bufio"
-	"fmt"
+	"bytes"
 	"io"
 	"log"
 	"os"
+	"path"
 	"path/filepath"
-	"strings"
+	"runtime"
+	"sync"
 
 	"github.com/russross/blackfriday"
 )
@@ -16,45 +18,71 @@ import (
 func markdownToHTML(inFolder, outFolder, templateFolder string) {
 	files, _ := os.ReadDir(inFolder)
 
+	var wg sync.WaitGroup
+	input := make(chan string, 128)
+
+	wg.Add(runtime.NumCPU())
+	for i := 0; i < runtime.NumCPU(); i++ {
+		go func() {
+			fileProcessor(input, inFolder, outFolder, templateFolder)
+			wg.Done()
+		}()
+	}
+
 	for _, file := range files {
 		// only markdown files
 		if filepath.Ext(file.Name()) == ".md" {
+			input <- file.Name()
+		}
+	}
+	close(input)
+	wg.Wait()
+}
 
-			// open the selected markdown file
-			markdownFile, _ := os.Open(inFolder + "/" + file.Name())
+func fileProcessor(input <-chan string, inFolder, outFolder, templateFolder string) {
+	header, _ := os.ReadFile(path.Join(templateFolder, "header.html"))
+	footer, _ := os.ReadFile(path.Join(templateFolder, "footer.html"))
+	// FIXME: do literally any kind of error handling
 
-			// create the html file
-			htmlFile, _ := os.Create(outFolder + "/" + file.Name() + ".html")
+	result := bytes.NewBuffer(make([]byte, 4096))
 
-			// read the md
-			reader := bufio.NewReader(markdownFile)
-			markdown, _ := io.ReadAll(reader)
+	for infile := range input {
+		result.Reset()
 
-			// send the md to blackfriday
-			html := blackfriday.MarkdownCommon(markdown)
+		// open the selected markdown file
+		markdownFile, _ := os.Open(path.Join(inFolder, infile))
 
-			// read in our templates
-			header, _ := os.ReadFile(templateFolder + "/header.html")
-			footer, _ := os.ReadFile(templateFolder + "/footer.html")
+		// create the html file
+		htmlFile, _ := os.Create(path.Join(outFolder, infile+".html"))
 
-			// assemble in order
-			completeHTML := string(header) + strings.TrimSpace(string(html)) + string(footer)
+		// read the md
+		reader := bufio.NewReader(markdownFile)
+		markdown, _ := io.ReadAll(reader)
 
-			// pass the assembled html into ScanForPluginCalls
-			htmlAfterPlugins, htmlAfterPluginsErr := ScanForPluginCalls(completeHTML)
-			if htmlAfterPluginsErr != nil {
-				log.Println("error inserting plugin content: ", htmlAfterPluginsErr)
-				log.Println("returning content without plugin...")
+		// send the md to blackfriday
+		html := blackfriday.MarkdownCommon(markdown)
 
-				// if there's an error, let's just take the html from before the error and use that.
-				htmlAfterPlugins = completeHTML
-			}
+		// assemble in order
+		result.Write(header)
+		result.Write(bytes.TrimSpace(html))
+		result.Write(footer)
 
-			// don't forget to close it when done
-			markdownFile.Close()
-			htmlFile.Close()
+		// pass the assembled html into ScanForPluginCalls
+		var resultCopy []byte
+		copy(resultCopy, result.Bytes())
+		htmlAfterPlugins, htmlAfterPluginsErr := ScanForPluginCalls(result.Bytes())
+		if htmlAfterPluginsErr != nil {
+			log.Println("error inserting plugin content: ", htmlAfterPluginsErr)
+			log.Println("returning content without plugin...")
 
-			fmt.Fprintln(htmlFile, htmlAfterPlugins)
+			// if there's an error, let's just take the html from before the error and use that.
+			htmlAfterPlugins = resultCopy
 		}
+
+		io.Copy(htmlFile, bytes.NewReader(htmlAfterPlugins))
+
+		// don't forget to close it when done
+		markdownFile.Close()
+		htmlFile.Close()
 	}
 }

plugins.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"archive/zip"
+	"bytes"
 	"io"
 	"os"
 	"path/filepath"
@@ -10,15 +11,13 @@ import (
 )
 
 func FindZips(folderPath string) error {
-
 	files, err := os.ReadDir(folderPath)
 	if err != nil {
 		return err
 	}
 
 	// loop through the files in the plugins dir
 	for _, file := range files {
-
 		// just look at the zips
 		if filepath.Ext(file.Name()) == ".zip" {
 
@@ -80,25 +79,23 @@ func FindZips(folderPath string) error {
 	return nil
 }
 
-func ScanForPluginCalls(html string) (string, error) {
-
+func ScanForPluginCalls(html []byte) ([]byte, error) {
 	// regexp to find the plugin call
 	re := regexp.MustCompile(`<!-- plugin "(.+)" -->`)
-	matches := re.FindAllStringSubmatch(html, -1)
+	matches := re.FindAllSubmatch(html, -1)
 	if len(matches) == 0 {
-
 		// we probably messed up the plugins call, or it didn't match anything.
 		return html, nil
 	}
 
 	// swap the plugin calls for the content of the plugin file
 	for _, match := range matches {
-		filePath := match[1]
+		filePath := string(match[1])
 		fileContent, err := os.ReadFile(filePath)
 		if err != nil {
-			return "", err
+			return nil, err
 		}
-		html = strings.Replace(html, match[0], string(fileContent), -1)
+		html = bytes.Replace(html, match[0], fileContent, -1)
 	}
 
 	return html, nil