Skip to content

Commit

Permalink
feat: PageIndex uses get_ftoc for cached ftocs to reduce load time
Browse files Browse the repository at this point in the history
  • Loading branch information
redimp committed Jan 23, 2025
1 parent 0d49d1b commit ea24cd0
Show file tree
Hide file tree
Showing 4 changed files with 91 additions and 6 deletions.
57 changes: 56 additions & 1 deletion otterwiki/helper.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,15 +11,20 @@

import os
import re
import os
from hashlib import sha256
import json
from datetime import datetime
from collections import namedtuple
from otterwiki.server import app, mail, storage, Preferences
from otterwiki.server import app, mail, storage, Preferences, db, app_renderer
from otterwiki.gitstorage import StorageError
from flask import flash, url_for, session
from threading import Thread
from flask_mail import Message
from itsdangerous import URLSafeTimedSerializer, BadSignature, SignatureExpired
from otterwiki.util import split_path, join_path, clean_slashes, titleSs
from otterwiki.renderer import OtterwikiRenderer
from otterwiki.models import Cache


class SerializeError(ValueError):
Expand Down Expand Up @@ -283,3 +288,53 @@ def patchset2urlmap(patchset, rev_b, rev_a=None):
}
url_map[file.path] = namedtuple('UrlData', d.keys())(*d.values())
return url_map


def sha256sum(s: str) -> str:
hash = sha256()
hash.update(s.encode())
return hash.hexdigest()


def update_ftoc_cache(filename, ftoc, mtime=None):
if mtime is None:
mtime = storage.mtime(filename)
hash = sha256sum(f"ftoc://{filename}")
value = json.dumps({"filename": filename, "ftoc": ftoc})
# check if key exists in Cache
c = Cache.query.filter(Cache.key == hash).first()
if c is None:
c = Cache()
c.key = hash
c.value = value
c.datetime = mtime
# and update in the database
db.session.add(c)
db.session.commit()


def get_ftoc(filename, mtime=None):
if mtime is None:
mtime = storage.mtime(filename)
hash = sha256sum(f"ftoc://{filename}")
# check if hash is in the Cache
result = Cache.query.filter(
db.and_(Cache.key == hash, Cache.datetime >= mtime)
).first()
if result is not None:
try:
value = json.loads(result.value)
try:
# check
if filename == value["filename"]:
return value["ftoc"]
except KeyError:
pass
except:
pass
content = storage.load(filename)
# parse file contents
_, ftoc = app_renderer.markdown(content)
update_ftoc_cache(filename, ftoc, mtime)

return ftoc
2 changes: 1 addition & 1 deletion otterwiki/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,6 @@ def __repr__(self):

class Cache(db.Model):
__tablename__ = "cache"
key = db.Column(db.String(64), primary_key=True)
key = db.Column(db.String(64), index=True, primary_key=True)
value = db.Column(db.Text)
datetime = db.Column(TimeStamp())
9 changes: 5 additions & 4 deletions otterwiki/wiki.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,8 @@
patchset2urlmap,
get_breadcrumbs,
upsert_pagecrumbs,
get_ftoc,
update_ftoc_cache,
)
from otterwiki.auth import has_permission, current_user
from otterwiki.plugins import chain_hooks
Expand Down Expand Up @@ -144,10 +146,8 @@ def __init__(self, path=None):
f,
full=False,
)
# read file
content = storage.load(f)
# parse file contents
_, ftoc = app_renderer.markdown(content)
ftoc = get_ftoc(f)

# add headers to page toc
# (4, '2 L <strong>bold</strong>', 1, '2 L bold', '2-l-bold')
for i, header in enumerate(ftoc):
Expand Down Expand Up @@ -549,6 +549,7 @@ def view(self):
htmlcontent, toc = app_renderer.markdown(
self.content, page_url=self.page_url
)
update_ftoc_cache(self.filename, ftoc=toc)

if len(toc) > 0:
# use first headline to overwrite pagename
Expand Down
29 changes: 29 additions & 0 deletions tests/test_helper.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@

import pytest
import flask
from datetime import datetime
import otterwiki
import otterwiki.gitstorage

Expand Down Expand Up @@ -252,3 +253,31 @@ def test_get_pagename_prefixes(test_client):
'Example',
'Random page',
]


def test_ftoc_cache(create_app, req_ctx):
assert create_app
assert req_ctx
from otterwiki.helper import get_ftoc, update_ftoc_cache

mtime = datetime.fromtimestamp(0)
filename = "ftoc.md"
ftoc = [
[0, "Ftoc", 1, "Ftoc", "ftoc"],
[1, "Header 2", 2, "Header 2", "header-2"],
[2, "Header 3", 3, "Header 3", "header-3"],
]
update_ftoc_cache(filename=filename, ftoc=ftoc, mtime=mtime)
assert get_ftoc(filename=filename, mtime=mtime) == ftoc
# store the file
assert True == create_app.storage.store(
filename,
content="# Header 1\n\n## Header 2",
author=("John", "[email protected]"),
message=f"added {filename}",
)

assert [
(0, 'Header 1', 1, 'Header 1', 'header-1'),
(1, 'Header 2', 2, 'Header 2', 'header-2'),
] == get_ftoc(filename)

0 comments on commit ea24cd0

Please sign in to comment.