New --plugins-dir=plugins/ option

datasette/app.py

@@ -31,6 +31,7 @@
     get_all_foreign_keys,
     is_url,
     InvalidSql,
+    module_from_path,
     path_from_row_pks,
     path_with_added_args,
     path_with_ext,
@@ -1032,7 +1033,7 @@ def __init__(
             self, files, num_threads=3, cache_headers=True, page_size=100,
             max_returned_rows=1000, sql_time_limit_ms=1000, cors=False,
             inspect_data=None, metadata=None, sqlite_extensions=None,
-            template_dir=None, static_mounts=None):
+            template_dir=None, plugins_dir=None, static_mounts=None):
         self.files = files
         self.num_threads = num_threads
         self.executor = futures.ThreadPoolExecutor(
@@ -1048,7 +1049,20 @@ def __init__(
         self.sqlite_functions = []
         self.sqlite_extensions = sqlite_extensions or []
         self.template_dir = template_dir
+        self.plugins_dir = plugins_dir
         self.static_mounts = static_mounts or []
+        # Execute plugins in constructor, to ensure they are available
+        # when the rest of `datasette inspect` executes
+        if self.plugins_dir:
+            for filename in os.listdir(self.plugins_dir):
+                filepath = os.path.join(self.plugins_dir, filename)
+                with open(filepath) as f:
+                    mod = module_from_path(filepath, name=filename)
+                    try:
+                        pm.register(mod)
+                    except ValueError:
+                        # Plugin already registered
+                        pass
 
     def app_css_hash(self):
         if not hasattr(self, '_app_css_hash'):

datasette/cli.py

@@ -58,13 +58,14 @@ def inspect(files, inspect_file, sqlite_extensions):
 @click.option('--force', is_flag=True, help='Pass --force option to now')
 @click.option('--branch', help='Install datasette from a GitHub branch e.g. master')
 @click.option('--template-dir', type=click.Path(exists=True, file_okay=False, dir_okay=True), help='Path to directory containing custom templates')
+@click.option('--plugins-dir', type=click.Path(exists=True, file_okay=False, dir_okay=True), help='Path to directory containing custom plugins')
 @click.option('--static', type=StaticMount(), help='mountpoint:path-to-directory for serving static files', multiple=True)
 @click.option('--title', help='Title for metadata')
 @click.option('--license', help='License label for metadata')
 @click.option('--license_url', help='License URL for metadata')
 @click.option('--source', help='Source label for metadata')
 @click.option('--source_url', help='Source URL for metadata')
-def publish(publisher, files, name, metadata, extra_options, force, branch, template_dir, static, **extra_metadata):
+def publish(publisher, files, name, metadata, extra_options, force, branch, template_dir, plugins_dir, static, **extra_metadata):
     """
     Publish specified SQLite database files to the internet along with a datasette API.
 
@@ -94,7 +95,7 @@ def _fail_if_publish_binary_not_installed(binary, publish_target, install_link):
 
     if publisher == 'now':
         _fail_if_publish_binary_not_installed('now', 'Zeit Now', 'https://zeit.co/now')
-        with temporary_docker_directory(files, name, metadata, extra_options, branch, template_dir, static, extra_metadata):
+        with temporary_docker_directory(files, name, metadata, extra_options, branch, template_dir, plugins_dir, static, extra_metadata):
             if force:
                 call(['now', '--force'])
             else:
@@ -110,7 +111,7 @@ def _fail_if_publish_binary_not_installed(binary, publish_target, install_link):
             click.confirm('Install it? (this will run `heroku plugins:install heroku-builds`)', abort=True)
             call(["heroku", "plugins:install", "heroku-builds"])
 
-        with temporary_heroku_directory(files, name, metadata, extra_options, branch, template_dir, static, extra_metadata):
+        with temporary_heroku_directory(files, name, metadata, extra_options, branch, template_dir, plugins_dir, static, extra_metadata):
             create_output = check_output(
                 ['heroku', 'apps:create', '--json']
             ).decode('utf8')
@@ -190,13 +191,14 @@ def skeleton(files, metadata, sqlite_extensions):
 @click.option('--extra-options', help='Extra options to pass to datasette serve')
 @click.option('--branch', help='Install datasette from a GitHub branch e.g. master')
 @click.option('--template-dir', type=click.Path(exists=True, file_okay=False, dir_okay=True), help='Path to directory containing custom templates')
+@click.option('--plugins-dir', type=click.Path(exists=True, file_okay=False, dir_okay=True), help='Path to directory containing custom plugins')
 @click.option('--static', type=StaticMount(), help='mountpoint:path-to-directory for serving static files', multiple=True)
 @click.option('--title', help='Title for metadata')
 @click.option('--license', help='License label for metadata')
 @click.option('--license_url', help='License URL for metadata')
 @click.option('--source', help='Source label for metadata')
 @click.option('--source_url', help='Source URL for metadata')
-def package(files, tag, metadata, extra_options, branch, template_dir, static, **extra_metadata):
+def package(files, tag, metadata, extra_options, branch, template_dir, plugins_dir, static, **extra_metadata):
     "Package specified SQLite files into a new datasette Docker container"
     if not shutil.which('docker'):
         click.secho(
@@ -207,7 +209,7 @@ def package(files, tag, metadata, extra_options, branch, template_dir, static, *
             err=True,
         )
         sys.exit(1)
-    with temporary_docker_directory(files, 'datasette', metadata, extra_options, branch, template_dir, static, extra_metadata):
+    with temporary_docker_directory(files, 'datasette', metadata, extra_options, branch, template_dir, plugins_dir, static, extra_metadata):
         args = ['docker', 'build']
         if tag:
             args.append('-t')
@@ -233,8 +235,9 @@ def package(files, tag, metadata, extra_options, branch, template_dir, static, *
 @click.option('--inspect-file', help='Path to JSON file created using "datasette inspect"')
 @click.option('-m', '--metadata', type=click.File(mode='r'), help='Path to JSON file containing license/source metadata')
 @click.option('--template-dir', type=click.Path(exists=True, file_okay=False, dir_okay=True), help='Path to directory containing custom templates')
+@click.option('--plugins-dir', type=click.Path(exists=True, file_okay=False, dir_okay=True), help='Path to directory containing custom plugins')
 @click.option('--static', type=StaticMount(), help='mountpoint:path-to-directory for serving static files', multiple=True)
-def serve(files, host, port, debug, reload, cors, page_size, max_returned_rows, sql_time_limit_ms, sqlite_extensions, inspect_file, metadata, template_dir, static):
+def serve(files, host, port, debug, reload, cors, page_size, max_returned_rows, sql_time_limit_ms, sqlite_extensions, inspect_file, metadata, template_dir, plugins_dir, static):
     """Serve up specified SQLite database files with a web UI"""
     if reload:
         import hupper
@@ -262,6 +265,7 @@ def serve(files, host, port, debug, reload, cors, page_size, max_returned_rows,
         metadata=metadata_data,
         sqlite_extensions=sqlite_extensions,
         template_dir=template_dir,
+        plugins_dir=plugins_dir,
         static_mounts=static,
     )
     # Force initial hashing/table counting

datasette/utils.py

@@ -1,6 +1,7 @@
 from contextlib import contextmanager
 import base64
 import hashlib
+import imp
 import json
 import os
 import re
@@ -182,14 +183,16 @@ def escape_sqlite(s):
         return '[{}]'.format(s)
 
 
-def make_dockerfile(files, metadata_file, extra_options, branch, template_dir, static):
+def make_dockerfile(files, metadata_file, extra_options, branch, template_dir, plugins_dir, static):
     cmd = ['"datasette"', '"serve"', '"--host"', '"0.0.0.0"']
     cmd.append('"' + '", "'.join(files) + '"')
     cmd.extend(['"--cors"', '"--port"', '"8001"', '"--inspect-file"', '"inspect-data.json"'])
     if metadata_file:
         cmd.extend(['"--metadata"', '"{}"'.format(metadata_file)])
     if template_dir:
         cmd.extend(['"--template-dir"', '"templates/"'])
+    if plugins_dir:
+        cmd.extend(['"--plugins-dir"', '"plugins/"'])
     if static:
         for mount_point, _ in static:
             cmd.extend(['"--static"', '"{}:{}"'.format(mount_point, mount_point)])
@@ -216,7 +219,7 @@ def make_dockerfile(files, metadata_file, extra_options, branch, template_dir, s
 
 
 @contextmanager
-def temporary_docker_directory(files, name, metadata, extra_options, branch, template_dir, static, extra_metadata=None):
+def temporary_docker_directory(files, name, metadata, extra_options, branch, template_dir, plugins_dir, static, extra_metadata=None):
     extra_metadata = extra_metadata or {}
     tmp = tempfile.TemporaryDirectory()
     # We create a datasette folder in there to get a nicer now deploy name
@@ -242,6 +245,7 @@ def temporary_docker_directory(files, name, metadata, extra_options, branch, tem
             extra_options,
             branch,
             template_dir,
+            plugins_dir,
             static,
         )
         os.chdir(datasette_dir)
@@ -255,6 +259,11 @@ def temporary_docker_directory(files, name, metadata, extra_options, branch, tem
                 os.path.join(saved_cwd, template_dir),
                 os.path.join(datasette_dir, 'templates')
             )
+        if plugins_dir:
+            link_or_copy_directory(
+                os.path.join(saved_cwd, plugins_dir),
+                os.path.join(datasette_dir, 'plugins')
+            )
         for mount_point, path in static:
             link_or_copy_directory(
                 os.path.join(saved_cwd, path),
@@ -267,7 +276,7 @@ def temporary_docker_directory(files, name, metadata, extra_options, branch, tem
 
 
 @contextmanager
-def temporary_heroku_directory(files, name, metadata, extra_options, branch, template_dir, static, extra_metadata=None):
+def temporary_heroku_directory(files, name, metadata, extra_options, branch, template_dir, plugins_dir, static, extra_metadata=None):
     # FIXME: lots of duplicated code from above
 
     extra_metadata = extra_metadata or {}
@@ -314,6 +323,13 @@ def temporary_heroku_directory(files, name, metadata, extra_options, branch, tem
                 os.path.join(tmp.name, 'templates')
             )
             extras.extend(['--template-dir', 'templates/'])
+        if plugins_dir:
+            link_or_copy_directory(
+                os.path.join(saved_cwd, plugins_dir),
+                os.path.join(tmp.name, 'plugins')
+            )
+            extras.extend(['--plugins-dir', 'plugins/'])
+
         if metadata:
             extras.extend(['--metadata', 'metadata.json'])
         for mount_point, path in static:
@@ -625,3 +641,13 @@ def link_or_copy_directory(src, dst):
         shutil.copytree(src, dst, copy_function=os.link)
     except OSError:
         shutil.copytree(src, dst)
+
+
+def module_from_path(path, name):
+    # Adapted from http://sayspy.blogspot.com/2011/07/how-to-import-module-from-just-file.html
+    mod = imp.new_module(name)
+    mod.__file__ = path
+    with open(path, 'r') as file:
+        code = compile(file.read(), path, 'exec', dont_inherit=True)
+    exec(code, mod.__dict__)
+    return mod

setup.py

@@ -1,5 +1,4 @@
 from setuptools import setup, find_packages
-from datasette.version import __version__
 import os
 
 
@@ -10,13 +9,22 @@ def get_long_description():
         return fp.read()
 
 
+def get_version():
+    path = os.path.join(
+        os.path.dirname(os.path.abspath(__file__)), 'datasette', 'version.py'
+    )
+    g = {}
+    exec(open(path).read(), g)
+    return g['__version__']
+
+
 setup(
     name='datasette',
     description='An instant JSON API for your SQLite databases',
     long_description=get_long_description(),
     long_description_content_type='text/markdown',
     author='Simon Willison',
-    version=__version__,
+    version=get_version(),
     license='Apache License, Version 2.0',
     url='https://github.com/simonw/datasette',
     packages=find_packages(),

tests/fixtures.py

@@ -15,12 +15,16 @@ def app_client():
         conn = sqlite3.connect(filepath)
         conn.executescript(TABLES)
         os.chdir(os.path.dirname(filepath))
+        plugins_dir = os.path.join(tmpdir, 'plugins')
+        os.mkdir(plugins_dir)
+        open(os.path.join(plugins_dir, 'my_plugin.py'), 'w').write(PLUGIN)
         ds = Datasette(
             [filepath],
             page_size=50,
             max_returned_rows=100,
             sql_time_limit_ms=20,
             metadata=METADATA,
+            plugins_dir=plugins_dir,
         )
         ds.sqlite_functions.append(
             ('sleep', 1, lambda n: time.sleep(float(n))),
@@ -90,6 +94,20 @@ def generate_sortable_rows(num):
     }
 }
 
+PLUGIN = '''
+from datasette import hookimpl
+import pint
+
+ureg = pint.UnitRegistry()
+
+
+@hookimpl
+def prepare_connection(conn):
+    def convert_units(amount, from_, to_):
+        "select convert_units(100, 'm', 'ft');"
+        return (amount * ureg(from_)).to(to_).to_tuple()[0]
+    conn.create_function('convert_units', 3, convert_units)
+'''
 
 TABLES = '''
 CREATE TABLE simple_primary_key (

tests/test_api.py

@@ -588,8 +588,10 @@ def test_row_foreign_key_tables(app_client):
 
 
 def test_unit_filters(app_client):
-    response = app_client.get('/test_tables/units.json?distance__lt=75km&frequency__gt=1kHz',
-                              gather_request=False)
+    response = app_client.get(
+        '/test_tables/units.json?distance__lt=75km&frequency__gt=1kHz',
+        gather_request=False
+    )
     assert response.status == 200
     data = response.json
 
@@ -598,3 +600,11 @@ def test_unit_filters(app_client):
 
     assert len(data['rows']) == 1
     assert data['rows'][0][0] == 2
+
+
+def test_plugins_dir_plugin(app_client):
+    response = app_client.get(
+        "/test_tables.json?sql=select+convert_units(100%2C+'m'%2C+'ft')",
+        gather_request=False
+    )
+    assert pytest.approx(328.0839) == response.json['rows'][0][0]

tests/test_utils.py

@@ -195,6 +195,7 @@ def test_temporary_docker_directory_uses_hard_link():
             extra_options=None,
             branch=None,
             template_dir=None,
+            plugins_dir=None,
             static=[],
         ) as temp_docker:
             hello = os.path.join(temp_docker, 'hello')
@@ -218,6 +219,7 @@ def test_temporary_docker_directory_uses_copy_if_hard_link_fails(mock_link):
             extra_options=None,
             branch=None,
             template_dir=None,
+            plugins_dir=None,
             static=[],
         ) as temp_docker:
             hello = os.path.join(temp_docker, 'hello')