--crossdb option for joining across databases, refs #283

datasette/app.py

@@ -85,6 +85,9 @@
 
 app_root = Path(__file__).parent.parent
 
+# https://github.com/simonw/datasette/issues/283#issuecomment-781591015
+SQLITE_LIMIT_ATTACHED = 10
+
 Setting = collections.namedtuple("Setting", ("name", "default", "help"))
 SETTINGS = (
     Setting("default_page_size", 100, "Default page size for the table view"),
@@ -194,6 +197,7 @@ def __init__(
         version_note=None,
         config_dir=None,
         pdb=False,
+        crossdb=False,
     ):
         assert config_dir is None or isinstance(
             config_dir, Path
@@ -217,7 +221,8 @@ def __init__(
         self.inspect_data = inspect_data
         self.immutables = set(immutables or [])
         self.databases = collections.OrderedDict()
-        if memory or not self.files:
+        self.crossdb = crossdb
+        if memory or crossdb or not self.files:
             self.add_database(Database(self, is_memory=True), name="_memory")
         # memory_name is a random string so that each Datasette instance gets its own
         # unique in-memory named database - otherwise unit tests can fail with weird
@@ -499,6 +504,19 @@ def _prepare_connection(self, conn, database):
             conn.execute(f"PRAGMA cache_size=-{self.setting('cache_size_kb')}")
         # pylint: disable=no-member
         pm.hook.prepare_connection(conn=conn, database=database, datasette=self)
+        # If self.crossdb and this is _memory, connect the first SQLITE_LIMIT_ATTACHED databases
+        if self.crossdb and database == "_memory":
+            count = 0
+            for db_name, db in self.databases.items():
+                if count >= SQLITE_LIMIT_ATTACHED or db.is_memory:
+                    continue
+                sql = 'ATTACH DATABASE "file:{path}?{qs}" AS [{name}];'.format(
+                    path=db.path,
+                    qs="mode=ro" if db.is_mutable else "immutable=1",
+                    name=db_name,
+                )
+                conn.execute(sql)
+                count += 1
 
     def add_message(self, request, message, type=INFO):
         if not hasattr(request, "_messages"):

datasette/cli.py

@@ -410,6 +410,11 @@ def uninstall(packages, yes):
     is_flag=True,
     help="Create database files if they do not exist",
 )
+@click.option(
+    "--crossdb",
+    is_flag=True,
+    help="Enable cross-database joins using the /_memory database",
+)
 @click.option(
     "--ssl-keyfile",
     help="SSL key file",
@@ -442,6 +447,7 @@ def serve(
     pdb,
     open_browser,
     create,
+    crossdb,
     ssl_keyfile,
     ssl_certfile,
     return_instance=False,
@@ -499,6 +505,7 @@ def serve(
         secret=secret,
         version_note=version_note,
         pdb=pdb,
+        crossdb=crossdb,
     )
 
     # if files is a single directory, use that as config_dir=

datasette/database.py

@@ -78,7 +78,7 @@ def connect(self, write=False):
                 conn.execute("PRAGMA query_only=1")
             return conn
         if self.is_memory:
-            return sqlite3.connect(":memory:")
+            return sqlite3.connect(":memory:", uri=True)
         # mode=ro or immutable=1?
         if self.is_mutable:
             qs = "?mode=ro"

tests/test_cli.py

@@ -147,6 +147,7 @@ def test_metadata_yaml():
         get=None,
         help_config=False,
         pdb=False,
+        crossdb=False,
         open_browser=False,
         create=False,
         ssl_keyfile=None,