DuckDB-Wasm

DuckDB-Wasm is an in-process analytical SQL database for the browser. It is powered by WebAssembly, speaks Arrow fluently, reads Parquet, CSV and JSON files backed by Filesystem APIs or HTTP requests and has been tested with Chrome, Firefox, Safari and Node.js. Read the launch blog post.

Try it out at shell.duckdb.org and on Observable and read the API documentation.

DuckDB-Wasm is fast! If you're here for performance numbers, head over to our benchmarks.

Instantiation

cdn(jsdelivr)

1import * as duckdb from '@malloydata/duckdb-wasm';
2
3const JSDELIVR_BUNDLES = duckdb.getJsDelivrBundles();
4
5// Select a bundle based on browser checks
6const bundle = await duckdb.selectBundle(JSDELIVR_BUNDLES);
7
8const worker_url = URL.createObjectURL(
9    new Blob([`importScripts("${bundle.mainWorker!}");`], { type: 'text/javascript' }),
10);
11
12// Instantiate the asynchronus version of DuckDB-wasm
13const worker = new Worker(worker_url);
14const logger = new duckdb.ConsoleLogger();
15const db = new duckdb.AsyncDuckDB(logger, worker);
16await db.instantiate(bundle.mainModule, bundle.pthreadWorker);
17URL.revokeObjectURL(worker_url);

webpack

1import * as duckdb from '@malloydata/duckdb-wasm';
2import duckdb_wasm from '@malloydata/duckdb-wasm/dist/duckdb-mvp.wasm';
3import duckdb_wasm_next from '@malloydata/duckdb-wasm/dist/duckdb-eh.wasm';
4const MANUAL_BUNDLES: duckdb.DuckDBBundles = {
5    mvp: {
6        mainModule: duckdb_wasm,
7        mainWorker: new URL('@malloydata/duckdb-wasm/dist/duckdb-browser-mvp.worker.js', import.meta.url).toString(),
8    },
9    eh: {
10        mainModule: duckdb_wasm_next,
11        mainWorker: new URL('@malloydata/duckdb-wasm/dist/duckdb-browser-eh.worker.js', import.meta.url).toString(),
12    },
13};
14// Select a bundle based on browser checks
15const bundle = await duckdb.selectBundle(MANUAL_BUNDLES);
16// Instantiate the asynchronus version of DuckDB-wasm
17const worker = new Worker(bundle.mainWorker!);
18const logger = new duckdb.ConsoleLogger();
19const db = new duckdb.AsyncDuckDB(logger, worker);
20await db.instantiate(bundle.mainModule, bundle.pthreadWorker);

vite

1import * as duckdb from '@malloydata/duckdb-wasm';
2import duckdb_wasm from '@malloydata/duckdb-wasm/dist/duckdb-mvp.wasm?url';
3import mvp_worker from '@malloydata/duckdb-wasm/dist/duckdb-browser-mvp.worker.js?url';
4import duckdb_wasm_eh from '@malloydata/duckdb-wasm/dist/duckdb-eh.wasm?url';
5import eh_worker from '@malloydata/duckdb-wasm/dist/duckdb-browser-eh.worker.js?url';
6
7const MANUAL_BUNDLES: duckdb.DuckDBBundles = {
8    mvp: {
9        mainModule: duckdb_wasm,
10        mainWorker: mvp_worker,
11    },
12    eh: {
13        mainModule: duckdb_wasm_eh,
14        mainWorker: eh_worker,
15    },
16};
17// Select a bundle based on browser checks
18const bundle = await duckdb.selectBundle(MANUAL_BUNDLES);
19// Instantiate the asynchronus version of DuckDB-wasm
20const worker = new Worker(bundle.mainWorker!);
21const logger = new duckdb.ConsoleLogger();
22const db = new duckdb.AsyncDuckDB(logger, worker);
23await db.instantiate(bundle.mainModule, bundle.pthreadWorker);

static served(manually download the files from https://cdn.jsdelivr.net/npm/@malloydata/duckdb-wasm/dist/)

1import * as duckdb from '@malloydata/duckdb-wasm';
2
3const MANUAL_BUNDLES: duckdb.DuckDBBundles = {
4    mvp: {
5        mainModule: 'change/me/../duckdb-mvp.wasm',
6        mainWorker: 'change/me/../duckdb-browser-mvp.worker.js',
7    },
8    eh: {
9        mainModule: 'change/m/../duckdb-eh.wasm',
10        mainWorker: 'change/m/../duckdb-browser-eh.worker.js',
11    },
12};
13// Select a bundle based on browser checks
14const bundle = await duckdb.selectBundle(JSDELIVR_BUNDLES);
15// Instantiate the asynchronus version of DuckDB-wasm
16const worker = new Worker(bundle.mainWorker!);
17const logger = new duckdb.ConsoleLogger();
18const db = new duckdb.AsyncDuckDB(logger, worker);
19await db.instantiate(bundle.mainModule, bundle.pthreadWorker);

Data Import

1// Data can be inserted from an existing arrow.Table
2await c.insertArrowTable(existingTable, { name: 'arrow_table' });
3// ..., from a raw Arrow IPC stream
4const c = await db.connect();
5const streamResponse = await fetch(`someapi`);
6const streamReader = streamResponse.body.getReader();
7const streamInserts = [];
8while (true) {
9    const { value, done } = await streamReader.read();
10    if (done) break;
11    streamInserts.push(c.insertArrowFromIPCStream(value, { name: 'streamed' }));
12}
13await Promise.all(streamInserts);
14
15// ..., from CSV files
16// (interchangeable: registerFile{Text,Buffer,URL,Handle})
17await db.registerFileText(`data.csv`, '1|foo\n2|bar\n');
18// ... with typed insert options
19await db.insertCSVFromPath('data.csv', {
20    schema: 'main',
21    name: 'foo',
22    detect: false,
23    header: false,
24    delimiter: '|',
25    columns: {
26        col1: new arrow.Int32(),
27        col2: new arrow.Utf8(),
28    },
29});
30
31// ..., from JSON documents in row-major format
32await db.registerFileText(
33    'rows.json',
34    `[
35    { "col1": 1, "col2": "foo" },
36    { "col1": 2, "col2": "bar" },
37]`,
38);
39// ... or column-major format
40await db.registerFileText(
41    'columns.json',
42    `{
43    "col1": [1, 2],
44    "col2": ["foo", "bar"]
45}`,
46);
47// ... with typed insert options
48await c.insertJSONFromPath('rows.json', { name: 'rows' });
49await c.insertJSONFromPath('columns.json', { name: 'columns' });
50
51// ... from a Javascript array
52const myArray = [
53  {
54    name: 'Dan',
55    age: 32,
56    numberOfPets: 3
57  }
58];
59const encoder = new TextEncoder();
60const buffer = encoder.encode(myArray);
61await db().registerFileBuffer(myTableName, buffer);
62await c.insertJSONFromPath(myTableName, {
63  schema: 'main',
64  name: 'foo',
65});
66
67// ..., from Parquet files
68const pickedFile: File = letUserPickFile();
69await db.registerFileHandle('local.parquet', pickedFile, DuckDBDataProtocol.BROWSER_FILEREADER, true);
70await db.registerFileURL('remote.parquet', 'https://origin/remote.parquet', DuckDBDataProtocol.HTTP, false);
71const res = await fetch('https://origin/remote.parquet');
72await db.registerFileBuffer('buffer.parquet', new Uint8Array(await res.arrayBuffer()));
73
74// ..., by specifying URLs in the SQL text
75await c.query(`
76    CREATE TABLE direct AS
77        SELECT * FROM "https://origin/remote.parquet"
78`);
79// ..., or by executing raw insert statements
80await c.query(`INSERT INTO existing_table
81    VALUES (1, "foo"), (2, "bar")`);
82
83// Close the connection to release memory
84await c.close();

Query Execution

1// Either materialize the query result
2await conn.query<{ v: arrow.Int }>(`
3    SELECT * FROM generate_series(1, 100) t(v)
4`);
5// ..., or fetch the result chunks lazily
6for await (const batch of await conn.send<{ v: arrow.Int }>(`
7    SELECT * FROM generate_series(1, 100) t(v)
8`)) {
9    // ...
10}
11// Close the connection to release memory
12await conn.close();

Prepared Statements

1// Prepare query
2const stmt = await conn.prepare(`SELECT v + ? FROM generate_series(0, 10000) as t(v);`);
3// ... and run the query with materialized results
4await stmt.query(234);
5// ... or result chunks
6for await (const batch of await stmt.send(234)) {
7    // ...
8}
9// Close the statement to release memory
10await stmt.close();
11// Closing the connection will release statements as well
12await conn.close();