VSCode TextMate

An interpreter for grammar files as defined by TextMate. TextMate grammars use the oniguruma dialect (https://github.com/kkos/oniguruma). Supports loading grammar files from JSON or PLIST format. This library is used in VS Code. Cross - grammar injections are currently not supported.

Installing

1npm install vscode-textmate

Using

1const fs = require('fs');
2const path = require('path');
3const vsctm = require('vscode-textmate');
4const oniguruma = require('vscode-oniguruma');
5
6/**
7 * Utility to read a file as a promise
8 */
9function readFile(path) {
10    return new Promise((resolve, reject) => {
11        fs.readFile(path, (error, data) => error ? reject(error) : resolve(data));
12    })
13}
14
15const wasmBin = fs.readFileSync(path.join(__dirname, './node_modules/vscode-oniguruma/release/onig.wasm')).buffer;
16const vscodeOnigurumaLib = oniguruma.loadWASM(wasmBin).then(() => {
17    return {
18        createOnigScanner(patterns) { return new oniguruma.OnigScanner(patterns); },
19        createOnigString(s) { return new oniguruma.OnigString(s); }
20    };
21});
22
23// Create a registry that can create a grammar from a scope name.
24const registry = new vsctm.Registry({
25    onigLib: vscodeOnigurumaLib,
26    loadGrammar: (scopeName) => {
27        if (scopeName === 'source.js') {
28            // https://github.com/textmate/javascript.tmbundle/blob/master/Syntaxes/JavaScript.plist
29            return readFile('./JavaScript.plist').then(data => vsctm.parseRawGrammar(data.toString()))
30        }
31        console.log(`Unknown scope name: ${scopeName}`);
32        return null;
33    }
34});
35
36// Load the JavaScript grammar and any other grammars included by it async.
37registry.loadGrammar('source.js').then(grammar => {
38    const text = [
39        `function sayHello(name) {`,
40        `\treturn "Hello, " + name;`,
41        `}`
42    ];
43    let ruleStack = vsctm.INITIAL;
44    for (let i = 0; i < text.length; i++) {
45        const line = text[i];
46        const lineTokens = grammar.tokenizeLine(line, ruleStack);
47        console.log(`\nTokenizing line: ${line}`);
48        for (let j = 0; j < lineTokens.tokens.length; j++) {
49            const token = lineTokens.tokens[j];
50            console.log(` - token from ${token.startIndex} to ${token.endIndex} ` +
51              `(${line.substring(token.startIndex, token.endIndex)}) ` +
52              `with scopes ${token.scopes.join(', ')}`
53            );
54        }
55        ruleStack = lineTokens.ruleStack;
56    }
57});
58
59/* OUTPUT:
60
61Unknown scope name: source.js.regexp
62
63Tokenizing line: function sayHello(name) {
64 - token from 0 to 8 (function) with scopes source.js, meta.function.js, storage.type.function.js
65 - token from 8 to 9 ( ) with scopes source.js, meta.function.js
66 - token from 9 to 17 (sayHello) with scopes source.js, meta.function.js, entity.name.function.js
67 - token from 17 to 18 (() with scopes source.js, meta.function.js, punctuation.definition.parameters.begin.js
68 - token from 18 to 22 (name) with scopes source.js, meta.function.js, variable.parameter.function.js
69 - token from 22 to 23 ()) with scopes source.js, meta.function.js, punctuation.definition.parameters.end.js
70 - token from 23 to 24 ( ) with scopes source.js
71 - token from 24 to 25 ({) with scopes source.js, punctuation.section.scope.begin.js
72
73Tokenizing line:        return "Hello, " + name;
74 - token from 0 to 1 (  ) with scopes source.js
75 - token from 1 to 7 (return) with scopes source.js, keyword.control.js
76 - token from 7 to 8 ( ) with scopes source.js
77 - token from 8 to 9 (") with scopes source.js, string.quoted.double.js, punctuation.definition.string.begin.js
78 - token from 9 to 16 (Hello, ) with scopes source.js, string.quoted.double.js
79 - token from 16 to 17 (") with scopes source.js, string.quoted.double.js, punctuation.definition.string.end.js
80 - token from 17 to 18 ( ) with scopes source.js
81 - token from 18 to 19 (+) with scopes source.js, keyword.operator.arithmetic.js
82 - token from 19 to 20 ( ) with scopes source.js
83 - token from 20 to 24 (name) with scopes source.js, support.constant.dom.js
84 - token from 24 to 25 (;) with scopes source.js, punctuation.terminator.statement.js
85
86Tokenizing line: }
87 - token from 0 to 1 (}) with scopes source.js, punctuation.section.scope.end.js
88
89*/
90

For grammar authors

See vscode-tmgrammar-test that can help you write unit tests against your grammar.

API doc

See the main.ts file

Developing

• Clone the repository
• Run npm install
• Compile in the background with npm run watch
• Run tests with npm test
• Run benchmark with npm run benchmark
• Troubleshoot a grammar with npm run inspect -- PATH_TO_GRAMMAR PATH_TO_FILE

Code of Conduct

This project has adopted the Microsoft Open Source Code of Conduct. For more information see the Code of Conduct FAQ or contact opencode@microsoft.com with any additional questions or comments.

License

MIT