diff --git a/.vscode/settings.json b/.vscode/settings.json index 5ce3aadf..7e02aff0 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -12,5 +12,9 @@ }, "lcov.sourceMaps": true, "typescript.tsdk": "./node_modules/typescript/lib", - "tslint.enable": true + "tslint.enable": true, + + "mochaExplorer.ui": "tdd", + "testExplorer.useNativeTesting": true, + "mochaExplorer.files": "out/tests/all.test.js" } \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 4daf509d..eed3d0f3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,6 +11,7 @@ "devDependencies": { "@types/mocha": "^9.1.0", "@types/node": "^16.6.1", + "copy-webpack-plugin": "^9.1.0", "mocha": "^9.2.2", "typescript": "^4.3.5", "vscode-oniguruma": "^1.5.1", @@ -27,6 +28,41 @@ "node": ">=10.0.0" } }, + "node_modules/@nodelib/fs.scandir": { + "version": "2.1.5", + "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", + "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==", + "dev": true, + "dependencies": { + "@nodelib/fs.stat": "2.0.5", + "run-parallel": "^1.1.9" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/@nodelib/fs.stat": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz", + "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==", + "dev": true, + "engines": { + "node": ">= 8" + } + }, + "node_modules/@nodelib/fs.walk": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz", + "integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==", + "dev": true, + "dependencies": { + "@nodelib/fs.scandir": "2.1.5", + "fastq": "^1.6.0" + }, + "engines": { + "node": ">= 8" + } + }, "node_modules/@types/eslint": { "version": "7.28.0", "resolved": "https://registry.npmjs.org/@types/eslint/-/eslint-7.28.0.tgz", @@ -369,6 +405,15 @@ "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", "dev": true }, + "node_modules/array-union": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/array-union/-/array-union-2.1.0.tgz", + "integrity": "sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw==", + "dev": true, + "engines": { + "node": ">=8" + } + }, "node_modules/balanced-match": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", @@ -588,6 +633,42 @@ "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=", "dev": true }, + "node_modules/copy-webpack-plugin": { + "version": "9.1.0", + "resolved": "https://registry.npmjs.org/copy-webpack-plugin/-/copy-webpack-plugin-9.1.0.tgz", + "integrity": "sha512-rxnR7PaGigJzhqETHGmAcxKnLZSR5u1Y3/bcIv/1FnqXedcL/E2ewK7ZCNrArJKCiSv8yVXhTqetJh8inDvfsA==", + "dev": true, + "dependencies": { + "fast-glob": "^3.2.7", + "glob-parent": "^6.0.1", + "globby": "^11.0.3", + "normalize-path": "^3.0.0", + "schema-utils": "^3.1.1", + "serialize-javascript": "^6.0.0" + }, + "engines": { + "node": ">= 12.13.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + }, + "peerDependencies": { + "webpack": "^5.1.0" + } + }, + "node_modules/copy-webpack-plugin/node_modules/glob-parent": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", + "integrity": "sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==", + "dev": true, + "dependencies": { + "is-glob": "^4.0.3" + }, + "engines": { + "node": ">=10.13.0" + } + }, "node_modules/cross-spawn": { "version": "7.0.3", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", @@ -646,6 +727,18 @@ "node": ">=0.3.1" } }, + "node_modules/dir-glob": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-3.0.1.tgz", + "integrity": "sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA==", + "dev": true, + "dependencies": { + "path-type": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/electron-to-chromium": { "version": "1.3.811", "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.3.811.tgz", @@ -791,6 +884,22 @@ "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", "dev": true }, + "node_modules/fast-glob": { + "version": "3.2.11", + "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.2.11.tgz", + "integrity": "sha512-xrO3+1bxSo3ZVHAnqzyuewYT6aMFHRAd4Kcs92MAonjwQZLsK9d0SF1IyQ3k5PoirxTW0Oe/RqFgMQ6TcNE5Ew==", + "dev": true, + "dependencies": { + "@nodelib/fs.stat": "^2.0.2", + "@nodelib/fs.walk": "^1.2.3", + "glob-parent": "^5.1.2", + "merge2": "^1.3.0", + "micromatch": "^4.0.4" + }, + "engines": { + "node": ">=8.6.0" + } + }, "node_modules/fast-json-stable-stringify": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", @@ -803,6 +912,15 @@ "integrity": "sha512-On2N+BpYJ15xIC974QNVuYGMOlEVt4s0EOI3wwMqOmK1fdDY+FN/zltPV8vosq4ad4c/gJ1KHScUn/6AWIgiow==", "dev": true }, + "node_modules/fastq": { + "version": "1.13.0", + "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.13.0.tgz", + "integrity": "sha512-YpkpUnK8od0o1hmeSc7UUs/eB/vIPWJYjKck2QKIzAf71Vm1AAQ3EbuZB3g2JIy+pg+ERD0vqI79KyZiB2e2Nw==", + "dev": true, + "dependencies": { + "reusify": "^1.0.4" + } + }, "node_modules/fill-range": { "version": "7.0.1", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", @@ -922,6 +1040,26 @@ "integrity": "sha512-lkX1HJXwyMcprw/5YUZc2s7DrpAiHB21/V+E1rHUrVNokkvB6bqMzT0VfV6/86ZNabt1k14YOIaT7nDvOX3Iiw==", "dev": true }, + "node_modules/globby": { + "version": "11.1.0", + "resolved": "https://registry.npmjs.org/globby/-/globby-11.1.0.tgz", + "integrity": "sha512-jhIXaOzy1sb8IyocaruWSn1TjmnBVs8Ayhcy83rmxNJ8q2uWKCAj3CnJY+KpGSXCueAPc0i05kVvVKtP1t9S3g==", + "dev": true, + "dependencies": { + "array-union": "^2.1.0", + "dir-glob": "^3.0.1", + "fast-glob": "^3.2.9", + "ignore": "^5.2.0", + "merge2": "^1.4.1", + "slash": "^3.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/graceful-fs": { "version": "4.2.8", "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.8.tgz", @@ -976,6 +1114,15 @@ "node": ">=10.17.0" } }, + "node_modules/ignore": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.2.0.tgz", + "integrity": "sha512-CmxgYGiEPCLhfLnpPp1MoRmifwEIOgjcHXxOBjv7mY96c+eWScsOP9c112ZyLdWHi0FxHjI+4uVhKYp/gcdRmQ==", + "dev": true, + "engines": { + "node": ">= 4" + } + }, "node_modules/import-local": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/import-local/-/import-local-3.0.2.tgz", @@ -1230,6 +1377,28 @@ "integrity": "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==", "dev": true }, + "node_modules/merge2": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", + "integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==", + "dev": true, + "engines": { + "node": ">= 8" + } + }, + "node_modules/micromatch": { + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.5.tgz", + "integrity": "sha512-DMy+ERcEW2q8Z2Po+WNXuw3c5YaUSFjAO5GsJqfEl7UjvtIuFKO6ZrKvcItdy98dwFI2N1tg3zNIdKaQT+aNdA==", + "dev": true, + "dependencies": { + "braces": "^3.0.2", + "picomatch": "^2.3.1" + }, + "engines": { + "node": ">=8.6" + } + }, "node_modules/mime-db": { "version": "1.49.0", "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.49.0.tgz", @@ -1532,6 +1701,15 @@ "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==", "dev": true }, + "node_modules/path-type": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz", + "integrity": "sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==", + "dev": true, + "engines": { + "node": ">=8" + } + }, "node_modules/picomatch": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", @@ -1565,6 +1743,26 @@ "node": ">=6" } }, + "node_modules/queue-microtask": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", + "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, "node_modules/randombytes": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/randombytes/-/randombytes-2.1.0.tgz", @@ -1641,6 +1839,39 @@ "node": ">=8" } }, + "node_modules/reusify": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz", + "integrity": "sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==", + "dev": true, + "engines": { + "iojs": ">=1.0.0", + "node": ">=0.10.0" + } + }, + "node_modules/run-parallel": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", + "integrity": "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "dependencies": { + "queue-microtask": "^1.2.2" + } + }, "node_modules/safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", @@ -1727,6 +1958,15 @@ "integrity": "sha512-VUJ49FC8U1OxwZLxIbTTrDvLnf/6TDgxZcK8wxR8zs13xpx7xbG60ndBlhNrFi2EMuFRoeDoJO7wthSLq42EjA==", "dev": true }, + "node_modules/slash": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz", + "integrity": "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==", + "dev": true, + "engines": { + "node": ">=8" + } + }, "node_modules/source-map": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", @@ -2169,6 +2409,32 @@ "integrity": "sha512-Fxt+AfXgjMoin2maPIYzFZnQjAXjAL0PHscM5pRTtatFqB+vZxAM9tLp2Optnuw3QOQC40jTNeGYFOMvyf7v9g==", "dev": true }, + "@nodelib/fs.scandir": { + "version": "2.1.5", + "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", + "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==", + "dev": true, + "requires": { + "@nodelib/fs.stat": "2.0.5", + "run-parallel": "^1.1.9" + } + }, + "@nodelib/fs.stat": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz", + "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==", + "dev": true + }, + "@nodelib/fs.walk": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz", + "integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==", + "dev": true, + "requires": { + "@nodelib/fs.scandir": "2.1.5", + "fastq": "^1.6.0" + } + }, "@types/eslint": { "version": "7.28.0", "resolved": "https://registry.npmjs.org/@types/eslint/-/eslint-7.28.0.tgz", @@ -2469,6 +2735,12 @@ "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", "dev": true }, + "array-union": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/array-union/-/array-union-2.1.0.tgz", + "integrity": "sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw==", + "dev": true + }, "balanced-match": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", @@ -2635,6 +2907,31 @@ "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=", "dev": true }, + "copy-webpack-plugin": { + "version": "9.1.0", + "resolved": "https://registry.npmjs.org/copy-webpack-plugin/-/copy-webpack-plugin-9.1.0.tgz", + "integrity": "sha512-rxnR7PaGigJzhqETHGmAcxKnLZSR5u1Y3/bcIv/1FnqXedcL/E2ewK7ZCNrArJKCiSv8yVXhTqetJh8inDvfsA==", + "dev": true, + "requires": { + "fast-glob": "^3.2.7", + "glob-parent": "^6.0.1", + "globby": "^11.0.3", + "normalize-path": "^3.0.0", + "schema-utils": "^3.1.1", + "serialize-javascript": "^6.0.0" + }, + "dependencies": { + "glob-parent": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", + "integrity": "sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==", + "dev": true, + "requires": { + "is-glob": "^4.0.3" + } + } + } + }, "cross-spawn": { "version": "7.0.3", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", @@ -2675,6 +2972,15 @@ "integrity": "sha512-/VTCrvm5Z0JGty/BWHljh+BAiw3IK+2j87NGMu8Nwc/f48WoDAC395uomO9ZD117ZOBaHmkX1oyLvkVM/aIT3w==", "dev": true }, + "dir-glob": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-3.0.1.tgz", + "integrity": "sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA==", + "dev": true, + "requires": { + "path-type": "^4.0.0" + } + }, "electron-to-chromium": { "version": "1.3.811", "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.3.811.tgz", @@ -2783,6 +3089,19 @@ "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", "dev": true }, + "fast-glob": { + "version": "3.2.11", + "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.2.11.tgz", + "integrity": "sha512-xrO3+1bxSo3ZVHAnqzyuewYT6aMFHRAd4Kcs92MAonjwQZLsK9d0SF1IyQ3k5PoirxTW0Oe/RqFgMQ6TcNE5Ew==", + "dev": true, + "requires": { + "@nodelib/fs.stat": "^2.0.2", + "@nodelib/fs.walk": "^1.2.3", + "glob-parent": "^5.1.2", + "merge2": "^1.3.0", + "micromatch": "^4.0.4" + } + }, "fast-json-stable-stringify": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", @@ -2795,6 +3114,15 @@ "integrity": "sha512-On2N+BpYJ15xIC974QNVuYGMOlEVt4s0EOI3wwMqOmK1fdDY+FN/zltPV8vosq4ad4c/gJ1KHScUn/6AWIgiow==", "dev": true }, + "fastq": { + "version": "1.13.0", + "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.13.0.tgz", + "integrity": "sha512-YpkpUnK8od0o1hmeSc7UUs/eB/vIPWJYjKck2QKIzAf71Vm1AAQ3EbuZB3g2JIy+pg+ERD0vqI79KyZiB2e2Nw==", + "dev": true, + "requires": { + "reusify": "^1.0.4" + } + }, "fill-range": { "version": "7.0.1", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", @@ -2880,6 +3208,20 @@ "integrity": "sha512-lkX1HJXwyMcprw/5YUZc2s7DrpAiHB21/V+E1rHUrVNokkvB6bqMzT0VfV6/86ZNabt1k14YOIaT7nDvOX3Iiw==", "dev": true }, + "globby": { + "version": "11.1.0", + "resolved": "https://registry.npmjs.org/globby/-/globby-11.1.0.tgz", + "integrity": "sha512-jhIXaOzy1sb8IyocaruWSn1TjmnBVs8Ayhcy83rmxNJ8q2uWKCAj3CnJY+KpGSXCueAPc0i05kVvVKtP1t9S3g==", + "dev": true, + "requires": { + "array-union": "^2.1.0", + "dir-glob": "^3.0.1", + "fast-glob": "^3.2.9", + "ignore": "^5.2.0", + "merge2": "^1.4.1", + "slash": "^3.0.0" + } + }, "graceful-fs": { "version": "4.2.8", "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.8.tgz", @@ -2919,6 +3261,12 @@ "integrity": "sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw==", "dev": true }, + "ignore": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.2.0.tgz", + "integrity": "sha512-CmxgYGiEPCLhfLnpPp1MoRmifwEIOgjcHXxOBjv7mY96c+eWScsOP9c112ZyLdWHi0FxHjI+4uVhKYp/gcdRmQ==", + "dev": true + }, "import-local": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/import-local/-/import-local-3.0.2.tgz", @@ -3104,6 +3452,22 @@ "integrity": "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==", "dev": true }, + "merge2": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", + "integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==", + "dev": true + }, + "micromatch": { + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.5.tgz", + "integrity": "sha512-DMy+ERcEW2q8Z2Po+WNXuw3c5YaUSFjAO5GsJqfEl7UjvtIuFKO6ZrKvcItdy98dwFI2N1tg3zNIdKaQT+aNdA==", + "dev": true, + "requires": { + "braces": "^3.0.2", + "picomatch": "^2.3.1" + } + }, "mime-db": { "version": "1.49.0", "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.49.0.tgz", @@ -3321,6 +3685,12 @@ "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==", "dev": true }, + "path-type": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz", + "integrity": "sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==", + "dev": true + }, "picomatch": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", @@ -3342,6 +3712,12 @@ "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==", "dev": true }, + "queue-microtask": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", + "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==", + "dev": true + }, "randombytes": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/randombytes/-/randombytes-2.1.0.tgz", @@ -3402,6 +3778,21 @@ "integrity": "sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==", "dev": true }, + "reusify": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz", + "integrity": "sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==", + "dev": true + }, + "run-parallel": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", + "integrity": "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==", + "dev": true, + "requires": { + "queue-microtask": "^1.2.2" + } + }, "safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", @@ -3458,6 +3849,12 @@ "integrity": "sha512-VUJ49FC8U1OxwZLxIbTTrDvLnf/6TDgxZcK8wxR8zs13xpx7xbG60ndBlhNrFi2EMuFRoeDoJO7wthSLq42EjA==", "dev": true }, + "slash": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz", + "integrity": "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==", + "dev": true + }, "source-map": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", diff --git a/package.json b/package.json index c742e278..a1106b9a 100644 --- a/package.json +++ b/package.json @@ -27,8 +27,9 @@ "prepublishOnly": "tsc && webpack --progress && node scripts/release.js" }, "devDependencies": { - "@types/node": "^16.6.1", "@types/mocha": "^9.1.0", + "@types/node": "^16.6.1", + "copy-webpack-plugin": "^9.1.0", "mocha": "^9.2.2", "typescript": "^4.3.5", "vscode-oniguruma": "^1.5.1", diff --git a/scripts/release.js b/scripts/release.js index 33c80c73..47471922 100644 --- a/scripts/release.js +++ b/scripts/release.js @@ -3,10 +3,7 @@ const path = require('path'); const OUT_FOLDER = path.join(__dirname, '../out'); const RELEASE_FOLDER = path.join(__dirname, '../release'); -`` + if (!fs.existsSync(RELEASE_FOLDER)) { fs.mkdirSync(RELEASE_FOLDER); } - -fs.writeFileSync(path.join(RELEASE_FOLDER, 'main.d.ts'), fs.readFileSync(path.join(OUT_FOLDER, 'main.d.ts'))); -fs.writeFileSync(path.join(RELEASE_FOLDER, 'types.d.ts'), fs.readFileSync(path.join(OUT_FOLDER, 'types.d.ts'))); diff --git a/src/encodedTokenAttributes.ts b/src/encodedTokenAttributes.ts new file mode 100644 index 00000000..e431f89f --- /dev/null +++ b/src/encodedTokenAttributes.ts @@ -0,0 +1,192 @@ +/*--------------------------------------------------------- + * Copyright (C) Microsoft Corporation. All rights reserved. + *--------------------------------------------------------*/ + +import { FontStyle } from "./theme"; + +export type EncodedTokenAttributes = number; + +export namespace EncodedTokenAttributes { + export function toBinaryStr(encodedTokenAttributes: EncodedTokenAttributes): string { + let r = encodedTokenAttributes.toString(2); + while (r.length < 32) { + r = "0" + r; + } + return r; + } + + export function print(encodedTokenAttributes: EncodedTokenAttributes): void { + const languageId = EncodedTokenAttributes.getLanguageId(encodedTokenAttributes); + const tokenType = EncodedTokenAttributes.getTokenType(encodedTokenAttributes); + const fontStyle = EncodedTokenAttributes.getFontStyle(encodedTokenAttributes); + const foreground = EncodedTokenAttributes.getForeground(encodedTokenAttributes); + const background = EncodedTokenAttributes.getBackground(encodedTokenAttributes); + + console.log({ + languageId: languageId, + tokenType: tokenType, + fontStyle: fontStyle, + foreground: foreground, + background: background, + }); + } + + export function getLanguageId(encodedTokenAttributes: EncodedTokenAttributes): number { + return ( + (encodedTokenAttributes & EncodedTokenDataConsts.LANGUAGEID_MASK) >>> + EncodedTokenDataConsts.LANGUAGEID_OFFSET + ); + } + + export function getTokenType(encodedTokenAttributes: EncodedTokenAttributes): StandardTokenType { + return ( + (encodedTokenAttributes & EncodedTokenDataConsts.TOKEN_TYPE_MASK) >>> + EncodedTokenDataConsts.TOKEN_TYPE_OFFSET + ); + } + + export function containsBalancedBrackets(encodedTokenAttributes: EncodedTokenAttributes): boolean { + return (encodedTokenAttributes & EncodedTokenDataConsts.BALANCED_BRACKETS_MASK) !== 0; + } + + export function getFontStyle(encodedTokenAttributes: EncodedTokenAttributes): number { + return ( + (encodedTokenAttributes & EncodedTokenDataConsts.FONT_STYLE_MASK) >>> + EncodedTokenDataConsts.FONT_STYLE_OFFSET + ); + } + + export function getForeground(encodedTokenAttributes: EncodedTokenAttributes): number { + return ( + (encodedTokenAttributes & EncodedTokenDataConsts.FOREGROUND_MASK) >>> + EncodedTokenDataConsts.FOREGROUND_OFFSET + ); + } + + export function getBackground(encodedTokenAttributes: EncodedTokenAttributes): number { + return ( + (encodedTokenAttributes & EncodedTokenDataConsts.BACKGROUND_MASK) >>> + EncodedTokenDataConsts.BACKGROUND_OFFSET + ); + } + + /** + * Updates the fields in `metadata`. + * A value of `0`, `NotSet` or `null` indicates that the corresponding field should be left as is. + */ + export function set( + encodedTokenAttributes: EncodedTokenAttributes, + languageId: number, + tokenType: OptionalStandardTokenType, + containsBalancedBrackets: boolean | null, + fontStyle: FontStyle, + foreground: number, + background: number + ): number { + let _languageId = EncodedTokenAttributes.getLanguageId(encodedTokenAttributes); + let _tokenType = EncodedTokenAttributes.getTokenType(encodedTokenAttributes); + let _containsBalancedBracketsBit: 0 | 1 = + EncodedTokenAttributes.containsBalancedBrackets(encodedTokenAttributes) ? 1 : 0; + let _fontStyle = EncodedTokenAttributes.getFontStyle(encodedTokenAttributes); + let _foreground = EncodedTokenAttributes.getForeground(encodedTokenAttributes); + let _background = EncodedTokenAttributes.getBackground(encodedTokenAttributes); + + if (languageId !== 0) { + _languageId = languageId; + } + if (tokenType !== OptionalStandardTokenType.NotSet) { + _tokenType = fromOptionalTokenType(tokenType); + } + if (containsBalancedBrackets !== null) { + _containsBalancedBracketsBit = containsBalancedBrackets ? 1 : 0; + } + if (fontStyle !== FontStyle.NotSet) { + _fontStyle = fontStyle; + } + if (foreground !== 0) { + _foreground = foreground; + } + if (background !== 0) { + _background = background; + } + + return ( + ((_languageId << EncodedTokenDataConsts.LANGUAGEID_OFFSET) | + (_tokenType << EncodedTokenDataConsts.TOKEN_TYPE_OFFSET) | + (_containsBalancedBracketsBit << + EncodedTokenDataConsts.BALANCED_BRACKETS_OFFSET) | + (_fontStyle << EncodedTokenDataConsts.FONT_STYLE_OFFSET) | + (_foreground << EncodedTokenDataConsts.FOREGROUND_OFFSET) | + (_background << EncodedTokenDataConsts.BACKGROUND_OFFSET)) >>> + 0 + ); + } +} + +/** + * Helpers to manage the "collapsed" metadata of an entire StackElement stack. + * The following assumptions have been made: + * - languageId < 256 => needs 8 bits + * - unique color count < 512 => needs 9 bits + * + * The binary format is: + * - ------------------------------------------- + * 3322 2222 2222 1111 1111 1100 0000 0000 + * 1098 7654 3210 9876 5432 1098 7654 3210 + * - ------------------------------------------- + * xxxx xxxx xxxx xxxx xxxx xxxx xxxx xxxx + * bbbb bbbb ffff ffff fFFF FBTT LLLL LLLL + * - ------------------------------------------- + * - L = LanguageId (8 bits) + * - T = StandardTokenType (2 bits) + * - B = Balanced bracket (1 bit) + * - F = FontStyle (4 bits) + * - f = foreground color (9 bits) + * - b = background color (9 bits) + */ +const enum EncodedTokenDataConsts { + LANGUAGEID_MASK = 0b00000000000000000000000011111111, + TOKEN_TYPE_MASK = 0b00000000000000000000001100000000, + BALANCED_BRACKETS_MASK = 0b00000000000000000000010000000000, + FONT_STYLE_MASK = 0b00000000000000000111100000000000, + FOREGROUND_MASK = 0b00000000111111111000000000000000, + BACKGROUND_MASK = 0b11111111000000000000000000000000, + + LANGUAGEID_OFFSET = 0, + TOKEN_TYPE_OFFSET = 8, + BALANCED_BRACKETS_OFFSET = 10, + FONT_STYLE_OFFSET = 11, + FOREGROUND_OFFSET = 15, + BACKGROUND_OFFSET = 24 +} + +export const enum StandardTokenType { + Other = 0, + Comment = 1, + String = 2, + RegEx = 3 +} + +export function toOptionalTokenType(standardType: StandardTokenType): OptionalStandardTokenType { + return standardType as any as OptionalStandardTokenType; +} + +function fromOptionalTokenType( + standardType: + | OptionalStandardTokenType.Other + | OptionalStandardTokenType.Comment + | OptionalStandardTokenType.String + | OptionalStandardTokenType.RegEx +): StandardTokenType { + return standardType as any as StandardTokenType; +} + +// Must have the same values as `StandardTokenType`! +export const enum OptionalStandardTokenType { + Other = 0, + Comment = 1, + String = 2, + RegEx = 3, + // Indicates that no token type is set. + NotSet = 8 +} diff --git a/src/grammar.ts b/src/grammar.ts deleted file mode 100644 index 8bde6d87..00000000 --- a/src/grammar.ts +++ /dev/null @@ -1,1767 +0,0 @@ -/*--------------------------------------------------------- - * Copyright (C) Microsoft Corporation. All rights reserved. - *--------------------------------------------------------*/ - -import { clone, mergeObjects } from './utils'; -import { IOnigLib, IOnigCaptureIndex, OnigString, OnigScanner, FindOption } from './onigLib'; -import { IRuleRegistry, IRuleFactoryHelper, RuleFactory, Rule, CaptureRule, BeginEndRule, BeginWhileRule, MatchRule, CompiledRule } from './rule'; -import { createMatchers, Matcher } from './matcher'; -import { IGrammar, ITokenizeLineResult, ITokenizeLineResult2, IToken, IEmbeddedLanguagesMap, StandardTokenType, StackElement as StackElementDef, ITokenTypeMap } from './main'; -import { DebugFlags, UseOnigurumaFindOptions } from './debug'; -import { FontStyle, ThemeTrieElementRule } from './theme'; -import { OptionalStandardTokenType, StackElementMetadata, toOptionalTokenType } from './metadata'; -import { IRawGrammar, IRawRule, IRawRepository } from './rawGrammar'; - -declare let performance: { now: () => number } | undefined; -const performanceNow = (function () { - if (typeof performance === 'undefined') { - // performance.now() is not available in this environment, so use Date.now() - return () => Date.now(); - } else { - return () => performance!.now(); - } -})(); - -export function createGrammar( - scopeName: string, - grammar: IRawGrammar, - initialLanguage: number, - embeddedLanguages: IEmbeddedLanguagesMap | null, - tokenTypes: ITokenTypeMap | null, - balancedBracketSelectors: BalancedBracketSelectors | null, - grammarRepository: IGrammarRepository & IThemeProvider, - onigLib: IOnigLib -): Grammar { - return new Grammar( - scopeName, - grammar, - initialLanguage, - embeddedLanguages, - tokenTypes, - balancedBracketSelectors, - grammarRepository, - onigLib - ); //TODO -} - -export interface IThemeProvider { - themeMatch(scopeName: string): ThemeTrieElementRule[]; - getDefaults(): ThemeTrieElementRule; -} - -export interface IGrammarRepository { - lookup(scopeName: string): IRawGrammar | undefined; - injections(scopeName: string): string[]; -} - -export interface IScopeNameSet { - [scopeName: string]: boolean; -} - -export class FullScopeDependency { - constructor( - public readonly scopeName: string - ) { } -} - -export class PartialScopeDependency { - constructor( - public readonly scopeName: string, - public readonly include: string - ) { } - - public toKey(): string { - return `${this.scopeName}#${this.include}`; - } -} - -export type ScopeDependency = FullScopeDependency | PartialScopeDependency; - -export class ScopeDependencyCollector { - - public readonly full: FullScopeDependency[]; - public readonly partial: PartialScopeDependency[]; - - public readonly visitedRule: Set; - private readonly _seenFull: Set; - private readonly _seenPartial: Set; - - constructor() { - this.full = []; - this.partial = []; - this.visitedRule = new Set(); - this._seenFull = new Set(); - this._seenPartial = new Set(); - } - - public add(dep: ScopeDependency): void { - if (dep instanceof FullScopeDependency) { - if (!this._seenFull.has(dep.scopeName)) { - this._seenFull.add(dep.scopeName); - this.full.push(dep); - } - } else { - if (!this._seenPartial.has(dep.toKey())) { - this._seenPartial.add(dep.toKey()); - this.partial.push(dep); - } - } - } -} - -/** - * Fill in `result` all external included scopes in `patterns` - */ -function _extractIncludedScopesInPatterns(result: ScopeDependencyCollector, baseGrammar: IRawGrammar, selfGrammar: IRawGrammar, patterns: IRawRule[], repository: IRawRepository | undefined): void { - for (const pattern of patterns) { - if (result.visitedRule.has(pattern)) { - continue; - } - result.visitedRule.add(pattern); - - const patternRepository = (pattern.repository ? mergeObjects({}, repository, pattern.repository) : repository); - - if (Array.isArray(pattern.patterns)) { - _extractIncludedScopesInPatterns(result, baseGrammar, selfGrammar, pattern.patterns, patternRepository); - } - - const include = pattern.include; - - if (!include) { - continue; - } - - if (include === '$base' || include === baseGrammar.scopeName) { - collectDependencies(result, baseGrammar, baseGrammar); - } else if (include === '$self' || include === selfGrammar.scopeName) { - collectDependencies(result, baseGrammar, selfGrammar); - } else if (include.charAt(0) === '#') { - collectSpecificDependencies(result, baseGrammar, selfGrammar, include.substring(1), patternRepository); - } else { - - const sharpIndex = include.indexOf('#'); - if (sharpIndex >= 0) { - const scopeName = include.substring(0, sharpIndex); - const includedName = include.substring(sharpIndex + 1); - if (scopeName === baseGrammar.scopeName) { - collectSpecificDependencies(result, baseGrammar, baseGrammar, includedName, patternRepository); - } else if (scopeName === selfGrammar.scopeName) { - collectSpecificDependencies(result, baseGrammar, selfGrammar, includedName, patternRepository); - } else { - result.add(new PartialScopeDependency(scopeName, include.substring(sharpIndex + 1))); - } - } else { - result.add(new FullScopeDependency(include)); - } - - } - } -} - -export class ScopeDependencyProcessor { - - public readonly seenFullScopeRequests = new Set(); - public readonly seenPartialScopeRequests = new Set(); - public Q: ScopeDependency[]; - - constructor( - public readonly repo: IGrammarRepository, - public readonly initialScopeName: string - ) { - this.seenFullScopeRequests.add(this.initialScopeName); - this.Q = [new FullScopeDependency(this.initialScopeName)]; - } - - public processQueue(): void { - const q = this.Q; - this.Q = []; - - const deps = new ScopeDependencyCollector(); - for (const dep of q) { - collectDependenciesForDep(this.repo, this.initialScopeName, deps, dep); - } - - for (const dep of deps.full) { - if (this.seenFullScopeRequests.has(dep.scopeName)) { - // already processed - continue; - } - this.seenFullScopeRequests.add(dep.scopeName); - this.Q.push(dep); - } - - for (const dep of deps.partial) { - if (this.seenFullScopeRequests.has(dep.scopeName)) { - // already processed in full - continue; - } - if (this.seenPartialScopeRequests.has(dep.toKey())) { - // already processed - continue; - } - this.seenPartialScopeRequests.add(dep.toKey()); - this.Q.push(dep); - } - } -} - -function collectDependenciesForDep(repo: IGrammarRepository, initialScopeName: string, result: ScopeDependencyCollector, dep: FullScopeDependency | PartialScopeDependency) { - const grammar = repo.lookup(dep.scopeName); - if (!grammar) { - if (dep.scopeName === initialScopeName) { - throw new Error(`No grammar provided for <${initialScopeName}>`); - } - return; - } - - if (dep instanceof FullScopeDependency) { - collectDependencies(result, repo.lookup(initialScopeName)!, grammar); - } else { - collectSpecificDependencies(result, repo.lookup(initialScopeName)!, grammar, dep.include); - } - - const injections = repo.injections(dep.scopeName); - if (injections) { - for (const injection of injections) { - result.add(new FullScopeDependency(injection)); - } - } -} - -/** - * Collect a specific dependency from the grammar's repository - */ -function collectSpecificDependencies(result: ScopeDependencyCollector, baseGrammar: IRawGrammar, selfGrammar: IRawGrammar, include: string, repository: IRawRepository | undefined = selfGrammar.repository): void { - if (repository && repository[include]) { - const rule = repository[include]; - _extractIncludedScopesInPatterns(result, baseGrammar, selfGrammar, [rule], repository); - } -} - -/** - * Collects the list of all external included scopes in `grammar`. - */ -function collectDependencies(result: ScopeDependencyCollector, baseGrammar: IRawGrammar, selfGrammar: IRawGrammar): void { - if (selfGrammar.patterns && Array.isArray(selfGrammar.patterns)) { - _extractIncludedScopesInPatterns(result, baseGrammar, selfGrammar, selfGrammar.patterns, selfGrammar.repository); - } - if (selfGrammar.injections) { - let injections: IRawRule[] = []; - for (let injection in selfGrammar.injections) { - injections.push(selfGrammar.injections[injection]); - } - _extractIncludedScopesInPatterns(result, baseGrammar, selfGrammar, injections, selfGrammar.repository); - } -} - -export interface Injection { - readonly debugSelector: string; - readonly matcher: Matcher; - readonly priority: -1 | 0 | 1; // 0 is the default. -1 for 'L' and 1 for 'R' - readonly ruleId: number; - readonly grammar: IRawGrammar; -} - -function scopesAreMatching(thisScopeName: string, scopeName: string): boolean { - if (!thisScopeName) { - return false; - } - if (thisScopeName === scopeName) { - return true; - } - const len = scopeName.length; - return thisScopeName.length > len && thisScopeName.substr(0, len) === scopeName && thisScopeName[len] === '.'; -} - -function nameMatcher(identifers: string[], scopes: string[]) { - if (scopes.length < identifers.length) { - return false; - } - let lastIndex = 0; - return identifers.every(identifier => { - for (let i = lastIndex; i < scopes.length; i++) { - if (scopesAreMatching(scopes[i], identifier)) { - lastIndex = i + 1; - return true; - } - } - return false; - }); -} - -function collectInjections(result: Injection[], selector: string, rule: IRawRule, ruleFactoryHelper: IRuleFactoryHelper, grammar: IRawGrammar): void { - const matchers = createMatchers(selector, nameMatcher); - const ruleId = RuleFactory.getCompiledRuleId(rule, ruleFactoryHelper, grammar.repository); - for (const matcher of matchers) { - result.push({ - debugSelector: selector, - matcher: matcher.matcher, - ruleId: ruleId, - grammar: grammar, - priority: matcher.priority - }); - } -} - -export class ScopeMetadata { - public readonly scopeName: string; - public readonly languageId: number; - public readonly tokenType: OptionalStandardTokenType; - public readonly themeData: ThemeTrieElementRule[] | null; - - constructor(scopeName: string, languageId: number, tokenType: OptionalStandardTokenType, themeData: ThemeTrieElementRule[] | null) { - this.scopeName = scopeName; - this.languageId = languageId; - this.tokenType = tokenType; - this.themeData = themeData; - } -} - -class ScopeMetadataProvider { - - private readonly _initialLanguage: number; - private readonly _themeProvider: IThemeProvider; - private _cache: Map; - private _defaultMetaData: ScopeMetadata; - private readonly _embeddedLanguages: IEmbeddedLanguagesMap; - private readonly _embeddedLanguagesRegex: RegExp | null; - - constructor(initialLanguage: number, themeProvider: IThemeProvider, embeddedLanguages: IEmbeddedLanguagesMap | null) { - this._initialLanguage = initialLanguage; - this._themeProvider = themeProvider; - this._cache = new Map(); - this._defaultMetaData = new ScopeMetadata( - '', - this._initialLanguage, - OptionalStandardTokenType.NotSet, - [this._themeProvider.getDefaults()] - ); - - // embeddedLanguages handling - this._embeddedLanguages = Object.create(null); - - if (embeddedLanguages) { - // If embeddedLanguages are configured, fill in `this._embeddedLanguages` - const scopes = Object.keys(embeddedLanguages); - for (let i = 0, len = scopes.length; i < len; i++) { - const scope = scopes[i]; - const language = embeddedLanguages[scope]; - if (typeof language !== 'number' || language === 0) { - console.warn('Invalid embedded language found at scope ' + scope + ': <<' + language + '>>'); - // never hurts to be too careful - continue; - } - this._embeddedLanguages[scope] = language; - } - } - - // create the regex - const escapedScopes = Object.keys(this._embeddedLanguages).map((scopeName) => ScopeMetadataProvider._escapeRegExpCharacters(scopeName)); - if (escapedScopes.length === 0) { - // no scopes registered - this._embeddedLanguagesRegex = null; - } else { - escapedScopes.sort(); - escapedScopes.reverse(); - this._embeddedLanguagesRegex = new RegExp(`^((${escapedScopes.join(')|(')}))($|\\.)`, ''); - } - } - - public onDidChangeTheme(): void { - this._cache = new Map(); - this._defaultMetaData = new ScopeMetadata( - '', - this._initialLanguage, - OptionalStandardTokenType.NotSet, - [this._themeProvider.getDefaults()] - ); - } - - public getDefaultMetadata(): ScopeMetadata { - return this._defaultMetaData; - } - - /** - * Escapes regular expression characters in a given string - */ - private static _escapeRegExpCharacters(value: string): string { - return value.replace(/[\-\\\{\}\*\+\?\|\^\$\.\,\[\]\(\)\#\s]/g, '\\$&'); - } - - private static _NULL_SCOPE_METADATA = new ScopeMetadata('', 0, 0, null); - public getMetadataForScope(scopeName: string | null): ScopeMetadata { - if (scopeName === null) { - return ScopeMetadataProvider._NULL_SCOPE_METADATA; - } - let value = this._cache.get(scopeName); - if (value) { - return value; - } - value = this._doGetMetadataForScope(scopeName); - this._cache.set(scopeName, value); - return value; - } - - private _doGetMetadataForScope(scopeName: string): ScopeMetadata { - const languageId = this._scopeToLanguage(scopeName); - const standardTokenType = this._toStandardTokenType(scopeName); - const themeData = this._themeProvider.themeMatch(scopeName); - - return new ScopeMetadata(scopeName, languageId, standardTokenType, themeData); - } - - /** - * Given a produced TM scope, return the language that token describes or null if unknown. - * e.g. source.html => html, source.css.embedded.html => css, punctuation.definition.tag.html => null - */ - private _scopeToLanguage(scope: string): number { - if (!scope) { - return 0; - } - if (!this._embeddedLanguagesRegex) { - // no scopes registered - return 0; - } - const m = scope.match(this._embeddedLanguagesRegex); - if (!m) { - // no scopes matched - return 0; - } - - const language = this._embeddedLanguages[m[1]] || 0; - if (!language) { - return 0; - } - - return language; - } - - private static STANDARD_TOKEN_TYPE_REGEXP = /\b(comment|string|regex|meta\.embedded)\b/; - private _toStandardTokenType(tokenType: string): OptionalStandardTokenType { - const m = tokenType.match(ScopeMetadataProvider.STANDARD_TOKEN_TYPE_REGEXP); - if (!m) { - return OptionalStandardTokenType.NotSet; - } - switch (m[1]) { - case 'comment': - return OptionalStandardTokenType.Comment; - case 'string': - return OptionalStandardTokenType.String; - case 'regex': - return OptionalStandardTokenType.RegEx; - case 'meta.embedded': - return OptionalStandardTokenType.Other; - } - throw new Error('Unexpected match for standard token type!'); - } -} - -export class Grammar implements IGrammar, IRuleFactoryHelper, IOnigLib { - - private readonly _scopeName: string; - private _rootId: number; - private _lastRuleId: number; - private readonly _ruleId2desc: Rule[]; - private readonly _includedGrammars: { [scopeName: string]: IRawGrammar; }; - private readonly _grammarRepository: IGrammarRepository; - private readonly _grammar: IRawGrammar; - private _injections: Injection[] | null; - private readonly _scopeMetadataProvider: ScopeMetadataProvider; - private readonly _tokenTypeMatchers: TokenTypeMatcher[]; - private readonly _onigLib: IOnigLib; - - constructor(scopeName: string, grammar: IRawGrammar, initialLanguage: number, embeddedLanguages: IEmbeddedLanguagesMap | null, tokenTypes: ITokenTypeMap | null, private readonly balancedBracketSelectors: BalancedBracketSelectors | null, grammarRepository: IGrammarRepository & IThemeProvider, onigLib: IOnigLib) { - this._scopeName = scopeName; - this._scopeMetadataProvider = new ScopeMetadataProvider(initialLanguage, grammarRepository, embeddedLanguages); - - this._onigLib = onigLib; - this._rootId = -1; - this._lastRuleId = 0; - this._ruleId2desc = [null!]; - this._includedGrammars = {}; - this._grammarRepository = grammarRepository; - this._grammar = initGrammar(grammar, null); - this._injections = null; - - this._tokenTypeMatchers = []; - if (tokenTypes) { - for (const selector of Object.keys(tokenTypes)) { - const matchers = createMatchers(selector, nameMatcher); - for (const matcher of matchers) { - this._tokenTypeMatchers.push({ - matcher: matcher.matcher, - type: tokenTypes[selector] - }); - } - } - } - } - - public dispose(): void { - for (const rule of this._ruleId2desc) { - if (rule) { - rule.dispose(); - } - } - } - - public createOnigScanner(sources: string[]): OnigScanner { - return this._onigLib.createOnigScanner(sources); - } - - public createOnigString(sources: string): OnigString { - return this._onigLib.createOnigString(sources); - } - - public onDidChangeTheme(): void { - this._scopeMetadataProvider.onDidChangeTheme(); - } - - public getMetadataForScope(scope: string): ScopeMetadata { - return this._scopeMetadataProvider.getMetadataForScope(scope); - } - - private _collectInjections(): Injection[] { - const grammarRepository: IGrammarRepository = { - lookup: (scopeName: string): IRawGrammar | undefined => { - if (scopeName === this._scopeName) { - return this._grammar; - } - return this.getExternalGrammar(scopeName); - }, - injections: (scopeName: string): string[] => { - return this._grammarRepository.injections(scopeName); - } - }; - - const dependencyProcessor = new ScopeDependencyProcessor(grammarRepository, this._scopeName); - // TODO: uncomment below to visit all scopes - // while (dependencyProcessor.Q.length > 0) { - // dependencyProcessor.processQueue(); - // } - - const result: Injection[] = []; - - dependencyProcessor.seenFullScopeRequests.forEach((scopeName) => { - const grammar = grammarRepository.lookup(scopeName); - if (!grammar) { - return; - } - - // add injections from the current grammar - const rawInjections = grammar.injections; - if (rawInjections) { - for (let expression in rawInjections) { - collectInjections(result, expression, rawInjections[expression], this, grammar); - } - } - - // add injection grammars contributed for the current scope - if (this._grammarRepository) { - const injectionScopeNames = this._grammarRepository.injections(scopeName); - if (injectionScopeNames) { - injectionScopeNames.forEach(injectionScopeName => { - const injectionGrammar = this.getExternalGrammar(injectionScopeName); - if (injectionGrammar) { - const selector = injectionGrammar.injectionSelector; - if (selector) { - collectInjections(result, selector, injectionGrammar, this, injectionGrammar); - } - } - }); - } - } - }); - - result.sort((i1, i2) => i1.priority - i2.priority); // sort by priority - - return result; - } - - public getInjections(): Injection[] { - if (this._injections === null) { - this._injections = this._collectInjections(); - - if (DebugFlags.InDebugMode && this._injections.length > 0) { - console.log(`Grammar ${this._scopeName} contains the following injections:`); - for (const injection of this._injections) { - console.log(` - ${injection.debugSelector}`); - } - } - } - return this._injections; - } - - public registerRule(factory: (id: number) => T): T { - const id = (++this._lastRuleId); - const result = factory(id); - this._ruleId2desc[id] = result; - return result; - } - - public getRule(patternId: number): Rule { - return this._ruleId2desc[patternId]; - } - - public getExternalGrammar(scopeName: string, repository?: IRawRepository): IRawGrammar | undefined { - if (this._includedGrammars[scopeName]) { - return this._includedGrammars[scopeName]; - } else if (this._grammarRepository) { - const rawIncludedGrammar = this._grammarRepository.lookup(scopeName); - if (rawIncludedGrammar) { - // console.log('LOADED GRAMMAR ' + pattern.include); - this._includedGrammars[scopeName] = initGrammar(rawIncludedGrammar, repository && repository.$base); - return this._includedGrammars[scopeName]; - } - } - return undefined; - } - - public tokenizeLine(lineText: string, prevState: StackElement | null, timeLimit: number = 0): ITokenizeLineResult { - const r = this._tokenize(lineText, prevState, false, timeLimit); - return { - tokens: r.lineTokens.getResult(r.ruleStack, r.lineLength), - ruleStack: r.ruleStack, - stoppedEarly: r.stoppedEarly - }; - } - - public tokenizeLine2(lineText: string, prevState: StackElement | null, timeLimit: number = 0): ITokenizeLineResult2 { - const r = this._tokenize(lineText, prevState, true, timeLimit); - return { - tokens: r.lineTokens.getBinaryResult(r.ruleStack, r.lineLength), - ruleStack: r.ruleStack, - stoppedEarly: r.stoppedEarly - }; - } - - private _tokenize(lineText: string, prevState: StackElement | null, emitBinaryTokens: boolean, timeLimit: number): { lineLength: number; lineTokens: LineTokens; ruleStack: StackElement; stoppedEarly: boolean; } { - if (this._rootId === -1) { - this._rootId = RuleFactory.getCompiledRuleId(this._grammar.repository.$self, this, this._grammar.repository); - } - - let isFirstLine: boolean; - if (!prevState || prevState === StackElement.NULL) { - isFirstLine = true; - const rawDefaultMetadata = this._scopeMetadataProvider.getDefaultMetadata(); - const defaultTheme = rawDefaultMetadata.themeData![0]; - const defaultMetadata = StackElementMetadata.set(0, rawDefaultMetadata.languageId, rawDefaultMetadata.tokenType, null, defaultTheme.fontStyle, defaultTheme.foreground, defaultTheme.background); - - const rootScopeName = this.getRule(this._rootId).getName(null, null); - const rawRootMetadata = this._scopeMetadataProvider.getMetadataForScope(rootScopeName); - const rootMetadata = ScopeListElement.mergeMetadata(defaultMetadata, null, rawRootMetadata); - - const scopeList = new ScopeListElement(null, rootScopeName === null ? 'unknown' : rootScopeName, rootMetadata); - - prevState = new StackElement(null, this._rootId, -1, -1, false, null, scopeList, scopeList); - } else { - isFirstLine = false; - prevState.reset(); - } - - lineText = lineText + '\n'; - const onigLineText = this.createOnigString(lineText); - const lineLength = onigLineText.content.length; - const lineTokens = new LineTokens(emitBinaryTokens, lineText, this._tokenTypeMatchers, this.balancedBracketSelectors); - const r = _tokenizeString(this, onigLineText, isFirstLine, 0, prevState, lineTokens, true, timeLimit); - - disposeOnigString(onigLineText); - - return { - lineLength: lineLength, - lineTokens: lineTokens, - ruleStack: r.stack, - stoppedEarly: r.stoppedEarly - }; - } -} - -function disposeOnigString(str: OnigString) { - if (typeof str.dispose === 'function') { - str.dispose(); - } -} - -function initGrammar(grammar: IRawGrammar, base: IRawRule | null | undefined): IRawGrammar { - grammar = clone(grammar); - - grammar.repository = grammar.repository || {}; - grammar.repository.$self = { - $vscodeTextmateLocation: grammar.$vscodeTextmateLocation, - patterns: grammar.patterns, - name: grammar.scopeName - }; - grammar.repository.$base = base || grammar.repository.$self; - return grammar; -} - -function handleCaptures(grammar: Grammar, lineText: OnigString, isFirstLine: boolean, stack: StackElement, lineTokens: LineTokens, captures: (CaptureRule | null)[], captureIndices: IOnigCaptureIndex[]): void { - if (captures.length === 0) { - return; - } - - const lineTextContent = lineText.content; - - const len = Math.min(captures.length, captureIndices.length); - const localStack: LocalStackElement[] = []; - const maxEnd = captureIndices[0].end; - - for (let i = 0; i < len; i++) { - const captureRule = captures[i]; - if (captureRule === null) { - // Not interested - continue; - } - - const captureIndex = captureIndices[i]; - - if (captureIndex.length === 0) { - // Nothing really captured - continue; - } - - if (captureIndex.start > maxEnd) { - // Capture going beyond consumed string - break; - } - - // pop captures while needed - while (localStack.length > 0 && localStack[localStack.length - 1].endPos <= captureIndex.start) { - // pop! - lineTokens.produceFromScopes(localStack[localStack.length - 1].scopes, localStack[localStack.length - 1].endPos); - localStack.pop(); - } - - if (localStack.length > 0) { - lineTokens.produceFromScopes(localStack[localStack.length - 1].scopes, captureIndex.start); - } else { - lineTokens.produce(stack, captureIndex.start); - } - - if (captureRule.retokenizeCapturedWithRuleId) { - // the capture requires additional matching - const scopeName = captureRule.getName(lineTextContent, captureIndices); - const nameScopesList = stack.contentNameScopesList.push(grammar, scopeName); - const contentName = captureRule.getContentName(lineTextContent, captureIndices); - const contentNameScopesList = nameScopesList.push(grammar, contentName); - - const stackClone = stack.push(captureRule.retokenizeCapturedWithRuleId, captureIndex.start, -1, false, null, nameScopesList, contentNameScopesList); - const onigSubStr = grammar.createOnigString(lineTextContent.substring(0, captureIndex.end)); - _tokenizeString(grammar, onigSubStr, (isFirstLine && captureIndex.start === 0), captureIndex.start, stackClone, lineTokens, false, /* no time limit */0); - disposeOnigString(onigSubStr); - continue; - } - - const captureRuleScopeName = captureRule.getName(lineTextContent, captureIndices); - if (captureRuleScopeName !== null) { - // push - const base = localStack.length > 0 ? localStack[localStack.length - 1].scopes : stack.contentNameScopesList; - const captureRuleScopesList = base.push(grammar, captureRuleScopeName); - localStack.push(new LocalStackElement(captureRuleScopesList, captureIndex.end)); - } - } - - while (localStack.length > 0) { - // pop! - lineTokens.produceFromScopes(localStack[localStack.length - 1].scopes, localStack[localStack.length - 1].endPos); - localStack.pop(); - } -} - -interface IMatchInjectionsResult { - readonly priorityMatch: boolean; - readonly captureIndices: IOnigCaptureIndex[]; - readonly matchedRuleId: number; -} - -function debugCompiledRuleToString(ruleScanner: CompiledRule): string { - const r: string[] = []; - for (let i = 0, len = ruleScanner.rules.length; i < len; i++) { - r.push(' - ' + ruleScanner.rules[i] + ': ' + ruleScanner.debugRegExps[i]); - } - return r.join('\n'); -} - -function getFindOptions(allowA: boolean, allowG: boolean): number { - let options = FindOption.None; - if (!allowA) { - options |= FindOption.NotBeginString; - } - if (!allowG) { - options |= FindOption.NotBeginPosition; - } - return options; -} - -function prepareRuleSearch(rule: Rule, grammar: Grammar, endRegexSource: string | null, allowA: boolean, allowG: boolean): { ruleScanner: CompiledRule; findOptions: number; } { - if (UseOnigurumaFindOptions) { - const ruleScanner = rule.compile(grammar, endRegexSource); - const findOptions = getFindOptions(allowA, allowG); - return { ruleScanner, findOptions }; - } - const ruleScanner = rule.compileAG(grammar, endRegexSource, allowA, allowG); - return { ruleScanner, findOptions: FindOption.None }; -} - -function prepareRuleWhileSearch(rule: BeginWhileRule, grammar: Grammar, endRegexSource: string | null, allowA: boolean, allowG: boolean): { ruleScanner: CompiledRule; findOptions: number; } { - if (UseOnigurumaFindOptions) { - const ruleScanner = rule.compileWhile(grammar, endRegexSource); - const findOptions = getFindOptions(allowA, allowG); - return { ruleScanner, findOptions }; - } - const ruleScanner = rule.compileWhileAG(grammar, endRegexSource, allowA, allowG); - return { ruleScanner, findOptions: FindOption.None }; -} - -function matchInjections(injections: Injection[], grammar: Grammar, lineText: OnigString, isFirstLine: boolean, linePos: number, stack: StackElement, anchorPosition: number): IMatchInjectionsResult | null { - // The lower the better - let bestMatchRating = Number.MAX_VALUE; - let bestMatchCaptureIndices: IOnigCaptureIndex[] | null = null; - let bestMatchRuleId: number; - let bestMatchResultPriority: number = 0; - - const scopes = stack.contentNameScopesList.generateScopes(); - - for (let i = 0, len = injections.length; i < len; i++) { - const injection = injections[i]; - if (!injection.matcher(scopes)) { - // injection selector doesn't match stack - continue; - } - const rule = grammar.getRule(injection.ruleId); - const { ruleScanner, findOptions } = prepareRuleSearch(rule, grammar, null, isFirstLine, linePos === anchorPosition); - const matchResult = ruleScanner.scanner.findNextMatchSync(lineText, linePos, findOptions); - if (!matchResult) { - continue; - } - - if (DebugFlags.InDebugMode) { - console.log(` matched injection: ${injection.debugSelector}`); - console.log(debugCompiledRuleToString(ruleScanner)); - } - - const matchRating = matchResult.captureIndices[0].start; - if (matchRating >= bestMatchRating) { - // Injections are sorted by priority, so the previous injection had a better or equal priority - continue; - } - - bestMatchRating = matchRating; - bestMatchCaptureIndices = matchResult.captureIndices; - bestMatchRuleId = ruleScanner.rules[matchResult.index]; - bestMatchResultPriority = injection.priority; - - if (bestMatchRating === linePos) { - // No more need to look at the rest of the injections. - break; - } - } - - if (bestMatchCaptureIndices) { - return { - priorityMatch: bestMatchResultPriority === -1, - captureIndices: bestMatchCaptureIndices, - matchedRuleId: bestMatchRuleId! - }; - } - - return null; -} - -interface IMatchResult { - readonly captureIndices: IOnigCaptureIndex[]; - readonly matchedRuleId: number; -} - -function matchRule(grammar: Grammar, lineText: OnigString, isFirstLine: boolean, linePos: number, stack: StackElement, anchorPosition: number): IMatchResult | null { - const rule = stack.getRule(grammar); - const { ruleScanner, findOptions } = prepareRuleSearch(rule, grammar, stack.endRule, isFirstLine, linePos === anchorPosition); - - let perfStart = 0; - if (DebugFlags.InDebugMode) { - perfStart = performanceNow(); - } - - const r = ruleScanner.scanner.findNextMatchSync(lineText, linePos, findOptions); - - if (DebugFlags.InDebugMode) { - const elapsedMillis = performanceNow() - perfStart; - if (elapsedMillis > 5) { - console.warn(`Rule ${rule.debugName} (${rule.id}) matching took ${elapsedMillis} against '${lineText}'`); - } - console.log(` scanning for (linePos: ${linePos}, anchorPosition: ${anchorPosition})`); - console.log(debugCompiledRuleToString(ruleScanner)); - if (r) { - console.log(`matched rule id: ${ruleScanner.rules[r.index]} from ${r.captureIndices[0].start} to ${r.captureIndices[0].end}`); - } - } - - if (r) { - return { - captureIndices: r.captureIndices, - matchedRuleId: ruleScanner.rules[r.index] - }; - } - return null; -} - -function matchRuleOrInjections(grammar: Grammar, lineText: OnigString, isFirstLine: boolean, linePos: number, stack: StackElement, anchorPosition: number): IMatchResult | null { - // Look for normal grammar rule - const matchResult = matchRule(grammar, lineText, isFirstLine, linePos, stack, anchorPosition); - - // Look for injected rules - const injections = grammar.getInjections(); - if (injections.length === 0) { - // No injections whatsoever => early return - return matchResult; - } - - const injectionResult = matchInjections(injections, grammar, lineText, isFirstLine, linePos, stack, anchorPosition); - if (!injectionResult) { - // No injections matched => early return - return matchResult; - } - - if (!matchResult) { - // Only injections matched => early return - return injectionResult; - } - - // Decide if `matchResult` or `injectionResult` should win - const matchResultScore = matchResult.captureIndices[0].start; - const injectionResultScore = injectionResult.captureIndices[0].start; - - if (injectionResultScore < matchResultScore || (injectionResult.priorityMatch && injectionResultScore === matchResultScore)) { - // injection won! - return injectionResult; - } - return matchResult; -} - -interface IWhileStack { - readonly stack: StackElement; - readonly rule: BeginWhileRule; -} - -interface IWhileCheckResult { - readonly stack: StackElement; - readonly linePos: number; - readonly anchorPosition: number; - readonly isFirstLine: boolean; -} - -/** - * Walk the stack from bottom to top, and check each while condition in this order. - * If any fails, cut off the entire stack above the failed while condition. While conditions - * may also advance the linePosition. - */ -function _checkWhileConditions(grammar: Grammar, lineText: OnigString, isFirstLine: boolean, linePos: number, stack: StackElement, lineTokens: LineTokens): IWhileCheckResult { - let anchorPosition = (stack.beginRuleCapturedEOL ? 0 : -1); - const whileRules: IWhileStack[] = []; - for (let node: StackElement | null = stack; node; node = node.pop()) { - const nodeRule = node.getRule(grammar); - if (nodeRule instanceof BeginWhileRule) { - whileRules.push({ - rule: nodeRule, - stack: node - }); - } - } - - for (let whileRule = whileRules.pop(); whileRule; whileRule = whileRules.pop()) { - const { ruleScanner, findOptions } = prepareRuleWhileSearch(whileRule.rule, grammar, whileRule.stack.endRule, isFirstLine, linePos === anchorPosition); - const r = ruleScanner.scanner.findNextMatchSync(lineText, linePos, findOptions); - if (DebugFlags.InDebugMode) { - console.log(' scanning for while rule'); - console.log(debugCompiledRuleToString(ruleScanner)); - } - - if (r) { - const matchedRuleId = ruleScanner.rules[r.index]; - if (matchedRuleId !== -2) { - // we shouldn't end up here - stack = whileRule.stack.pop()!; - break; - } - if (r.captureIndices && r.captureIndices.length) { - lineTokens.produce(whileRule.stack, r.captureIndices[0].start); - handleCaptures(grammar, lineText, isFirstLine, whileRule.stack, lineTokens, whileRule.rule.whileCaptures, r.captureIndices); - lineTokens.produce(whileRule.stack, r.captureIndices[0].end); - anchorPosition = r.captureIndices[0].end; - if (r.captureIndices[0].end > linePos) { - linePos = r.captureIndices[0].end; - isFirstLine = false; - } - } - } else { - if (DebugFlags.InDebugMode) { - console.log(' popping ' + whileRule.rule.debugName + ' - ' + whileRule.rule.debugWhileRegExp); - } - - stack = whileRule.stack.pop()!; - break; - } - } - - return { stack: stack, linePos: linePos, anchorPosition: anchorPosition, isFirstLine: isFirstLine }; -} - -class TokenizeStringResult { - constructor( - public readonly stack: StackElement, - public readonly stoppedEarly: boolean - ) { } -} - -/** - * Tokenize a string - * @param grammar - * @param lineText - * @param isFirstLine - * @param linePos - * @param stack - * @param lineTokens - * @param checkWhileConditions - * @param timeLimit Use `0` to indicate no time limit - * @returns the StackElement or StackElement.TIME_LIMIT_REACHED if the time limit has been reached - */ -function _tokenizeString(grammar: Grammar, lineText: OnigString, isFirstLine: boolean, linePos: number, stack: StackElement, lineTokens: LineTokens, checkWhileConditions: boolean, timeLimit: number): TokenizeStringResult { - const lineLength = lineText.content.length; - - let STOP = false; - let anchorPosition = -1; - - if (checkWhileConditions) { - const whileCheckResult = _checkWhileConditions(grammar, lineText, isFirstLine, linePos, stack, lineTokens); - stack = whileCheckResult.stack; - linePos = whileCheckResult.linePos; - isFirstLine = whileCheckResult.isFirstLine; - anchorPosition = whileCheckResult.anchorPosition; - } - - const startTime = Date.now(); - while (!STOP) { - if (timeLimit !== 0) { - const elapsedTime = Date.now() - startTime; - if (elapsedTime > timeLimit) { - return new TokenizeStringResult(stack, true); - } - } - scanNext(); // potentially modifies linePos && anchorPosition - } - - return new TokenizeStringResult(stack, false); - - function scanNext(): void { - if (DebugFlags.InDebugMode) { - console.log(''); - console.log(`@@scanNext ${linePos}: |${lineText.content.substr(linePos).replace(/\n$/, '\\n')}|`); - } - const r = matchRuleOrInjections(grammar, lineText, isFirstLine, linePos, stack, anchorPosition); - - if (!r) { - if (DebugFlags.InDebugMode) { - console.log(' no more matches.'); - } - // No match - lineTokens.produce(stack, lineLength); - STOP = true; - return; - } - - const captureIndices: IOnigCaptureIndex[] = r.captureIndices; - const matchedRuleId: number = r.matchedRuleId; - - const hasAdvanced = (captureIndices && captureIndices.length > 0) ? (captureIndices[0].end > linePos) : false; - - if (matchedRuleId === -1) { - // We matched the `end` for this rule => pop it - const poppedRule = stack.getRule(grammar); - - if (DebugFlags.InDebugMode) { - console.log(' popping ' + poppedRule.debugName + ' - ' + poppedRule.debugEndRegExp); - } - - lineTokens.produce(stack, captureIndices[0].start); - stack = stack.setContentNameScopesList(stack.nameScopesList); - handleCaptures(grammar, lineText, isFirstLine, stack, lineTokens, poppedRule.endCaptures, captureIndices); - lineTokens.produce(stack, captureIndices[0].end); - - // pop - const popped = stack; - stack = stack.pop()!; - anchorPosition = popped.getAnchorPos(); - - if (!hasAdvanced && popped.getEnterPos() === linePos) { - // Grammar pushed & popped a rule without advancing - if (DebugFlags.InDebugMode) { - console.error('[1] - Grammar is in an endless loop - Grammar pushed & popped a rule without advancing'); - } - - // See https://github.com/Microsoft/vscode-textmate/issues/12 - // Let's assume this was a mistake by the grammar author and the intent was to continue in this state - stack = popped; - - lineTokens.produce(stack, lineLength); - STOP = true; - return; - } - } else { - // We matched a rule! - const _rule = grammar.getRule(matchedRuleId); - - lineTokens.produce(stack, captureIndices[0].start); - - const beforePush = stack; - // push it on the stack rule - const scopeName = _rule.getName(lineText.content, captureIndices); - const nameScopesList = stack.contentNameScopesList.push(grammar, scopeName); - stack = stack.push(matchedRuleId, linePos, anchorPosition, captureIndices[0].end === lineLength, null, nameScopesList, nameScopesList); - - if (_rule instanceof BeginEndRule) { - const pushedRule = _rule; - if (DebugFlags.InDebugMode) { - console.log(' pushing ' + pushedRule.debugName + ' - ' + pushedRule.debugBeginRegExp); - } - - handleCaptures(grammar, lineText, isFirstLine, stack, lineTokens, pushedRule.beginCaptures, captureIndices); - lineTokens.produce(stack, captureIndices[0].end); - anchorPosition = captureIndices[0].end; - - const contentName = pushedRule.getContentName(lineText.content, captureIndices); - const contentNameScopesList = nameScopesList.push(grammar, contentName); - stack = stack.setContentNameScopesList(contentNameScopesList); - - if (pushedRule.endHasBackReferences) { - stack = stack.setEndRule(pushedRule.getEndWithResolvedBackReferences(lineText.content, captureIndices)); - } - - if (!hasAdvanced && beforePush.hasSameRuleAs(stack)) { - // Grammar pushed the same rule without advancing - if (DebugFlags.InDebugMode) { - console.error('[2] - Grammar is in an endless loop - Grammar pushed the same rule without advancing'); - } - stack = stack.pop()!; - lineTokens.produce(stack, lineLength); - STOP = true; - return; - } - } else if (_rule instanceof BeginWhileRule) { - const pushedRule = _rule; - if (DebugFlags.InDebugMode) { - console.log(' pushing ' + pushedRule.debugName); - } - - handleCaptures(grammar, lineText, isFirstLine, stack, lineTokens, pushedRule.beginCaptures, captureIndices); - lineTokens.produce(stack, captureIndices[0].end); - anchorPosition = captureIndices[0].end; - const contentName = pushedRule.getContentName(lineText.content, captureIndices); - const contentNameScopesList = nameScopesList.push(grammar, contentName); - stack = stack.setContentNameScopesList(contentNameScopesList); - - if (pushedRule.whileHasBackReferences) { - stack = stack.setEndRule(pushedRule.getWhileWithResolvedBackReferences(lineText.content, captureIndices)); - } - - if (!hasAdvanced && beforePush.hasSameRuleAs(stack)) { - // Grammar pushed the same rule without advancing - if (DebugFlags.InDebugMode) { - console.error('[3] - Grammar is in an endless loop - Grammar pushed the same rule without advancing'); - } - stack = stack.pop()!; - lineTokens.produce(stack, lineLength); - STOP = true; - return; - } - } else { - const matchingRule = _rule; - if (DebugFlags.InDebugMode) { - console.log(' matched ' + matchingRule.debugName + ' - ' + matchingRule.debugMatchRegExp); - } - - handleCaptures(grammar, lineText, isFirstLine, stack, lineTokens, matchingRule.captures, captureIndices); - lineTokens.produce(stack, captureIndices[0].end); - - // pop rule immediately since it is a MatchRule - stack = stack.pop()!; - - if (!hasAdvanced) { - // Grammar is not advancing, nor is it pushing/popping - if (DebugFlags.InDebugMode) { - console.error('[4] - Grammar is in an endless loop - Grammar is not advancing, nor is it pushing/popping'); - } - stack = stack.safePop(); - lineTokens.produce(stack, lineLength); - STOP = true; - return; - } - } - } - - if (captureIndices[0].end > linePos) { - // Advance stream - linePos = captureIndices[0].end; - isFirstLine = false; - } - } -} - -export class ScopeListElement { - - public readonly parent: ScopeListElement | null; - public readonly scope: string; - public readonly metadata: number; - - constructor(parent: ScopeListElement | null, scope: string, metadata: number) { - this.parent = parent; - this.scope = scope; - this.metadata = metadata; - } - - private static _equals(a: ScopeListElement | null, b: ScopeListElement | null): boolean { - do { - if (a === b) { - return true; - } - - if (!a && !b) { - // End of list reached for both - return true; - } - - if (!a || !b) { - // End of list reached only for one - return false; - } - - if (a.scope !== b.scope || a.metadata !== b.metadata) { - return false; - } - - // Go to previous pair - a = a.parent; - b = b.parent; - } while (true); - } - - public equals(other: ScopeListElement): boolean { - return ScopeListElement._equals(this, other); - } - - private static _matchesScope(scope: string, selector: string, selectorWithDot: string): boolean { - return (selector === scope || scope.substring(0, selectorWithDot.length) === selectorWithDot); - } - - private static _matches(target: ScopeListElement | null, parentScopes: string[] | null): boolean { - if (parentScopes === null) { - return true; - } - - const len = parentScopes.length; - let index = 0; - let selector = parentScopes[index]; - let selectorWithDot = selector + '.'; - - while (target) { - if (this._matchesScope(target.scope, selector, selectorWithDot)) { - index++; - if (index === len) { - return true; - } - selector = parentScopes[index]; - selectorWithDot = selector + '.'; - } - target = target.parent; - } - - return false; - } - - public static mergeMetadata(metadata: number, scopesList: ScopeListElement | null, source: ScopeMetadata): number { - if (source === null) { - return metadata; - } - - let fontStyle = FontStyle.NotSet; - let foreground = 0; - let background = 0; - - if (source.themeData !== null) { - // Find the first themeData that matches - for (let i = 0, len = source.themeData.length; i < len; i++) { - const themeData = source.themeData[i]; - - if (this._matches(scopesList, themeData.parentScopes)) { - fontStyle = themeData.fontStyle; - foreground = themeData.foreground; - background = themeData.background; - break; - } - } - } - - return StackElementMetadata.set(metadata, source.languageId, source.tokenType, null, fontStyle, foreground, background); - } - - private static _push(target: ScopeListElement, grammar: Grammar, scopes: string[]): ScopeListElement { - for (let i = 0, len = scopes.length; i < len; i++) { - const scope = scopes[i]; - const rawMetadata = grammar.getMetadataForScope(scope); - const metadata = ScopeListElement.mergeMetadata(target.metadata, target, rawMetadata); - target = new ScopeListElement(target, scope, metadata); - } - return target; - } - - public push(grammar: Grammar, scope: string | null): ScopeListElement { - if (scope === null) { - return this; - } - if (scope.indexOf(' ') >= 0) { - // there are multiple scopes to push - return ScopeListElement._push(this, grammar, scope.split(/ /g)); - } - // there is a single scope to push - return ScopeListElement._push(this, grammar, [scope]); - } - - private static _generateScopes(scopesList: ScopeListElement | null): string[] { - const result: string[] = []; - let resultLen = 0; - while (scopesList) { - result[resultLen++] = scopesList.scope; - scopesList = scopesList.parent; - } - result.reverse(); - return result; - } - - public generateScopes(): string[] { - return ScopeListElement._generateScopes(this); - } -} - -/** - * Represents a "pushed" state on the stack (as a linked list element). - */ -export class StackElement implements StackElementDef { - _stackElementBrand: void = undefined; - - public static NULL = new StackElement(null, 0, 0, 0, false, null, null!, null!); - - /** - * The position on the current line where this state was pushed. - * This is relevant only while tokenizing a line, to detect endless loops. - * Its value is meaningless across lines. - */ - private _enterPos: number; - - /** - * The captured anchor position when this stack element was pushed. - * This is relevant only while tokenizing a line, to restore the anchor position when popping. - * Its value is meaningless across lines. - */ - private _anchorPos: number; - - /** - * The previous state on the stack (or null for the root state). - */ - public readonly parent: StackElement | null; - /** - * The depth of the stack. - */ - public readonly depth: number; - - /** - * The state (rule) that this element represents. - */ - public readonly ruleId: number; - /** - * The state has entered and captured \n. This means that the next line should have an anchorPosition of 0. - */ - public readonly beginRuleCapturedEOL: boolean; - /** - * The "pop" (end) condition for this state in case that it was dynamically generated through captured text. - */ - public readonly endRule: string | null; - /** - * The list of scopes containing the "name" for this state. - */ - public readonly nameScopesList: ScopeListElement; - /** - * The list of scopes containing the "contentName" (besides "name") for this state. - * This list **must** contain as an element `scopeName`. - */ - public readonly contentNameScopesList: ScopeListElement; - - constructor(parent: StackElement | null, ruleId: number, enterPos: number, anchorPos: number, beginRuleCapturedEOL: boolean, endRule: string | null, nameScopesList: ScopeListElement, contentNameScopesList: ScopeListElement) { - this.parent = parent; - this.depth = (this.parent ? this.parent.depth + 1 : 1); - this.ruleId = ruleId; - this._enterPos = enterPos; - this._anchorPos = anchorPos; - this.beginRuleCapturedEOL = beginRuleCapturedEOL; - this.endRule = endRule; - this.nameScopesList = nameScopesList; - this.contentNameScopesList = contentNameScopesList; - } - - /** - * A structural equals check. Does not take into account `scopes`. - */ - private static _structuralEquals(a: StackElement | null, b: StackElement | null): boolean { - do { - if (a === b) { - return true; - } - - if (!a && !b) { - // End of list reached for both - return true; - } - - if (!a || !b) { - // End of list reached only for one - return false; - } - - if (a.depth !== b.depth || a.ruleId !== b.ruleId || a.endRule !== b.endRule) { - return false; - } - - // Go to previous pair - a = a.parent; - b = b.parent; - } while (true); - } - - private static _equals(a: StackElement, b: StackElement): boolean { - if (a === b) { - return true; - } - if (!this._structuralEquals(a, b)) { - return false; - } - return a.contentNameScopesList.equals(b.contentNameScopesList); - } - - public clone(): StackElement { - return this; - } - - public equals(other: StackElement): boolean { - if (other === null) { - return false; - } - return StackElement._equals(this, other); - } - - private static _reset(el: StackElement | null): void { - while (el) { - el._enterPos = -1; - el._anchorPos = -1; - el = el.parent; - } - } - - public reset(): void { - StackElement._reset(this); - } - - public pop(): StackElement | null { - return this.parent; - } - - public safePop(): StackElement { - if (this.parent) { - return this.parent; - } - return this; - } - - public push(ruleId: number, enterPos: number, anchorPos: number, beginRuleCapturedEOL: boolean, endRule: string | null, nameScopesList: ScopeListElement, contentNameScopesList: ScopeListElement): StackElement { - return new StackElement(this, ruleId, enterPos, anchorPos, beginRuleCapturedEOL, endRule, nameScopesList, contentNameScopesList); - } - - public getEnterPos(): number { - return this._enterPos; - } - - public getAnchorPos(): number { - return this._anchorPos; - } - - public getRule(grammar: IRuleRegistry): Rule { - return grammar.getRule(this.ruleId); - } - - private _writeString(res: string[], outIndex: number): number { - if (this.parent) { - outIndex = this.parent._writeString(res, outIndex); - } - - res[outIndex++] = `(${this.ruleId}, TODO-${this.nameScopesList}, TODO-${this.contentNameScopesList})`; - - return outIndex; - } - - public toString(): string { - const r: string[] = []; - this._writeString(r, 0); - return '[' + r.join(',') + ']'; - } - - public setContentNameScopesList(contentNameScopesList: ScopeListElement): StackElement { - if (this.contentNameScopesList === contentNameScopesList) { - return this; - } - return this.parent!.push(this.ruleId, this._enterPos, this._anchorPos, this.beginRuleCapturedEOL, this.endRule, this.nameScopesList, contentNameScopesList); - } - - public setEndRule(endRule: string): StackElement { - if (this.endRule === endRule) { - return this; - } - return new StackElement(this.parent, this.ruleId, this._enterPos, this._anchorPos, this.beginRuleCapturedEOL, endRule, this.nameScopesList, this.contentNameScopesList); - } - - public hasSameRuleAs(other: StackElement): boolean { - let el: StackElement | null = this; - while (el && el._enterPos === other._enterPos) { - if (el.ruleId === other.ruleId) { - return true; - } - el = el.parent; - } - return false; - } -} - -export class LocalStackElement { - public readonly scopes: ScopeListElement; - public readonly endPos: number; - - constructor(scopes: ScopeListElement, endPos: number) { - this.scopes = scopes; - this.endPos = endPos; - } -} - -interface TokenTypeMatcher { - readonly matcher: Matcher; - readonly type: StandardTokenType; -} - -export class BalancedBracketSelectors { - private readonly balancedBracketScopes: Matcher[]; - private readonly unbalancedBracketScopes: Matcher[]; - - private allowAny = false; - - constructor( - balancedBracketScopes: string[], - unbalancedBracketScopes: string[], - ) { - this.balancedBracketScopes = balancedBracketScopes.flatMap((selector) => { - if (selector === '*') { - this.allowAny = true; - return []; - } - return createMatchers(selector, nameMatcher).map((m) => m.matcher); - } - ); - this.unbalancedBracketScopes = unbalancedBracketScopes.flatMap((selector) => - createMatchers(selector, nameMatcher).map((m) => m.matcher) - ); - } - - public get matchesAlways(): boolean { - return this.allowAny && this.unbalancedBracketScopes.length === 0; - } - - public get matchesNever(): boolean { - return this.balancedBracketScopes.length === 0 && !this.allowAny; - } - - public match(scopes: string[]): boolean { - for (const excluder of this.unbalancedBracketScopes) { - if (excluder(scopes)) { - return false; - } - } - - for (const includer of this.balancedBracketScopes) { - if (includer(scopes)) { - return true; - } - } - return this.allowAny; - } -} - -class LineTokens { - private readonly _emitBinaryTokens: boolean; - /** - * defined only if `DebugFlags.InDebugMode`. - */ - private readonly _lineText: string | null; - /** - * used only if `_emitBinaryTokens` is false. - */ - private readonly _tokens: IToken[]; - /** - * used only if `_emitBinaryTokens` is true. - */ - private readonly _binaryTokens: number[]; - - private _lastTokenEndIndex: number; - - private readonly _tokenTypeOverrides: TokenTypeMatcher[]; - - constructor( - emitBinaryTokens: boolean, - lineText: string, - tokenTypeOverrides: TokenTypeMatcher[], - private readonly balancedBracketSelectors: BalancedBracketSelectors | null, - ) { - this._emitBinaryTokens = emitBinaryTokens; - this._tokenTypeOverrides = tokenTypeOverrides; - if (DebugFlags.InDebugMode) { - this._lineText = lineText; - } else { - this._lineText = null; - } - this._tokens = []; - this._binaryTokens = []; - this._lastTokenEndIndex = 0; - } - - public produce(stack: StackElement, endIndex: number): void { - this.produceFromScopes(stack.contentNameScopesList, endIndex); - } - - public produceFromScopes( - scopesList: ScopeListElement, - endIndex: number - ): void { - if (this._lastTokenEndIndex >= endIndex) { - return; - } - - if (this._emitBinaryTokens) { - let metadata = scopesList.metadata; - let containsBalancedBrackets = false; - if (this.balancedBracketSelectors?.matchesAlways) { - containsBalancedBrackets = true; - } - - if (this._tokenTypeOverrides.length > 0 || (this.balancedBracketSelectors && !this.balancedBracketSelectors.matchesAlways && !this.balancedBracketSelectors.matchesNever)) { - // Only generate scope array when required to improve performance - const scopes = scopesList.generateScopes(); - for (const tokenType of this._tokenTypeOverrides) { - if (tokenType.matcher(scopes)) { - metadata = StackElementMetadata.set( - metadata, - 0, - toOptionalTokenType(tokenType.type), - null, - FontStyle.NotSet, - 0, - 0 - ); - } - } - if (this.balancedBracketSelectors) { - containsBalancedBrackets = this.balancedBracketSelectors.match(scopes); - } - } - - if (containsBalancedBrackets) { - metadata = StackElementMetadata.set( - metadata, - 0, - OptionalStandardTokenType.NotSet, - containsBalancedBrackets, - FontStyle.NotSet, - 0, - 0 - ); - } - - if (this._binaryTokens.length > 0 && this._binaryTokens[this._binaryTokens.length - 1] === metadata) { - // no need to push a token with the same metadata - this._lastTokenEndIndex = endIndex; - return; - } - - if (DebugFlags.InDebugMode) { - const scopes = scopesList.generateScopes(); - console.log(' token: |' + this._lineText!.substring(this._lastTokenEndIndex, endIndex).replace(/\n$/, '\\n') + '|'); - for (let k = 0; k < scopes.length; k++) { - console.log(' * ' + scopes[k]); - } - } - - this._binaryTokens.push(this._lastTokenEndIndex); - this._binaryTokens.push(metadata); - - this._lastTokenEndIndex = endIndex; - return; - } - - const scopes = scopesList.generateScopes(); - - if (DebugFlags.InDebugMode) { - console.log(' token: |' + this._lineText!.substring(this._lastTokenEndIndex, endIndex).replace(/\n$/, '\\n') + '|'); - for (let k = 0; k < scopes.length; k++) { - console.log(' * ' + scopes[k]); - } - } - - this._tokens.push({ - startIndex: this._lastTokenEndIndex, - endIndex: endIndex, - // value: lineText.substring(lastTokenEndIndex, endIndex), - scopes: scopes - }); - - this._lastTokenEndIndex = endIndex; - } - - public getResult(stack: StackElement, lineLength: number): IToken[] { - if (this._tokens.length > 0 && this._tokens[this._tokens.length - 1].startIndex === lineLength - 1) { - // pop produced token for newline - this._tokens.pop(); - } - - if (this._tokens.length === 0) { - this._lastTokenEndIndex = -1; - this.produce(stack, lineLength); - this._tokens[this._tokens.length - 1].startIndex = 0; - } - - return this._tokens; - } - - public getBinaryResult(stack: StackElement, lineLength: number): Uint32Array { - if (this._binaryTokens.length > 0 && this._binaryTokens[this._binaryTokens.length - 2] === lineLength - 1) { - // pop produced token for newline - this._binaryTokens.pop(); - this._binaryTokens.pop(); - } - - if (this._binaryTokens.length === 0) { - this._lastTokenEndIndex = -1; - this.produce(stack, lineLength); - this._binaryTokens[this._binaryTokens.length - 2] = 0; - } - - const result = new Uint32Array(this._binaryTokens.length); - for (let i = 0, len = this._binaryTokens.length; i < len; i++) { - result[i] = this._binaryTokens[i]; - } - - return result; - } -} - diff --git a/src/grammar/basicScopesAttributeProvider.ts b/src/grammar/basicScopesAttributeProvider.ts new file mode 100644 index 00000000..5d32c45f --- /dev/null +++ b/src/grammar/basicScopesAttributeProvider.ts @@ -0,0 +1,111 @@ +/*--------------------------------------------------------- + * Copyright (C) Microsoft Corporation. All rights reserved. + *--------------------------------------------------------*/ + +import { OptionalStandardTokenType } from "../encodedTokenAttributes"; +import { IEmbeddedLanguagesMap } from "../main"; +import { ScopeName } from "../theme"; +import { CachedFn, escapeRegExpCharacters } from "../utils"; + +export class BasicScopeAttributes { + constructor( + public readonly languageId: number, + public readonly tokenType: OptionalStandardTokenType + ) { + } +} + +export class BasicScopeAttributesProvider { + private readonly _defaultAttributes: BasicScopeAttributes; + private readonly _embeddedLanguagesMatcher: ScopeMatcher; + + constructor(initialLanguageId: number, embeddedLanguages: IEmbeddedLanguagesMap | null) { + this._defaultAttributes = new BasicScopeAttributes(initialLanguageId, OptionalStandardTokenType.NotSet); + this._embeddedLanguagesMatcher = new ScopeMatcher(Object.entries(embeddedLanguages || {})); + } + + public getDefaultAttributes(): BasicScopeAttributes { + return this._defaultAttributes; + } + + public getBasicScopeAttributes(scopeName: ScopeName | null): BasicScopeAttributes { + if (scopeName === null) { + return BasicScopeAttributesProvider._NULL_SCOPE_METADATA; + } + return this._getBasicScopeAttributes.get(scopeName); + } + + private static readonly _NULL_SCOPE_METADATA = new BasicScopeAttributes(0, 0); + + private readonly _getBasicScopeAttributes = new CachedFn((scopeName) => { + const languageId = this._scopeToLanguage(scopeName); + const standardTokenType = this._toStandardTokenType(scopeName); + return new BasicScopeAttributes(languageId, standardTokenType); + }); + + /** + * Given a produced TM scope, return the language that token describes or null if unknown. + * e.g. source.html => html, source.css.embedded.html => css, punctuation.definition.tag.html => null + */ + private _scopeToLanguage(scope: ScopeName): number { + return this._embeddedLanguagesMatcher.match(scope) || 0; + } + + private _toStandardTokenType(scopeName: ScopeName): OptionalStandardTokenType { + const m = scopeName.match(BasicScopeAttributesProvider.STANDARD_TOKEN_TYPE_REGEXP); + if (!m) { + return OptionalStandardTokenType.NotSet; + } + switch (m[1]) { + case "comment": + return OptionalStandardTokenType.Comment; + case "string": + return OptionalStandardTokenType.String; + case "regex": + return OptionalStandardTokenType.RegEx; + case "meta.embedded": + return OptionalStandardTokenType.Other; + } + throw new Error("Unexpected match for standard token type!"); + } + + private static STANDARD_TOKEN_TYPE_REGEXP = /\b(comment|string|regex|meta\.embedded)\b/; +} + +class ScopeMatcher { + private readonly values: ReadonlyMap | null; + private readonly scopesRegExp: RegExp | null; + + constructor(values: [ScopeName, TValue][]) { + if (values.length === 0) { + this.values = null; + this.scopesRegExp = null; + } else { + this.values = new Map(values); + + // create the regex + const escapedScopes = values.map( + ([scopeName, value]) => escapeRegExpCharacters(scopeName) + ); + + escapedScopes.sort(); + escapedScopes.reverse(); // Longest scope first + this.scopesRegExp = new RegExp( + `^((${escapedScopes.join(")|(")}))($|\\.)`, + "" + ); + } + } + + public match(scope: ScopeName): TValue | undefined { + if (!this.scopesRegExp) { + return undefined; + } + const m = scope.match(this.scopesRegExp); + if (!m) { + // no scopes matched + return undefined; + } + return this.values!.get(m[1])!; + } +} diff --git a/src/grammar/grammar.ts b/src/grammar/grammar.ts new file mode 100644 index 00000000..c69b4d52 --- /dev/null +++ b/src/grammar/grammar.ts @@ -0,0 +1,1020 @@ +/*--------------------------------------------------------- + * Copyright (C) Microsoft Corporation. All rights reserved. + *--------------------------------------------------------*/ + +import { DebugFlags } from '../debug'; +import { EncodedTokenAttributes, OptionalStandardTokenType, StandardTokenType, toOptionalTokenType } from '../encodedTokenAttributes'; +import { IEmbeddedLanguagesMap, IGrammar, IToken, ITokenizeLineResult, ITokenizeLineResult2, ITokenTypeMap, StateStack as StackElementDef } from '../main'; +import { createMatchers, Matcher } from '../matcher'; +import { disposeOnigString, IOnigLib, OnigScanner, OnigString } from '../onigLib'; +import { IRawGrammar, IRawRepository, IRawRule } from '../rawGrammar'; +import { ruleIdFromNumber, IRuleFactoryHelper, IRuleRegistry, Rule, RuleFactory, RuleId, ruleIdToNumber } from '../rule'; +import { FontStyle, ScopeName, ScopePath, ScopeStack, StyleAttributes } from '../theme'; +import { clone } from '../utils'; +import { BasicScopeAttributes, BasicScopeAttributesProvider } from './basicScopesAttributeProvider'; +import { ScopeDependencyProcessor } from './grammarDependencies'; +import { _tokenizeString } from './tokenizeString'; + +export function createGrammar( + scopeName: ScopeName, + grammar: IRawGrammar, + initialLanguage: number, + embeddedLanguages: IEmbeddedLanguagesMap | null, + tokenTypes: ITokenTypeMap | null, + balancedBracketSelectors: BalancedBracketSelectors | null, + grammarRepository: IGrammarRepository & IThemeProvider, + onigLib: IOnigLib +): Grammar { + return new Grammar( + scopeName, + grammar, + initialLanguage, + embeddedLanguages, + tokenTypes, + balancedBracketSelectors, + grammarRepository, + onigLib + ); //TODO +} + +export interface IThemeProvider { + themeMatch(scopePath: ScopeStack): StyleAttributes | null; + getDefaults(): StyleAttributes; +} + +export interface IGrammarRepository { + lookup(scopeName: ScopeName): IRawGrammar | undefined; + injections(scopeName: ScopeName): ScopeName[]; +} + +export interface Injection { + readonly debugSelector: string; + readonly matcher: Matcher; + readonly priority: -1 | 0 | 1; // 0 is the default. -1 for 'L' and 1 for 'R' + readonly ruleId: RuleId; + readonly grammar: IRawGrammar; +} + +function collectInjections(result: Injection[], selector: string, rule: IRawRule, ruleFactoryHelper: IRuleFactoryHelper, grammar: IRawGrammar): void { + const matchers = createMatchers(selector, nameMatcher); + const ruleId = RuleFactory.getCompiledRuleId(rule, ruleFactoryHelper, grammar.repository); + for (const matcher of matchers) { + result.push({ + debugSelector: selector, + matcher: matcher.matcher, + ruleId: ruleId, + grammar: grammar, + priority: matcher.priority + }); + } +} + +function nameMatcher(identifers: ScopeName[], scopes: ScopeName[]): boolean { + if (scopes.length < identifers.length) { + return false; + } + let lastIndex = 0; + return identifers.every(identifier => { + for (let i = lastIndex; i < scopes.length; i++) { + if (scopesAreMatching(scopes[i], identifier)) { + lastIndex = i + 1; + return true; + } + } + return false; + }); +} + +function scopesAreMatching(thisScopeName: string, scopeName: string): boolean { + if (!thisScopeName) { + return false; + } + if (thisScopeName === scopeName) { + return true; + } + const len = scopeName.length; + return thisScopeName.length > len && thisScopeName.substr(0, len) === scopeName && thisScopeName[len] === '.'; +} + +export class Grammar implements IGrammar, IRuleFactoryHelper, IOnigLib { + private _rootId: RuleId | -1; + private _lastRuleId: number; + private readonly _ruleId2desc: Rule[]; + private readonly _includedGrammars: { [scopeName: string]: IRawGrammar }; + private readonly _grammarRepository: IGrammarRepository & IThemeProvider; + private readonly _grammar: IRawGrammar; + private _injections: Injection[] | null; + private readonly _basicScopeAttributesProvider: BasicScopeAttributesProvider; + private readonly _tokenTypeMatchers: TokenTypeMatcher[]; + + public get themeProvider(): IThemeProvider { return this._grammarRepository; } + + constructor( + private readonly _rootScopeName: ScopeName, + grammar: IRawGrammar, + initialLanguage: number, + embeddedLanguages: IEmbeddedLanguagesMap | null, + tokenTypes: ITokenTypeMap | null, + private readonly balancedBracketSelectors: BalancedBracketSelectors | null, + grammarRepository: IGrammarRepository & IThemeProvider, + private readonly _onigLib: IOnigLib + ) { + this._basicScopeAttributesProvider = new BasicScopeAttributesProvider( + initialLanguage, + embeddedLanguages + ); + + this._rootId = -1; + this._lastRuleId = 0; + this._ruleId2desc = [null!]; + this._includedGrammars = {}; + this._grammarRepository = grammarRepository; + this._grammar = initGrammar(grammar, null); + this._injections = null; + + this._tokenTypeMatchers = []; + if (tokenTypes) { + for (const selector of Object.keys(tokenTypes)) { + const matchers = createMatchers(selector, nameMatcher); + for (const matcher of matchers) { + this._tokenTypeMatchers.push({ + matcher: matcher.matcher, + type: tokenTypes[selector], + }); + } + } + } + } + + public dispose(): void { + for (const rule of this._ruleId2desc) { + if (rule) { + rule.dispose(); + } + } + } + + public createOnigScanner(sources: string[]): OnigScanner { + return this._onigLib.createOnigScanner(sources); + } + + public createOnigString(sources: string): OnigString { + return this._onigLib.createOnigString(sources); + } + + public getMetadataForScope(scope: string): BasicScopeAttributes { + return this._basicScopeAttributesProvider.getBasicScopeAttributes(scope); + } + + private _collectInjections(): Injection[] { + const grammarRepository: IGrammarRepository = { + lookup: (scopeName: string): IRawGrammar | undefined => { + if (scopeName === this._rootScopeName) { + return this._grammar; + } + return this.getExternalGrammar(scopeName); + }, + injections: (scopeName: string): string[] => { + return this._grammarRepository.injections(scopeName); + }, + }; + + const result: Injection[] = []; + + const scopeName = this._rootScopeName; + + const grammar = grammarRepository.lookup(scopeName); + if (grammar) { + // add injections from the current grammar + const rawInjections = grammar.injections; + if (rawInjections) { + for (let expression in rawInjections) { + collectInjections( + result, + expression, + rawInjections[expression], + this, + grammar + ); + } + } + + // add injection grammars contributed for the current scope + + const injectionScopeNames = this._grammarRepository.injections(scopeName); + if (injectionScopeNames) { + injectionScopeNames.forEach((injectionScopeName) => { + const injectionGrammar = + this.getExternalGrammar(injectionScopeName); + if (injectionGrammar) { + const selector = injectionGrammar.injectionSelector; + if (selector) { + collectInjections( + result, + selector, + injectionGrammar, + this, + injectionGrammar + ); + } + } + }); + } + } + + result.sort((i1, i2) => i1.priority - i2.priority); // sort by priority + + return result; + } + + public getInjections(): Injection[] { + if (this._injections === null) { + this._injections = this._collectInjections(); + + if (DebugFlags.InDebugMode && this._injections.length > 0) { + console.log( + `Grammar ${this._rootScopeName} contains the following injections:` + ); + for (const injection of this._injections) { + console.log(` - ${injection.debugSelector}`); + } + } + } + return this._injections; + } + + public registerRule(factory: (id: RuleId) => T): T { + const id = ++this._lastRuleId; + const result = factory(ruleIdFromNumber(id)); + this._ruleId2desc[id] = result; + return result; + } + + public getRule(ruleId: RuleId): Rule { + return this._ruleId2desc[ruleIdToNumber(ruleId)]; + } + + public getExternalGrammar( + scopeName: string, + repository?: IRawRepository + ): IRawGrammar | undefined { + if (this._includedGrammars[scopeName]) { + return this._includedGrammars[scopeName]; + } else if (this._grammarRepository) { + const rawIncludedGrammar = + this._grammarRepository.lookup(scopeName); + if (rawIncludedGrammar) { + // console.log('LOADED GRAMMAR ' + pattern.include); + this._includedGrammars[scopeName] = initGrammar( + rawIncludedGrammar, + repository && repository.$base + ); + return this._includedGrammars[scopeName]; + } + } + return undefined; + } + + public tokenizeLine( + lineText: string, + prevState: StateStack | null, + timeLimit: number = 0 + ): ITokenizeLineResult { + const r = this._tokenize(lineText, prevState, false, timeLimit); + return { + tokens: r.lineTokens.getResult(r.ruleStack, r.lineLength), + ruleStack: r.ruleStack, + stoppedEarly: r.stoppedEarly, + }; + } + + public tokenizeLine2( + lineText: string, + prevState: StateStack | null, + timeLimit: number = 0 + ): ITokenizeLineResult2 { + const r = this._tokenize(lineText, prevState, true, timeLimit); + return { + tokens: r.lineTokens.getBinaryResult(r.ruleStack, r.lineLength), + ruleStack: r.ruleStack, + stoppedEarly: r.stoppedEarly, + }; + } + + private _tokenize( + lineText: string, + prevState: StateStack | null, + emitBinaryTokens: boolean, + timeLimit: number + ): { + lineLength: number; + lineTokens: LineTokens; + ruleStack: StateStack; + stoppedEarly: boolean; + } { + if (this._rootId === -1) { + this._rootId = RuleFactory.getCompiledRuleId( + this._grammar.repository.$self, + this, + this._grammar.repository + ); + } + + let isFirstLine: boolean; + if (!prevState || prevState === StateStack.NULL) { + isFirstLine = true; + const rawDefaultMetadata = + this._basicScopeAttributesProvider.getDefaultAttributes(); + const defaultStyle = this.themeProvider.getDefaults(); + const defaultMetadata = EncodedTokenAttributes.set( + 0, + rawDefaultMetadata.languageId, + rawDefaultMetadata.tokenType, + null, + defaultStyle.fontStyle, + defaultStyle.foregroundId, + defaultStyle.backgroundId + ); + + const rootScopeName = this.getRule(this._rootId).getName( + null, + null + ); + + let scopeList: AttributedScopeStack; + if (rootScopeName) { + scopeList = AttributedScopeStack.createRootAndLookUpScopeName( + rootScopeName, + defaultMetadata, + this + ); + } else { + scopeList = AttributedScopeStack.createRoot( + "unknown", + defaultMetadata + ); + } + + prevState = new StateStack( + null, + this._rootId, + -1, + -1, + false, + null, + scopeList, + scopeList + ); + } else { + isFirstLine = false; + prevState.reset(); + } + + lineText = lineText + "\n"; + const onigLineText = this.createOnigString(lineText); + const lineLength = onigLineText.content.length; + const lineTokens = new LineTokens( + emitBinaryTokens, + lineText, + this._tokenTypeMatchers, + this.balancedBracketSelectors + ); + const r = _tokenizeString( + this, + onigLineText, + isFirstLine, + 0, + prevState, + lineTokens, + true, + timeLimit + ); + + disposeOnigString(onigLineText); + + return { + lineLength: lineLength, + lineTokens: lineTokens, + ruleStack: r.stack, + stoppedEarly: r.stoppedEarly, + }; + } +} + +function initGrammar(grammar: IRawGrammar, base: IRawRule | null | undefined): IRawGrammar { + grammar = clone(grammar); + + grammar.repository = grammar.repository || {}; + grammar.repository.$self = { + $vscodeTextmateLocation: grammar.$vscodeTextmateLocation, + patterns: grammar.patterns, + name: grammar.scopeName + }; + grammar.repository.$base = base || grammar.repository.$self; + return grammar; +} + +export class AttributedScopeStack { + public static createRoot(scopeName: ScopeName, tokenAttributes: EncodedTokenAttributes): AttributedScopeStack { + return new AttributedScopeStack(null, new ScopeStack(null, scopeName), tokenAttributes); + } + + public static createRootAndLookUpScopeName(scopeName: ScopeName, tokenAttributes: EncodedTokenAttributes, grammar: Grammar): AttributedScopeStack { + const rawRootMetadata = grammar.getMetadataForScope(scopeName); + const scopePath = new ScopeStack(null, scopeName); + const rootStyle = grammar.themeProvider.themeMatch(scopePath); + + const resolvedTokenAttributes = AttributedScopeStack.mergeAttributes( + tokenAttributes, + rawRootMetadata, + rootStyle + ); + + return new AttributedScopeStack(null, scopePath, resolvedTokenAttributes); + } + + public get scopeName(): ScopeName { return this.scopePath.scopeName; } + + private constructor( + public readonly parent: AttributedScopeStack | null, + public readonly scopePath: ScopeStack, + public readonly tokenAttributes: EncodedTokenAttributes + ) { + } + + public equals(other: AttributedScopeStack): boolean { + return AttributedScopeStack._equals(this, other); + } + + private static _equals( + a: AttributedScopeStack | null, + b: AttributedScopeStack | null + ): boolean { + do { + if (a === b) { + return true; + } + + if (!a && !b) { + // End of list reached for both + return true; + } + + if (!a || !b) { + // End of list reached only for one + return false; + } + + if (a.scopeName !== b.scopeName || a.tokenAttributes !== b.tokenAttributes) { + return false; + } + + // Go to previous pair + a = a.parent; + b = b.parent; + } while (true); + } + + private static mergeAttributes( + existingTokenAttributes: EncodedTokenAttributes, + basicScopeAttributes: BasicScopeAttributes, + styleAttributes: StyleAttributes | null + ): EncodedTokenAttributes { + let fontStyle = FontStyle.NotSet; + let foreground = 0; + let background = 0; + + if (styleAttributes !== null) { + fontStyle = styleAttributes.fontStyle; + foreground = styleAttributes.foregroundId; + background = styleAttributes.backgroundId; + } + + return EncodedTokenAttributes.set( + existingTokenAttributes, + basicScopeAttributes.languageId, + basicScopeAttributes.tokenType, + null, + fontStyle, + foreground, + background + ); + } + + public pushAttributed(scopePath: ScopePath | null, grammar: Grammar): AttributedScopeStack { + if (scopePath === null) { + return this; + } + + if (scopePath.indexOf(' ') === -1) { + // This is the common case and much faster + + return AttributedScopeStack._pushAttributed(this, scopePath, grammar); + } + + const scopes = scopePath.split(/ /g); + let result: AttributedScopeStack = this; + for (const scope of scopes) { + result = AttributedScopeStack._pushAttributed(result, scope, grammar); + } + return result; + + } + + private static _pushAttributed( + target: AttributedScopeStack, + scopeName: ScopeName, + grammar: Grammar, + ): AttributedScopeStack { + const rawMetadata = grammar.getMetadataForScope(scopeName); + + const newPath = target.scopePath.push(scopeName); + const scopeThemeMatchResult = + grammar.themeProvider.themeMatch(newPath); + const metadata = AttributedScopeStack.mergeAttributes( + target.tokenAttributes, + rawMetadata, + scopeThemeMatchResult + ); + return new AttributedScopeStack(target, newPath, metadata); + } + + public getScopeNames(): string[] { + return this.scopePath.getSegments(); + } +} + +/** + * Represents a "pushed" state on the stack (as a linked list element). + */ +export class StateStack implements StackElementDef { + _stackElementBrand: void = undefined; + + // TODO remove me + public static NULL = new StateStack( + null, + 0 as any, + 0, + 0, + false, + null, + null!, + null! + ); + + /** + * The position on the current line where this state was pushed. + * This is relevant only while tokenizing a line, to detect endless loops. + * Its value is meaningless across lines. + */ + private _enterPos: number; + + /** + * The captured anchor position when this stack element was pushed. + * This is relevant only while tokenizing a line, to restore the anchor position when popping. + * Its value is meaningless across lines. + */ + private _anchorPos: number; + + + /** + * The depth of the stack. + */ + public readonly depth: number; + + constructor( + /** + * The previous state on the stack (or null for the root state). + */ + public readonly parent: StateStack | null, + + /** + * The state (rule) that this element represents. + */ + private readonly ruleId: RuleId, + + enterPos: number, + anchorPos: number, + + /** + * The state has entered and captured \n. This means that the next line should have an anchorPosition of 0. + */ + public readonly beginRuleCapturedEOL: boolean, + + /** + * The "pop" (end) condition for this state in case that it was dynamically generated through captured text. + */ + public readonly endRule: string | null, + + /** + * The list of scopes containing the "name" for this state. + */ + public readonly nameScopesList: AttributedScopeStack, + + /** + * The list of scopes containing the "contentName" (besides "name") for this state. + * This list **must** contain as an element `scopeName`. + */ + public readonly contentNameScopesList: AttributedScopeStack + ) { + this.depth = this.parent ? this.parent.depth + 1 : 1; + this._enterPos = enterPos; + this._anchorPos = anchorPos; + } + + public equals(other: StateStack): boolean { + if (other === null) { + return false; + } + return StateStack._equals(this, other); + } + + private static _equals(a: StateStack, b: StateStack): boolean { + if (a === b) { + return true; + } + if (!this._structuralEquals(a, b)) { + return false; + } + return a.contentNameScopesList.equals(b.contentNameScopesList); + } + + /** + * A structural equals check. Does not take into account `scopes`. + */ + private static _structuralEquals( + a: StateStack | null, + b: StateStack | null + ): boolean { + do { + if (a === b) { + return true; + } + + if (!a && !b) { + // End of list reached for both + return true; + } + + if (!a || !b) { + // End of list reached only for one + return false; + } + + if ( + a.depth !== b.depth || + a.ruleId !== b.ruleId || + a.endRule !== b.endRule + ) { + return false; + } + + // Go to previous pair + a = a.parent; + b = b.parent; + } while (true); + } + + public clone(): StateStack { + return this; + } + + private static _reset(el: StateStack | null): void { + while (el) { + el._enterPos = -1; + el._anchorPos = -1; + el = el.parent; + } + } + + public reset(): void { + StateStack._reset(this); + } + + public pop(): StateStack | null { + return this.parent; + } + + public safePop(): StateStack { + if (this.parent) { + return this.parent; + } + return this; + } + + public push( + ruleId: RuleId, + enterPos: number, + anchorPos: number, + beginRuleCapturedEOL: boolean, + endRule: string | null, + nameScopesList: AttributedScopeStack, + contentNameScopesList: AttributedScopeStack + ): StateStack { + return new StateStack( + this, + ruleId, + enterPos, + anchorPos, + beginRuleCapturedEOL, + endRule, + nameScopesList, + contentNameScopesList + ); + } + + public getEnterPos(): number { + return this._enterPos; + } + + public getAnchorPos(): number { + return this._anchorPos; + } + + public getRule(grammar: IRuleRegistry): Rule { + return grammar.getRule(this.ruleId); + } + + public toString(): string { + const r: string[] = []; + this._writeString(r, 0); + return "[" + r.join(",") + "]"; + } + + private _writeString(res: string[], outIndex: number): number { + if (this.parent) { + outIndex = this.parent._writeString(res, outIndex); + } + + res[ + outIndex++ + ] = `(${this.ruleId}, TODO-${this.nameScopesList}, TODO-${this.contentNameScopesList})`; + + return outIndex; + } + + public withContentNameScopesList( + contentNameScopeStack: AttributedScopeStack + ): StateStack { + if (this.contentNameScopesList === contentNameScopeStack) { + return this; + } + return this.parent!.push( + this.ruleId, + this._enterPos, + this._anchorPos, + this.beginRuleCapturedEOL, + this.endRule, + this.nameScopesList, + contentNameScopeStack + ); + } + + public withEndRule(endRule: string): StateStack { + if (this.endRule === endRule) { + return this; + } + return new StateStack( + this.parent, + this.ruleId, + this._enterPos, + this._anchorPos, + this.beginRuleCapturedEOL, + endRule, + this.nameScopesList, + this.contentNameScopesList + ); + } + + // Used to warn of endless loops + public hasSameRuleAs(other: StateStack): boolean { + let el: StateStack | null = this; + while (el && el._enterPos === other._enterPos) { + if (el.ruleId === other.ruleId) { + return true; + } + el = el.parent; + } + return false; + } +} + + +interface TokenTypeMatcher { + readonly matcher: Matcher; + readonly type: StandardTokenType; +} + +export class BalancedBracketSelectors { + private readonly balancedBracketScopes: Matcher[]; + private readonly unbalancedBracketScopes: Matcher[]; + + private allowAny = false; + + constructor( + balancedBracketScopes: string[], + unbalancedBracketScopes: string[], + ) { + this.balancedBracketScopes = balancedBracketScopes.flatMap((selector) => { + if (selector === '*') { + this.allowAny = true; + return []; + } + return createMatchers(selector, nameMatcher).map((m) => m.matcher); + } + ); + this.unbalancedBracketScopes = unbalancedBracketScopes.flatMap((selector) => + createMatchers(selector, nameMatcher).map((m) => m.matcher) + ); + } + + public get matchesAlways(): boolean { + return this.allowAny && this.unbalancedBracketScopes.length === 0; + } + + public get matchesNever(): boolean { + return this.balancedBracketScopes.length === 0 && !this.allowAny; + } + + public match(scopes: string[]): boolean { + for (const excluder of this.unbalancedBracketScopes) { + if (excluder(scopes)) { + return false; + } + } + + for (const includer of this.balancedBracketScopes) { + if (includer(scopes)) { + return true; + } + } + return this.allowAny; + } +} + +export class LineTokens { + private readonly _emitBinaryTokens: boolean; + /** + * defined only if `DebugFlags.InDebugMode`. + */ + private readonly _lineText: string | null; + /** + * used only if `_emitBinaryTokens` is false. + */ + private readonly _tokens: IToken[]; + /** + * used only if `_emitBinaryTokens` is true. + */ + private readonly _binaryTokens: number[]; + + private _lastTokenEndIndex: number; + + private readonly _tokenTypeOverrides: TokenTypeMatcher[]; + + constructor( + emitBinaryTokens: boolean, + lineText: string, + tokenTypeOverrides: TokenTypeMatcher[], + private readonly balancedBracketSelectors: BalancedBracketSelectors | null, + ) { + this._emitBinaryTokens = emitBinaryTokens; + this._tokenTypeOverrides = tokenTypeOverrides; + if (DebugFlags.InDebugMode) { + this._lineText = lineText; + } else { + this._lineText = null; + } + this._tokens = []; + this._binaryTokens = []; + this._lastTokenEndIndex = 0; + } + + public produce(stack: StateStack, endIndex: number): void { + this.produceFromScopes(stack.contentNameScopesList, endIndex); + } + + public produceFromScopes( + scopesList: AttributedScopeStack, + endIndex: number + ): void { + if (this._lastTokenEndIndex >= endIndex) { + return; + } + + if (this._emitBinaryTokens) { + let metadata = scopesList.tokenAttributes; + let containsBalancedBrackets = false; + if (this.balancedBracketSelectors?.matchesAlways) { + containsBalancedBrackets = true; + } + + if (this._tokenTypeOverrides.length > 0 || (this.balancedBracketSelectors && !this.balancedBracketSelectors.matchesAlways && !this.balancedBracketSelectors.matchesNever)) { + // Only generate scope array when required to improve performance + const scopes = scopesList.getScopeNames(); + for (const tokenType of this._tokenTypeOverrides) { + if (tokenType.matcher(scopes)) { + metadata = EncodedTokenAttributes.set( + metadata, + 0, + toOptionalTokenType(tokenType.type), + null, + FontStyle.NotSet, + 0, + 0 + ); + } + } + if (this.balancedBracketSelectors) { + containsBalancedBrackets = this.balancedBracketSelectors.match(scopes); + } + } + + if (containsBalancedBrackets) { + metadata = EncodedTokenAttributes.set( + metadata, + 0, + OptionalStandardTokenType.NotSet, + containsBalancedBrackets, + FontStyle.NotSet, + 0, + 0 + ); + } + + if (this._binaryTokens.length > 0 && this._binaryTokens[this._binaryTokens.length - 1] === metadata) { + // no need to push a token with the same metadata + this._lastTokenEndIndex = endIndex; + return; + } + + if (DebugFlags.InDebugMode) { + const scopes = scopesList.getScopeNames(); + console.log(' token: |' + this._lineText!.substring(this._lastTokenEndIndex, endIndex).replace(/\n$/, '\\n') + '|'); + for (let k = 0; k < scopes.length; k++) { + console.log(' * ' + scopes[k]); + } + } + + this._binaryTokens.push(this._lastTokenEndIndex); + this._binaryTokens.push(metadata); + + this._lastTokenEndIndex = endIndex; + return; + } + + const scopes = scopesList.getScopeNames(); + + if (DebugFlags.InDebugMode) { + console.log(' token: |' + this._lineText!.substring(this._lastTokenEndIndex, endIndex).replace(/\n$/, '\\n') + '|'); + for (let k = 0; k < scopes.length; k++) { + console.log(' * ' + scopes[k]); + } + } + + this._tokens.push({ + startIndex: this._lastTokenEndIndex, + endIndex: endIndex, + // value: lineText.substring(lastTokenEndIndex, endIndex), + scopes: scopes + }); + + this._lastTokenEndIndex = endIndex; + } + + public getResult(stack: StateStack, lineLength: number): IToken[] { + if (this._tokens.length > 0 && this._tokens[this._tokens.length - 1].startIndex === lineLength - 1) { + // pop produced token for newline + this._tokens.pop(); + } + + if (this._tokens.length === 0) { + this._lastTokenEndIndex = -1; + this.produce(stack, lineLength); + this._tokens[this._tokens.length - 1].startIndex = 0; + } + + return this._tokens; + } + + public getBinaryResult(stack: StateStack, lineLength: number): Uint32Array { + if (this._binaryTokens.length > 0 && this._binaryTokens[this._binaryTokens.length - 2] === lineLength - 1) { + // pop produced token for newline + this._binaryTokens.pop(); + this._binaryTokens.pop(); + } + + if (this._binaryTokens.length === 0) { + this._lastTokenEndIndex = -1; + this.produce(stack, lineLength); + this._binaryTokens[this._binaryTokens.length - 2] = 0; + } + + const result = new Uint32Array(this._binaryTokens.length); + for (let i = 0, len = this._binaryTokens.length; i < len; i++) { + result[i] = this._binaryTokens[i]; + } + + return result; + } +} + diff --git a/src/grammar/grammarDependencies.ts b/src/grammar/grammarDependencies.ts new file mode 100644 index 00000000..4ad2b898 --- /dev/null +++ b/src/grammar/grammarDependencies.ts @@ -0,0 +1,295 @@ +/*--------------------------------------------------------- + * Copyright (C) Microsoft Corporation. All rights reserved. + *--------------------------------------------------------*/ + +import { IRawGrammar, IRawRepository, IRawRule } from '../rawGrammar'; +import { ScopeName } from '../theme'; +import { mergeObjects } from '../utils'; +import { IGrammarRepository } from './grammar'; + +export type AbsoluteRuleReference = TopLevelRuleReference | TopLevelRepositoryRuleReference; + +/** + * References the top level rule of a grammar with the given scope name. +*/ +export class TopLevelRuleReference { + constructor( + public readonly scopeName: ScopeName + ) { } + + public toKey(): string { + return this.scopeName; + } +} + +/** + * References a rule of a grammar in the top level repository section with the given name. +*/ +export class TopLevelRepositoryRuleReference { + constructor( + public readonly scopeName: ScopeName, + public readonly ruleName: string + ) { } + + public toKey(): string { + return `${this.scopeName}#${this.ruleName}`; + } +} + +export class ExternalReferenceCollector { + private readonly _references: AbsoluteRuleReference[] = []; + private readonly _seenReferenceKeys = new Set(); + + public get references(): readonly AbsoluteRuleReference[] { + return this._references; + } + + public readonly visitedRule = new Set(); + + public add(reference: AbsoluteRuleReference): void { + const key = reference.toKey(); + if (this._seenReferenceKeys.has(key)) { + return; + } + this._seenReferenceKeys.add(key); + this._references.push(reference); + } +} + +export class ScopeDependencyProcessor { + public readonly seenFullScopeRequests = new Set(); + public readonly seenPartialScopeRequests = new Set(); + public Q: AbsoluteRuleReference[]; + + constructor( + public readonly repo: IGrammarRepository, + public readonly initialScopeName: ScopeName + ) { + this.seenFullScopeRequests.add(this.initialScopeName); + this.Q = [new TopLevelRuleReference(this.initialScopeName)]; + } + + public processQueue(): void { + const q = this.Q; + this.Q = []; + + const deps = new ExternalReferenceCollector(); + for (const dep of q) { + collectReferencesOfReference(dep, this.initialScopeName, this.repo, deps); + } + + for (const dep of deps.references) { + if (dep instanceof TopLevelRuleReference) { + if (this.seenFullScopeRequests.has(dep.scopeName)) { + // already processed + continue; + } + this.seenFullScopeRequests.add(dep.scopeName); + this.Q.push(dep); + } else { + if (this.seenFullScopeRequests.has(dep.scopeName)) { + // already processed in full + continue; + } + if (this.seenPartialScopeRequests.has(dep.toKey())) { + // already processed + continue; + } + this.seenPartialScopeRequests.add(dep.toKey()); + this.Q.push(dep); + } + } + } +} + +function collectReferencesOfReference( + reference: TopLevelRuleReference | TopLevelRepositoryRuleReference, + baseGrammarScopeName: ScopeName, + repo: IGrammarRepository, + result: ExternalReferenceCollector, +) { + const selfGrammar = repo.lookup(reference.scopeName); + if (!selfGrammar) { + if (reference.scopeName === baseGrammarScopeName) { + throw new Error(`No grammar provided for <${baseGrammarScopeName}>`); + } + return; + } + + const baseGrammar = repo.lookup(baseGrammarScopeName)!; + + if (reference instanceof TopLevelRuleReference) { + collectExternalReferencesInTopLevelRule({ baseGrammar, selfGrammar }, result); + } else { + collectExternalReferencesInTopLevelRepositoryRule( + reference.ruleName, + { baseGrammar, selfGrammar, repository: selfGrammar.repository }, + result + ); + } + + const injections = repo.injections(reference.scopeName); + if (injections) { + for (const injection of injections) { + result.add(new TopLevelRuleReference(injection)); + } + } +} + +interface Context { + baseGrammar: IRawGrammar; + selfGrammar: IRawGrammar; +} + +interface ContextWithRepository { + baseGrammar: IRawGrammar; + selfGrammar: IRawGrammar; + repository: Record | undefined; +} + +function collectExternalReferencesInTopLevelRepositoryRule( + ruleName: string, + context: ContextWithRepository, + result: ExternalReferenceCollector +): void { + if (context.repository && context.repository[ruleName]) { + const rule = context.repository[ruleName]; + collectExternalReferencesInRules([rule], context, result); + } +} + +function collectExternalReferencesInTopLevelRule(context: Context, result: ExternalReferenceCollector): void { + if (context.selfGrammar.patterns && Array.isArray(context.selfGrammar.patterns)) { + collectExternalReferencesInRules( + context.selfGrammar.patterns, + { ...context, repository: context.selfGrammar.repository }, + result + ); + } + if (context.selfGrammar.injections) { + collectExternalReferencesInRules( + Object.values(context.selfGrammar.injections), + { ...context, repository: context.selfGrammar.repository }, + result + ); + } +} + +function collectExternalReferencesInRules( + rules: IRawRule[], + context: ContextWithRepository, + result: ExternalReferenceCollector, +): void { + for (const rule of rules) { + if (result.visitedRule.has(rule)) { + continue; + } + result.visitedRule.add(rule); + + const patternRepository = rule.repository ? mergeObjects({}, context.repository, rule.repository) : context.repository; + + if (Array.isArray(rule.patterns)) { + collectExternalReferencesInRules(rule.patterns, { ...context, repository: patternRepository }, result); + } + + const include = rule.include; + + if (!include) { + continue; + } + + const reference = parseInclude(include); + + switch (reference.kind) { + case IncludeReferenceKind.Base: + collectExternalReferencesInTopLevelRule({ ...context, selfGrammar: context.baseGrammar }, result); + break; + case IncludeReferenceKind.Self: + collectExternalReferencesInTopLevelRule(context, result); + break; + case IncludeReferenceKind.RelativeReference: + collectExternalReferencesInTopLevelRepositoryRule(reference.ruleName, { ...context, repository: patternRepository }, result); + break; + case IncludeReferenceKind.TopLevelReference: + case IncludeReferenceKind.TopLevelRepositoryReference: + const selfGrammar = + reference.scopeName === context.selfGrammar.scopeName + ? context.selfGrammar + : reference.scopeName === context.baseGrammar.scopeName + ? context.baseGrammar + : undefined; + if (selfGrammar) { + const newContext: ContextWithRepository = { baseGrammar: context.baseGrammar, selfGrammar, repository: patternRepository }; + if (reference.kind === IncludeReferenceKind.TopLevelRepositoryReference) { + collectExternalReferencesInTopLevelRepositoryRule(reference.ruleName, newContext, result); + } else { + collectExternalReferencesInTopLevelRule(newContext, result); + } + } else { + if (reference.kind === IncludeReferenceKind.TopLevelRepositoryReference) { + result.add(new TopLevelRepositoryRuleReference(reference.scopeName, reference.ruleName)); + } else { + result.add(new TopLevelRuleReference(reference.scopeName)); + } + } + break; + } + } +} + +export type IncludeReference = + | BaseReference + | SelfReference + | RelativeReference + | TopLevelReference + | TopLevelRepositoryReference; + +export const enum IncludeReferenceKind { + Base, + Self, + RelativeReference, + TopLevelReference, + TopLevelRepositoryReference, +} + +export class BaseReference { + public readonly kind = IncludeReferenceKind.Base; +} + +export class SelfReference { + public readonly kind = IncludeReferenceKind.Self; +} + +export class RelativeReference { + public readonly kind = IncludeReferenceKind.RelativeReference; + constructor(public readonly ruleName: string) {} +} + +export class TopLevelReference { + public readonly kind = IncludeReferenceKind.TopLevelReference; + constructor(public readonly scopeName: ScopeName) {} +} + +export class TopLevelRepositoryReference { + public readonly kind = IncludeReferenceKind.TopLevelRepositoryReference; + constructor(public readonly scopeName: ScopeName, public readonly ruleName: string) {} +} + +export function parseInclude(include: string): IncludeReference { + if (include === '$base') { + return new BaseReference(); + } else if (include === '$self') { + return new SelfReference(); + } + + const indexOfSharp = include.indexOf("#"); + if (indexOfSharp === -1) { + return new TopLevelReference(include); + } else if (indexOfSharp === 0) { + return new RelativeReference(include.substring(1)); + } else { + const scopeName = include.substring(0, indexOfSharp); + const ruleName = include.substring(indexOfSharp + 1); + return new TopLevelRepositoryReference(scopeName, ruleName); + } +} diff --git a/src/grammar/index.ts b/src/grammar/index.ts new file mode 100644 index 00000000..7f0446bb --- /dev/null +++ b/src/grammar/index.ts @@ -0,0 +1,5 @@ +/*--------------------------------------------------------- + * Copyright (C) Microsoft Corporation. All rights reserved. + *--------------------------------------------------------*/ + +export * from './grammar'; \ No newline at end of file diff --git a/src/grammar/tokenizeString.ts b/src/grammar/tokenizeString.ts new file mode 100644 index 00000000..41749986 --- /dev/null +++ b/src/grammar/tokenizeString.ts @@ -0,0 +1,642 @@ +/*--------------------------------------------------------- + * Copyright (C) Microsoft Corporation. All rights reserved. + *--------------------------------------------------------*/ + +import { DebugFlags, UseOnigurumaFindOptions } from '../debug'; +import type { LineTokens, StateStack } from './grammar'; +import { disposeOnigString, FindOption, IOnigCaptureIndex, OnigString } from '../onigLib'; +import { BeginEndRule, BeginWhileRule, CaptureRule, CompiledRule, endRuleId, MatchRule, Rule, RuleId, whileRuleId } from '../rule'; +import { performanceNow } from '../utils'; +import type { AttributedScopeStack, Grammar, Injection } from './grammar'; + +class TokenizeStringResult { + constructor( + public readonly stack: StateStack, + public readonly stoppedEarly: boolean + ) { } +} + +/** + * Tokenize a string + * @param grammar + * @param lineText + * @param isFirstLine + * @param linePos + * @param stack + * @param lineTokens + * @param checkWhileConditions + * @param timeLimit Use `0` to indicate no time limit + * @returns the StackElement or StackElement.TIME_LIMIT_REACHED if the time limit has been reached + */ + export function _tokenizeString( + grammar: Grammar, + lineText: OnigString, + isFirstLine: boolean, + linePos: number, + stack: StateStack, + lineTokens: LineTokens, + checkWhileConditions: boolean, + timeLimit: number +): TokenizeStringResult { + const lineLength = lineText.content.length; + + let STOP = false; + let anchorPosition = -1; + + if (checkWhileConditions) { + const whileCheckResult = _checkWhileConditions( + grammar, + lineText, + isFirstLine, + linePos, + stack, + lineTokens + ); + stack = whileCheckResult.stack; + linePos = whileCheckResult.linePos; + isFirstLine = whileCheckResult.isFirstLine; + anchorPosition = whileCheckResult.anchorPosition; + } + + const startTime = Date.now(); + while (!STOP) { + if (timeLimit !== 0) { + const elapsedTime = Date.now() - startTime; + if (elapsedTime > timeLimit) { + return new TokenizeStringResult(stack, true); + } + } + scanNext(); // potentially modifies linePos && anchorPosition + } + + return new TokenizeStringResult(stack, false); + + function scanNext(): void { + if (DebugFlags.InDebugMode) { + console.log(""); + console.log( + `@@scanNext ${linePos}: |${lineText.content + .substr(linePos) + .replace(/\n$/, "\\n")}|` + ); + } + const r = matchRuleOrInjections( + grammar, + lineText, + isFirstLine, + linePos, + stack, + anchorPosition + ); + + if (!r) { + if (DebugFlags.InDebugMode) { + console.log(" no more matches."); + } + // No match + lineTokens.produce(stack, lineLength); + STOP = true; + return; + } + + const captureIndices: IOnigCaptureIndex[] = r.captureIndices; + const matchedRuleId = r.matchedRuleId; + + const hasAdvanced = + captureIndices && captureIndices.length > 0 + ? captureIndices[0].end > linePos + : false; + + if (matchedRuleId === endRuleId) { + // We matched the `end` for this rule => pop it + const poppedRule = stack.getRule(grammar); + + if (DebugFlags.InDebugMode) { + console.log( + " popping " + + poppedRule.debugName + + " - " + + poppedRule.debugEndRegExp + ); + } + + lineTokens.produce(stack, captureIndices[0].start); + stack = stack.withContentNameScopesList(stack.nameScopesList); + handleCaptures( + grammar, + lineText, + isFirstLine, + stack, + lineTokens, + poppedRule.endCaptures, + captureIndices + ); + lineTokens.produce(stack, captureIndices[0].end); + + // pop + const popped = stack; + stack = stack.parent!; + anchorPosition = popped.getAnchorPos(); + + if (!hasAdvanced && popped.getEnterPos() === linePos) { + // Grammar pushed & popped a rule without advancing + if (DebugFlags.InDebugMode) { + console.error( + "[1] - Grammar is in an endless loop - Grammar pushed & popped a rule without advancing" + ); + } + + // See https://github.com/Microsoft/vscode-textmate/issues/12 + // Let's assume this was a mistake by the grammar author and the intent was to continue in this state + stack = popped; + + lineTokens.produce(stack, lineLength); + STOP = true; + return; + } + } else { + // We matched a rule! + const _rule = grammar.getRule(matchedRuleId); + + lineTokens.produce(stack, captureIndices[0].start); + + const beforePush = stack; + // push it on the stack rule + const scopeName = _rule.getName(lineText.content, captureIndices); + const nameScopesList = stack.contentNameScopesList.pushAttributed( + scopeName, + grammar + ); + stack = stack.push( + matchedRuleId, + linePos, + anchorPosition, + captureIndices[0].end === lineLength, + null, + nameScopesList, + nameScopesList + ); + + if (_rule instanceof BeginEndRule) { + const pushedRule = _rule; + if (DebugFlags.InDebugMode) { + console.log( + " pushing " + + pushedRule.debugName + + " - " + + pushedRule.debugBeginRegExp + ); + } + + handleCaptures( + grammar, + lineText, + isFirstLine, + stack, + lineTokens, + pushedRule.beginCaptures, + captureIndices + ); + lineTokens.produce(stack, captureIndices[0].end); + anchorPosition = captureIndices[0].end; + + const contentName = pushedRule.getContentName( + lineText.content, + captureIndices + ); + const contentNameScopesList = nameScopesList.pushAttributed( + contentName, + grammar + ); + stack = stack.withContentNameScopesList(contentNameScopesList); + + if (pushedRule.endHasBackReferences) { + stack = stack.withEndRule( + pushedRule.getEndWithResolvedBackReferences( + lineText.content, + captureIndices + ) + ); + } + + if (!hasAdvanced && beforePush.hasSameRuleAs(stack)) { + // Grammar pushed the same rule without advancing + if (DebugFlags.InDebugMode) { + console.error( + "[2] - Grammar is in an endless loop - Grammar pushed the same rule without advancing" + ); + } + stack = stack.pop()!; + lineTokens.produce(stack, lineLength); + STOP = true; + return; + } + } else if (_rule instanceof BeginWhileRule) { + const pushedRule = _rule; + if (DebugFlags.InDebugMode) { + console.log(" pushing " + pushedRule.debugName); + } + + handleCaptures( + grammar, + lineText, + isFirstLine, + stack, + lineTokens, + pushedRule.beginCaptures, + captureIndices + ); + lineTokens.produce(stack, captureIndices[0].end); + anchorPosition = captureIndices[0].end; + const contentName = pushedRule.getContentName( + lineText.content, + captureIndices + ); + const contentNameScopesList = nameScopesList.pushAttributed( + contentName, + grammar + ); + stack = stack.withContentNameScopesList(contentNameScopesList); + + if (pushedRule.whileHasBackReferences) { + stack = stack.withEndRule( + pushedRule.getWhileWithResolvedBackReferences( + lineText.content, + captureIndices + ) + ); + } + + if (!hasAdvanced && beforePush.hasSameRuleAs(stack)) { + // Grammar pushed the same rule without advancing + if (DebugFlags.InDebugMode) { + console.error( + "[3] - Grammar is in an endless loop - Grammar pushed the same rule without advancing" + ); + } + stack = stack.pop()!; + lineTokens.produce(stack, lineLength); + STOP = true; + return; + } + } else { + const matchingRule = _rule; + if (DebugFlags.InDebugMode) { + console.log( + " matched " + + matchingRule.debugName + + " - " + + matchingRule.debugMatchRegExp + ); + } + + handleCaptures( + grammar, + lineText, + isFirstLine, + stack, + lineTokens, + matchingRule.captures, + captureIndices + ); + lineTokens.produce(stack, captureIndices[0].end); + + // pop rule immediately since it is a MatchRule + stack = stack.pop()!; + + if (!hasAdvanced) { + // Grammar is not advancing, nor is it pushing/popping + if (DebugFlags.InDebugMode) { + console.error( + "[4] - Grammar is in an endless loop - Grammar is not advancing, nor is it pushing/popping" + ); + } + stack = stack.safePop(); + lineTokens.produce(stack, lineLength); + STOP = true; + return; + } + } + } + + if (captureIndices[0].end > linePos) { + // Advance stream + linePos = captureIndices[0].end; + isFirstLine = false; + } + } +} + +/** + * Walk the stack from bottom to top, and check each while condition in this order. + * If any fails, cut off the entire stack above the failed while condition. While conditions + * may also advance the linePosition. + */ +function _checkWhileConditions(grammar: Grammar, lineText: OnigString, isFirstLine: boolean, linePos: number, stack: StateStack, lineTokens: LineTokens): IWhileCheckResult { + let anchorPosition = (stack.beginRuleCapturedEOL ? 0 : -1); + + interface IWhileStack { + readonly stack: StateStack; + readonly rule: BeginWhileRule; + } + + const whileRules: IWhileStack[] = []; + for (let node: StateStack | null = stack; node; node = node.pop()) { + const nodeRule = node.getRule(grammar); + if (nodeRule instanceof BeginWhileRule) { + whileRules.push({ + rule: nodeRule, + stack: node + }); + } + } + + for (let whileRule = whileRules.pop(); whileRule; whileRule = whileRules.pop()) { + const { ruleScanner, findOptions } = prepareRuleWhileSearch(whileRule.rule, grammar, whileRule.stack.endRule, isFirstLine, linePos === anchorPosition); + const r = ruleScanner.findNextMatchSync(lineText, linePos, findOptions); + if (DebugFlags.InDebugMode) { + console.log(' scanning for while rule'); + console.log(ruleScanner.toString()); + } + + if (r) { + const matchedRuleId = r.ruleId; + if (matchedRuleId !== whileRuleId) { + // we shouldn't end up here + stack = whileRule.stack.pop()!; + break; + } + if (r.captureIndices && r.captureIndices.length) { + lineTokens.produce(whileRule.stack, r.captureIndices[0].start); + handleCaptures(grammar, lineText, isFirstLine, whileRule.stack, lineTokens, whileRule.rule.whileCaptures, r.captureIndices); + lineTokens.produce(whileRule.stack, r.captureIndices[0].end); + anchorPosition = r.captureIndices[0].end; + if (r.captureIndices[0].end > linePos) { + linePos = r.captureIndices[0].end; + isFirstLine = false; + } + } + } else { + if (DebugFlags.InDebugMode) { + console.log(' popping ' + whileRule.rule.debugName + ' - ' + whileRule.rule.debugWhileRegExp); + } + + stack = whileRule.stack.pop()!; + break; + } + } + + return { stack: stack, linePos: linePos, anchorPosition: anchorPosition, isFirstLine: isFirstLine }; +} + +interface IWhileCheckResult { + readonly stack: StateStack; + readonly linePos: number; + readonly anchorPosition: number; + readonly isFirstLine: boolean; +} + +function matchRuleOrInjections(grammar: Grammar, lineText: OnigString, isFirstLine: boolean, linePos: number, stack: StateStack, anchorPosition: number): IMatchResult | null { + // Look for normal grammar rule + const matchResult = matchRule(grammar, lineText, isFirstLine, linePos, stack, anchorPosition); + + // Look for injected rules + const injections = grammar.getInjections(); + if (injections.length === 0) { + // No injections whatsoever => early return + return matchResult; + } + + const injectionResult = matchInjections(injections, grammar, lineText, isFirstLine, linePos, stack, anchorPosition); + if (!injectionResult) { + // No injections matched => early return + return matchResult; + } + + if (!matchResult) { + // Only injections matched => early return + return injectionResult; + } + + // Decide if `matchResult` or `injectionResult` should win + const matchResultScore = matchResult.captureIndices[0].start; + const injectionResultScore = injectionResult.captureIndices[0].start; + + if (injectionResultScore < matchResultScore || (injectionResult.priorityMatch && injectionResultScore === matchResultScore)) { + // injection won! + return injectionResult; + } + return matchResult; +} + +interface IMatchResult { + readonly captureIndices: IOnigCaptureIndex[]; + readonly matchedRuleId: RuleId | typeof endRuleId; +} + +function matchRule(grammar: Grammar, lineText: OnigString, isFirstLine: boolean, linePos: number, stack: StateStack, anchorPosition: number): IMatchResult | null { + const rule = stack.getRule(grammar); + const { ruleScanner, findOptions } = prepareRuleSearch(rule, grammar, stack.endRule, isFirstLine, linePos === anchorPosition); + + let perfStart = 0; + if (DebugFlags.InDebugMode) { + perfStart = performanceNow(); + } + + const r = ruleScanner.findNextMatchSync(lineText, linePos, findOptions); + + if (DebugFlags.InDebugMode) { + const elapsedMillis = performanceNow() - perfStart; + if (elapsedMillis > 5) { + console.warn(`Rule ${rule.debugName} (${rule.id}) matching took ${elapsedMillis} against '${lineText}'`); + } + console.log(` scanning for (linePos: ${linePos}, anchorPosition: ${anchorPosition})`); + console.log(ruleScanner.toString()); + if (r) { + console.log(`matched rule id: ${r.ruleId} from ${r.captureIndices[0].start} to ${r.captureIndices[0].end}`); + } + } + + if (r) { + return { + captureIndices: r.captureIndices, + matchedRuleId: r.ruleId + }; + } + return null; +} + +function matchInjections(injections: Injection[], grammar: Grammar, lineText: OnigString, isFirstLine: boolean, linePos: number, stack: StateStack, anchorPosition: number): IMatchInjectionsResult | null { + // The lower the better + let bestMatchRating = Number.MAX_VALUE; + let bestMatchCaptureIndices: IOnigCaptureIndex[] | null = null; + let bestMatchRuleId: RuleId | typeof endRuleId; + let bestMatchResultPriority: number = 0; + + const scopes = stack.contentNameScopesList.getScopeNames(); + + for (let i = 0, len = injections.length; i < len; i++) { + const injection = injections[i]; + if (!injection.matcher(scopes)) { + // injection selector doesn't match stack + continue; + } + const rule = grammar.getRule(injection.ruleId); + const { ruleScanner, findOptions } = prepareRuleSearch(rule, grammar, null, isFirstLine, linePos === anchorPosition); + const matchResult = ruleScanner.findNextMatchSync(lineText, linePos, findOptions); + if (!matchResult) { + continue; + } + + if (DebugFlags.InDebugMode) { + console.log(` matched injection: ${injection.debugSelector}`); + console.log(ruleScanner.toString()); + } + + const matchRating = matchResult.captureIndices[0].start; + if (matchRating >= bestMatchRating) { + // Injections are sorted by priority, so the previous injection had a better or equal priority + continue; + } + + bestMatchRating = matchRating; + bestMatchCaptureIndices = matchResult.captureIndices; + bestMatchRuleId = matchResult.ruleId; + bestMatchResultPriority = injection.priority; + + if (bestMatchRating === linePos) { + // No more need to look at the rest of the injections. + break; + } + } + + if (bestMatchCaptureIndices) { + return { + priorityMatch: bestMatchResultPriority === -1, + captureIndices: bestMatchCaptureIndices, + matchedRuleId: bestMatchRuleId! + }; + } + + return null; +} + +interface IMatchInjectionsResult { + readonly priorityMatch: boolean; + readonly captureIndices: IOnigCaptureIndex[]; + readonly matchedRuleId: RuleId | typeof endRuleId; +} + +function prepareRuleSearch(rule: Rule, grammar: Grammar, endRegexSource: string | null, allowA: boolean, allowG: boolean): { ruleScanner: CompiledRule; findOptions: number; } { + if (UseOnigurumaFindOptions) { + const ruleScanner = rule.compile(grammar, endRegexSource); + const findOptions = getFindOptions(allowA, allowG); + return { ruleScanner, findOptions }; + } + const ruleScanner = rule.compileAG(grammar, endRegexSource, allowA, allowG); + return { ruleScanner, findOptions: FindOption.None }; +} + +function prepareRuleWhileSearch(rule: BeginWhileRule, grammar: Grammar, endRegexSource: string | null, allowA: boolean, allowG: boolean): { ruleScanner: CompiledRule; findOptions: number; } { + if (UseOnigurumaFindOptions) { + const ruleScanner = rule.compileWhile(grammar, endRegexSource); + const findOptions = getFindOptions(allowA, allowG); + return { ruleScanner, findOptions }; + } + const ruleScanner = rule.compileWhileAG(grammar, endRegexSource, allowA, allowG); + return { ruleScanner, findOptions: FindOption.None }; +} + +function getFindOptions(allowA: boolean, allowG: boolean): number { + let options = FindOption.None; + if (!allowA) { + options |= FindOption.NotBeginString; + } + if (!allowG) { + options |= FindOption.NotBeginPosition; + } + return options; +} + +function handleCaptures(grammar: Grammar, lineText: OnigString, isFirstLine: boolean, stack: StateStack, lineTokens: LineTokens, captures: (CaptureRule | null)[], captureIndices: IOnigCaptureIndex[]): void { + if (captures.length === 0) { + return; + } + + const lineTextContent = lineText.content; + + const len = Math.min(captures.length, captureIndices.length); + const localStack: LocalStackElement[] = []; + const maxEnd = captureIndices[0].end; + + for (let i = 0; i < len; i++) { + const captureRule = captures[i]; + if (captureRule === null) { + // Not interested + continue; + } + + const captureIndex = captureIndices[i]; + + if (captureIndex.length === 0) { + // Nothing really captured + continue; + } + + if (captureIndex.start > maxEnd) { + // Capture going beyond consumed string + break; + } + + // pop captures while needed + while (localStack.length > 0 && localStack[localStack.length - 1].endPos <= captureIndex.start) { + // pop! + lineTokens.produceFromScopes(localStack[localStack.length - 1].scopes, localStack[localStack.length - 1].endPos); + localStack.pop(); + } + + if (localStack.length > 0) { + lineTokens.produceFromScopes(localStack[localStack.length - 1].scopes, captureIndex.start); + } else { + lineTokens.produce(stack, captureIndex.start); + } + + if (captureRule.retokenizeCapturedWithRuleId) { + // the capture requires additional matching + const scopeName = captureRule.getName(lineTextContent, captureIndices); + const nameScopesList = stack.contentNameScopesList.pushAttributed(scopeName, grammar); + const contentName = captureRule.getContentName(lineTextContent, captureIndices); + const contentNameScopesList = nameScopesList.pushAttributed(contentName, grammar); + + const stackClone = stack.push(captureRule.retokenizeCapturedWithRuleId, captureIndex.start, -1, false, null, nameScopesList, contentNameScopesList); + const onigSubStr = grammar.createOnigString(lineTextContent.substring(0, captureIndex.end)); + _tokenizeString(grammar, onigSubStr, (isFirstLine && captureIndex.start === 0), captureIndex.start, stackClone, lineTokens, false, /* no time limit */0); + disposeOnigString(onigSubStr); + continue; + } + + const captureRuleScopeName = captureRule.getName(lineTextContent, captureIndices); + if (captureRuleScopeName !== null) { + // push + const base = localStack.length > 0 ? localStack[localStack.length - 1].scopes : stack.contentNameScopesList; + const captureRuleScopesList = base.pushAttributed(captureRuleScopeName, grammar); + localStack.push(new LocalStackElement(captureRuleScopesList, captureIndex.end)); + } + } + + while (localStack.length > 0) { + // pop! + lineTokens.produceFromScopes(localStack[localStack.length - 1].scopes, localStack[localStack.length - 1].endPos); + localStack.pop(); + } +} + +export class LocalStackElement { + public readonly scopes: AttributedScopeStack; + public readonly endPos: number; + + constructor(scopes: AttributedScopeStack, endPos: number) { + this.scopes = scopes; + this.endPos = endPos; + } +} diff --git a/src/json.ts b/src/json.ts index 95646b28..44883991 100644 --- a/src/json.ts +++ b/src/json.ts @@ -13,7 +13,7 @@ export interface ILocation { readonly char: number; } -export function parse(source: string, filename: string | null, withMetadata: boolean): any { +export function parseJSON(source: string, filename: string | null, withMetadata: boolean): any { let streamState = new JSONStreamState(source); let token = new JSONToken(); let state = JSONState.ROOT_STATE; diff --git a/src/main.ts b/src/main.ts index 8d739dad..4f823206 100644 --- a/src/main.ts +++ b/src/main.ts @@ -2,36 +2,17 @@ * Copyright (C) Microsoft Corporation. All rights reserved. *--------------------------------------------------------*/ -import { BalancedBracketSelectors, ScopeDependencyProcessor, StackElement as StackElementImpl } from './grammar'; +import { BalancedBracketSelectors, StateStack as StackElementImpl } from './grammar'; import * as grammarReader from './parseRawGrammar'; import { IOnigLib } from './onigLib'; import { IRawGrammar } from './rawGrammar'; import { SyncRegistry } from './registry'; -import { Theme } from './theme'; +import { IRawTheme, ScopeName, Theme } from './theme'; +import { StandardTokenType } from './encodedTokenAttributes'; +import { ScopeDependencyProcessor } from './grammar/grammarDependencies' export * from './onigLib'; -/** - * A single theme setting. - */ -export interface IRawThemeSetting { - readonly name?: string; - readonly scope?: string | string[]; - readonly settings: { - readonly fontStyle?: string; - readonly foreground?: string; - readonly background?: string; - }; -} - -/** - * A TextMate theme. - */ -export interface IRawTheme { - readonly name?: string; - readonly settings: IRawThemeSetting[]; -} - /** * A registry helper that can locate grammar file paths given scope names. */ @@ -39,8 +20,8 @@ export interface RegistryOptions { onigLib: Promise; theme?: IRawTheme; colorMap?: string[]; - loadGrammar(scopeName: string): Promise; - getInjections?(scopeName: string): string[] | undefined; + loadGrammar(scopeName: ScopeName): Promise; + getInjections?(scopeName: ScopeName): ScopeName[] | undefined; } /** @@ -57,13 +38,6 @@ export interface ITokenTypeMap { [selector: string]: StandardTokenType; } -export const enum StandardTokenType { - Other = 0, - Comment = 1, - String = 2, - RegEx = 3 -} - export interface IGrammarConfiguration { embeddedLanguages?: IEmbeddedLanguagesMap; tokenTypes?: ITokenTypeMap; @@ -75,14 +49,16 @@ export interface IGrammarConfiguration { * The registry that will hold all grammars. */ export class Registry { - private readonly _options: RegistryOptions; private readonly _syncRegistry: SyncRegistry; private readonly _ensureGrammarCache: Map>; constructor(options: RegistryOptions) { this._options = options; - this._syncRegistry = new SyncRegistry(Theme.createFromRawTheme(options.theme, options.colorMap), options.onigLib); + this._syncRegistry = new SyncRegistry( + Theme.createFromRawTheme(options.theme, options.colorMap), + options.onigLib + ); this._ensureGrammarCache = new Map>(); } @@ -108,7 +84,11 @@ export class Registry { * Load the grammar for `scopeName` and all referenced included grammars asynchronously. * Please do not use language id 0. */ - public loadGrammarWithEmbeddedLanguages(initialScopeName: string, initialLanguage: number, embeddedLanguages: IEmbeddedLanguagesMap): Promise { + public loadGrammarWithEmbeddedLanguages( + initialScopeName: ScopeName, + initialLanguage: number, + embeddedLanguages: IEmbeddedLanguagesMap + ): Promise { return this.loadGrammarWithConfiguration(initialScopeName, initialLanguage, { embeddedLanguages }); } @@ -116,7 +96,11 @@ export class Registry { * Load the grammar for `scopeName` and all referenced included grammars asynchronously. * Please do not use language id 0. */ - public loadGrammarWithConfiguration(initialScopeName: string, initialLanguage: number, configuration: IGrammarConfiguration): Promise { + public loadGrammarWithConfiguration( + initialScopeName: ScopeName, + initialLanguage: number, + configuration: IGrammarConfiguration + ): Promise { return this._loadGrammar( initialScopeName, initialLanguage, @@ -132,40 +116,57 @@ export class Registry { /** * Load the grammar for `scopeName` and all referenced included grammars asynchronously. */ - public loadGrammar(initialScopeName: string): Promise { + public loadGrammar(initialScopeName: ScopeName): Promise { return this._loadGrammar(initialScopeName, 0, null, null, null); } - private async _doLoadSingleGrammar(scopeName: string): Promise { - const grammar = await this._options.loadGrammar(scopeName); - if (grammar) { - const injections = (typeof this._options.getInjections === 'function' ? this._options.getInjections(scopeName) : undefined); - this._syncRegistry.addGrammar(grammar, injections); + private async _loadGrammar( + initialScopeName: ScopeName, + initialLanguage: number, + embeddedLanguages: IEmbeddedLanguagesMap | null | undefined, + tokenTypes: ITokenTypeMap | null | undefined, + balancedBracketSelectors: BalancedBracketSelectors | null + ): Promise { + const dependencyProcessor = new ScopeDependencyProcessor(this._syncRegistry, initialScopeName); + while (dependencyProcessor.Q.length > 0) { + await Promise.all(dependencyProcessor.Q.map((request) => this._loadSingleGrammar(request.scopeName))); + dependencyProcessor.processQueue(); } + + return this._grammarForScopeName( + initialScopeName, + initialLanguage, + embeddedLanguages, + tokenTypes, + balancedBracketSelectors + ); } - private async _loadSingleGrammar(scopeName: string): Promise { + private async _loadSingleGrammar(scopeName: ScopeName): Promise { if (!this._ensureGrammarCache.has(scopeName)) { this._ensureGrammarCache.set(scopeName, this._doLoadSingleGrammar(scopeName)); } return this._ensureGrammarCache.get(scopeName); } - private async _loadGrammar(initialScopeName: string, initialLanguage: number, embeddedLanguages: IEmbeddedLanguagesMap | null | undefined, tokenTypes: ITokenTypeMap | null | undefined, balancedBracketSelectors: BalancedBracketSelectors | null): Promise { - - const dependencyProcessor = new ScopeDependencyProcessor(this._syncRegistry, initialScopeName); - while (dependencyProcessor.Q.length > 0) { - await Promise.all(dependencyProcessor.Q.map(request => this._loadSingleGrammar(request.scopeName))); - dependencyProcessor.processQueue(); + private async _doLoadSingleGrammar(scopeName: ScopeName): Promise { + const grammar = await this._options.loadGrammar(scopeName); + if (grammar) { + const injections = + typeof this._options.getInjections === "function" ? this._options.getInjections(scopeName) : undefined; + this._syncRegistry.addGrammar(grammar, injections); } - - return this._grammarForScopeName(initialScopeName, initialLanguage, embeddedLanguages, tokenTypes, balancedBracketSelectors); } /** * Adds a rawGrammar. */ - public async addGrammar(rawGrammar: IRawGrammar, injections: string[] = [], initialLanguage: number = 0, embeddedLanguages: IEmbeddedLanguagesMap | null = null): Promise { + public async addGrammar( + rawGrammar: IRawGrammar, + injections: string[] = [], + initialLanguage: number = 0, + embeddedLanguages: IEmbeddedLanguagesMap | null = null + ): Promise { this._syncRegistry.addGrammar(rawGrammar, injections); return (await this._grammarForScopeName(rawGrammar.scopeName, initialLanguage, embeddedLanguages))!; } @@ -173,13 +174,23 @@ export class Registry { /** * Get the grammar for `scopeName`. The grammar must first be created via `loadGrammar` or `addGrammar`. */ - private _grammarForScopeName(scopeName: string, initialLanguage: number = 0, embeddedLanguages: IEmbeddedLanguagesMap | null = null, tokenTypes: ITokenTypeMap | null = null, balancedBracketSelectors: BalancedBracketSelectors | null = null): Promise { - return this._syncRegistry.grammarForScopeName(scopeName, initialLanguage, embeddedLanguages, tokenTypes, balancedBracketSelectors); + private _grammarForScopeName( + scopeName: string, + initialLanguage: number = 0, + embeddedLanguages: IEmbeddedLanguagesMap | null = null, + tokenTypes: ITokenTypeMap | null = null, + balancedBracketSelectors: BalancedBracketSelectors | null = null + ): Promise { + return this._syncRegistry.grammarForScopeName( + scopeName, + initialLanguage, + embeddedLanguages, + tokenTypes, + balancedBracketSelectors + ); } } - - /** * A grammar */ @@ -187,7 +198,7 @@ export interface IGrammar { /** * Tokenize `lineText` using previous line state `prevState`. */ - tokenizeLine(lineText: string, prevState: StackElement | null, timeLimit?: number): ITokenizeLineResult; + tokenizeLine(lineText: string, prevState: StateStack | null, timeLimit?: number): ITokenizeLineResult; /** * Tokenize `lineText` using previous line state `prevState`. @@ -199,7 +210,7 @@ export interface IGrammar { * - background color * e.g. for getting the languageId: `(metadata & MetadataConsts.LANGUAGEID_MASK) >>> MetadataConsts.LANGUAGEID_OFFSET` */ - tokenizeLine2(lineText: string, prevState: StackElement | null, timeLimit?: number): ITokenizeLineResult2; + tokenizeLine2(lineText: string, prevState: StateStack | null, timeLimit?: number): ITokenizeLineResult2; } export interface ITokenizeLineResult { @@ -207,7 +218,7 @@ export interface ITokenizeLineResult { /** * The `prevState` to be passed on to the next line tokenization. */ - readonly ruleStack: StackElement; + readonly ruleStack: StateStack; /** * Did tokenization stop early due to reaching the time limit. */ @@ -225,7 +236,7 @@ export interface ITokenizeLineResult2 { /** * The `prevState` to be passed on to the next line tokenization. */ - readonly ruleStack: StackElement; + readonly ruleStack: StateStack; /** * Did tokenization stop early due to reaching the time limit. */ @@ -241,14 +252,14 @@ export interface IToken { /** * **IMPORTANT** - Immutable! */ -export interface StackElement { +export interface StateStack { _stackElementBrand: void; readonly depth: number; - clone(): StackElement; - equals(other: StackElement): boolean; + clone(): StateStack; + equals(other: StateStack): boolean; } -export const INITIAL: StackElement = StackElementImpl.NULL; +export const INITIAL: StateStack = StackElementImpl.NULL; export const parseRawGrammar: (content: string, filePath?: string) => IRawGrammar = grammarReader.parseRawGrammar; diff --git a/src/metadata.ts b/src/metadata.ts deleted file mode 100644 index a04a9b5a..00000000 --- a/src/metadata.ts +++ /dev/null @@ -1,159 +0,0 @@ -/*--------------------------------------------------------- - * Copyright (C) Microsoft Corporation. All rights reserved. - *--------------------------------------------------------*/ - -import { StandardTokenType } from "./main"; -import { FontStyle } from "./theme"; - -/** - * Helpers to manage the "collapsed" metadata of an entire StackElement stack. - * The following assumptions have been made: - * - languageId < 256 => needs 8 bits - * - unique color count < 512 => needs 9 bits - * - * The binary format is: - * - ------------------------------------------- - * 3322 2222 2222 1111 1111 1100 0000 0000 - * 1098 7654 3210 9876 5432 1098 7654 3210 - * - ------------------------------------------- - * xxxx xxxx xxxx xxxx xxxx xxxx xxxx xxxx - * bbbb bbbb ffff ffff fFFF FBTT LLLL LLLL - * - ------------------------------------------- - * - L = LanguageId (8 bits) - * - T = StandardTokenType (2 bits) - * - B = Balanced bracket (1 bit) - * - F = FontStyle (4 bits) - * - f = foreground color (9 bits) - * - b = background color (9 bits) - */ - export const enum MetadataConsts { - LANGUAGEID_MASK = 0b00000000000000000000000011111111, - TOKEN_TYPE_MASK = 0b00000000000000000000001100000000, - BALANCED_BRACKETS_MASK = 0b00000000000000000000010000000000, - FONT_STYLE_MASK = 0b00000000000000000111100000000000, - FOREGROUND_MASK = 0b00000000111111111000000000000000, - BACKGROUND_MASK = 0b11111111000000000000000000000000, - - LANGUAGEID_OFFSET = 0, - TOKEN_TYPE_OFFSET = 8, - BALANCED_BRACKETS_OFFSET = 10, - FONT_STYLE_OFFSET = 11, - FOREGROUND_OFFSET = 15, - BACKGROUND_OFFSET = 24 -} - -export class StackElementMetadata { - - public static toBinaryStr(metadata: number): string { - let r = metadata.toString(2); - while (r.length < 32) { - r = '0' + r; - } - return r; - } - - public static printMetadata(metadata: number): void { - const languageId = StackElementMetadata.getLanguageId(metadata); - const tokenType = StackElementMetadata.getTokenType(metadata); - const fontStyle = StackElementMetadata.getFontStyle(metadata); - const foreground = StackElementMetadata.getForeground(metadata); - const background = StackElementMetadata.getBackground(metadata); - - console.log({ - languageId: languageId, - tokenType: tokenType, - fontStyle: fontStyle, - foreground: foreground, - background: background, - }); - } - - public static getLanguageId(metadata: number): number { - return (metadata & MetadataConsts.LANGUAGEID_MASK) >>> MetadataConsts.LANGUAGEID_OFFSET; - } - - public static getTokenType(metadata: number): StandardTokenType { - return (metadata & MetadataConsts.TOKEN_TYPE_MASK) >>> MetadataConsts.TOKEN_TYPE_OFFSET; - } - - public static containsBalancedBrackets(metadata: number): boolean { - return (metadata & MetadataConsts.BALANCED_BRACKETS_MASK) !== 0; - } - - public static getFontStyle(metadata: number): number { - return (metadata & MetadataConsts.FONT_STYLE_MASK) >>> MetadataConsts.FONT_STYLE_OFFSET; - } - - public static getForeground(metadata: number): number { - return (metadata & MetadataConsts.FOREGROUND_MASK) >>> MetadataConsts.FOREGROUND_OFFSET; - } - - public static getBackground(metadata: number): number { - return (metadata & MetadataConsts.BACKGROUND_MASK) >>> MetadataConsts.BACKGROUND_OFFSET; - } - - /** - * Updates the fields in `metadata`. - * A value of `0`, `NotSet` or `null` indicates that the corresponding field should be left as is. - */ - public static set(metadata: number, languageId: number, tokenType: OptionalStandardTokenType, containsBalancedBrackets: boolean | null, fontStyle: FontStyle, foreground: number, background: number): number { - let _languageId = StackElementMetadata.getLanguageId(metadata); - let _tokenType = StackElementMetadata.getTokenType(metadata); - let _containsBalancedBracketsBit: 0 | 1 = StackElementMetadata.containsBalancedBrackets(metadata) ? 1 : 0; - let _fontStyle = StackElementMetadata.getFontStyle(metadata); - let _foreground = StackElementMetadata.getForeground(metadata); - let _background = StackElementMetadata.getBackground(metadata); - - if (languageId !== 0) { - _languageId = languageId; - } - if (tokenType !== OptionalStandardTokenType.NotSet) { - _tokenType = fromOptionalTokenType(tokenType); - } - if (containsBalancedBrackets !== null) { - _containsBalancedBracketsBit = containsBalancedBrackets ? 1 : 0; - } - if (fontStyle !== FontStyle.NotSet) { - _fontStyle = fontStyle; - } - if (foreground !== 0) { - _foreground = foreground; - } - if (background !== 0) { - _background = background; - } - - return ( - (_languageId << MetadataConsts.LANGUAGEID_OFFSET) - | (_tokenType << MetadataConsts.TOKEN_TYPE_OFFSET) - | (_containsBalancedBracketsBit << MetadataConsts.BALANCED_BRACKETS_OFFSET) - | (_fontStyle << MetadataConsts.FONT_STYLE_OFFSET) - | (_foreground << MetadataConsts.FOREGROUND_OFFSET) - | (_background << MetadataConsts.BACKGROUND_OFFSET) - ) >>> 0; - } -} - -export function toOptionalTokenType(standardType: StandardTokenType): OptionalStandardTokenType { - return standardType as any as OptionalStandardTokenType; -} - -function fromOptionalTokenType( - standardType: - | OptionalStandardTokenType.Other - | OptionalStandardTokenType.Comment - | OptionalStandardTokenType.String - | OptionalStandardTokenType.RegEx -): StandardTokenType { - return standardType as any as StandardTokenType; -} - -// Must have the same values as `StandardTokenType`! -export const enum OptionalStandardTokenType { - Other = 0, - Comment = 1, - String = 2, - RegEx = 3, - // Indicates that no token type is set. - NotSet = 8 -} diff --git a/src/onigLib.ts b/src/onigLib.ts index 4932e8d6..dc18448b 100644 --- a/src/onigLib.ts +++ b/src/onigLib.ts @@ -49,3 +49,9 @@ export interface OnigString { readonly content: string; dispose?(): void; } + +export function disposeOnigString(str: OnigString) { + if (typeof str.dispose === 'function') { + str.dispose(); + } +} \ No newline at end of file diff --git a/src/parseRawGrammar.ts b/src/parseRawGrammar.ts index a8bc43c6..f64614d4 100644 --- a/src/parseRawGrammar.ts +++ b/src/parseRawGrammar.ts @@ -5,7 +5,7 @@ import { IRawGrammar } from './rawGrammar'; import * as plist from './plist'; import { DebugFlags } from './debug'; -import { parse as manualParseJSON } from './json'; +import { parseJSON } from './json'; export function parseRawGrammar(content: string, filePath: string | null = null): IRawGrammar { if (filePath !== null && /\.json$/.test(filePath)) { @@ -16,7 +16,7 @@ export function parseRawGrammar(content: string, filePath: string | null = null) function parseJSONGrammar(contents: string, filename: string | null): IRawGrammar { if (DebugFlags.InDebugMode) { - return manualParseJSON(contents, filename, true); + return parseJSON(contents, filename, true); } return JSON.parse(contents); } @@ -25,5 +25,5 @@ function parsePLISTGrammar(contents: string, filename: string | null): IRawGramm if (DebugFlags.InDebugMode) { return plist.parseWithLocation(contents, filename, '$vscodeTextmateLocation'); } - return plist.parse(contents); + return plist.parsePLIST(contents); } diff --git a/src/plist.ts b/src/plist.ts index e08963fa..27e5d5bb 100644 --- a/src/plist.ts +++ b/src/plist.ts @@ -30,7 +30,7 @@ export function parseWithLocation(content: string, filename: string | null, loca /** * A very fast plist parser */ -export function parse(content: string): any { +export function parsePLIST(content: string): any { return _parse(content, null, null); } diff --git a/src/rawGrammar.ts b/src/rawGrammar.ts index e0af7dce..1361051f 100644 --- a/src/rawGrammar.ts +++ b/src/rawGrammar.ts @@ -2,6 +2,8 @@ * Copyright (C) Microsoft Corporation. All rights reserved. *--------------------------------------------------------*/ +import { RuleId } from "./rule"; + export interface ILocation { readonly filename: string; readonly line: number; @@ -33,7 +35,7 @@ export interface IRawRepositoryMap { export type IRawRepository = IRawRepositoryMap & ILocatable; export interface IRawRule extends ILocatable { - id?: number; + id?: RuleId; readonly include?: string; diff --git a/src/registry.ts b/src/registry.ts index 1ef75637..600608c9 100644 --- a/src/registry.ts +++ b/src/registry.ts @@ -2,42 +2,30 @@ * Copyright (C) Microsoft Corporation. All rights reserved. *--------------------------------------------------------*/ -import { BalancedBracketSelectors, createGrammar, Grammar, IGrammarRepository } from './grammar'; +import { BalancedBracketSelectors, createGrammar, Grammar, IGrammarRepository, IThemeProvider, AttributedScopeStack } from './grammar'; import { IRawGrammar } from './rawGrammar'; import { IGrammar, IEmbeddedLanguagesMap, ITokenTypeMap } from './main'; -import { Theme, ThemeTrieElementRule } from './theme'; +import { ScopeStack, Theme, StyleAttributes, ThemeTrieElementRule, ScopeName } from './theme'; import { IOnigLib } from './onigLib'; -export class SyncRegistry implements IGrammarRepository { - - private readonly _grammars: { [scopeName: string]: Grammar; }; - private readonly _rawGrammars: { [scopeName: string]: IRawGrammar; }; - private readonly _injectionGrammars: { [scopeName: string]: string[]; }; +export class SyncRegistry implements IGrammarRepository, IThemeProvider { + private readonly _grammars = new Map(); + private readonly _rawGrammars = new Map(); + private readonly _injectionGrammars = new Map(); private _theme: Theme; - private readonly _onigLibPromise: Promise; - constructor(theme: Theme, onigLibPromise: Promise) { + constructor(theme: Theme, private readonly _onigLibPromise: Promise) { this._theme = theme; - this._grammars = {}; - this._rawGrammars = {}; - this._injectionGrammars = {}; - this._onigLibPromise = onigLibPromise; } public dispose(): void { - for (const scopeName in this._grammars) { - if (this._grammars.hasOwnProperty(scopeName)) { - this._grammars[scopeName].dispose(); - } + for (const grammar of this._grammars.values()) { + grammar.dispose(); } } public setTheme(theme: Theme): void { this._theme = theme; - Object.keys(this._grammars).forEach((scopeName) => { - let grammar = this._grammars[scopeName]; - grammar.onDidChangeTheme(); - }); } public getColorMap(): string[] { @@ -47,53 +35,68 @@ export class SyncRegistry implements IGrammarRepository { /** * Add `grammar` to registry and return a list of referenced scope names */ - public addGrammar(grammar: IRawGrammar, injectionScopeNames?: string[]): void { - this._rawGrammars[grammar.scopeName] = grammar; + public addGrammar(grammar: IRawGrammar, injectionScopeNames?: ScopeName[]): void { + this._rawGrammars.set(grammar.scopeName, grammar); if (injectionScopeNames) { - this._injectionGrammars[grammar.scopeName] = injectionScopeNames; + this._injectionGrammars.set(grammar.scopeName, injectionScopeNames); } } /** * Lookup a raw grammar. */ - public lookup(scopeName: string): IRawGrammar | undefined { - return this._rawGrammars[scopeName]; + public lookup(scopeName: ScopeName): IRawGrammar | undefined { + return this._rawGrammars.get(scopeName)!; } /** * Returns the injections for the given grammar */ - public injections(targetScope: string): string[] { - return this._injectionGrammars[targetScope]; + public injections(targetScope: ScopeName): ScopeName[] { + return this._injectionGrammars.get(targetScope)!; } /** * Get the default theme settings */ - public getDefaults(): ThemeTrieElementRule { + public getDefaults(): StyleAttributes { return this._theme.getDefaults(); } /** * Match a scope in the theme. */ - public themeMatch(scopeName: string): ThemeTrieElementRule[] { - return this._theme.match(scopeName); + public themeMatch(scopePath: ScopeStack): StyleAttributes | null { + return this._theme.match(scopePath); } /** * Lookup a grammar. */ - public async grammarForScopeName(scopeName: string, initialLanguage: number, embeddedLanguages: IEmbeddedLanguagesMap | null, tokenTypes: ITokenTypeMap | null, balancedBracketSelectors: BalancedBracketSelectors | null): Promise { - if (!this._grammars[scopeName]) { - let rawGrammar = this._rawGrammars[scopeName]; + public async grammarForScopeName( + scopeName: ScopeName, + initialLanguage: number, + embeddedLanguages: IEmbeddedLanguagesMap | null, + tokenTypes: ITokenTypeMap | null, + balancedBracketSelectors: BalancedBracketSelectors | null + ): Promise { + if (!this._grammars.has(scopeName)) { + let rawGrammar = this._rawGrammars.get(scopeName)!; if (!rawGrammar) { return null; } - this._grammars[scopeName] = createGrammar(scopeName, rawGrammar, initialLanguage, embeddedLanguages, tokenTypes, balancedBracketSelectors, this, await this._onigLibPromise); + this._grammars.set(scopeName, createGrammar( + scopeName, + rawGrammar, + initialLanguage, + embeddedLanguages, + tokenTypes, + balancedBracketSelectors, + this, + await this._onigLibPromise + )); } - return this._grammars[scopeName]; + return this._grammars.get(scopeName)!; } } diff --git a/src/rule.ts b/src/rule.ts index faf36296..6f30da68 100644 --- a/src/rule.ts +++ b/src/rule.ts @@ -2,16 +2,35 @@ * Copyright (C) Microsoft Corporation. All rights reserved. *--------------------------------------------------------*/ -import { RegexSource, mergeObjects, basename } from './utils'; -import { IOnigLib, OnigScanner, IOnigCaptureIndex } from './onigLib'; +import { RegexSource, mergeObjects, basename, escapeRegExpCharacters, OrMask } from './utils'; +import { IOnigLib, OnigScanner, IOnigCaptureIndex, FindOption, IOnigMatch, OnigString } from './onigLib'; import { ILocation, IRawGrammar, IRawRepository, IRawRule, IRawCaptures } from './rawGrammar'; +import { IncludeReferenceKind, parseInclude } from './grammar/grammarDependencies'; const HAS_BACK_REFERENCES = /\\(\d+)/; const BACK_REFERENCING_END = /\\(\d+)/g; +const ruleIdSymbol = Symbol('RuleId'); +export type RuleId = { __brand: typeof ruleIdSymbol }; + +// This is a special constant to indicate that the end regexp matched. +export const endRuleId = -1; + +// This is a special constant to indicate that the while regexp matched. +export const whileRuleId = -2; + + +export function ruleIdFromNumber(id: number): RuleId { + return id as any as RuleId; +} + +export function ruleIdToNumber(id: RuleId): number { + return id as any as number; +} + export interface IRuleRegistry { - getRule(patternId: number): Rule; - registerRule(factory: (id: number) => T): T; + getRule(ruleId: RuleId): Rule; + registerRule(factory: (id: RuleId) => T): T; } export interface IGrammarRegistry { @@ -21,29 +40,10 @@ export interface IGrammarRegistry { export interface IRuleFactoryHelper extends IRuleRegistry, IGrammarRegistry { } -export class CompiledRule { - - public readonly debugRegExps: string[]; - public readonly rules: number[]; - public readonly scanner: OnigScanner; - - constructor(onigLib: IOnigLib, regExps: string[], rules: number[]) { - this.debugRegExps = regExps; - this.rules = rules; - this.scanner = onigLib.createOnigScanner(regExps); - } - - public dispose(): void { - if (typeof this.scanner.dispose === 'function') { - this.scanner.dispose(); - } - } -} - export abstract class Rule { public readonly $location: ILocation | undefined; - public readonly id: number; + public readonly id: RuleId; private readonly _nameIsCapturing: boolean; private readonly _name: string | null; @@ -51,7 +51,7 @@ export abstract class Rule { private readonly _contentNameIsCapturing: boolean; private readonly _contentName: string | null; - constructor($location: ILocation | undefined, id: number, name: string | null | undefined, contentName: string | null | undefined) { + constructor($location: ILocation | undefined, id: RuleId, name: string | null | undefined, contentName: string | null | undefined) { this.$location = $location; this.id = id; this._name = name || null; @@ -81,7 +81,7 @@ export abstract class Rule { return RegexSource.replaceCaptures(this._contentName, lineText, captureIndices); } - public abstract collectPatternsRecursive(grammar: IRuleRegistry, out: RegExpSourceList, isFirst: boolean): void; + public abstract collectPatterns(grammar: IRuleRegistry, out: RegExpSourceList): void; public abstract compile(grammar: IRuleRegistry & IOnigLib, endRegexSource: string | null): CompiledRule; @@ -89,15 +89,15 @@ export abstract class Rule { } export interface ICompilePatternsResult { - readonly patterns: number[]; + readonly patterns: RuleId[]; readonly hasMissingPatterns: boolean; } export class CaptureRule extends Rule { - public readonly retokenizeCapturedWithRuleId: number; + public readonly retokenizeCapturedWithRuleId: RuleId | 0; - constructor($location: ILocation | undefined, id: number, name: string | null | undefined, contentName: string | null | undefined, retokenizeCapturedWithRuleId: number) { + constructor($location: ILocation | undefined, id: RuleId, name: string | null | undefined, contentName: string | null | undefined, retokenizeCapturedWithRuleId: RuleId | 0) { super($location, id, name, contentName); this.retokenizeCapturedWithRuleId = retokenizeCapturedWithRuleId; } @@ -106,7 +106,7 @@ export class CaptureRule extends Rule { // nothing to dispose } - public collectPatternsRecursive(grammar: IRuleRegistry, out: RegExpSourceList, isFirst: boolean) { + public collectPatterns(grammar: IRuleRegistry, out: RegExpSourceList) { throw new Error('Not supported!'); } @@ -119,444 +119,138 @@ export class CaptureRule extends Rule { } } -interface IRegExpSourceAnchorCache { - readonly A0_G0: string; - readonly A0_G1: string; - readonly A1_G0: string; - readonly A1_G1: string; -} - -export class RegExpSource { - - public source: string; - public readonly ruleId: number; - public hasAnchor: boolean; - public readonly hasBackReferences: boolean; - private _anchorCache: IRegExpSourceAnchorCache | null; - - constructor(regExpSource: string, ruleId: number, handleAnchors: boolean = true) { - if (handleAnchors) { - if (regExpSource) { - const len = regExpSource.length; - let lastPushedPos = 0; - let output: string[] = []; - - let hasAnchor = false; - for (let pos = 0; pos < len; pos++) { - const ch = regExpSource.charAt(pos); - - if (ch === '\\') { - if (pos + 1 < len) { - const nextCh = regExpSource.charAt(pos + 1); - if (nextCh === 'z') { - output.push(regExpSource.substring(lastPushedPos, pos)); - output.push('$(?!\\n)(? ' + this.source + ', ' + this.hasAnchor); + public get debugMatchRegExp(): string { + return `${this._match.source}`; } - public clone(): RegExpSource { - return new RegExpSource(this.source, this.ruleId, true); + public collectPatterns(grammar: IRuleRegistry, out: RegExpSourceList) { + out.push(this._match); } - public setSource(newSource: string): void { - if (this.source === newSource) { - return; - } - this.source = newSource; + public compile(grammar: IRuleRegistry & IOnigLib, endRegexSource: string): CompiledRule { + return this._getCachedCompiledPatterns(grammar).compile(grammar); + } - if (this.hasAnchor) { - this._anchorCache = this._buildAnchorCache(); - } + public compileAG(grammar: IRuleRegistry & IOnigLib, endRegexSource: string, allowA: boolean, allowG: boolean): CompiledRule { + return this._getCachedCompiledPatterns(grammar).compileAG(grammar, allowA, allowG); } - public resolveBackReferences(lineText: string, captureIndices: IOnigCaptureIndex[]): string { - let capturedValues = captureIndices.map((capture) => { - return lineText.substring(capture.start, capture.end); - }); - BACK_REFERENCING_END.lastIndex = 0; - return this.source.replace(BACK_REFERENCING_END, (match, g1) => { - return escapeRegExpCharacters(capturedValues[parseInt(g1, 10)] || ''); - }); + private _getCachedCompiledPatterns(grammar: IRuleRegistry & IOnigLib): RegExpSourceList { + if (!this._cachedCompiledPatterns) { + this._cachedCompiledPatterns = new RegExpSourceList(); + this.collectPatterns(grammar, this._cachedCompiledPatterns); + } + return this._cachedCompiledPatterns; } +} - private _buildAnchorCache(): IRegExpSourceAnchorCache { - let A0_G0_result: string[] = []; - let A0_G1_result: string[] = []; - let A1_G0_result: string[] = []; - let A1_G1_result: string[] = []; +export class IncludeOnlyRule extends Rule { + public readonly hasMissingPatterns: boolean; + public readonly patterns: RuleId[]; + private _cachedCompiledPatterns: RegExpSourceList | null; - let pos: number, - len: number, - ch: string, - nextCh: string; + constructor($location: ILocation | undefined, id: RuleId, name: string | null | undefined, contentName: string | null | undefined, patterns: ICompilePatternsResult) { + super($location, id, name, contentName); + this.patterns = patterns.patterns; + this.hasMissingPatterns = patterns.hasMissingPatterns; + this._cachedCompiledPatterns = null; + } - for (pos = 0, len = this.source.length; pos < len; pos++) { - ch = this.source.charAt(pos); - A0_G0_result[pos] = ch; - A0_G1_result[pos] = ch; - A1_G0_result[pos] = ch; - A1_G1_result[pos] = ch; + public dispose(): void { + if (this._cachedCompiledPatterns) { + this._cachedCompiledPatterns.dispose(); + this._cachedCompiledPatterns = null; + } + } - if (ch === '\\') { - if (pos + 1 < len) { - nextCh = this.source.charAt(pos + 1); - if (nextCh === 'A') { - A0_G0_result[pos + 1] = '\uFFFF'; - A0_G1_result[pos + 1] = '\uFFFF'; - A1_G0_result[pos + 1] = 'A'; - A1_G1_result[pos + 1] = 'A'; - } else if (nextCh === 'G') { - A0_G0_result[pos + 1] = '\uFFFF'; - A0_G1_result[pos + 1] = 'G'; - A1_G0_result[pos + 1] = '\uFFFF'; - A1_G1_result[pos + 1] = 'G'; - } else { - A0_G0_result[pos + 1] = nextCh; - A0_G1_result[pos + 1] = nextCh; - A1_G0_result[pos + 1] = nextCh; - A1_G1_result[pos + 1] = nextCh; - } - pos++; - } - } + public collectPatterns(grammar: IRuleRegistry, out: RegExpSourceList) { + for (const pattern of this.patterns) { + const rule = grammar.getRule(pattern); + rule.collectPatterns(grammar, out); } + } - return { - A0_G0: A0_G0_result.join(''), - A0_G1: A0_G1_result.join(''), - A1_G0: A1_G0_result.join(''), - A1_G1: A1_G1_result.join('') - }; + public compile(grammar: IRuleRegistry & IOnigLib, endRegexSource: string): CompiledRule { + return this._getCachedCompiledPatterns(grammar).compile(grammar); } - public resolveAnchors(allowA: boolean, allowG: boolean): string { - if (!this.hasAnchor || !this._anchorCache) { - return this.source; - } + public compileAG(grammar: IRuleRegistry & IOnigLib, endRegexSource: string, allowA: boolean, allowG: boolean): CompiledRule { + return this._getCachedCompiledPatterns(grammar).compileAG(grammar, allowA, allowG); + } - if (allowA) { - if (allowG) { - return this._anchorCache.A1_G1; - } else { - return this._anchorCache.A1_G0; - } - } else { - if (allowG) { - return this._anchorCache.A0_G1; - } else { - return this._anchorCache.A0_G0; - } + private _getCachedCompiledPatterns(grammar: IRuleRegistry & IOnigLib): RegExpSourceList { + if (!this._cachedCompiledPatterns) { + this._cachedCompiledPatterns = new RegExpSourceList(); + this.collectPatterns(grammar, this._cachedCompiledPatterns); } + return this._cachedCompiledPatterns; } } -interface IRegExpSourceListAnchorCache { - A0_G0: CompiledRule | null; - A0_G1: CompiledRule | null; - A1_G0: CompiledRule | null; - A1_G1: CompiledRule | null; -} - -export class RegExpSourceList { - - private readonly _items: RegExpSource[]; - private _hasAnchors: boolean; - private _cached: CompiledRule | null; - private _anchorCache: IRegExpSourceListAnchorCache; +export class BeginEndRule extends Rule { + private readonly _begin: RegExpSource; + public readonly beginCaptures: (CaptureRule | null)[]; + private readonly _end: RegExpSource; + public readonly endHasBackReferences: boolean; + public readonly endCaptures: (CaptureRule | null)[]; + public readonly applyEndPatternLast: boolean; + public readonly hasMissingPatterns: boolean; + public readonly patterns: RuleId[]; + private _cachedCompiledPatterns: RegExpSourceList | null; - constructor() { - this._items = []; - this._hasAnchors = false; - this._cached = null; - this._anchorCache = { - A0_G0: null, - A0_G1: null, - A1_G0: null, - A1_G1: null - }; + constructor($location: ILocation | undefined, id: RuleId, name: string | null | undefined, contentName: string | null | undefined, begin: string, beginCaptures: (CaptureRule | null)[], end: string | undefined, endCaptures: (CaptureRule | null)[], applyEndPatternLast: boolean | undefined, patterns: ICompilePatternsResult) { + super($location, id, name, contentName); + this._begin = new RegExpSource(begin, this.id); + this.beginCaptures = beginCaptures; + this._end = new RegExpSource(end ? end : '\uFFFF', -1); + this.endHasBackReferences = this._end.hasBackReferences; + this.endCaptures = endCaptures; + this.applyEndPatternLast = applyEndPatternLast || false; + this.patterns = patterns.patterns; + this.hasMissingPatterns = patterns.hasMissingPatterns; + this._cachedCompiledPatterns = null; } public dispose(): void { - this._disposeCaches(); + if (this._cachedCompiledPatterns) { + this._cachedCompiledPatterns.dispose(); + this._cachedCompiledPatterns = null; + } } - private _disposeCaches(): void { - if (this._cached) { - this._cached.dispose(); - this._cached = null; - } - if (this._anchorCache.A0_G0) { - this._anchorCache.A0_G0.dispose(); - this._anchorCache.A0_G0 = null; - } - if (this._anchorCache.A0_G1) { - this._anchorCache.A0_G1.dispose(); - this._anchorCache.A0_G1 = null; - } - if (this._anchorCache.A1_G0) { - this._anchorCache.A1_G0.dispose(); - this._anchorCache.A1_G0 = null; - } - if (this._anchorCache.A1_G1) { - this._anchorCache.A1_G1.dispose(); - this._anchorCache.A1_G1 = null; - } + public get debugBeginRegExp(): string { + return `${this._begin.source}`; } - public push(item: RegExpSource): void { - this._items.push(item); - this._hasAnchors = this._hasAnchors || item.hasAnchor; + public get debugEndRegExp(): string { + return `${this._end.source}`; } - public unshift(item: RegExpSource): void { - this._items.unshift(item); - this._hasAnchors = this._hasAnchors || item.hasAnchor; + public getEndWithResolvedBackReferences(lineText: string, captureIndices: IOnigCaptureIndex[]): string { + return this._end.resolveBackReferences(lineText, captureIndices); } - public length(): number { - return this._items.length; - } - - public setSource(index: number, newSource: string): void { - if (this._items[index].source !== newSource) { - // bust the cache - this._disposeCaches(); - this._items[index].setSource(newSource); - } - } - - public compile(onigLib: IOnigLib): CompiledRule { - if (!this._cached) { - let regExps = this._items.map(e => e.source); - this._cached = new CompiledRule(onigLib, regExps, this._items.map(e => e.ruleId)); - } - return this._cached; - } - - public compileAG(onigLib: IOnigLib, allowA: boolean, allowG: boolean): CompiledRule { - if (!this._hasAnchors) { - return this.compile(onigLib); - } else { - if (allowA) { - if (allowG) { - if (!this._anchorCache.A1_G1) { - this._anchorCache.A1_G1 = this._resolveAnchors(onigLib, allowA, allowG); - } - return this._anchorCache.A1_G1; - } else { - if (!this._anchorCache.A1_G0) { - this._anchorCache.A1_G0 = this._resolveAnchors(onigLib, allowA, allowG); - } - return this._anchorCache.A1_G0; - } - } else { - if (allowG) { - if (!this._anchorCache.A0_G1) { - this._anchorCache.A0_G1 = this._resolveAnchors(onigLib, allowA, allowG); - } - return this._anchorCache.A0_G1; - } else { - if (!this._anchorCache.A0_G0) { - this._anchorCache.A0_G0 = this._resolveAnchors(onigLib, allowA, allowG); - } - return this._anchorCache.A0_G0; - } - } - } - } - - private _resolveAnchors(onigLib: IOnigLib, allowA: boolean, allowG: boolean): CompiledRule { - let regExps = this._items.map(e => e.resolveAnchors(allowA, allowG)); - return new CompiledRule(onigLib, regExps, this._items.map(e => e.ruleId)); - } -} - -export class MatchRule extends Rule { - private readonly _match: RegExpSource; - public readonly captures: (CaptureRule | null)[]; - private _cachedCompiledPatterns: RegExpSourceList | null; - - constructor($location: ILocation | undefined, id: number, name: string | undefined, match: string, captures: (CaptureRule | null)[]) { - super($location, id, name, null); - this._match = new RegExpSource(match, this.id); - this.captures = captures; - this._cachedCompiledPatterns = null; - } - - public dispose(): void { - if (this._cachedCompiledPatterns) { - this._cachedCompiledPatterns.dispose(); - this._cachedCompiledPatterns = null; - } - } - - public get debugMatchRegExp(): string { - return `${this._match.source}`; - } - - public collectPatternsRecursive(grammar: IRuleRegistry, out: RegExpSourceList, isFirst: boolean) { - out.push(this._match); - } - - public compile(grammar: IRuleRegistry & IOnigLib, endRegexSource: string): CompiledRule { - return this._getCachedCompiledPatterns(grammar).compile(grammar); - } - - public compileAG(grammar: IRuleRegistry & IOnigLib, endRegexSource: string, allowA: boolean, allowG: boolean): CompiledRule { - return this._getCachedCompiledPatterns(grammar).compileAG(grammar, allowA, allowG); - } - - private _getCachedCompiledPatterns(grammar: IRuleRegistry & IOnigLib): RegExpSourceList { - if (!this._cachedCompiledPatterns) { - this._cachedCompiledPatterns = new RegExpSourceList(); - this.collectPatternsRecursive(grammar, this._cachedCompiledPatterns, true); - } - return this._cachedCompiledPatterns; - } -} - -export class IncludeOnlyRule extends Rule { - public readonly hasMissingPatterns: boolean; - public readonly patterns: number[]; - private _cachedCompiledPatterns: RegExpSourceList | null; - - constructor($location: ILocation | undefined, id: number, name: string | null | undefined, contentName: string | null | undefined, patterns: ICompilePatternsResult) { - super($location, id, name, contentName); - this.patterns = patterns.patterns; - this.hasMissingPatterns = patterns.hasMissingPatterns; - this._cachedCompiledPatterns = null; - } - - public dispose(): void { - if (this._cachedCompiledPatterns) { - this._cachedCompiledPatterns.dispose(); - this._cachedCompiledPatterns = null; - } - } - - public collectPatternsRecursive(grammar: IRuleRegistry, out: RegExpSourceList, isFirst: boolean) { - let i: number, - len: number, - rule: Rule; - - for (i = 0, len = this.patterns.length; i < len; i++) { - rule = grammar.getRule(this.patterns[i]); - rule.collectPatternsRecursive(grammar, out, false); - } - } - - public compile(grammar: IRuleRegistry & IOnigLib, endRegexSource: string): CompiledRule { - return this._getCachedCompiledPatterns(grammar).compile(grammar); - } - - public compileAG(grammar: IRuleRegistry & IOnigLib, endRegexSource: string, allowA: boolean, allowG: boolean): CompiledRule { - return this._getCachedCompiledPatterns(grammar).compileAG(grammar, allowA, allowG); - } - - private _getCachedCompiledPatterns(grammar: IRuleRegistry & IOnigLib): RegExpSourceList { - if (!this._cachedCompiledPatterns) { - this._cachedCompiledPatterns = new RegExpSourceList(); - this.collectPatternsRecursive(grammar, this._cachedCompiledPatterns, true); - } - return this._cachedCompiledPatterns; - } -} - -function escapeRegExpCharacters(value: string): string { - return value.replace(/[\-\\\{\}\*\+\?\|\^\$\.\,\[\]\(\)\#\s]/g, '\\$&'); -} - -export class BeginEndRule extends Rule { - private readonly _begin: RegExpSource; - public readonly beginCaptures: (CaptureRule | null)[]; - private readonly _end: RegExpSource; - public readonly endHasBackReferences: boolean; - public readonly endCaptures: (CaptureRule | null)[]; - public readonly applyEndPatternLast: boolean; - public readonly hasMissingPatterns: boolean; - public readonly patterns: number[]; - private _cachedCompiledPatterns: RegExpSourceList | null; - - constructor($location: ILocation | undefined, id: number, name: string | null | undefined, contentName: string | null | undefined, begin: string, beginCaptures: (CaptureRule | null)[], end: string | undefined, endCaptures: (CaptureRule | null)[], applyEndPatternLast: boolean | undefined, patterns: ICompilePatternsResult) { - super($location, id, name, contentName); - this._begin = new RegExpSource(begin, this.id); - this.beginCaptures = beginCaptures; - this._end = new RegExpSource(end ? end : '\uFFFF', -1); - this.endHasBackReferences = this._end.hasBackReferences; - this.endCaptures = endCaptures; - this.applyEndPatternLast = applyEndPatternLast || false; - this.patterns = patterns.patterns; - this.hasMissingPatterns = patterns.hasMissingPatterns; - this._cachedCompiledPatterns = null; - } - - public dispose(): void { - if (this._cachedCompiledPatterns) { - this._cachedCompiledPatterns.dispose(); - this._cachedCompiledPatterns = null; - } - } - - public get debugBeginRegExp(): string { - return `${this._begin.source}`; - } - - public get debugEndRegExp(): string { - return `${this._end.source}`; - } - - public getEndWithResolvedBackReferences(lineText: string, captureIndices: IOnigCaptureIndex[]): string { - return this._end.resolveBackReferences(lineText, captureIndices); - } - - public collectPatternsRecursive(grammar: IRuleRegistry, out: RegExpSourceList, isFirst: boolean) { - if (isFirst) { - let i: number, - len: number, - rule: Rule; - - for (i = 0, len = this.patterns.length; i < len; i++) { - rule = grammar.getRule(this.patterns[i]); - rule.collectPatternsRecursive(grammar, out, false); - } - } else { - out.push(this._begin); - } + public collectPatterns(grammar: IRuleRegistry, out: RegExpSourceList) { + out.push(this._begin); } public compile(grammar: IRuleRegistry & IOnigLib, endRegexSource: string): CompiledRule { @@ -571,7 +265,10 @@ export class BeginEndRule extends Rule { if (!this._cachedCompiledPatterns) { this._cachedCompiledPatterns = new RegExpSourceList(); - this.collectPatternsRecursive(grammar, this._cachedCompiledPatterns, true); + for (const pattern of this.patterns) { + const rule = grammar.getRule(pattern); + rule.collectPatterns(grammar, this._cachedCompiledPatterns); + } if (this.applyEndPatternLast) { this._cachedCompiledPatterns.push(this._end.hasBackReferences ? this._end.clone() : this._end); @@ -594,19 +291,19 @@ export class BeginWhileRule extends Rule { private readonly _begin: RegExpSource; public readonly beginCaptures: (CaptureRule | null)[]; public readonly whileCaptures: (CaptureRule | null)[]; - private readonly _while: RegExpSource; + private readonly _while: RegExpSource; public readonly whileHasBackReferences: boolean; public readonly hasMissingPatterns: boolean; - public readonly patterns: number[]; + public readonly patterns: RuleId[]; private _cachedCompiledPatterns: RegExpSourceList | null; - private _cachedCompiledWhilePatterns: RegExpSourceList | null; + private _cachedCompiledWhilePatterns: RegExpSourceList | null; - constructor($location: ILocation | undefined, id: number, name: string | null | undefined, contentName: string | null | undefined, begin: string, beginCaptures: (CaptureRule | null)[], _while: string, whileCaptures: (CaptureRule | null)[], patterns: ICompilePatternsResult) { + constructor($location: ILocation | undefined, id: RuleId, name: string | null | undefined, contentName: string | null | undefined, begin: string, beginCaptures: (CaptureRule | null)[], _while: string, whileCaptures: (CaptureRule | null)[], patterns: ICompilePatternsResult) { super($location, id, name, contentName); this._begin = new RegExpSource(begin, this.id); this.beginCaptures = beginCaptures; this.whileCaptures = whileCaptures; - this._while = new RegExpSource(_while, -2); + this._while = new RegExpSource(_while, whileRuleId); this.whileHasBackReferences = this._while.hasBackReferences; this.patterns = patterns.patterns; this.hasMissingPatterns = patterns.hasMissingPatterns; @@ -637,19 +334,8 @@ export class BeginWhileRule extends Rule { return this._while.resolveBackReferences(lineText, captureIndices); } - public collectPatternsRecursive(grammar: IRuleRegistry, out: RegExpSourceList, isFirst: boolean) { - if (isFirst) { - let i: number, - len: number, - rule: Rule; - - for (i = 0, len = this.patterns.length; i < len; i++) { - rule = grammar.getRule(this.patterns[i]); - rule.collectPatternsRecursive(grammar, out, false); - } - } else { - out.push(this._begin); - } + public collectPatterns(grammar: IRuleRegistry, out: RegExpSourceList) { + out.push(this._begin); } public compile(grammar: IRuleRegistry & IOnigLib, endRegexSource: string): CompiledRule { @@ -663,22 +349,26 @@ export class BeginWhileRule extends Rule { private _getCachedCompiledPatterns(grammar: IRuleRegistry & IOnigLib): RegExpSourceList { if (!this._cachedCompiledPatterns) { this._cachedCompiledPatterns = new RegExpSourceList(); - this.collectPatternsRecursive(grammar, this._cachedCompiledPatterns, true); + + for (const pattern of this.patterns) { + const rule = grammar.getRule(pattern); + rule.collectPatterns(grammar, this._cachedCompiledPatterns); + } } return this._cachedCompiledPatterns; } - public compileWhile(grammar: IRuleRegistry & IOnigLib, endRegexSource: string | null): CompiledRule { + public compileWhile(grammar: IRuleRegistry & IOnigLib, endRegexSource: string | null): CompiledRule { return this._getCachedCompiledWhilePatterns(grammar, endRegexSource).compile(grammar); } - public compileWhileAG(grammar: IRuleRegistry & IOnigLib, endRegexSource: string | null, allowA: boolean, allowG: boolean): CompiledRule { + public compileWhileAG(grammar: IRuleRegistry & IOnigLib, endRegexSource: string | null, allowA: boolean, allowG: boolean): CompiledRule { return this._getCachedCompiledWhilePatterns(grammar, endRegexSource).compileAG(grammar, allowA, allowG); } - private _getCachedCompiledWhilePatterns(grammar: IRuleRegistry & IOnigLib, endRegexSource: string | null): RegExpSourceList { + private _getCachedCompiledWhilePatterns(grammar: IRuleRegistry & IOnigLib, endRegexSource: string | null): RegExpSourceList { if (!this._cachedCompiledWhilePatterns) { - this._cachedCompiledWhilePatterns = new RegExpSourceList(); + this._cachedCompiledWhilePatterns = new RegExpSourceList(); this._cachedCompiledWhilePatterns.push(this._while.hasBackReferences ? this._while.clone() : this._while); } if (this._while.hasBackReferences) { @@ -690,13 +380,13 @@ export class BeginWhileRule extends Rule { export class RuleFactory { - public static createCaptureRule(helper: IRuleFactoryHelper, $location: ILocation | undefined, name: string | null | undefined, contentName: string | null | undefined, retokenizeCapturedWithRuleId: number): CaptureRule { + public static createCaptureRule(helper: IRuleFactoryHelper, $location: ILocation | undefined, name: string | null | undefined, contentName: string | null | undefined, retokenizeCapturedWithRuleId: RuleId | 0): CaptureRule { return helper.registerRule((id) => { return new CaptureRule($location, id, name, contentName, retokenizeCapturedWithRuleId); }); } - public static getCompiledRuleId(desc: IRawRule, helper: IRuleFactoryHelper, repository: IRawRepository): number { + public static getCompiledRuleId(desc: IRawRule, helper: IRuleFactoryHelper, repository: IRawRepository): RuleId { if (!desc.id) { helper.registerRule((id) => { desc.id = id; @@ -783,7 +473,7 @@ export class RuleFactory { continue; } const numericCaptureId = parseInt(captureId, 10); - let retokenizeCapturedWithRuleId = 0; + let retokenizeCapturedWithRuleId: RuleId | 0 = 0; if (captures[captureId].patterns) { retokenizeCapturedWithRuleId = RuleFactory.getCompiledRuleId(captures[captureId], helper, repository); } @@ -795,61 +485,67 @@ export class RuleFactory { } private static _compilePatterns(patterns: IRawRule[] | undefined, helper: IRuleFactoryHelper, repository: IRawRepository): ICompilePatternsResult { - let r: number[] = []; + let r: RuleId[] = []; if (patterns) { for (let i = 0, len = patterns.length; i < len; i++) { const pattern = patterns[i]; - let patternId = -1; + let ruleId: RuleId | -1 = -1; if (pattern.include) { - if (pattern.include.charAt(0) === '#') { - // Local include found in `repository` - let localIncludedRule = repository[pattern.include.substr(1)]; - if (localIncludedRule) { - patternId = RuleFactory.getCompiledRuleId(localIncludedRule, helper, repository); - } else { - // console.warn('CANNOT find rule for scopeName: ' + pattern.include + ', I am: ', repository['$base'].name); - } - } else if (pattern.include === '$base' || pattern.include === '$self') { - // Special include also found in `repository` - patternId = RuleFactory.getCompiledRuleId(repository[pattern.include], helper, repository); - } else { - let externalGrammarName: string | null = null; - let externalGrammarInclude: string | null = null; - let sharpIndex = pattern.include.indexOf('#'); - - if (sharpIndex >= 0) { - externalGrammarName = pattern.include.substring(0, sharpIndex); - externalGrammarInclude = pattern.include.substring(sharpIndex + 1); - } else { - externalGrammarName = pattern.include; - } - // External include - const externalGrammar = helper.getExternalGrammar(externalGrammarName, repository); - - if (externalGrammar) { - if (externalGrammarInclude) { - let externalIncludedRule = externalGrammar.repository[externalGrammarInclude]; - if (externalIncludedRule) { - patternId = RuleFactory.getCompiledRuleId(externalIncludedRule, helper, externalGrammar.repository); + + const reference = parseInclude(pattern.include); + + switch (reference.kind) { + case IncludeReferenceKind.Base: + case IncludeReferenceKind.Self: + ruleId = RuleFactory.getCompiledRuleId(repository[pattern.include], helper, repository); + break; + + case IncludeReferenceKind.RelativeReference: + // Local include found in `repository` + let localIncludedRule = repository[reference.ruleName]; + if (localIncludedRule) { + ruleId = RuleFactory.getCompiledRuleId(localIncludedRule, helper, repository); + } else { + // console.warn('CANNOT find rule for scopeName: ' + pattern.include + ', I am: ', repository['$base'].name); + } + break; + + case IncludeReferenceKind.TopLevelReference: + case IncludeReferenceKind.TopLevelRepositoryReference: + + const externalGrammarName = reference.scopeName; + const externalGrammarInclude = + reference.kind === IncludeReferenceKind.TopLevelRepositoryReference + ? reference.ruleName + : null; + + // External include + const externalGrammar = helper.getExternalGrammar(externalGrammarName, repository); + + if (externalGrammar) { + if (externalGrammarInclude) { + let externalIncludedRule = externalGrammar.repository[externalGrammarInclude]; + if (externalIncludedRule) { + ruleId = RuleFactory.getCompiledRuleId(externalIncludedRule, helper, externalGrammar.repository); + } else { + // console.warn('CANNOT find rule for scopeName: ' + pattern.include + ', I am: ', repository['$base'].name); + } } else { - // console.warn('CANNOT find rule for scopeName: ' + pattern.include + ', I am: ', repository['$base'].name); + ruleId = RuleFactory.getCompiledRuleId(externalGrammar.repository.$self, helper, externalGrammar.repository); } } else { - patternId = RuleFactory.getCompiledRuleId(externalGrammar.repository.$self, helper, externalGrammar.repository); + // console.warn('CANNOT find grammar for scopeName: ' + pattern.include + ', I am: ', repository['$base'].name); } - } else { - // console.warn('CANNOT find grammar for scopeName: ' + pattern.include + ', I am: ', repository['$base'].name); - } - + break; } } else { - patternId = RuleFactory.getCompiledRuleId(pattern, helper, repository); + ruleId = RuleFactory.getCompiledRuleId(pattern, helper, repository); } - if (patternId !== -1) { - const rule = helper.getRule(patternId); + if (ruleId !== -1) { + const rule = helper.getRule(ruleId); let skipRule = false; @@ -864,7 +560,7 @@ export class RuleFactory { continue; } - r.push(patternId); + r.push(ruleId); } } } @@ -875,3 +571,329 @@ export class RuleFactory { }; } } + +interface IRegExpSourceAnchorCache { + readonly A0_G0: string; + readonly A0_G1: string; + readonly A1_G0: string; + readonly A1_G1: string; +} + +export class RegExpSource { + + public source: string; + public readonly ruleId: TRuleId; + public hasAnchor: boolean; + public readonly hasBackReferences: boolean; + private _anchorCache: IRegExpSourceAnchorCache | null; + + constructor(regExpSource: string, ruleId: TRuleId) { + if (regExpSource) { + const len = regExpSource.length; + let lastPushedPos = 0; + let output: string[] = []; + + let hasAnchor = false; + for (let pos = 0; pos < len; pos++) { + const ch = regExpSource.charAt(pos); + + if (ch === '\\') { + if (pos + 1 < len) { + const nextCh = regExpSource.charAt(pos + 1); + if (nextCh === 'z') { + output.push(regExpSource.substring(lastPushedPos, pos)); + output.push('$(?!\\n)(? ' + this.source + ', ' + this.hasAnchor); + } + + public clone(): RegExpSource { + return new RegExpSource(this.source, this.ruleId); + } + + public setSource(newSource: string): void { + if (this.source === newSource) { + return; + } + this.source = newSource; + + if (this.hasAnchor) { + this._anchorCache = this._buildAnchorCache(); + } + } + + public resolveBackReferences(lineText: string, captureIndices: IOnigCaptureIndex[]): string { + let capturedValues = captureIndices.map((capture) => { + return lineText.substring(capture.start, capture.end); + }); + BACK_REFERENCING_END.lastIndex = 0; + return this.source.replace(BACK_REFERENCING_END, (match, g1) => { + return escapeRegExpCharacters(capturedValues[parseInt(g1, 10)] || ''); + }); + } + + private _buildAnchorCache(): IRegExpSourceAnchorCache { + let A0_G0_result: string[] = []; + let A0_G1_result: string[] = []; + let A1_G0_result: string[] = []; + let A1_G1_result: string[] = []; + + let pos: number, + len: number, + ch: string, + nextCh: string; + + for (pos = 0, len = this.source.length; pos < len; pos++) { + ch = this.source.charAt(pos); + A0_G0_result[pos] = ch; + A0_G1_result[pos] = ch; + A1_G0_result[pos] = ch; + A1_G1_result[pos] = ch; + + if (ch === '\\') { + if (pos + 1 < len) { + nextCh = this.source.charAt(pos + 1); + if (nextCh === 'A') { + A0_G0_result[pos + 1] = '\uFFFF'; + A0_G1_result[pos + 1] = '\uFFFF'; + A1_G0_result[pos + 1] = 'A'; + A1_G1_result[pos + 1] = 'A'; + } else if (nextCh === 'G') { + A0_G0_result[pos + 1] = '\uFFFF'; + A0_G1_result[pos + 1] = 'G'; + A1_G0_result[pos + 1] = '\uFFFF'; + A1_G1_result[pos + 1] = 'G'; + } else { + A0_G0_result[pos + 1] = nextCh; + A0_G1_result[pos + 1] = nextCh; + A1_G0_result[pos + 1] = nextCh; + A1_G1_result[pos + 1] = nextCh; + } + pos++; + } + } + } + + return { + A0_G0: A0_G0_result.join(''), + A0_G1: A0_G1_result.join(''), + A1_G0: A1_G0_result.join(''), + A1_G1: A1_G1_result.join('') + }; + } + + public resolveAnchors(allowA: boolean, allowG: boolean): string { + if (!this.hasAnchor || !this._anchorCache) { + return this.source; + } + + if (allowA) { + if (allowG) { + return this._anchorCache.A1_G1; + } else { + return this._anchorCache.A1_G0; + } + } else { + if (allowG) { + return this._anchorCache.A0_G1; + } else { + return this._anchorCache.A0_G0; + } + } + } +} + +interface IRegExpSourceListAnchorCache { + A0_G0: CompiledRule | null; + A0_G1: CompiledRule | null; + A1_G0: CompiledRule | null; + A1_G1: CompiledRule | null; +} + +export class RegExpSourceList { + + private readonly _items: RegExpSource[]; + private _hasAnchors: boolean; + private _cached: CompiledRule | null; + private _anchorCache: IRegExpSourceListAnchorCache; + + constructor() { + this._items = []; + this._hasAnchors = false; + this._cached = null; + this._anchorCache = { + A0_G0: null, + A0_G1: null, + A1_G0: null, + A1_G1: null + }; + } + + public dispose(): void { + this._disposeCaches(); + } + + private _disposeCaches(): void { + if (this._cached) { + this._cached.dispose(); + this._cached = null; + } + if (this._anchorCache.A0_G0) { + this._anchorCache.A0_G0.dispose(); + this._anchorCache.A0_G0 = null; + } + if (this._anchorCache.A0_G1) { + this._anchorCache.A0_G1.dispose(); + this._anchorCache.A0_G1 = null; + } + if (this._anchorCache.A1_G0) { + this._anchorCache.A1_G0.dispose(); + this._anchorCache.A1_G0 = null; + } + if (this._anchorCache.A1_G1) { + this._anchorCache.A1_G1.dispose(); + this._anchorCache.A1_G1 = null; + } + } + + public push(item: RegExpSource): void { + this._items.push(item); + this._hasAnchors = this._hasAnchors || item.hasAnchor; + } + + public unshift(item: RegExpSource): void { + this._items.unshift(item); + this._hasAnchors = this._hasAnchors || item.hasAnchor; + } + + public length(): number { + return this._items.length; + } + + public setSource(index: number, newSource: string): void { + if (this._items[index].source !== newSource) { + // bust the cache + this._disposeCaches(); + this._items[index].setSource(newSource); + } + } + + public compile(onigLib: IOnigLib): CompiledRule { + if (!this._cached) { + let regExps = this._items.map(e => e.source); + this._cached = new CompiledRule(onigLib, regExps, this._items.map(e => e.ruleId)); + } + return this._cached; + } + + public compileAG(onigLib: IOnigLib, allowA: boolean, allowG: boolean): CompiledRule { + if (!this._hasAnchors) { + return this.compile(onigLib); + } else { + if (allowA) { + if (allowG) { + if (!this._anchorCache.A1_G1) { + this._anchorCache.A1_G1 = this._resolveAnchors(onigLib, allowA, allowG); + } + return this._anchorCache.A1_G1; + } else { + if (!this._anchorCache.A1_G0) { + this._anchorCache.A1_G0 = this._resolveAnchors(onigLib, allowA, allowG); + } + return this._anchorCache.A1_G0; + } + } else { + if (allowG) { + if (!this._anchorCache.A0_G1) { + this._anchorCache.A0_G1 = this._resolveAnchors(onigLib, allowA, allowG); + } + return this._anchorCache.A0_G1; + } else { + if (!this._anchorCache.A0_G0) { + this._anchorCache.A0_G0 = this._resolveAnchors(onigLib, allowA, allowG); + } + return this._anchorCache.A0_G0; + } + } + } + } + + private _resolveAnchors(onigLib: IOnigLib, allowA: boolean, allowG: boolean): CompiledRule { + let regExps = this._items.map(e => e.resolveAnchors(allowA, allowG)); + return new CompiledRule(onigLib, regExps, this._items.map(e => e.ruleId)); + } +} + +export class CompiledRule { + private readonly scanner: OnigScanner; + + constructor(onigLib: IOnigLib, private readonly regExps: string[], private readonly rules: TRuleId[]) { + this.scanner = onigLib.createOnigScanner(regExps); + } + + public dispose(): void { + if (typeof this.scanner.dispose === "function") { + this.scanner.dispose(); + } + } + + toString(): string { + const r: string[] = []; + for (let i = 0, len = this.rules.length; i < len; i++) { + r.push(" - " + this.rules[i] + ": " + this.regExps[i]); + } + return r.join("\n"); + } + + findNextMatchSync( + string: string | OnigString, + startPosition: number, + options: OrMask + ): IFindNextMatchResult | null { + const result = this.scanner.findNextMatchSync(string, startPosition, options); + if (!result) { + return null; + } + + return { + ruleId: this.rules[result.index], + captureIndices: result.captureIndices, + }; + } +} + +export interface IFindNextMatchResult { + ruleId: TRuleId; + captureIndices: IOnigCaptureIndex[]; +} diff --git a/src/tests/grammar.test.ts b/src/tests/grammar.test.ts index a7d53599..1fb50a17 100644 --- a/src/tests/grammar.test.ts +++ b/src/tests/grammar.test.ts @@ -3,18 +3,19 @@ *--------------------------------------------------------*/ import * as assert from 'assert'; -import { StandardTokenType } from '../main'; -import { StackElementMetadata, OptionalStandardTokenType } from '../metadata'; +import { EncodedTokenAttributes, OptionalStandardTokenType, StandardTokenType } from '../encodedTokenAttributes'; +import { Registry } from '../main'; import { FontStyle } from '../theme'; +import { getOniguruma } from './onigLibs'; function assertEquals(metadata: number, languageId: number, tokenType: StandardTokenType, containsBalancedBrackets: boolean, fontStyle: FontStyle, foreground: number, background: number): void { const actual = { - languageId: StackElementMetadata.getLanguageId(metadata), - tokenType: StackElementMetadata.getTokenType(metadata), - containsBalancedBrackets: StackElementMetadata.containsBalancedBrackets(metadata), - fontStyle: StackElementMetadata.getFontStyle(metadata), - foreground: StackElementMetadata.getForeground(metadata), - background: StackElementMetadata.getBackground(metadata), + languageId: EncodedTokenAttributes.getLanguageId(metadata), + tokenType: EncodedTokenAttributes.getTokenType(metadata), + containsBalancedBrackets: EncodedTokenAttributes.containsBalancedBrackets(metadata), + fontStyle: EncodedTokenAttributes.getFontStyle(metadata), + foreground: EncodedTokenAttributes.getForeground(metadata), + background: EncodedTokenAttributes.getBackground(metadata), }; const expected = { @@ -26,70 +27,70 @@ function assertEquals(metadata: number, languageId: number, tokenType: StandardT background, }; - assert.deepStrictEqual(actual, expected, 'equals for ' + StackElementMetadata.toBinaryStr(metadata)); + assert.deepStrictEqual(actual, expected, 'equals for ' + EncodedTokenAttributes.toBinaryStr(metadata)); } test('StackElementMetadata works', () => { - let value = StackElementMetadata.set(0, 1, OptionalStandardTokenType.RegEx, false, FontStyle.Underline | FontStyle.Bold, 101, 102); + let value = EncodedTokenAttributes.set(0, 1, OptionalStandardTokenType.RegEx, false, FontStyle.Underline | FontStyle.Bold, 101, 102); assertEquals(value, 1, StandardTokenType.RegEx, false, FontStyle.Underline | FontStyle.Bold, 101, 102); }); test('StackElementMetadata can overwrite languageId', () => { - let value = StackElementMetadata.set(0, 1, OptionalStandardTokenType.RegEx, false, FontStyle.Underline | FontStyle.Bold, 101, 102); + let value = EncodedTokenAttributes.set(0, 1, OptionalStandardTokenType.RegEx, false, FontStyle.Underline | FontStyle.Bold, 101, 102); assertEquals(value, 1, StandardTokenType.RegEx, false, FontStyle.Underline | FontStyle.Bold, 101, 102); - value = StackElementMetadata.set(value, 2, OptionalStandardTokenType.NotSet, false, FontStyle.NotSet, 0, 0); + value = EncodedTokenAttributes.set(value, 2, OptionalStandardTokenType.NotSet, false, FontStyle.NotSet, 0, 0); assertEquals(value, 2, StandardTokenType.RegEx, false, FontStyle.Underline | FontStyle.Bold, 101, 102); }); test('StackElementMetadata can overwrite tokenType', () => { - let value = StackElementMetadata.set(0, 1, OptionalStandardTokenType.RegEx, false, FontStyle.Underline | FontStyle.Bold, 101, 102); + let value = EncodedTokenAttributes.set(0, 1, OptionalStandardTokenType.RegEx, false, FontStyle.Underline | FontStyle.Bold, 101, 102); assertEquals(value, 1, StandardTokenType.RegEx, false, FontStyle.Underline | FontStyle.Bold, 101, 102); - value = StackElementMetadata.set(value, 0, OptionalStandardTokenType.Comment, false, FontStyle.NotSet, 0, 0); + value = EncodedTokenAttributes.set(value, 0, OptionalStandardTokenType.Comment, false, FontStyle.NotSet, 0, 0); assertEquals(value, 1, StandardTokenType.Comment, false, FontStyle.Underline | FontStyle.Bold, 101, 102); }); test('StackElementMetadata can overwrite font style', () => { - let value = StackElementMetadata.set(0, 1, OptionalStandardTokenType.RegEx, false, FontStyle.Underline | FontStyle.Bold, 101, 102); + let value = EncodedTokenAttributes.set(0, 1, OptionalStandardTokenType.RegEx, false, FontStyle.Underline | FontStyle.Bold, 101, 102); assertEquals(value, 1, StandardTokenType.RegEx, false, FontStyle.Underline | FontStyle.Bold, 101, 102); - value = StackElementMetadata.set(value, 0, OptionalStandardTokenType.NotSet, false, FontStyle.None, 0, 0); + value = EncodedTokenAttributes.set(value, 0, OptionalStandardTokenType.NotSet, false, FontStyle.None, 0, 0); assertEquals(value, 1, StandardTokenType.RegEx, false, FontStyle.None, 101, 102); }); test('StackElementMetadata can overwrite font style with strikethrough', () => { - let value = StackElementMetadata.set(0, 1, OptionalStandardTokenType.RegEx, false, FontStyle.Strikethrough, 101, 102); + let value = EncodedTokenAttributes.set(0, 1, OptionalStandardTokenType.RegEx, false, FontStyle.Strikethrough, 101, 102); assertEquals(value, 1, StandardTokenType.RegEx, false, FontStyle.Strikethrough, 101, 102); - value = StackElementMetadata.set(value, 0, OptionalStandardTokenType.NotSet, false, FontStyle.None, 0, 0); + value = EncodedTokenAttributes.set(value, 0, OptionalStandardTokenType.NotSet, false, FontStyle.None, 0, 0); assertEquals(value, 1, StandardTokenType.RegEx, false, FontStyle.None, 101, 102); }); test('StackElementMetadata can overwrite foreground', () => { - let value = StackElementMetadata.set(0, 1, OptionalStandardTokenType.RegEx, false, FontStyle.Underline | FontStyle.Bold, 101, 102); + let value = EncodedTokenAttributes.set(0, 1, OptionalStandardTokenType.RegEx, false, FontStyle.Underline | FontStyle.Bold, 101, 102); assertEquals(value, 1, StandardTokenType.RegEx, false, FontStyle.Underline | FontStyle.Bold, 101, 102); - value = StackElementMetadata.set(value, 0, OptionalStandardTokenType.NotSet, false, FontStyle.NotSet, 5, 0); + value = EncodedTokenAttributes.set(value, 0, OptionalStandardTokenType.NotSet, false, FontStyle.NotSet, 5, 0); assertEquals(value, 1, StandardTokenType.RegEx, false, FontStyle.Underline | FontStyle.Bold, 5, 102); }); test('StackElementMetadata can overwrite background', () => { - let value = StackElementMetadata.set(0, 1, OptionalStandardTokenType.RegEx, false, FontStyle.Underline | FontStyle.Bold, 101, 102); + let value = EncodedTokenAttributes.set(0, 1, OptionalStandardTokenType.RegEx, false, FontStyle.Underline | FontStyle.Bold, 101, 102); assertEquals(value, 1, StandardTokenType.RegEx, false, FontStyle.Underline | FontStyle.Bold, 101, 102); - value = StackElementMetadata.set(value, 0, OptionalStandardTokenType.NotSet, false, FontStyle.NotSet, 0, 7); + value = EncodedTokenAttributes.set(value, 0, OptionalStandardTokenType.NotSet, false, FontStyle.NotSet, 0, 7); assertEquals(value, 1, StandardTokenType.RegEx, false, FontStyle.Underline | FontStyle.Bold, 101, 7); }); test('StackElementMetadata can overwrite balanced backet bit', () => { - let value = StackElementMetadata.set(0, 1, OptionalStandardTokenType.RegEx, false, FontStyle.Underline | FontStyle.Bold, 101, 102); + let value = EncodedTokenAttributes.set(0, 1, OptionalStandardTokenType.RegEx, false, FontStyle.Underline | FontStyle.Bold, 101, 102); assertEquals(value, 1, StandardTokenType.RegEx, false, FontStyle.Underline | FontStyle.Bold, 101, 102); - value = StackElementMetadata.set(value, 0, OptionalStandardTokenType.NotSet, true, FontStyle.NotSet, 0, 0); + value = EncodedTokenAttributes.set(value, 0, OptionalStandardTokenType.NotSet, true, FontStyle.NotSet, 0, 0); assertEquals(value, 1, StandardTokenType.RegEx, true, FontStyle.Underline | FontStyle.Bold, 101, 102); - value = StackElementMetadata.set(value, 0, OptionalStandardTokenType.NotSet, false, FontStyle.NotSet, 0, 0); + value = EncodedTokenAttributes.set(value, 0, OptionalStandardTokenType.NotSet, false, FontStyle.NotSet, 0, 0); assertEquals(value, 1, StandardTokenType.RegEx, false, FontStyle.Underline | FontStyle.Bold, 101, 102); }); @@ -100,6 +101,42 @@ test('StackElementMetadata can work at max values', () => { const maxForeground = 511; const maxBackground = 254; - let value = StackElementMetadata.set(0, maxLangId, maxTokenType, true, maxFontStyle, maxForeground, maxBackground); + let value = EncodedTokenAttributes.set(0, maxLangId, maxTokenType, true, maxFontStyle, maxForeground, maxBackground); assertEquals(value, maxLangId, maxTokenType, true, maxFontStyle, maxForeground, maxBackground); }); + +test.skip('Shadowed rules are resolved correctly', async function () { + const registry = new Registry({ loadGrammar: async () => undefined, onigLib: getOniguruma() }); + try { + const grammar = await registry.addGrammar({ + scopeName: 'source.test', + repository: { + $base: undefined!, + $self: undefined!, + foo: { include: '#bar', }, + bar: { match: 'bar1', name: 'outer' } + }, + patterns: [ + { + patterns: [{ include: '#foo' }], + repository: { + $base: undefined!, + $self: undefined!, + bar: { match: 'bar1', name: 'inner' } + } + }, + // When you move this up, the test passes + { + begin: 'begin', + patterns: [{ include: '#foo' }], + end: 'end' + }, + ] + }); + const result = grammar.tokenizeLine('bar1', null, undefined); + // TODO this should be inner! + assert.deepStrictEqual(result.tokens, [{ startIndex: 0, endIndex: 4, scopes: ["source.test", "outer"] }]); + } finally { + registry.dispose(); + } +}); diff --git a/src/tests/inspect.ts b/src/tests/inspect.ts index afa74322..c84e43fe 100644 --- a/src/tests/inspect.ts +++ b/src/tests/inspect.ts @@ -4,7 +4,7 @@ import * as fs from 'fs'; import { Registry, IGrammar, parseRawGrammar } from '../main'; -import { StackElement as StackElementImpl, Grammar as GrammarImpl } from '../grammar'; +import { StateStack as StackElementImpl, Grammar as GrammarImpl } from '../grammar'; import * as debug from '../debug'; import { getOniguruma } from './onigLibs'; @@ -66,11 +66,11 @@ Promise.all(grammarPromises).then(_grammars => { if (!stackElement._instanceId) { stackElement._instanceId = (++lastElementId); } - let ruleDesc = (grammar).getRule(stackElement.ruleId); + let ruleDesc = stackElement.getRule(grammar as GrammarImpl); if (!ruleDesc) { - list.push(' * no rule description found for rule id: ' + stackElement.ruleId); + list.push(' * no rule description found'); } else { - list.push(' * ' + ruleDesc.debugName + ' -- [' + ruleDesc.id + ',' + stackElement._instanceId + '] "' + stackElement.nameScopesList.generateScopes() + '", "' + stackElement.contentNameScopesList.generateScopes() + '"'); + list.push(' * ' + ruleDesc.debugName + ' -- [' + ruleDesc.id + ',' + stackElement._instanceId + '] "' + stackElement.nameScopesList.getScopeNames() + '", "' + stackElement.contentNameScopesList.getScopeNames() + '"'); } stackElement = stackElement.parent; } diff --git a/src/tests/json.test.ts b/src/tests/json.test.ts index 2484f598..fd190dc9 100644 --- a/src/tests/json.test.ts +++ b/src/tests/json.test.ts @@ -3,11 +3,11 @@ *--------------------------------------------------------*/ import * as assert from 'assert'; -import { parse as JSONparse } from '../json'; +import { parseJSON } from '../json'; function isValid(json: string): void { let expected = JSON.parse(json); - let actual = JSONparse(json, null, false); + let actual = parseJSON(json, null, false); assert.deepStrictEqual(actual, expected); // let actual2 = JSONparse(json, true); @@ -17,7 +17,7 @@ function isValid(json: string): void { function isInvalid(json: string): void { let hadErr = false; try { - JSONparse(json, null, false); + parseJSON(json, null, false); } catch (err) { hadErr = true; } diff --git a/src/tests/themedTokenizer.ts b/src/tests/themedTokenizer.ts index bc67d022..1b7b07f6 100644 --- a/src/tests/themedTokenizer.ts +++ b/src/tests/themedTokenizer.ts @@ -2,8 +2,8 @@ * Copyright (C) Microsoft Corporation. All rights reserved. *--------------------------------------------------------*/ -import { IGrammar, StackElement } from '../main'; -import { StackElementMetadata } from '../metadata'; +import { IGrammar, StateStack } from '../main'; +import { EncodedTokenAttributes } from '../encodedTokenAttributes'; export interface IThemedToken { content: string; @@ -14,7 +14,7 @@ export function tokenizeWithTheme(colorMap: string[], fileContents: string, gram const lines = fileContents.split(/\r\n|\r|\n/); - let ruleStack: StackElement | null = null; + let ruleStack: StateStack | null = null; let actual: IThemedToken[] = [], actualLen = 0; for (let i = 0, len = lines.length; i < len; i++) { @@ -29,7 +29,7 @@ export function tokenizeWithTheme(colorMap: string[], fileContents: string, gram continue; } const metadata = result.tokens[2 * j + 1]; - const foreground = StackElementMetadata.getForeground(metadata); + const foreground = EncodedTokenAttributes.getForeground(metadata); const foregroundColor = colorMap[foreground]; actual[actualLen++] = { diff --git a/src/tests/themes.test.ts b/src/tests/themes.test.ts index a27f51a7..830fdf29 100644 --- a/src/tests/themes.test.ts +++ b/src/tests/themes.test.ts @@ -5,18 +5,21 @@ import * as fs from 'fs'; import * as path from 'path'; import * as assert from 'assert'; -import { Registry, IRawTheme } from '../main'; -import { ScopeListElement, ScopeMetadata } from '../grammar'; +import { Registry } from '../main'; import { Theme, ThemeTrieElement, ThemeTrieElementRule, - parseTheme, ParsedThemeRule, FontStyle, ColorMap + parseTheme, ParsedThemeRule, FontStyle, ColorMap, + IRawTheme, + ScopeStack, + fontStyleToString, + StyleAttributes } from '../theme'; -import * as plist from '../plist'; import { ThemeTest } from './themeTest'; import { getOniguruma } from './onigLibs'; import { Resolver, IGrammarRegistration, ILanguageRegistration } from './resolver'; -import { StackElementMetadata } from '../metadata'; +import { EncodedTokenAttributes } from '../encodedTokenAttributes'; import { strArrCmp, strcmp } from '../utils'; +import { parsePLIST } from '../plist'; const THEMES_TEST_PATH = path.join(__dirname, '../../test-cases/themes'); @@ -45,7 +48,7 @@ class ThemeInfo { if (/\.json$/.test(filename)) { return JSON.parse(fileContents); } - return plist.parse(fileContents); + return parsePLIST(fileContents); } public create(resolver: Resolver): ThemeData { @@ -121,29 +124,15 @@ class ThemeInfo { })(); test('Theme matching gives higher priority to deeper matches', () => { - let theme = Theme.createFromRawTheme({ + const theme = Theme.createFromRawTheme({ settings: [ { settings: { foreground: '#100000', background: '#200000' } }, { scope: 'punctuation.definition.string.begin.html', settings: { foreground: '#300000' } }, { scope: 'meta.tag punctuation.definition.string', settings: { foreground: '#400000' } }, - // { scope: 'a', settings: { foreground: '#500000' } }, ] }); - - let colorMap = new ColorMap(); - const _NOT_SET = 0; - const _A = colorMap.getId('#100000'); - const _B = colorMap.getId('#200000'); - const _C = colorMap.getId('#400000'); - const _D = colorMap.getId('#300000'); - - let actual = theme.match('punctuation.definition.string.begin.html'); - // console.log(actual); process.exit(0); - - assert.deepStrictEqual(actual, [ - new ThemeTrieElementRule(5, null, FontStyle.NotSet, _D, _NOT_SET), - new ThemeTrieElementRule(3, ['meta.tag'], FontStyle.NotSet, _C, _NOT_SET), - ]); + const actual = theme.match(ScopeStack.from('punctuation.definition.string.begin.html')); + assert.deepStrictEqual(theme.getColorMap()[actual!.foregroundId], '#300000'); }); test('Theme matching gives higher priority to parent matches 1', () => { @@ -156,21 +145,12 @@ test('Theme matching gives higher priority to parent matches 1', () => { ] }); - let colorMap = new ColorMap(); - const _NOT_SET = 0; - const _A = colorMap.getId('#100000'); - const _B = colorMap.getId('#200000'); - const _C = colorMap.getId('#500000'); - const _D = colorMap.getId('#300000'); - const _E = colorMap.getId('#400000'); - - let actual = theme.match('a.b'); + const map = theme.getColorMap(); - assert.deepStrictEqual(actual, [ - new ThemeTrieElementRule(2, ['d'], FontStyle.NotSet, _E, _NOT_SET), - new ThemeTrieElementRule(1, ['c'], FontStyle.NotSet, _D, _NOT_SET), - new ThemeTrieElementRule(1, null, FontStyle.NotSet, _C, _NOT_SET), - ]); + assert.deepStrictEqual( + map[theme.match(ScopeStack.from('d', 'a.b'))!.foregroundId], + '#400000', + ); }); test('Theme matching gives higher priority to parent matches 2', () => { @@ -183,15 +163,20 @@ test('Theme matching gives higher priority to parent matches 2', () => { ] }); - let root = new ScopeListElement(null, 'text.html.cshtml', 0); - let parent = new ScopeListElement(root, 'meta.tag.structure.any.html', 0); - let r = ScopeListElement.mergeMetadata(0, parent, new ScopeMetadata('entity.name.tag.structure.any.html', 0, 0, theme.match('entity.name.tag.structure.any.html'))); - let colorMap = theme.getColorMap(); - assert.strictEqual(colorMap[StackElementMetadata.getForeground(r)], '#300000'); + const result = theme.match( + ScopeStack.from( + "text.html.cshtml", + "meta.tag.structure.any.html", + "entity.name.tag.structure.any.html", + ) + ); + + const colorMap = theme.getColorMap(); + assert.strictEqual(colorMap[result!.foregroundId], '#300000'); }); -test('Theme matching can match', () => { - let theme = Theme.createFromRawTheme({ +suite('Theme matching can match', () => { + const theme = Theme.createFromRawTheme({ settings: [ { settings: { foreground: '#F8F8F2', background: '#272822' } }, { scope: 'source, something', settings: { background: '#100000' } }, @@ -206,89 +191,57 @@ test('Theme matching can match', () => { ] }); - let colorMap = new ColorMap(); - const _NOT_SET = 0; - const _A = colorMap.getId('#F8F8F2'); - const _B = colorMap.getId('#272822'); - const _C = colorMap.getId('#200000'); - const _D = colorMap.getId('#300000'); - const _E = colorMap.getId('#400000'); - const _F = colorMap.getId('#500000'); - const _G = colorMap.getId('#100000'); - const _H = colorMap.getId('#600000'); - - function assertMatch(scopeName: string, expected: ThemeTrieElementRule[]): void { - let actual = theme.match(scopeName); - assert.deepStrictEqual(actual, expected, 'when matching <<' + scopeName + '>>'); - } - - function assertSimpleMatch(scopeName: string, scopeDepth: number, fontStyle: FontStyle, foreground: number, background: number): void { - assertMatch(scopeName, [ - new ThemeTrieElementRule(scopeDepth, null, fontStyle, foreground, background) - ]); - } + const map = theme.getColorMap(); - function assertNoMatch(scopeName: string): void { - assertMatch(scopeName, [ - new ThemeTrieElementRule(0, null, FontStyle.NotSet, _NOT_SET, _NOT_SET) - ]); + function match(...path: string[]) { + const result = theme.match(ScopeStack.from(...path)); + if (!result) { + return null; + } + let obj: any = { + fontStyle: fontStyleToString(result.fontStyle) + }; + if (result.foregroundId !== 0) { + obj.foreground = map[result.foregroundId]; + } + if (result.backgroundId !== 0) { + obj.background = map[result.backgroundId]; + } + return obj; } - // matches defaults - assertNoMatch(''); - assertNoMatch('bazz'); - assertNoMatch('asdfg'); - - // matches source - assertSimpleMatch('source', 1, FontStyle.NotSet, _NOT_SET, _G); - assertSimpleMatch('source.ts', 1, FontStyle.NotSet, _NOT_SET, _G); - assertSimpleMatch('source.tss', 1, FontStyle.NotSet, _NOT_SET, _G); - - // matches something - assertSimpleMatch('something', 1, FontStyle.NotSet, _NOT_SET, _G); - assertSimpleMatch('something.ts', 1, FontStyle.NotSet, _NOT_SET, _G); - assertSimpleMatch('something.tss', 1, FontStyle.NotSet, _NOT_SET, _G); - - // matches baz - assertSimpleMatch('baz', 1, FontStyle.NotSet, _NOT_SET, _C); - assertSimpleMatch('baz.ts', 1, FontStyle.NotSet, _NOT_SET, _C); - assertSimpleMatch('baz.tss', 1, FontStyle.NotSet, _NOT_SET, _C); - - // matches constant - assertSimpleMatch('constant', 1, FontStyle.Italic, _D, _NOT_SET); - assertSimpleMatch('constant.string', 1, FontStyle.Italic, _D, _NOT_SET); - assertSimpleMatch('constant.hex', 1, FontStyle.Italic, _D, _NOT_SET); - - // matches constant.numeric - assertSimpleMatch('constant.numeric', 2, FontStyle.Italic, _E, _NOT_SET); - assertSimpleMatch('constant.numeric.baz', 2, FontStyle.Italic, _E, _NOT_SET); - - // matches constant.numeric.hex - assertSimpleMatch('constant.numeric.hex', 3, FontStyle.Bold, _E, _NOT_SET); - assertSimpleMatch('constant.numeric.hex.baz', 3, FontStyle.Bold, _E, _NOT_SET); - - // matches constant.numeric.oct - assertSimpleMatch('constant.numeric.oct', 3, FontStyle.Bold | FontStyle.Italic | FontStyle.Underline, _E, _NOT_SET); - assertSimpleMatch('constant.numeric.oct.baz', 3, FontStyle.Bold | FontStyle.Italic | FontStyle.Underline, _E, _NOT_SET); - - // matches constant.numeric.dec - assertSimpleMatch('constant.numeric.dec', 3, FontStyle.None, _F, _NOT_SET); - assertSimpleMatch('constant.numeric.dec.baz', 3, FontStyle.None, _F, _NOT_SET); - - // matches storage.object.bar - assertSimpleMatch('storage.object.bar', 3, FontStyle.None, _H, _NOT_SET); - assertSimpleMatch('storage.object.bar.baz', 3, FontStyle.None, _H, _NOT_SET); - - // does not match storage.object.bar - assertSimpleMatch('storage.object.bart', 0, FontStyle.NotSet, _NOT_SET, _NOT_SET); - assertSimpleMatch('storage.object', 0, FontStyle.NotSet, _NOT_SET, _NOT_SET); - assertSimpleMatch('storage', 0, FontStyle.NotSet, _NOT_SET, _NOT_SET); - - - assertMatch('bar', [ - new ThemeTrieElementRule(1, ['selector', 'source.css'], FontStyle.Bold, _NOT_SET, _C), - new ThemeTrieElementRule(1, null, FontStyle.NotSet, _NOT_SET, _C), - ]); + test('simpleMatch1', () => assert.deepStrictEqual(match('source'), { background: "#100000", fontStyle: "not set", })); + test('simpleMatch2', () => assert.deepStrictEqual(match('source.ts'), { background: "#100000", fontStyle: "not set", })); + test('simpleMatch3', () => assert.deepStrictEqual(match('source.tss'), { background: "#100000", fontStyle: "not set", })); + test('simpleMatch4', () => assert.deepStrictEqual(match('something'), { background: "#100000", fontStyle: "not set", })); + test('simpleMatch5', () => assert.deepStrictEqual(match('something.ts'), { background: "#100000", fontStyle: "not set", })); + test('simpleMatch6', () => assert.deepStrictEqual(match('something.tss'), { background: "#100000", fontStyle: "not set", })); + test('simpleMatch7', () => assert.deepStrictEqual(match('baz'), { background: "#200000", fontStyle: "not set", })); + test('simpleMatch8', () => assert.deepStrictEqual(match('baz.ts'), { background: "#200000", fontStyle: "not set", })); + test('simpleMatch9', () => assert.deepStrictEqual(match('baz.tss'), { background: "#200000", fontStyle: "not set", })); + test('simpleMatch10', () => assert.deepStrictEqual(match('constant'), { foreground: "#300000", fontStyle: "italic", })); + test('simpleMatch11', () => assert.deepStrictEqual(match('constant.string'), { foreground: "#300000", fontStyle: "italic", })); + test('simpleMatch12', () => assert.deepStrictEqual(match('constant.hex'), { foreground: "#300000", fontStyle: "italic", })); + test('simpleMatch13', () => assert.deepStrictEqual(match('constant.numeric'), { foreground: "#400000", fontStyle: "italic", })); + test('simpleMatch14', () => assert.deepStrictEqual(match('constant.numeric.baz'), { foreground: "#400000", fontStyle: "italic", })); + test('simpleMatch15', () => assert.deepStrictEqual(match('constant.numeric.hex'), { foreground: "#400000", fontStyle: "bold", })); + test('simpleMatch16', () => assert.deepStrictEqual(match('constant.numeric.hex.baz'), { foreground: "#400000", fontStyle: "bold", })); + test('simpleMatch17', () => assert.deepStrictEqual(match('constant.numeric.oct'), { foreground: "#400000", fontStyle: "italic bold underline", })); + test('simpleMatch18', () => assert.deepStrictEqual(match('constant.numeric.oct.baz'), { foreground: "#400000", fontStyle: "italic bold underline", })); + test('simpleMatch19', () => assert.deepStrictEqual(match('constant.numeric.dec'), { foreground: "#500000", fontStyle: "none", })); + test('simpleMatch20', () => assert.deepStrictEqual(match('constant.numeric.dec.baz'), { foreground: "#500000", fontStyle: "none", })); + test('simpleMatch21', () => assert.deepStrictEqual(match('storage.object.bar'), { foreground: "#600000", fontStyle: "none", })); + test('simpleMatch22', () => assert.deepStrictEqual(match('storage.object.bar.baz'), { foreground: "#600000", fontStyle: "none", })); + test('simpleMatch23', () => assert.deepStrictEqual(match('storage.object.bart'), { fontStyle: "not set", })); + test('simpleMatch24', () => assert.deepStrictEqual(match('storage.object'), { fontStyle: "not set", })); + test('simpleMatch25', () => assert.deepStrictEqual(match('storage'), { fontStyle: "not set", })); + + test('defaultMatch1', () => assert.deepStrictEqual(match(''), { fontStyle: "not set", })); + test('defaultMatch2', () => assert.deepStrictEqual(match('bazz'), { fontStyle: "not set", })); + test('defaultMatch3', () => assert.deepStrictEqual(match('asdfg'), { fontStyle: "not set", })); + + test('multiMatch1', () => assert.deepStrictEqual(match('bar'), { background: "#200000", fontStyle: "not set", })); + test('multiMatch2', () => assert.deepStrictEqual(match('source.css', 'selector', 'bar'), { background: "#200000", fontStyle: "bold", })); }); test('Theme matching Microsoft/vscode#23460', () => { @@ -320,39 +273,14 @@ test('Theme matching Microsoft/vscode#23460', () => { ] }); - let colorMap = new ColorMap(); - const _NOT_SET = 0; - const _A = colorMap.getId('#aec2e0'); - const _B = colorMap.getId('#14191f'); - const _C = colorMap.getId('#FF410D'); - const _D = colorMap.getId('#ffffff'); - - function assertMatch(scopeName: string, expected: ThemeTrieElementRule[]): void { - let actual = theme.match(scopeName); - assert.deepStrictEqual(actual, expected, 'when matching <<' + scopeName + '>>'); - } - - // string.quoted.double.json - // meta.structure.dictionary.value.json - // meta.structure.dictionary.json - // source.json - assertMatch('string.quoted.double.json', [ - new ThemeTrieElementRule(4, ['meta.structure.dictionary.value.json'], FontStyle.NotSet, _C, _NOT_SET), - new ThemeTrieElementRule(4, ['meta.structure.dictionary.json'], FontStyle.NotSet, _D, _NOT_SET), - new ThemeTrieElementRule(0, null, FontStyle.NotSet, _NOT_SET, _NOT_SET), - ]); - - let parent3 = new ScopeListElement(null, 'source.json', 0); - let parent2 = new ScopeListElement(parent3, 'meta.structure.dictionary.json', 0); - let parent1 = new ScopeListElement(parent2, 'meta.structure.dictionary.value.json', 0); - - let r = ScopeListElement.mergeMetadata( - 0, - parent1, - new ScopeMetadata('string.quoted.double.json', 0, 0, theme.match('string.quoted.double.json')) + const path = ScopeStack.from( + "source.json", + "meta.structure.dictionary.json", + "meta.structure.dictionary.value.json", + "string.quoted.double.json" ); - let colorMap2 = theme.getColorMap(); - assert.strictEqual(colorMap2[StackElementMetadata.getForeground(r)], '#FF410D'); + const result = theme.match(path); + assert.strictEqual(theme.getColorMap()[result!.foregroundId], '#FF410D'); }); test('Theme parsing can parse', () => { @@ -420,6 +348,14 @@ test('Theme resolving strArrCmp works', () => { assertStrArrCmp('014', ['a', 'c'], ['a', 'b'], 1); }); +function assertThemeEqual(actual: Theme, expected: Theme): void { + // Don't compare cache objects + assert.deepStrictEqual( + [actual["_colorMap"], actual["_defaults"], actual["_root"]], + [expected["_colorMap"], actual["_defaults"], actual["_root"]] + ); +} + test('Theme resolving always has defaults', () => { let actual = Theme.createFromParsedTheme([]); let colorMap = new ColorMap(); @@ -428,10 +364,10 @@ test('Theme resolving always has defaults', () => { const _B = colorMap.getId('#ffffff'); let expected = new Theme( colorMap, - new ThemeTrieElementRule(0, null, FontStyle.None, _A, _B), + new StyleAttributes(FontStyle.None, _A, _B), new ThemeTrieElement(new ThemeTrieElementRule(0, null, FontStyle.NotSet, _NOT_SET, _NOT_SET)) ); - assert.deepStrictEqual(actual, expected); + assertThemeEqual(actual, expected); }); test('Theme resolving respects incoming defaults 1', () => { @@ -444,10 +380,10 @@ test('Theme resolving respects incoming defaults 1', () => { const _B = colorMap.getId('#ffffff'); let expected = new Theme( colorMap, - new ThemeTrieElementRule(0, null, FontStyle.None, _A, _B), + new StyleAttributes(FontStyle.None, _A, _B), new ThemeTrieElement(new ThemeTrieElementRule(0, null, FontStyle.NotSet, _NOT_SET, _NOT_SET)) ); - assert.deepStrictEqual(actual, expected); + assertThemeEqual(actual, expected); }); test('Theme resolving respects incoming defaults 2', () => { @@ -460,10 +396,10 @@ test('Theme resolving respects incoming defaults 2', () => { const _B = colorMap.getId('#ffffff'); let expected = new Theme( colorMap, - new ThemeTrieElementRule(0, null, FontStyle.None, _A, _B), + new StyleAttributes(FontStyle.None, _A, _B), new ThemeTrieElement(new ThemeTrieElementRule(0, null, FontStyle.NotSet, _NOT_SET, _NOT_SET)) ); - assert.deepStrictEqual(actual, expected); + assertThemeEqual(actual, expected); }); test('Theme resolving respects incoming defaults 3', () => { @@ -476,10 +412,10 @@ test('Theme resolving respects incoming defaults 3', () => { const _B = colorMap.getId('#ffffff'); let expected = new Theme( colorMap, - new ThemeTrieElementRule(0, null, FontStyle.Bold, _A, _B), + new StyleAttributes(FontStyle.Bold, _A, _B), new ThemeTrieElement(new ThemeTrieElementRule(0, null, FontStyle.NotSet, _NOT_SET, _NOT_SET)) ); - assert.deepStrictEqual(actual, expected); + assertThemeEqual(actual, expected); }); test('Theme resolving respects incoming defaults 4', () => { @@ -492,10 +428,10 @@ test('Theme resolving respects incoming defaults 4', () => { const _B = colorMap.getId('#ffffff'); let expected = new Theme( colorMap, - new ThemeTrieElementRule(0, null, FontStyle.None, _A, _B), + new StyleAttributes(FontStyle.None, _A, _B), new ThemeTrieElement(new ThemeTrieElementRule(0, null, FontStyle.NotSet, _NOT_SET, _NOT_SET)) ); - assert.deepStrictEqual(actual, expected); + assertThemeEqual(actual, expected); }); test('Theme resolving respects incoming defaults 5', () => { @@ -508,10 +444,10 @@ test('Theme resolving respects incoming defaults 5', () => { const _B = colorMap.getId('#ff0000'); let expected = new Theme( colorMap, - new ThemeTrieElementRule(0, null, FontStyle.None, _A, _B), + new StyleAttributes(FontStyle.None, _A, _B), new ThemeTrieElement(new ThemeTrieElementRule(0, null, FontStyle.NotSet, _NOT_SET, _NOT_SET)) ); - assert.deepStrictEqual(actual, expected); + assertThemeEqual(actual, expected); }); test('Theme resolving can merge incoming defaults', () => { @@ -526,10 +462,10 @@ test('Theme resolving can merge incoming defaults', () => { const _B = colorMap.getId('#ff0000'); let expected = new Theme( colorMap, - new ThemeTrieElementRule(0, null, FontStyle.Bold, _A, _B), + new StyleAttributes(FontStyle.Bold, _A, _B), new ThemeTrieElement(new ThemeTrieElementRule(0, null, FontStyle.NotSet, _NOT_SET, _NOT_SET)) ); - assert.deepStrictEqual(actual, expected); + assertThemeEqual(actual, expected); }); test('Theme resolving defaults are inherited', () => { @@ -544,12 +480,12 @@ test('Theme resolving defaults are inherited', () => { const _C = colorMap.getId('#ff0000'); let expected = new Theme( colorMap, - new ThemeTrieElementRule(0, null, FontStyle.None, _A, _B), + new StyleAttributes(FontStyle.None, _A, _B), new ThemeTrieElement(new ThemeTrieElementRule(0, null, FontStyle.NotSet, _NOT_SET, _NOT_SET), [], { 'var': new ThemeTrieElement(new ThemeTrieElementRule(1, null, FontStyle.NotSet, _C, _NOT_SET)) }) ); - assert.deepStrictEqual(actual, expected); + assertThemeEqual(actual, expected); }); test('Theme resolving same rules get merged', () => { @@ -565,12 +501,12 @@ test('Theme resolving same rules get merged', () => { const _C = colorMap.getId('#ff0000'); let expected = new Theme( colorMap, - new ThemeTrieElementRule(0, null, FontStyle.None, _A, _B), + new StyleAttributes(FontStyle.None, _A, _B), new ThemeTrieElement(new ThemeTrieElementRule(0, null, FontStyle.NotSet, _NOT_SET, _NOT_SET), [], { 'var': new ThemeTrieElement(new ThemeTrieElementRule(1, null, FontStyle.Bold, _C, _NOT_SET)) }) ); - assert.deepStrictEqual(actual, expected); + assertThemeEqual(actual, expected); }); test('Theme resolving rules are inherited 1', () => { @@ -587,14 +523,14 @@ test('Theme resolving rules are inherited 1', () => { const _D = colorMap.getId('#00ff00'); let expected = new Theme( colorMap, - new ThemeTrieElementRule(0, null, FontStyle.None, _A, _B), + new StyleAttributes(FontStyle.None, _A, _B), new ThemeTrieElement(new ThemeTrieElementRule(0, null, FontStyle.NotSet, _NOT_SET, _NOT_SET), [], { 'var': new ThemeTrieElement(new ThemeTrieElementRule(1, null, FontStyle.Bold, _C, _NOT_SET), [], { 'identifier': new ThemeTrieElement(new ThemeTrieElementRule(2, null, FontStyle.Bold, _D, _NOT_SET)) }) }) ); - assert.deepStrictEqual(actual, expected); + assertThemeEqual(actual, expected); }); test('Theme resolving rules are inherited 2', () => { @@ -619,7 +555,7 @@ test('Theme resolving rules are inherited 2', () => { const _G = colorMap.getId('#00ff00'); let expected = new Theme( colorMap, - new ThemeTrieElementRule(0, null, FontStyle.None, _A, _B), + new StyleAttributes(FontStyle.None, _A, _B), new ThemeTrieElement(new ThemeTrieElementRule(0, null, FontStyle.NotSet, _NOT_SET, _NOT_SET), [], { 'var': new ThemeTrieElement(new ThemeTrieElementRule(1, null, FontStyle.Bold, _F, _NOT_SET), [], { 'identifier': new ThemeTrieElement(new ThemeTrieElementRule(2, null, FontStyle.Bold, _G, _NOT_SET)) @@ -633,7 +569,7 @@ test('Theme resolving rules are inherited 2', () => { }) }) ); - assert.deepStrictEqual(actual, expected); + assertThemeEqual(actual, expected); }); test('Theme resolving rules with parent scopes', () => { @@ -653,7 +589,7 @@ test('Theme resolving rules with parent scopes', () => { const _E = colorMap.getId('#200000'); let expected = new Theme( colorMap, - new ThemeTrieElementRule(0, null, FontStyle.None, _A, _B), + new StyleAttributes(FontStyle.None, _A, _B), new ThemeTrieElement(new ThemeTrieElementRule(0, null, FontStyle.NotSet, _NOT_SET, _NOT_SET), [], { 'var': new ThemeTrieElement( new ThemeTrieElementRule(1, null, FontStyle.Bold, _C, 0), @@ -667,7 +603,7 @@ test('Theme resolving rules with parent scopes', () => { ) }) ); - assert.deepStrictEqual(actual, expected); + assertThemeEqual(actual, expected); }); test('Theme resolving issue #38: ignores rules with invalid colors', () => { diff --git a/src/tests/tokenization.test.ts b/src/tests/tokenization.test.ts index 320c5cc8..2239a0d1 100644 --- a/src/tests/tokenization.test.ts +++ b/src/tests/tokenization.test.ts @@ -5,7 +5,7 @@ import * as fs from 'fs'; import * as path from 'path'; import * as assert from 'assert'; -import { Registry, IGrammar, RegistryOptions, StackElement, parseRawGrammar } from '../main'; +import { Registry, IGrammar, RegistryOptions, StateStack, parseRawGrammar } from '../main'; import { IOnigLib } from '../onigLib'; import { getOniguruma } from './onigLibs'; import { IRawGrammar } from '../rawGrammar'; @@ -71,13 +71,13 @@ function assertTokenizationSuite(testLocation: string): void { if (!grammar) { throw new Error('I HAVE NO GRAMMAR FOR TEST'); } - let prevState: StackElement | null = null; + let prevState: StateStack | null = null; for (let i = 0; i < test.lines.length; i++) { prevState = assertLineTokenization(grammar, test.lines[i], prevState); } } - function assertLineTokenization(grammar: IGrammar, testCase: IRawTestLine, prevState: StackElement | null): StackElement { + function assertLineTokenization(grammar: IGrammar, testCase: IRawTestLine, prevState: StateStack | null): StateStack { let actual = grammar.tokenizeLine(testCase.line, prevState); let actualTokens: IRawToken[] = actual.tokens.map((token) => { diff --git a/src/theme.ts b/src/theme.ts index aa723791..6296109d 100644 --- a/src/theme.ts +++ b/src/theme.ts @@ -2,46 +2,172 @@ * Copyright (C) Microsoft Corporation. All rights reserved. *--------------------------------------------------------*/ -import { IRawTheme } from './main'; -import { isValidHexColor, OrMask, strArrCmp, strcmp } from './utils'; +import { CachedFn, isValidHexColor, OrMask, strArrCmp, strcmp } from './utils'; export class Theme { - public static createFromRawTheme(source: IRawTheme | undefined, colorMap?: string[]): Theme { + public static createFromRawTheme( + source: IRawTheme | undefined, + colorMap?: string[] + ): Theme { return this.createFromParsedTheme(parseTheme(source), colorMap); } - public static createFromParsedTheme(source: ParsedThemeRule[], colorMap?: string[]): Theme { + public static createFromParsedTheme( + source: ParsedThemeRule[], + colorMap?: string[] + ): Theme { return resolveParsedThemeRules(source, colorMap); } - private readonly _colorMap: ColorMap; - private readonly _root: ThemeTrieElement; - private readonly _defaults: ThemeTrieElementRule; - private readonly _cache: { [scopeName: string]: ThemeTrieElementRule[]; }; + private readonly _cachedMatchRoot = new CachedFn( + (scopeName) => this._root.match(scopeName) + ); - constructor(colorMap: ColorMap, defaults: ThemeTrieElementRule, root: ThemeTrieElement) { - this._colorMap = colorMap; - this._root = root; - this._defaults = defaults; - this._cache = {}; - } + constructor( + private readonly _colorMap: ColorMap, + private readonly _defaults: StyleAttributes, + private readonly _root: ThemeTrieElement + ) {} public getColorMap(): string[] { return this._colorMap.getColorMap(); } - public getDefaults(): ThemeTrieElementRule { + public getDefaults(): StyleAttributes { return this._defaults; } - public match(scopeName: string): ThemeTrieElementRule[] { - if (!this._cache.hasOwnProperty(scopeName)) { - this._cache[scopeName] = this._root.match(scopeName); + public match(scopePath: ScopeStack | null): StyleAttributes | null { + if (scopePath === null) { + return this._defaults; + } + const scopeName = scopePath.scopeName; + const matchingTrieElements = this._cachedMatchRoot.get(scopeName); + + const effectiveRule = matchingTrieElements.find((v) => + _scopePathMatchesParentScopes(scopePath.parent, v.parentScopes) + ); + if (!effectiveRule) { + return null; } - return this._cache[scopeName]; + + return new StyleAttributes( + effectiveRule.fontStyle, + effectiveRule.foreground, + effectiveRule.background + ); } } +/** + * Identifiers with a binary dot operator. + * Examples: `baz` or `foo.bar` +*/ +export type ScopeName = string; + +/** + * An expression language of ScopeNames with a binary space (to indicate nesting) operator. + * Examples: `foo.bar boo.baz` +*/ +export type ScopePath = string; + +/** + * An expression language of ScopePathStr with a binary comma (to indicate alternatives) operator. + * Examples: `foo.bar boo.baz,quick quack` +*/ +export type ScopePattern = string; + +/** + * A TextMate theme. + */ + export interface IRawTheme { + readonly name?: string; + readonly settings: IRawThemeSetting[]; +} + +/** + * A single theme setting. + */ + export interface IRawThemeSetting { + readonly name?: string; + readonly scope?: ScopePattern | ScopePattern[]; + readonly settings: { + readonly fontStyle?: string; + readonly foreground?: string; + readonly background?: string; + }; +} + +export class ScopeStack { + public static from(first: ScopeName, ...segments: ScopeName[]): ScopeStack; + public static from(...segments: ScopeName[]): ScopeStack | null; + public static from(...segments: ScopeName[]): ScopeStack | null { + let result: ScopeStack | null = null; + for (let i = 0; i < segments.length; i++) { + result = new ScopeStack(result, segments[i]); + } + return result; + } + + constructor( + public readonly parent: ScopeStack | null, + public readonly scopeName: ScopeName + ) {} + + public push(scopeName: ScopeName): ScopeStack { + return new ScopeStack(this, scopeName); + } + + public getSegments(): ScopeName[] { + let item: ScopeStack | null = this; + const result: ScopeName[] = []; + while (item) { + result.push(item.scopeName); + item = item.parent; + } + result.reverse(); + return result; + } + + public toString() { + return this.getSegments().join(' '); + } +} + +function _scopePathMatchesParentScopes(scopePath: ScopeStack | null, parentScopes: ScopeName[] | null): boolean { + if (parentScopes === null) { + return true; + } + + let index = 0; + let scopePattern = parentScopes[index]; + + while (scopePath) { + if (_matchesScope(scopePath.scopeName, scopePattern)) { + index++; + if (index === parentScopes.length) { + return true; + } + scopePattern = parentScopes[index]; + } + scopePath = scopePath.parent; + } + + return false; +} + +function _matchesScope(scopeName: ScopeName, scopePattern: ScopeName): boolean { + return scopePattern === scopeName || (scopeName.startsWith(scopePattern) && scopeName[scopePattern.length] === '.'); +} + +export class StyleAttributes { + constructor( + public readonly fontStyle: OrMask, + public readonly foregroundId: number, + public readonly backgroundId: number + ) {} +} + /** * Parse a raw theme into rules. */ @@ -78,7 +204,7 @@ export function parseTheme(source: IRawTheme | undefined): ParsedThemeRule[] { scopes = ['']; } - let fontStyle: number = FontStyle.NotSet; + let fontStyle: OrMask = FontStyle.NotSet; if (typeof entry.settings.fontStyle === 'string') { fontStyle = FontStyle.None; @@ -140,8 +266,8 @@ export function parseTheme(source: IRawTheme | undefined): ParsedThemeRule[] { export class ParsedThemeRule { constructor( - public readonly scope: string, - public readonly parentScopes: string[] | null, + public readonly scope: ScopeName, + public readonly parentScopes: ScopeName[] | null, public readonly index: number, public readonly fontStyle: OrMask, public readonly foreground: string | null, @@ -159,6 +285,30 @@ export const enum FontStyle { Strikethrough = 8 } +export function fontStyleToString(fontStyle: OrMask) { + if (fontStyle === FontStyle.NotSet) { + return 'not set'; + } + + let style = ''; + if (fontStyle & FontStyle.Italic) { + style += 'italic '; + } + if (fontStyle & FontStyle.Bold) { + style += 'bold '; + } + if (fontStyle & FontStyle.Underline) { + style += 'underline '; + } + if (fontStyle & FontStyle.Strikethrough) { + style += 'strikethrough '; + } + if (style === '') { + style = 'none'; + } + return style.trim(); +} + /** * Resolve rules (i.e. inheritance). */ @@ -194,7 +344,7 @@ function resolveParsedThemeRules(parsedThemeRules: ParsedThemeRule[], _colorMap: } } let colorMap = new ColorMap(_colorMap); - let defaults = new ThemeTrieElementRule(0, null, defaultFontStyle, colorMap.getId(defaultForeground), colorMap.getId(defaultBackground)); + let defaults = new StyleAttributes(defaultFontStyle, colorMap.getId(defaultForeground), colorMap.getId(defaultBackground)); let root = new ThemeTrieElement(new ThemeTrieElementRule(0, null, FontStyle.NotSet, 0, 0), []); for (let i = 0, len = parsedThemeRules.length; i < len; i++) { @@ -253,12 +403,12 @@ export class ColorMap { export class ThemeTrieElementRule { scopeDepth: number; - parentScopes: string[] | null; + parentScopes: ScopeName[] | null; fontStyle: number; foreground: number; background: number; - constructor(scopeDepth: number, parentScopes: string[] | null, fontStyle: number, foreground: number, background: number) { + constructor(scopeDepth: number, parentScopes: ScopeName[] | null, fontStyle: number, foreground: number, background: number) { this.scopeDepth = scopeDepth; this.parentScopes = parentScopes; this.fontStyle = fontStyle; @@ -302,28 +452,21 @@ export interface ITrieChildrenMap { } export class ThemeTrieElement { - - private readonly _mainRule: ThemeTrieElementRule; private readonly _rulesWithParentScopes: ThemeTrieElementRule[]; - private readonly _children: ITrieChildrenMap; constructor( - mainRule: ThemeTrieElementRule, + private readonly _mainRule: ThemeTrieElementRule, rulesWithParentScopes: ThemeTrieElementRule[] = [], - children: ITrieChildrenMap = {} + private readonly _children: ITrieChildrenMap = {} ) { - this._mainRule = mainRule; this._rulesWithParentScopes = rulesWithParentScopes; - this._children = children; } private static _sortBySpecificity(arr: ThemeTrieElementRule[]): ThemeTrieElementRule[] { if (arr.length === 1) { return arr; } - arr.sort(this._cmpBySpecificity); - return arr; } @@ -347,7 +490,7 @@ export class ThemeTrieElement { return b.scopeDepth - a.scopeDepth; } - public match(scope: string): ThemeTrieElementRule[] { + public match(scope: ScopeName): ThemeTrieElementRule[] { if (scope === '') { return ThemeTrieElement._sortBySpecificity(([]).concat(this._mainRule).concat(this._rulesWithParentScopes)); } @@ -370,7 +513,7 @@ export class ThemeTrieElement { return ThemeTrieElement._sortBySpecificity(([]).concat(this._mainRule).concat(this._rulesWithParentScopes)); } - public insert(scopeDepth: number, scope: string, parentScopes: string[] | null, fontStyle: number, foreground: number, background: number): void { + public insert(scopeDepth: number, scope: ScopeName, parentScopes: ScopeName[] | null, fontStyle: number, foreground: number, background: number): void { if (scope === '') { this._doInsertHere(scopeDepth, parentScopes, fontStyle, foreground, background); return; @@ -398,7 +541,7 @@ export class ThemeTrieElement { child.insert(scopeDepth + 1, tail, parentScopes, fontStyle, foreground, background); } - private _doInsertHere(scopeDepth: number, parentScopes: string[] | null, fontStyle: number, foreground: number, background: number): void { + private _doInsertHere(scopeDepth: number, parentScopes: ScopeName[] | null, fontStyle: number, foreground: number, background: number): void { if (parentScopes === null) { // Merge into the main rule diff --git a/src/utils.ts b/src/utils.ts index 0dc5ee3f..46ae13b5 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -152,3 +152,36 @@ export function isValidHexColor(hex: string): boolean { return false; } + +/** + * Escapes regular expression characters in a given string + */ +export function escapeRegExpCharacters(value: string): string { + return value.replace(/[\-\\\{\}\*\+\?\|\^\$\.\,\[\]\(\)\#\s]/g, '\\$&'); +} + +export class CachedFn { + private readonly cache = new Map(); + constructor(private readonly fn: (key: TKey) => TValue) { + } + + public get(key: TKey): TValue { + if (this.cache.has(key)) { + return this.cache.get(key)!; + } + const value = this.fn(key); + this.cache.set(key, value); + return value; + } +} + +declare let performance: { now: () => number } | undefined; +export const performanceNow = + typeof performance === "undefined" + // performance.now() is not available in this environment, so use Date.now() + ? function () { + return Date.now(); + } + : function () { + return performance!.now(); + }; \ No newline at end of file diff --git a/tsconfig.json b/tsconfig.json index 8ce19c4e..4af4deeb 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -1,6 +1,6 @@ { "compilerOptions": { - "target": "ES5", + "target": "es2020", "module": "commonjs", "outDir": "out", "noImplicitAny": true, diff --git a/webpack.config.js b/webpack.config.js index e73d7f81..6d4d7806 100644 --- a/webpack.config.js +++ b/webpack.config.js @@ -3,6 +3,7 @@ *--------------------------------------------------------*/ const path = require('path'); +const CopyPlugin = require("copy-webpack-plugin"); module.exports = { entry: './out/main.js', @@ -17,5 +18,12 @@ module.exports = { devtool: 'source-map', resolve: { extensions: ['.js'] - } + }, + plugins: [ + new CopyPlugin({ + patterns: [ + { context: './out', from: '**/*.d.ts', to: '.' }, + ], + }), + ], };