diff --git a/experiments/js-parse-and-output/README.md b/experiments/js-parse-and-output/README.md new file mode 100644 index 000000000..9c6e7fb90 --- /dev/null +++ b/experiments/js-parse-and-output/README.md @@ -0,0 +1,29 @@ +# Parsing Javascript and Generating it Back + +## What Purpose? + +This would be really useful for refactoring tools. I also want +automatic comments to be placed when certain features are used, +such as the [Sequence](../../src/backend/src/codex/Sequence.js) +class, since its usefulness won't be immediately apparent where +it appears in the source code. + +I turns out the state of affairs with respect to generating a +CST for javascript... [kind of sucks](https://github.com/benjamn/recast/issues/1412). +I hope that further discussion on the issue I linked renders the +previous statement ironic. + +## So, What Next? + +The options I see are: +1. Add support to recast to make use of @babel/parser tokens when + the `tokens: true` option is set. +2. Add a format-preserving outputter to @babel/outputter. + [this is being worked on](https://github.com/babel/rfcs/pull/15) +3. Wait for someone else to do either of the previous two things. +4. Write a CST parser for javascript. + +I'm going to start with option #3. It's very disappointing that +I don't have time to do #4, because I don't very much like solutions +#1 and #2; I like my CSTs to be more cohesive - a pyramid of meaning - +rather than lexer output dumped at the end of an AST. diff --git a/experiments/js-parse-and-output/package.json b/experiments/js-parse-and-output/package.json new file mode 100644 index 000000000..e0ee5f726 --- /dev/null +++ b/experiments/js-parse-and-output/package.json @@ -0,0 +1,20 @@ +{ + "name": "js-parse-and-output", + "version": "1.0.0", + "description": "", + "main": "test.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "keywords": [], + "author": "", + "license": "AGPL-3.0-only", + "dependencies": { + "@babel/generator": "^7.24.7", + "@babel/parser": "^7.24.7", + "acorn": "^8.12.1", + "acorn-static-class-features": "^1.0.0", + "escodegen": "^2.1.0", + "recast": "^0.23.9" + } +} diff --git a/experiments/js-parse-and-output/test.js b/experiments/js-parse-and-output/test.js new file mode 100644 index 000000000..3fa6568a5 --- /dev/null +++ b/experiments/js-parse-and-output/test.js @@ -0,0 +1,37 @@ +const babelParser = require('@babel/parser'); +const generate = (require('@babel/generator')).default; +const fs = require('fs'); + +const recast = require('recast'); + +const example = fs.readFileSync('./src/backend/src/filesystem/ll_operations/ll_read.js'); + +{ + const ast = recast.parse(example, { + parser: { + parse (source) { + return babelParser.parse(source, { + ranges: true, + tokens: true, + }); + }, + }, + }); + const { code } = recast.print(ast); +} + +{ + const ast = babelParser.parse('' + example, { + tokens: true, + }); + + console.log(JSON.stringify(ast, undefined, ' ')); +} + +/* +const { code } = generate(ast, { + retainLines: true, +}); +*/ + +// console.log(code); \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 7c07f6b96..73908b772 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,7 +9,9 @@ "version": "1.0.0", "license": "AGPL-3.0-only", "workspaces": [ - "src/*" + "src/*", + "tools/*", + "experiments/js-parse-and-output" ], "dependencies": { "json-colorizer": "^3.0.1", @@ -224,6 +226,18 @@ "extraneous": true, "license": "AGPL-3.0-only" }, + "experiments/js-parse-and-output": { + "version": "1.0.0", + "license": "AGPL-3.0-only", + "dependencies": { + "@babel/generator": "^7.24.7", + "@babel/parser": "^7.24.7", + "acorn": "^8.12.1", + "acorn-static-class-features": "^1.0.0", + "escodegen": "^2.1.0", + "recast": "^0.23.9" + } + }, "node_modules/@aashutoshrathi/word-wrap": { "version": "1.2.6", "resolved": "https://registry.npmjs.org/@aashutoshrathi/word-wrap/-/word-wrap-1.2.6.tgz", @@ -323,7 +337,6 @@ "version": "7.24.7", "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.24.7.tgz", "integrity": "sha512-oipXieGC3i45Y1A41t4tAqpnEZWgB/lC6Ehh6+rOviR5XWpTtMmLN+fGjz9vOiNRt0p6RtO6DtD0pdU3vpqdSA==", - "dev": true, "license": "MIT", "dependencies": { "@babel/types": "^7.24.7", @@ -467,7 +480,6 @@ "version": "7.24.7", "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.24.7.tgz", "integrity": "sha512-7MbVt6xrwFQbunH2DNQsAP5sTGxfqQtErvBIvIMi6EQnbgUOuVYanvREcmFrOPhoXBrTtjhhP+lW+o5UfK+tDg==", - "dev": true, "license": "MIT", "engines": { "node": ">=6.9.0" @@ -477,7 +489,6 @@ "version": "7.24.7", "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.24.7.tgz", "integrity": "sha512-rR+PBcQ1SMQDDyF6X0wxtG8QyLCgUB0eRAGguqRLfkCA87l7yAP7ehq8SNj96OOGTO8OBV70KhuFYcIkHXOg0w==", - "dev": true, "license": "MIT", "engines": { "node": ">=6.9.0" @@ -582,7 +593,6 @@ "version": "7.24.7", "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.24.7.tgz", "integrity": "sha512-9uUYRm6OqQrCqQdG1iCBwBPZgN8ciDBro2nIOFaiRz1/BCxaI7CNvQbDHvsArAC7Tw9Hda/B3U+6ui9u4HWXPw==", - "dev": true, "license": "MIT", "bin": { "parser": "bin/babel-parser.js" @@ -655,7 +665,6 @@ "version": "7.24.7", "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.24.7.tgz", "integrity": "sha512-XEFXSlxiG5td2EJRe8vOmRbaXVgfcBlszKujvVmWIK/UpywWljQCfzAv3RQCGujWQ1RD4YYWEAqDXfuJiy8f5Q==", - "dev": true, "license": "MIT", "dependencies": { "@babel/helper-string-parser": "^7.24.7", @@ -1668,7 +1677,6 @@ "version": "0.3.5", "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.5.tgz", "integrity": "sha512-IzL8ZoEDIBRWEzlCcRhOaCupYyN5gdIK+Q6fbFdPDg6HqX6jpkItn7DFIpW9LQzXG6Df9sA7+OKnq0qlz/GaQg==", - "dev": true, "dependencies": { "@jridgewell/set-array": "^1.2.1", "@jridgewell/sourcemap-codec": "^1.4.10", @@ -1682,7 +1690,6 @@ "version": "3.1.2", "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", - "dev": true, "engines": { "node": ">=6.0.0" } @@ -1691,7 +1698,6 @@ "version": "1.2.1", "resolved": "https://registry.npmjs.org/@jridgewell/set-array/-/set-array-1.2.1.tgz", "integrity": "sha512-R8gLRTZeyp03ymzP/6Lil/28tGeGEzhx1q2k703KGWRAI1VdvPIXdG70VJc2pAMw3NA6JKL5hhFu1sJX0Mnn/A==", - "dev": true, "engines": { "node": ">=6.0.0" } @@ -1709,14 +1715,12 @@ "node_modules/@jridgewell/sourcemap-codec": { "version": "1.4.15", "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.15.tgz", - "integrity": "sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg==", - "dev": true + "integrity": "sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg==" }, "node_modules/@jridgewell/trace-mapping": { "version": "0.3.25", "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.25.tgz", "integrity": "sha512-vNk6aEwybGtawWmy/PzwnGDOjCkLWSD2wqvjGGAgOAwCGWySYXfYoxt00IJkTF+8Lb57DwOb3Aa0o9CApepiYQ==", - "dev": true, "dependencies": { "@jridgewell/resolve-uri": "^3.1.0", "@jridgewell/sourcemap-codec": "^1.4.14" @@ -4813,9 +4817,9 @@ } }, "node_modules/acorn": { - "version": "8.11.3", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.11.3.tgz", - "integrity": "sha512-Y9rRfJG5jcKOE0CLisYbojUjIrIEE7AGMzA/Sm4BslANhbS+cDMpgBdcPT91oJ7OuJ9hYJBx59RjbhxVnrF8Xg==", + "version": "8.12.1", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.12.1.tgz", + "integrity": "sha512-tcpGyI9zbizT9JbV6oYE477V6mTlXvvi0T0G3SNIYE2apm/G5huBa1+K89VGeovbg+jycCrfhl3ADxErOuO6Jg==", "bin": { "acorn": "bin/acorn" }, @@ -4840,6 +4844,31 @@ "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" } }, + "node_modules/acorn-private-class-elements": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/acorn-private-class-elements/-/acorn-private-class-elements-1.0.0.tgz", + "integrity": "sha512-zYNcZtxKgVCg1brS39BEou86mIao1EV7eeREG+6WMwKbuYTeivRRs6S2XdWnboRde6G9wKh2w+WBydEyJsJ6mg==", + "engines": { + "node": ">=4.8.2" + }, + "peerDependencies": { + "acorn": "^6.1.0 || ^7 || ^8" + } + }, + "node_modules/acorn-static-class-features": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/acorn-static-class-features/-/acorn-static-class-features-1.0.0.tgz", + "integrity": "sha512-XZJECjbmMOKvMHiNzbiPXuXpLAJfN3dAKtfIYbk1eHiWdsutlek+gS7ND4B8yJ3oqvHo1NxfafnezVmq7NXK0A==", + "dependencies": { + "acorn-private-class-elements": "^1.0.0" + }, + "engines": { + "node": ">=4.8.2" + }, + "peerDependencies": { + "acorn": "^6.1.0 || ^7 || ^8" + } + }, "node_modules/agent-base": { "version": "6.0.2", "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", @@ -5114,6 +5143,17 @@ "node": "*" } }, + "node_modules/ast-types": { + "version": "0.16.1", + "resolved": "https://registry.npmjs.org/ast-types/-/ast-types-0.16.1.tgz", + "integrity": "sha512-6t10qk83GOG8p0vKmaCr8eiilZwO171AvbROMtvvNiwrTly62t+7XkA8RdIIVbpMhCASAsxgAzdRSwh6nw/5Dg==", + "dependencies": { + "tslib": "^2.0.1" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/async": { "version": "2.6.4", "resolved": "https://registry.npmjs.org/async/-/async-2.6.4.tgz", @@ -6852,6 +6892,34 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/escodegen": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-2.1.0.tgz", + "integrity": "sha512-2NlIDTwUWJN0mRPQOdtQBzbUHvdGY2P1VXSyU83Q3xKxM7WHX2Ql8dKq782Q9TgQUNOLEzEYu9bzLNj1q88I5w==", + "dependencies": { + "esprima": "^4.0.1", + "estraverse": "^5.2.0", + "esutils": "^2.0.2" + }, + "bin": { + "escodegen": "bin/escodegen.js", + "esgenerate": "bin/esgenerate.js" + }, + "engines": { + "node": ">=6.0" + }, + "optionalDependencies": { + "source-map": "~0.6.1" + } + }, + "node_modules/escodegen/node_modules/estraverse": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", + "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", + "engines": { + "node": ">=4.0" + } + }, "node_modules/eslint": { "version": "9.1.1", "resolved": "https://registry.npmjs.org/eslint/-/eslint-9.1.1.tgz", @@ -6995,7 +7063,6 @@ "version": "4.0.1", "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", - "dev": true, "license": "BSD-2-Clause", "bin": { "esparse": "bin/esparse.js", @@ -7067,7 +7134,6 @@ "version": "2.0.3", "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", - "dev": true, "engines": { "node": ">=0.10.0" } @@ -7309,6 +7375,10 @@ "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==", "license": "MIT" }, + "node_modules/file-walker": { + "resolved": "tools/file-walker", + "link": true + }, "node_modules/fill-range": { "version": "7.1.1", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", @@ -8893,6 +8963,10 @@ "integrity": "sha512-WZzeDOEtTOBK4Mdsar0IqEU5sMr3vSV2RqkAIzUEV2BHnUfKGyswWFPFwK5EeDo93K3FohSHbLAjj0s1Wzd+dg==", "license": "BSD-3-Clause" }, + "node_modules/js-parse-and-output": { + "resolved": "experiments/js-parse-and-output", + "link": true + }, "node_modules/js-sha256": { "version": "0.9.0", "resolved": "https://registry.npmjs.org/js-sha256/-/js-sha256-0.9.0.tgz", @@ -8922,7 +8996,6 @@ "version": "2.5.2", "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-2.5.2.tgz", "integrity": "sha512-OYu7XEzjkCQ3C5Ps3QIZsQfNpqoJyZZA99wd9aWd05NCtC5pWOkShK2mkL6HXQR6/Cy2lbNdPlZBpuQHXE63gA==", - "dev": true, "license": "MIT", "bin": { "jsesc": "bin/jsesc" @@ -11178,6 +11251,21 @@ "node": ">=8.10.0" } }, + "node_modules/recast": { + "version": "0.23.9", + "resolved": "https://registry.npmjs.org/recast/-/recast-0.23.9.tgz", + "integrity": "sha512-Hx/BGIbwj+Des3+xy5uAtAbdCyqK9y9wbBcDFDYanLS9JnMqf7OeF87HQwUimE87OEc72mr6tkKUKMBBL+hF9Q==", + "dependencies": { + "ast-types": "^0.16.1", + "esprima": "~4.0.0", + "source-map": "~0.6.1", + "tiny-invariant": "^1.3.3", + "tslib": "^2.0.1" + }, + "engines": { + "node": ">= 4" + } + }, "node_modules/rechoir": { "version": "0.8.0", "resolved": "https://registry.npmjs.org/rechoir/-/rechoir-0.8.0.tgz", @@ -12444,6 +12532,11 @@ "integrity": "sha512-IjZc9KIotudix8bMaBW6QvMuq64BrJWFs1+4V0lXwWGQZwH+LnX87doAYhem4caOEusRP9/g6jVDQmZ8XOk1nw==", "license": "MIT" }, + "node_modules/tiny-invariant": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/tiny-invariant/-/tiny-invariant-1.3.3.tgz", + "integrity": "sha512-+FbBPE1o9QAYvviau/qC5SE3caw21q3xkvWKBtja5vgqOWIHHJ3ioaq1VPfn/Szqctz2bU/oYeKd9/z5BL+PVg==" + }, "node_modules/tinycolor2": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/tinycolor2/-/tinycolor2-1.6.0.tgz", @@ -12454,7 +12547,6 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/to-fast-properties/-/to-fast-properties-2.0.0.tgz", "integrity": "sha512-/OaKK0xYrs3DmxRYqL/yDc+FxFUVYhDlXMhRmv3z915w2HF1tnN1omB354j8VUGO/hbRzyD6Y3sA7v7GS/ceog==", - "dev": true, "license": "MIT", "engines": { "node": ">=4" @@ -13897,6 +13989,10 @@ "src/useapi": { "version": "1.0.0", "license": "AGPL-3.0-only" + }, + "tools/file-walker": { + "version": "1.0.0", + "license": "AGPL-3.0-only" } } } diff --git a/package.json b/package.json index 2ac95fb53..dc4be8bc8 100644 --- a/package.json +++ b/package.json @@ -33,7 +33,8 @@ }, "workspaces": [ "src/*", - "tools/*" + "tools/*", + "experiments/js-parse-and-output" ], "nodemonConfig": { "ext": "js, json, mjs, jsx, svg, css",