Add helper scripts for updating headers and symbols.js (#7)
diff --git a/.github/workflows/sync-headers.yml b/.github/workflows/sync-headers.yml new file mode 100644 index 0000000..5a9459f --- /dev/null +++ b/.github/workflows/sync-headers.yml
@@ -0,0 +1,52 @@ +name: Header Sync + +on: + workflow_dispatch: null + schedule: + - cron: "0 0 * * *" + +permissions: + contents: write + pull-requests: write + +jobs: + build: + runs-on: ubuntu-latest + name: Update headers from nodejs/node + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: 18 + - shell: bash + id: check-changes + name: Check Changes + run: | + COMMIT_MESSAGE=$(npm run --silent update-headers) + VERSION=${COMMIT_MESSAGE##* } + echo $COMMIT_MESSAGE + npm run --silent write-symbols + CHANGED_FILES=$(git diff --name-only) + BRANCH_NAME="update-headers/${VERSION}" + if [ -z "$CHANGED_FILES" ]; then + echo "No changes exist. Nothing to do." + else + echo "Changes exist. Checking if branch exists: $BRANCH_NAME" + if git ls-remote --exit-code --heads $GITHUB_SERVER_URL/$GITHUB_REPOSITORY $BRANCH_NAME >/dev/null; then + echo "Branch exists. Nothing to do." + else + echo "Branch does not exists." + echo "BRANCH_NAME=$BRANCH_NAME" >> $GITHUB_OUTPUT + echo "COMMIT_MESSAGE=$COMMIT_MESSAGE" >> $GITHUB_OUTPUT + fi + fi + - name: Create Pull Request + uses: peter-evans/create-pull-request@v4 + if: ${{ steps.check-changes.outputs.BRANCH_NAME }} + with: + branch: ${{ steps.check-changes.outputs.BRANCH_NAME }} + commit-message: ${{ steps.check-changes.outputs.COMMIT_MESSAGE }} + title: ${{ steps.check-changes.outputs.COMMIT_MESSAGE }} + author: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> + body: null + delete-branch: true
diff --git a/.npmignore b/.npmignore new file mode 100644 index 0000000..127db70 --- /dev/null +++ b/.npmignore
@@ -0,0 +1,2 @@ +scripts/ +.github/
diff --git a/package.json b/package.json index 1680474..877d7b8 100644 --- a/package.json +++ b/package.json
@@ -46,6 +46,8 @@ "url": "git://github.com/nodejs/node-api-headers.git" }, "scripts": { + "update-headers": "node --no-warnings scripts/update-headers.js", + "write-symbols": "node --no-warnings scripts/write-symbols.js" }, "version": "0.0.2", "support": true
diff --git a/scripts/clang-utils.js b/scripts/clang-utils.js new file mode 100644 index 0000000..2f2b032 --- /dev/null +++ b/scripts/clang-utils.js
@@ -0,0 +1,50 @@ +'use strict'; + +const { spawn } = require('child_process'); + +/** + * @param {Array<string>} [args] + * @returns {Promise<{exitCode: number | null, stdout: string, stderr: string}>} + */ +async function runClang(args = []) { + try { + const { exitCode, stdout, stderr } = await new Promise((resolve, reject) => { + const spawned = spawn('clang', + ['-Xclang', ...args] + ); + + let stdout = ''; + let stderr = ''; + + spawned.stdout?.on('data', (data) => { + stdout += data.toString('utf-8'); + }); + spawned.stderr?.on('data', (data) => { + stderr += data.toString('utf-8'); + }); + + spawned.on('exit', function (exitCode) { + resolve({ exitCode, stdout, stderr }); + }); + + spawned.on('error', function (err) { + reject(err); + }); + }); + + if (exitCode !== 0) { + throw new Error(`clang exited with non-zero exit code ${exitCode}. stderr: ${stderr ? stderr : '<empty>'}`); + } + + return { exitCode, stdout, stderr }; + } catch (err) { + if (err.code === 'ENOENT') { + throw new Error('This tool requires clang to be installed.'); + } + throw err; + } +} + +module.exports = { + runClang +};
diff --git a/scripts/update-headers.js b/scripts/update-headers.js new file mode 100644 index 0000000..03e7c1f --- /dev/null +++ b/scripts/update-headers.js
@@ -0,0 +1,189 @@ +'use strict'; + +const { writeFile } = require('fs/promises'); +const { Readable } = require('stream'); +const { resolve } = require('path'); +const { parseArgs } = require('util') +const { createInterface } = require('readline'); +const { inspect } = require('util'); +const { runClang } = require('./clang-utils'); + +/** + * @returns {Promise<string>} Version string, eg. `'v19.6.0'`. + */ +async function getLatestReleaseVersion() { + const response = await fetch('https://nodejs.org/download/release/index.json'); + const json = await response.json(); + return json[0].version; +} + +/** + * @param {NodeJS.ReadableStream} stream + * @param {string} destination + * @param {boolean} verbose + * @returns {Promise<void>} The `writeFile` Promise. + */ +function removeExperimentals(stream, destination, verbose = false) { + return new Promise((resolve, reject) => { + const debug = (...args) => { + if (verbose) { + console.log(...args); + } + }; + const rl = createInterface(stream); + + /** @type {Array<'write' | 'ignore'>} */ + let mode = ['write']; + + /** @type {Array<string>} */ + const macroStack = []; + + /** @type {RegExpMatchArray | null} */ + let matches; + + let lineNumber = 0; + let toWrite = ''; + + rl.on('line', function lineHandler(line) { + ++lineNumber; + if (matches = line.match(/^\s*#if(n)?def\s+([A-Za-z_][A-Za-z0-9_]*)/)) { + const negated = Boolean(matches[1]); + const identifier = matches[2]; + macroStack.push(identifier); + + debug(`Line ${lineNumber} Pushed ${identifier}`); + + if (identifier === 'NAPI_EXPERIMENTAL') { + if (negated) { + mode.push('write'); + } else { + mode.push('ignore'); + } + return; + } else { + mode.push('write'); + } + + } + else if (matches = line.match(/^\s*#if\s+(.+)$/)) { + const identifier = matches[1]; + macroStack.push(identifier); + mode.push('write'); + + debug(`Line ${lineNumber} Pushed ${identifier}`); + } + else if (line.match(/^#else(?:\s+|$)/)) { + const identifier = macroStack[macroStack.length - 1]; + + debug(`Line ${lineNumber} Peeked ${identifier}`); + + if (!identifier) { + rl.off('line', lineHandler); + reject(new Error(`Macro stack is empty handling #else on line ${lineNumber}`)); + return; + } + + if (identifier === 'NAPI_EXPERIMENTAL') { + const lastMode = mode[mode.length - 1]; + mode[mode.length - 1] = (lastMode === 'ignore') ? 'write' : 'ignore'; + return; + } + } + else if (line.match(/^\s*#endif(?:\s+|$)/)) { + const identifier = macroStack.pop(); + mode.pop(); + + debug(`Line ${lineNumber} Popped ${identifier}`); + + if (!identifier) { + rl.off('line', lineHandler); + reject(new Error(`Macro stack is empty handling #endif on line ${lineNumber}`)); + } + + if (identifier === 'NAPI_EXPERIMENTAL') { + return; + } + } + + if (mode.length === 0) { + rl.off('line', lineHandler); + reject(new Error(`Write mode empty handling #endif on line ${lineNumber}`)); + return; + } + + if (mode[mode.length - 1] === 'write') { + toWrite += `${line}\n`; + } + }); + + rl.on('close', () => { + if (macroStack.length > 0) { + reject(new Error(`Macro stack is not empty at EOF: ${inspect(macroStack)}`)); + } + else if (mode.length > 1) { + reject(new Error(`Write mode greater than 1 at EOF: ${inspect(mode)}`)); + } + else if (toWrite.match(/^\s*#if(?:n)?def\s+NAPI_EXPERIMENTAL/m)) { + reject(new Error(`Output has match for NAPI_EXPERIMENTAL`)); + } + else { + resolve(writeFile(destination, toWrite)); + } + }); + }); +} + +/** + * Validate syntax for a file using clang. + * @param {string} path Path for file to validate with clang. + */ +async function validateSyntax(path) { + try { + await runClang(['-fsyntax-only', path]); + } catch (e) { + throw new Error(`Syntax validation failed for ${path}: ${e}`); + } +} + +async function main() { + const { values: { tag, verbose } } = parseArgs({ + options: { + tag: { + type: 'string', + short: 't', + default: await getLatestReleaseVersion() + }, + verbose: { + type: 'boolean', + short: 'v', + }, + }, + }); + + console.log(`Update headers from nodejs/node tag ${tag}`); + + const files = ['js_native_api_types.h', 'js_native_api.h', 'node_api_types.h', 'node_api.h']; + + for (const filename of files) { + const url = `https://raw.githubusercontent.com/nodejs/node/${tag}/src/${filename}`; + const path = resolve(__dirname, '..', 'include', filename); + + if (verbose) { + console.log(` ${url} -> ${path}`); + } + + const response = await fetch(url); + if (!response.ok) { + throw new Error(`Fetch of ${url} returned ${response.status} ${response.statusText}`); + } + + await removeExperimentals(Readable.fromWeb(response.body), path, verbose); + + await validateSyntax(path); + } +} + +main().catch(e => { + console.error(e); + process.exitCode = 1; +});
diff --git a/scripts/write-symbols.js b/scripts/write-symbols.js new file mode 100644 index 0000000..d3621c1 --- /dev/null +++ b/scripts/write-symbols.js
@@ -0,0 +1,154 @@ +'use strict'; + +const { resolve: resolvePath } = require('path'); +const { writeFile } = require('fs/promises'); +const { runClang } = require('./clang-utils'); + +/** @typedef {{ js_native_api_symbols: string[]; node_api_symbols: string[]; }} SymbolInfo */ + +/** + * @param {number} [version] + * @returns {Promise<SymbolInfo>} + */ +async function getSymbolsForVersion(version) { + try { + const { stdout } = await runClang([ '-ast-dump=json', '-fsyntax-only', '-fno-diagnostics-color', `-DNAPI_VERSION=${version}`, resolvePath(__dirname, '..', 'include', 'node_api.h')]) + + const ast = JSON.parse(stdout); + + /** @type {SymbolInfo} */ + const symbols = { js_native_api_symbols: [], node_api_symbols: [] }; + + for (const statement of ast.inner) { + if (statement.kind !== 'FunctionDecl') { + continue; + } + + const name = statement.name; + const file = statement.loc.includedFrom?.file; + + if (file) { + symbols.js_native_api_symbols.push(name); + } else { + symbols.node_api_symbols.push(name); + } + } + + symbols.js_native_api_symbols.sort(); + symbols.node_api_symbols.sort(); + + return symbols; + } catch (err) { + if (err.code === 'ENOENT') { + throw new Error('This tool requires clang to be installed.'); + } + throw err; + } +} + +/** @returns {Promise<{maxVersion: number, symbols: {[x: string]: SymbolInfo}}>} */ +async function getAllSymbols() { + /** @type {{[x: string]: SymbolInfo}} */ + const allSymbols = {}; + let version = 1; + + console.log('Processing symbols from clang:') + while (true) { + const symbols = await getSymbolsForVersion(version); + + if (version > 1) { + const previousSymbols = allSymbols[`v${version - 1}`]; + if (previousSymbols.js_native_api_symbols.length == symbols.js_native_api_symbols.length && previousSymbols.node_api_symbols.length === symbols.node_api_symbols.length) { + --version; + break; + } + } + allSymbols[`v${version}`] = symbols; + console.log(` v${version}: ${symbols.js_native_api_symbols.length} js_native_api_symbols, ${symbols.node_api_symbols.length} node_api_symbols`); + ++version; + } + + return { + maxVersion: version, + symbols: allSymbols + }; +} + +/** + * @param {SymbolInfo} previousSymbols + * @param {SymbolInfo} currentSymbols + * @returns {SymbolInfo} + */ +function getUniqueSymbols(previousSymbols, currentSymbols) { + /** @type {SymbolInfo} */ + const symbols = { js_native_api_symbols: [], node_api_symbols: [] }; + for (const symbol of currentSymbols.js_native_api_symbols) { + if (!previousSymbols.js_native_api_symbols.includes(symbol)) { + symbols.js_native_api_symbols.push(symbol); + } + } + for (const symbol of currentSymbols.node_api_symbols) { + if (!previousSymbols.node_api_symbols.includes(symbol)) { + symbols.node_api_symbols.push(symbol); + } + } + return symbols; +} + +/** + * @param {string[]} strings + */ +function joinStrings(strings, prependNewLine = false) { + if (strings.length === 0) return ''; + return `${prependNewLine ? ',\n ' : ''}'${strings.join("',\n '")}'`; +} + +async function getSymbolData() { + const { maxVersion, symbols } = await getAllSymbols(); + + let data = `'use strict' + +const v1 = { + js_native_api_symbols: [ + ${joinStrings(symbols.v1.js_native_api_symbols)} + ], + node_api_symbols: [ + ${joinStrings(symbols.v1.node_api_symbols)} + ] +} +`; + + for (let version = 2; version <= maxVersion; ++version) { + const newSymbols = getUniqueSymbols(symbols[`v${version - 1}`], symbols[`v${version}`]); + + data += ` +const v${version} = { + js_native_api_symbols: [ + ...v${version - 1}.js_native_api_symbols${joinStrings(newSymbols.js_native_api_symbols, true)} + ], + node_api_symbols: [ + ...v${version - 1}.node_api_symbols${joinStrings(newSymbols.node_api_symbols, true)} + ] +} +`; + } + + data += ` +module.exports = { + ${new Array(maxVersion).fill(undefined).map((_, i) => `v${i + 1}`).join(',\n ')} +} +` + return data; +} + +async function main() { + const path = resolvePath(__dirname, '../symbols.js'); + const data = await getSymbolData(); + console.log(`Writing symbols to ${path}`) + return writeFile(path, data); +} + +main().catch(e => { + console.error(e); + process.exitCode = 1; +});