From 5128444e3bf470c4ab32c6e4ad81cb2b88f0528c Mon Sep 17 00:00:00 2001 From: Orta Date: Tue, 22 Jun 2021 14:09:07 +0100 Subject: [PATCH 1/4] Add the ability to upload to @types/x --- .github/workflows/deploy.yml | 33 +++++++++ .gitignore | 1 + deploy/README.md | 3 + deploy/createTypesPackages.mjs | 122 +++++++++++++++++++++++++++++++ deploy/deployChangedPackages.mjs | 84 +++++++++++++++++++++ deploy/template/LICENSE | 21 ++++++ deploy/template/README.md | 12 +++ deploy/template/package.json | 15 ++++ 8 files changed, 291 insertions(+) create mode 100644 .github/workflows/deploy.yml create mode 100644 deploy/README.md create mode 100644 deploy/createTypesPackages.mjs create mode 100644 deploy/deployChangedPackages.mjs create mode 100644 deploy/template/LICENSE create mode 100644 deploy/template/README.md create mode 100644 deploy/template/package.json diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 000000000..bbf753812 --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,33 @@ +name: Deploy to npm + +on: + push: + branches: + - master + + workflow_dispatch: + +jobs: + deploy: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-node@v1 + with: + node-version: "15.x" + registry-url: "https://registry.npmjs.org" + + - run: npm install + - run: npm run build + - run: npm test + + - name: Create packages for .d.ts files + run: node deploy/createTypesPackages.mjs + + # Deploy anything which differs from the npm version of a tsconfig + - name: 'Deploy built packages to NPM' + run: node deploy/deployChangedPackages.mjs + env: + NODE_AUTH_TOKEN: ${{ secrets.NODE_AUTH_TOKEN }} + \ No newline at end of file diff --git a/.gitignore b/.gitignore index 72bc839d1..cad483484 100644 --- a/.gitignore +++ b/.gitignore @@ -286,3 +286,4 @@ inputfiles/browser.webidl.json package-lock.json yarn.lock TypeScript +deploy/generated \ No newline at end of file diff --git a/deploy/README.md b/deploy/README.md new file mode 100644 index 000000000..81c992372 --- /dev/null +++ b/deploy/README.md @@ -0,0 +1,3 @@ +## Deploys + +We want to generate @types/xyz \ No newline at end of file diff --git a/deploy/createTypesPackages.mjs b/deploy/createTypesPackages.mjs new file mode 100644 index 000000000..826c0ff49 --- /dev/null +++ b/deploy/createTypesPackages.mjs @@ -0,0 +1,122 @@ +// @ts-check + +// node deploy/createTypesPackages.mjs + +// prettier-ignore +const packages = [ + { + name: "@types/web", + description: "Types for the DOM, and other web technologies in browsers", + files: [ + { from: "../generated/dom.generated.d.ts", to: "index.d.ts" } + ], + }, + ]; + +// Note: You can add 'version: "1.0.0"' to a package above +// to set the major or minor, otherwise it will always bump +// the patch. + +import { join, dirname } from "path"; +import fs from "fs"; +import { fileURLToPath } from "url"; +import pkg from "prettier"; +const { format } = pkg; +import { execSync } from "child_process"; + +// eslint-disable-next-line @typescript-eslint/ban-ts-comment +// @ts-ignore +const __filename = fileURLToPath(import.meta.url); +const __dirname = dirname(__filename); + +const go = async () => { + const gitSha = execSync("git rev-parse HEAD").toString().trim().slice(0, 7); + + const generatedDir = join(__dirname, "generated"); + const templateDir = join(__dirname, "template"); + + for (const pkg of packages) { + const folderName = pkg.name.replace("@", "").replace("/", "-"); + const packagePath = join(generatedDir, folderName); + + if (fs.existsSync(packagePath)) fs.rmSync(packagePath, { recursive: true }); + fs.mkdirSync(packagePath, { recursive: true }); + + // Migrate in the template files + for (const templateFile of fs.readdirSync(templateDir)) { + if (templateFile.startsWith(".")) continue; + + const templatedFile = join(templateDir, templateFile); + fs.copyFileSync(templatedFile, join(packagePath, templateFile)); + } + + // Add the reference files in the config above + pkg.files.forEach((fileRef) => { + fs.copyFileSync( + join(__filename, "..", fileRef.from), + join(packagePath, fileRef.to) + ); + }); + + // Setup the files + await updatePackageJSON(packagePath, pkg, gitSha); + + console.log("Built:", pkg.name); + } +}; + +go(); + +async function updatePackageJSON(packagePath, pkg, gitSha) { + const pkgJSONPath = join(packagePath, "package.json"); + const packageText = fs.readFileSync(pkgJSONPath, "utf8"); + const packageJSON = JSON.parse(packageText); + packageJSON.name = pkg.name; + packageJSON.description = pkg.description; + + // Bump the last version of the number from npm, + // or use the _version in tsconfig if it's higher, + // or default to 0.0.1 + let version = pkg.version || "0.0.1"; + try { + const npmResponse = await fetch( + `https://registry.npmjs.org/${packageJSON.name}` + ); + const npmPackage = await npmResponse.json(); + + const semverMarkers = npmPackage["dist-tags"].latest.split("."); + const bumpedVersion = `${semverMarkers[0]}.${semverMarkers[1]}.${ + Number(semverMarkers[2]) + 1 + }`; + + if (isBumpedVersionHigher(version, bumpedVersion)) { + version = bumpedVersion; + } + } catch (error) { + // NOOP, this is for the first deploy, which will set it to 0.0.1 + } + + packageJSON.version = version; + packageJSON.domLibGeneratorSha = gitSha; + + fs.writeFileSync( + pkgJSONPath, + format(JSON.stringify(packageJSON), { filepath: pkgJSONPath }) + ); +} + +/** + * @param packageJSONVersion {string} + * @param bumpedVersion {string} + */ +function isBumpedVersionHigher(packageJSONVersion, bumpedVersion) { + const semverMarkersPackageJSON = packageJSONVersion.split("."); + const semverMarkersBumped = bumpedVersion.split("."); + for (let i = 0; i < 3; i++) { + if (Number(semverMarkersPackageJSON[i]) > Number(semverMarkersBumped[i])) { + return false; + } + } + + return true; +} diff --git a/deploy/deployChangedPackages.mjs b/deploy/deployChangedPackages.mjs new file mode 100644 index 000000000..42de38a5e --- /dev/null +++ b/deploy/deployChangedPackages.mjs @@ -0,0 +1,84 @@ +// @ts-check + +// node deploy/deployChangedPackages.mjs +// Builds on the results of createTypesPackages.mjs and deploys the +// ones which have changed. + +import * as fs from "fs"; +import { join, dirname } from "path"; +import { fileURLToPath } from "url"; +import fetch from "node-fetch"; +import { spawnSync } from "child_process"; + +// eslint-disable-next-line @typescript-eslint/ban-ts-comment +// @ts-ignore +const __filename = fileURLToPath(import.meta.url); +const __dirname = dirname(__filename); + +const go = async () => { + const uploaded = []; + + // Loop through generated packages, deploying versions for anything which has different + // .d.ts files from the version available on npm. + const generatedDir = join(__dirname, "generated"); + for (const dirName of fs.readdirSync(generatedDir)) { + const localPackageJSONPath = join(generatedDir, dirName, "package.json"); + const newTSConfig = fs.readFileSync(localPackageJSONPath, "utf-8"); + const pkgJSON = JSON.parse(newTSConfig); + + const dtsFiles = fs + .readdirSync(join(generatedDir, dirName)) + .filter((f) => f.endsWith(".d.ts")); + + // Look through each .d.ts file included in a package to + // determine if anything has changed + let upload = false; + for (const file of dtsFiles) { + const generatedDTSPath = join(generatedDir, dirName, file); + const generatedDTSContent = fs.readFileSync(generatedDTSPath, "utf8"); + try { + const unpkgURL = `https://unpkg.com/${pkgJSON.name}/${file}`; + const npmDTSReq = await fetch(unpkgURL); + const npmDTSText = await npmDTSReq.text(); + upload = upload || npmDTSText !== generatedDTSContent; + } catch (error) { + // Not here, definitely needs to be uploaded + upload = true; + } + } + + // Publish via npm + if (upload) { + if (process.env.NODE_AUTH_TOKEN) { + const publish = spawnSync("npm", ["publish", "--access", "public"], { + cwd: join("packages", dirName), + }); + + if (publish.status) { + console.log(publish.stdout?.toString()); + console.log(publish.stderr?.toString()); + process.exit(publish.status); + } else { + console.log(publish.stdout?.toString()); + } + } + + uploaded.push(dirName); + } + } + + // Warn if we did a dry run. + if (!process.env.NODE_AUTH_TOKEN) { + console.log( + "Did a dry run because process.env.NODE_AUTH_TOKEN is not set." + ); + } + + if (uploaded.length) { + console.log("Uploaded: ", uploaded.join(", ")); + } else { + console.log("No uploads"); + } +}; + +go(); diff --git a/deploy/template/LICENSE b/deploy/template/LICENSE new file mode 100644 index 000000000..9e841e7a2 --- /dev/null +++ b/deploy/template/LICENSE @@ -0,0 +1,21 @@ + MIT License + + Copyright (c) Microsoft Corporation. + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE diff --git a/deploy/template/README.md b/deploy/template/README.md new file mode 100644 index 000000000..89b58ef29 --- /dev/null +++ b/deploy/template/README.md @@ -0,0 +1,12 @@ +# Installation +> `npm install --save {{name}}` + +# Summary +{{description}} + + +### Additional Details + * Last updated: {{date}} + * Dependencies: none + * Global values: none + diff --git a/deploy/template/package.json b/deploy/template/package.json new file mode 100644 index 000000000..69cd74af7 --- /dev/null +++ b/deploy/template/package.json @@ -0,0 +1,15 @@ +{ + "name": "@types/xyz", + "version": "0.0.x", + "description": "TypeScript definitions for xyz", + "license": "MIT", + "contributors": [], + "main": "", + "types": "index.d.ts", + "repository": { + "type": "git", + "url": "https://github.com/microsoft/TypeScript-DOM-Lib-Generator.git" + }, + "scripts": {}, + "dependencies": {} +} \ No newline at end of file From 82f1a37b88df08726404969f4424ea8984c72e71 Mon Sep 17 00:00:00 2001 From: Orta Date: Wed, 23 Jun 2021 14:15:52 +0100 Subject: [PATCH 2/4] Feedback and a good README --- .github/workflows/deploy.yml | 1 + deploy/createTypesPackages.mjs | 39 +++++++++++++++++++------------- deploy/deployChangedPackages.mjs | 36 ++++++++++++++++++++++++++--- deploy/readmes/web.md | 33 +++++++++++++++++++++++++++ deploy/template/README.md | 12 ---------- package.json | 2 ++ 6 files changed, 92 insertions(+), 31 deletions(-) create mode 100644 deploy/readmes/web.md delete mode 100644 deploy/template/README.md diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index bbf753812..b748d7f96 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -30,4 +30,5 @@ jobs: run: node deploy/deployChangedPackages.mjs env: NODE_AUTH_TOKEN: ${{ secrets.NODE_AUTH_TOKEN }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file diff --git a/deploy/createTypesPackages.mjs b/deploy/createTypesPackages.mjs index 826c0ff49..bb9c6322a 100644 --- a/deploy/createTypesPackages.mjs +++ b/deploy/createTypesPackages.mjs @@ -7,6 +7,7 @@ const packages = [ { name: "@types/web", description: "Types for the DOM, and other web technologies in browsers", + readme: "./readmes/web.md", files: [ { from: "../generated/dom.generated.d.ts", to: "index.d.ts" } ], @@ -20,6 +21,7 @@ const packages = [ import { join, dirname } from "path"; import fs from "fs"; import { fileURLToPath } from "url"; +import semver from "semver"; import pkg from "prettier"; const { format } = pkg; import { execSync } from "child_process"; @@ -58,9 +60,11 @@ const go = async () => { ); }); - // Setup the files - await updatePackageJSON(packagePath, pkg, gitSha); + // Setup the files in the repo + const newPkgJSON = await updatePackageJSON(packagePath, pkg, gitSha); + copyREADME(pkg, newPkgJSON, join(packagePath, "README.md")); + // Done console.log("Built:", pkg.name); } }; @@ -89,7 +93,7 @@ async function updatePackageJSON(packagePath, pkg, gitSha) { Number(semverMarkers[2]) + 1 }`; - if (isBumpedVersionHigher(version, bumpedVersion)) { + if (semver.gt(version, bumpedVersion)) { version = bumpedVersion; } } catch (error) { @@ -103,20 +107,23 @@ async function updatePackageJSON(packagePath, pkg, gitSha) { pkgJSONPath, format(JSON.stringify(packageJSON), { filepath: pkgJSONPath }) ); + + return packageJSON; } -/** - * @param packageJSONVersion {string} - * @param bumpedVersion {string} - */ -function isBumpedVersionHigher(packageJSONVersion, bumpedVersion) { - const semverMarkersPackageJSON = packageJSONVersion.split("."); - const semverMarkersBumped = bumpedVersion.split("."); - for (let i = 0; i < 3; i++) { - if (Number(semverMarkersPackageJSON[i]) > Number(semverMarkersBumped[i])) { - return false; - } - } +// Copies the README and adds some rudimentary templating to the file. +function copyREADME(pkg, pkgJSON, writePath) { + let readme = fs.readFileSync(join(__filename, "..", pkg.readme), "utf-8"); + + const htmlEncodedTag = + encodeURIComponent(pkgJSON.name) + "%40" + pkgJSON.version; + + readme = readme + .replace("{{version}}", pkgJSON.version) + .replace( + "{{release_href}}", + `https://github.com/microsoft/TypeScript-DOM-lib-generator/releases/tag/${htmlEncodedTag}` + ); - return true; + fs.writeFileSync(writePath, readme); } diff --git a/deploy/deployChangedPackages.mjs b/deploy/deployChangedPackages.mjs index 42de38a5e..b2fdd027d 100644 --- a/deploy/deployChangedPackages.mjs +++ b/deploy/deployChangedPackages.mjs @@ -1,6 +1,7 @@ // @ts-check // node deploy/deployChangedPackages.mjs + // Builds on the results of createTypesPackages.mjs and deploys the // ones which have changed. @@ -9,13 +10,24 @@ import { join, dirname } from "path"; import { fileURLToPath } from "url"; import fetch from "node-fetch"; import { spawnSync } from "child_process"; +import { Octokit } from "@octokit/core"; // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-ignore const __filename = fileURLToPath(import.meta.url); const __dirname = dirname(__filename); +const verify = () => { + const authToken = process.env.GITHUB_TOKEN || process.env.GITHUB_API_TOKEN; + if (!authToken) + throw new Error( + "There isn't an ENV var set up for creating a GitHub release, expected GITHUB_TOKEN." + ); +}; + const go = async () => { + verify(); + const uploaded = []; // Loop through generated packages, deploying versions for anything which has different @@ -36,14 +48,18 @@ const go = async () => { for (const file of dtsFiles) { const generatedDTSPath = join(generatedDir, dirName, file); const generatedDTSContent = fs.readFileSync(generatedDTSPath, "utf8"); + const unpkgURL = `https://unpkg.com/${pkgJSON.name}/${file}`; try { - const unpkgURL = `https://unpkg.com/${pkgJSON.name}/${file}`; const npmDTSReq = await fetch(unpkgURL); const npmDTSText = await npmDTSReq.text(); upload = upload || npmDTSText !== generatedDTSContent; } catch (error) { - // Not here, definitely needs to be uploaded - upload = true; + // Could not find a previous build + console.log( + `Could not get the file ${file} inside the npm package ${pkgJSON.name} from unpkg at ${unpkgURL}` + ); + process.exitCode = 1; + upload = false; } } @@ -60,6 +76,8 @@ const go = async () => { process.exit(publish.status); } else { console.log(publish.stdout?.toString()); + + await createRelease(`${pkgJSON.name}@${pkgJSON.version}`); } } @@ -81,4 +99,16 @@ const go = async () => { } }; +async function createRelease(tag) { + const authToken = process.env.GITHUB_TOKEN || process.env.GITHUB_API_TOKEN; + const octokit = new Octokit({ auth: authToken }); + + await octokit.request("POST /repos/{owner}/{repo}/releases", { + owner: "microsoft", + repo: "TypeScript-DOM-lib-generator", + tag_name: tag, + target_commitish: process.env.GITHUB_SHA, + }); +} + go(); diff --git a/deploy/readmes/web.md b/deploy/readmes/web.md new file mode 100644 index 000000000..a0e8b5c49 --- /dev/null +++ b/deploy/readmes/web.md @@ -0,0 +1,33 @@ +### `@types/web` - Types for the DOM and most web-related APIs + +This module contains the DOM types for the majority of the web APIs used in a web browser. + +The APIs inside `@types/web` are generated from the specifications for CSS, HTML and JavaScript. Given the size and state of constant change in web browsers, `@types/web` only has APIs which have passed a certain level of standardization and are available in at least two different browser engines. + +`@types/web` is also included inside TypeScript, available as `dom` in the [`lib`](https://www.typescriptlang.org/tsconfig#lib) section and included in projects by default. By using `@types/web` you can lock your the web APIs used in your projects, easing the process of updating TypeScript and offering more control in your environment. + +## Installation + +To use `@types/web` you need to do two things: + +1. Install the dependency: `npm install @types/web --save-dev`, `yarn add @types/web --dev` or `pnpm add @types/web --dev`. + +1. Update your [`tsconfig.json`](https://www.typescriptlang.org/tsconfig). There are two cases to consider depending on if you have `lib` defined in your `tsconfig.json` or not. + + 1. **Without "lib"** - You will need to add `"lib": []`. The value you want to add inside your lib should correlate to your [`"target"`](https://www.typescriptlang.org/tsconfig#target). For example if you had `"target": "es2017"`, then you would add `"lib": ["es2017"]` + 1. **With "lib"** - You should remove `"dom"`. + +That's all. + +## SemVer + +This project does not respect semantic versioning as almost every change could potentially break a project, though we try to minimize removing types. +`@types/web` follow the specifications, so when they mark a function/object/API/type as deprecated or removed - that is respected. + +## TypeScript Version Support + +Prior to `@types/web` the web APIs were deployed with a version of TypeScript, and backwards compatibility has not been a concern. Now the web APIs and TypeScript can be de-coupled, then we expect to eventually hit a point where we take backwards compatibility in mind. For now, `@types/web` officially supports TypeScript 4.4 and above. It very likely will work with TypeScript versions much earlier that that however. + +## Deploy Metadata + +You can read what changed in version {{version}} at {{release_href}}. \ No newline at end of file diff --git a/deploy/template/README.md b/deploy/template/README.md deleted file mode 100644 index 89b58ef29..000000000 --- a/deploy/template/README.md +++ /dev/null @@ -1,12 +0,0 @@ -# Installation -> `npm install --save {{name}}` - -# Summary -{{description}} - - -### Additional Details - * Last updated: {{date}} - * Dependencies: none - * Global values: none - diff --git a/package.json b/package.json index 95009df7b..0021730ee 100644 --- a/package.json +++ b/package.json @@ -17,6 +17,7 @@ }, "dependencies": { "@mdn/browser-compat-data": "2.0.7", + "@octokit/core": "^3.5.1", "@types/jsdom": "^16.2.10", "@types/node": "^15.6.1", "@types/node-fetch": "^2.5.10", @@ -33,6 +34,7 @@ "parse-diff": "^0.8.1", "prettier": "^2.3.0", "print-diff": "^1.0.0", + "semver": "^7.3.5", "styleless-innertext": "^1.1.3", "typescript": "^4.3.0-dev.20210327", "webidl2": "^24.1.1" From e4577bc7195bcb1a548a504ccfb59e49d21c1ae3 Mon Sep 17 00:00:00 2001 From: Orta Date: Fri, 25 Jun 2021 14:22:46 +0100 Subject: [PATCH 3/4] Adds iterable --- deploy/createTypesPackages.mjs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/deploy/createTypesPackages.mjs b/deploy/createTypesPackages.mjs index bb9c6322a..3f6d04b39 100644 --- a/deploy/createTypesPackages.mjs +++ b/deploy/createTypesPackages.mjs @@ -9,7 +9,8 @@ const packages = [ description: "Types for the DOM, and other web technologies in browsers", readme: "./readmes/web.md", files: [ - { from: "../generated/dom.generated.d.ts", to: "index.d.ts" } + { from: "../generated/dom.generated.d.ts", to: "index.d.ts" }, + { from: "../generated/dom.iterable.generated.d.ts", to: "index.iterable.d.ts" } ], }, ]; From 41abedf40e0ef4856a8adbdf2c98b4ebce6d7d59 Mon Sep 17 00:00:00 2001 From: Orta Date: Fri, 25 Jun 2021 14:29:08 +0100 Subject: [PATCH 4/4] Deploy when there's not a file on unpkg --- deploy/deployChangedPackages.mjs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/deploy/deployChangedPackages.mjs b/deploy/deployChangedPackages.mjs index b2fdd027d..2134f6b21 100644 --- a/deploy/deployChangedPackages.mjs +++ b/deploy/deployChangedPackages.mjs @@ -55,11 +55,10 @@ const go = async () => { upload = upload || npmDTSText !== generatedDTSContent; } catch (error) { // Could not find a previous build - console.log( - `Could not get the file ${file} inside the npm package ${pkgJSON.name} from unpkg at ${unpkgURL}` - ); - process.exitCode = 1; - upload = false; + console.log(` +Could not get the file ${file} inside the npm package ${pkgJSON.name} from unpkg at ${unpkgURL} +Assuming that this means we need to upload this package.`); + upload = true; } }