feat(changelog): get from jsDelivr filelist if possible (#640)

Co-authored-by: Haroen Viaene <[email protected]>

Author: Samuel Bodin

src/__tests__/changelog.test.ts

@@ -1,4 +1,5 @@
-import { getChangelogs, baseUrlMap } from '../changelog';
+import { getChangelogs, baseUrlMap, getChangelog } from '../changelog';
+import * as jsDelivr from '../jsDelivr';
 
 jest.mock('got', () => {
   const gotSnapshotUrls = new Set([
@@ -16,6 +17,12 @@ jest.mock('got', () => {
   };
 });
 
+const spy = jest
+  .spyOn(jsDelivr, 'getFilesList')
+  .mockImplementation((): Promise<any[]> => {
+    return Promise.resolve([]);
+  });
+
 describe('should test baseUrlMap', () => {
   it('should work with paths', () => {
     const bitbucketRepo = {
@@ -210,3 +217,76 @@ it('should work with HISTORY.md', async () => {
     'https://raw.githubusercontent.com/expressjs/body-parser/master/HISTORY.md'
   );
 });
+
+describe('jsDelivr', () => {
+  it('should early return when finding changelog', async () => {
+    spy.mockResolvedValue([
+      { name: '/package.json', hash: '', time: '1', size: 1 },
+      { name: '/CHANGELOG.md', hash: '', time: '1', size: 1 },
+    ]);
+
+    const { changelogFilename } = await getChangelog({
+      name: 'foo',
+      version: '1.0.0',
+      repository: {
+        url: '',
+        host: 'github.com',
+        user: 'expressjs',
+        project: 'body-parser',
+        path: '',
+        head: 'master',
+        branch: 'master',
+      },
+    });
+    expect(jsDelivr.getFilesList).toHaveBeenCalled();
+    expect(changelogFilename).toEqual(
+      'https://cdn.jsdelivr.net/npm/[email protected]/CHANGELOG.md'
+    );
+  });
+
+  it('should early return when finding changelog in nested file', async () => {
+    spy.mockResolvedValue([
+      { name: '/pkg/CHANGELOG.md', hash: '', time: '1', size: 1 },
+    ]);
+
+    const { changelogFilename } = await getChangelog({
+      name: 'foo',
+      version: '1.0.0',
+      repository: {
+        url: '',
+        host: 'github.com',
+        user: 'expressjs',
+        project: 'body-parser',
+        path: '',
+        head: 'master',
+        branch: 'master',
+      },
+    });
+    expect(jsDelivr.getFilesList).toHaveBeenCalled();
+    expect(changelogFilename).toEqual(
+      'https://cdn.jsdelivr.net/npm/[email protected]/pkg/CHANGELOG.md'
+    );
+  });
+
+  it('should not register a file looking like a changelog', async () => {
+    spy.mockResolvedValue([
+      { name: '/dist/changelog.js', hash: '', time: '1', size: 1 },
+    ]);
+
+    const { changelogFilename } = await getChangelog({
+      name: 'foo',
+      version: '1.0.0',
+      repository: {
+        url: '',
+        host: 'github.com',
+        user: 'hello',
+        project: 'foo',
+        path: '',
+        head: 'master',
+        branch: 'master',
+      },
+    });
+    expect(jsDelivr.getFilesList).toHaveBeenCalled();
+    expect(changelogFilename).toEqual(null);
+  });
+});

src/changelog.ts

@@ -1,29 +1,41 @@
+import path from 'path';
+
 import race from 'promise-rat-race';
 
 import type { RawPkg, Repo } from './@types/pkg';
 import { config } from './config';
+import * as jsDelivr from './jsDelivr/index';
 import { datadog } from './utils/datadog';
 import { request } from './utils/request';
 
 export const baseUrlMap = new Map<
   string,
   (opts: Pick<Repo, 'user' | 'project' | 'path' | 'branch'>) => string
 >();
-baseUrlMap.set('github.com', ({ user, project, path, branch }): string => {
-  return `https://raw.githubusercontent.com/${user}/${project}/${
-    path ? '' : branch
-  }${`${path.replace('/tree/', '')}`}`;
-});
-baseUrlMap.set('gitlab.com', ({ user, project, path, branch }): string => {
-  return `https://gitlab.com/${user}/${project}${
-    path ? path.replace('tree', 'raw') : `/raw/${branch}`
-  }`;
-});
-baseUrlMap.set('bitbucket.org', ({ user, project, path, branch }): string => {
-  return `https://bitbucket.org/${user}/${project}${
-    path ? path.replace('src', 'raw') : `/raw/${branch}`
-  }`;
-});
+baseUrlMap.set(
+  'github.com',
+  ({ user, project, path: pathName, branch }): string => {
+    return `https://raw.githubusercontent.com/${user}/${project}/${
+      pathName ? '' : branch
+    }${`${pathName.replace('/tree/', '')}`}`;
+  }
+);
+baseUrlMap.set(
+  'gitlab.com',
+  ({ user, project, path: pathName, branch }): string => {
+    return `https://gitlab.com/${user}/${project}${
+      pathName ? pathName.replace('tree', 'raw') : `/raw/${branch}`
+    }`;
+  }
+);
+baseUrlMap.set(
+  'bitbucket.org',
+  ({ user, project, path: pathName, branch }): string => {
+    return `https://bitbucket.org/${user}/${project}${
+      pathName ? pathName.replace('src', 'raw') : `/raw/${branch}`
+    }`;
+  }
+);
 
 const fileOptions = [
   'CHANGELOG.md',
@@ -43,8 +55,14 @@ const fileOptions = [
   'history.md',
   'HISTORY',
   'history',
+  'RELEASES.md',
+  'RELEASES',
 ];
 
+// https://regex101.com/r/zU2gjr/1
+const fileRegex =
+  /^(((changelogs?)|changes|history|(releases?)))((.(md|markdown))?$)/i;
+
 async function handledGot(file: string): Promise<string> {
   const result = await request(file, { method: 'HEAD' });
 
@@ -76,13 +94,27 @@ async function raceFromPaths(files: string[]): Promise<{
   }
 }
 
-function getChangelog({
-  repository,
-  name,
-  version,
-}: Pick<RawPkg, 'repository' | 'name' | 'version'>): Promise<{
+export async function getChangelog(
+  pkg: Pick<RawPkg, 'repository' | 'name' | 'version'>
+): Promise<{
   changelogFilename: string | null;
 }> {
+  // Do a quick call to jsDelivr
+  // Only work if the package has published their changelog along with the code
+  const filesList = await jsDelivr.getFilesList(pkg);
+  for (const file of filesList) {
+    const name = path.basename(file.name);
+    if (!fileRegex.test(name)) {
+      // eslint-disable-next-line no-continue
+      continue;
+    }
+
+    return { changelogFilename: jsDelivr.getFullURL(pkg, file) };
+  }
+
+  const { repository, name, version } = pkg;
+
+  // Rollback to brute-force the source code
   const unpkgFiles = fileOptions.map(
     (file) => `${config.unpkgRoot}/${name}@${version}/${file}`
   );

src/jsDelivr/index.ts

@@ -6,12 +6,13 @@ import { request } from '../utils/request';
 
 type Hit = { type: 'npm'; name: string; hits: number };
 type File = { name: string; hash: string; time: string; size: number };
-const hits = new Map<string, number>();
+
+export const hits = new Map<string, number>();
 
 /**
  * Load downloads hits.
  */
-async function loadHits(): Promise<void> {
+export async function loadHits(): Promise<void> {
   const start = Date.now();
   log.info('📦  Loading hits from jsDelivr');
 
@@ -34,7 +35,7 @@ async function loadHits(): Promise<void> {
 /**
  * Get download hits.
  */
-function getHits(pkgs: Array<Pick<RawPkg, 'name'>>): Array<{
+export function getHits(pkgs: Array<Pick<RawPkg, 'name'>>): Array<{
   jsDelivrHits: number;
   _searchInternal: { jsDelivrPopularity: number };
 }> {
@@ -59,7 +60,7 @@ function getHits(pkgs: Array<Pick<RawPkg, 'name'>>): Array<{
 /**
  * Get packages files list.
  */
-async function getAllFilesList(
+export async function getAllFilesList(
   pkgs: Array<Pick<RawPkg, 'name' | 'version'>>
 ): Promise<File[][]> {
   const start = Date.now();
@@ -73,7 +74,7 @@ async function getAllFilesList(
 /**
  * Get one package files list.
  */
-async function getFilesList(
+export async function getFilesList(
   pkg: Pick<RawPkg, 'name' | 'version'>
 ): Promise<File[]> {
   const start = Date.now();
@@ -100,4 +101,9 @@ async function getFilesList(
   return files;
 }
 
-export { hits, loadHits, getHits, getAllFilesList, getFilesList };
+export function getFullURL(
+  pkg: Pick<RawPkg, 'name' | 'version'>,
+  file: File
+): string {
+  return `https://cdn.jsdelivr.net/npm/${pkg.name}@${pkg.version}${file.name}`;
+}