refactor: build multiple engine-host to help with use cases

Also, separation of concern for engine host has been improved, and a simple base implementation
has been added. Tools should use this (or any subclasses provided) to implement their own
EngineHost. This includes Google, which will use the FileSystem based one, and the Workbench
which will use the Registry one.

Author: hansl

packages/schematics/src/engine/engine.ts

@@ -20,7 +20,7 @@ import {MergeStrategy} from '../tree/interface';
 import {NullTree} from '../tree/null';
 import {branch, empty} from '../tree/static';
 
-import {Url, parse, format} from 'url';
+import {Url} from 'url';
 import 'rxjs/add/operator/map';
 
 

packages/schematics/src/exception/exception.ts

@@ -32,4 +32,8 @@ export class MergeConflictException extends BaseException {
   constructor(path: string) {
     super(`A merge conflicted on path "${path}".`);
   }
-}
+}
+
+export class UnimplementedException extends BaseException {
+  constructor() { super('This function is unimplemented.'); }
+}

packages/schematics/tooling/export-ref_spec.ts

@@ -36,7 +36,7 @@ describe('ExportStringRef', () => {
 
   it('works on package names', () => {
     // META
-    const ref = new ExportStringRef('@angular/schematics-cli#ExportStringRef');
+    const ref = new ExportStringRef('@angular/schematics/tooling#ExportStringRef');
     expect(ref.ref).toEqual(ExportStringRef);
     expect(ref.path).toBe(__dirname);
     expect(ref.module).toBe(path.join(__dirname, 'index.ts'));

packages/schematics/tooling/file-system-engine-host-base.ts

@@ -0,0 +1,154 @@
+import {FileSystemHost} from './file-system-host';
+import {
+  Collection,
+  CollectionDescription,
+  EngineHost,
+  FileSystemTree,
+  RuleFactory,
+  SchematicDescription,
+  Source,
+  TypedSchematicContext,
+} from '../src/index';
+
+import {dirname, join, resolve} from 'path';
+import {Url} from 'url';
+import {readJsonFile} from './file-system-utility';
+
+
+export interface FileSystemCollectionDescription {
+  readonly path: string;
+  readonly version?: string;
+  readonly schematics: { [name: string]: FileSystemSchematicJsonDescription };
+}
+
+
+export interface FileSystemSchematicJsonDescription {
+  readonly factory: string;
+  readonly description: string;
+  readonly schema?: string;
+}
+
+export interface FileSystemSchematicDescription extends FileSystemSchematicJsonDescription {
+  // Processed by the EngineHost.
+  readonly path: string;
+  readonly schemaJson?: Object;
+  // Using `any` here is okay because the type isn't resolved when we read this value,
+  // but rather when the Engine asks for it.
+  readonly factoryFn: RuleFactory<any>;
+}
+
+
+/**
+ * Used to simplify typings.
+ */
+export declare type FileSystemCollection
+    = Collection<FileSystemCollectionDescription, FileSystemSchematicDescription>;
+export declare type FileSystemCollectionDesc
+    = CollectionDescription<FileSystemCollectionDescription>;
+export declare type FileSystemSchematicDesc
+    = SchematicDescription<FileSystemCollectionDescription, FileSystemSchematicDescription>;
+export declare type FileSystemSchematicContext
+    = TypedSchematicContext<FileSystemCollectionDescription, FileSystemSchematicDescription>;
+
+
+/**
+ * A EngineHost base class that uses the file system to resolve collections. This is the base of
+ * all other EngineHost provided by the tooling part of the Schematics library.
+ */
+export abstract class FileSystemEngineHostBase implements
+    EngineHost<FileSystemCollectionDescription, FileSystemSchematicDescription> {
+  protected abstract _resolveCollectionPath(_name: string): string | null;
+  protected abstract _resolveReferenceString(
+      _name: string, _parentPath: string): { ref: RuleFactory<any>, path: string } | null;
+
+  listSchematics(collection: FileSystemCollection) {
+    return Object.keys(collection.description.schematics);
+  }
+
+  /**
+   *
+   * @param name
+   * @return {{path: string}}
+   */
+  createCollectionDescription(name: string): FileSystemCollectionDesc | null {
+    try {
+      const path = this._resolveCollectionPath(name);
+      if (!path) {
+        return null;
+      }
+
+      const description: FileSystemCollectionDesc = readJsonFile(path);
+      if (!description.name) {
+        return null;
+      }
+
+      return {
+        ...description,
+        path,
+      };
+    } catch (e) {
+      return null;
+    }
+  }
+
+  createSchematicDescription(
+      name: string, collection: FileSystemCollectionDesc): FileSystemSchematicDesc | null {
+    if (!(name in collection.schematics)) {
+      return null;
+    }
+
+    const collectionPath = dirname(collection.path);
+    const description = collection.schematics[name];
+
+    if (!description) {
+      return null;
+    }
+
+    // Use any on this ref as we don't have the OptionT here, but we don't need it (we only need
+    // the path).
+    const resolvedRef = this._resolveReferenceString(description.factory, collectionPath);
+    if (!resolvedRef) {
+      return null;
+    }
+
+    const { path } = resolvedRef;
+    let schema = description.schema;
+    let schemaJson = undefined;
+    if (schema) {
+      schema = join(collectionPath, schema);
+      schemaJson = readJsonFile(schema);
+    }
+
+    return {
+      ...description,
+      schema,
+      schemaJson,
+      name,
+      path,
+      factoryFn: resolvedRef.ref,
+      collection
+    };
+  }
+
+  createSourceFromUrl(url: Url): Source | null {
+    switch (url.protocol) {
+      case null:
+      case 'file:':
+        return (context: FileSystemSchematicContext) => {
+          // Resolve all file:///a/b/c/d from the schematic's own path, and not the current
+          // path.
+          const root = resolve(dirname(context.schematic.description.path), url.path);
+          return new FileSystemTree(new FileSystemHost(root), true);
+        };
+    }
+
+    return null;
+  }
+
+  getSchematicRuleFactory<OptionT>(
+    schematic: FileSystemSchematicDesc,
+    _collection: FileSystemCollectionDesc): RuleFactory<OptionT> {
+    return schematic.factoryFn;
+  }
+
+}

packages/schematics/tooling/file-system-engine-host.ts

@@ -0,0 +1,35 @@
+import {ExportStringRef} from './export-ref';
+import {FileSystemEngineHostBase} from './file-system-engine-host-base';
+import {RuleFactory} from '../src/engine/interface';
+
+import {join} from 'path';
+import {existsSync} from 'fs';
+
+
+/**
+ * A simple EngineHost that uses a root with one directory per collection inside of it. The
+ * collection declaration follows the same rules as the regular FileSystemEngineHostBase.
+ */
+export class FileSystemEngineHost extends FileSystemEngineHostBase {
+  constructor(protected _root: string) { super(); }
+
+  protected _resolveCollectionPath(name: string): string | null {
+    // Allow `${_root}/${name}.json` as a collection.
+    if (existsSync(join(this._root, name + '.json'))) {
+      return join(this._root, name + '.json');
+    }
+
+    // Allow `${_root}/${name}/collection.json.
+    if (existsSync(join(this._root, name, 'collection.json'))) {
+      return join(this._root, name, 'collection.json');
+    }
+
+    return null;
+  }
+
+  protected _resolveReferenceString(refString: string, parentPath: string) {
+    // Use the same kind of export strings as NodeModule.
+    const ref = new ExportStringRef<RuleFactory<any>>(refString, parentPath);
+    return { ref: ref.ref, path: ref.module };
+  }
+}

packages/schematics/tooling/file-system-utility.ts

@@ -0,0 +1,95 @@
+import {existsSync, readFileSync} from 'fs';
+
+
+/**
+ * Read a file and returns its content. This supports different file encoding.
+ */
+export function readFile(fileName: string): string | null {
+  if (!existsSync(fileName)) {
+    return null;
+  }
+  const buffer = readFileSync(fileName);
+  let len = buffer.length;
+  if (len >= 2 && buffer[0] === 0xFE && buffer[1] === 0xFF) {
+    // Big endian UTF-16 byte order mark detected. Since big endian is not supported by node.js,
+    // flip all byte pairs and treat as little endian.
+    len &= ~1;
+    for (let i = 0; i < len; i += 2) {
+      const temp = buffer[i];
+      buffer[i] = buffer[i + 1];
+      buffer[i + 1] = temp;
+    }
+    return buffer.toString('utf16le', 2);
+  }
+  if (len >= 2 && buffer[0] === 0xFF && buffer[1] === 0xFE) {
+    // Little endian UTF-16 byte order mark detected
+    return buffer.toString('utf16le', 2);
+  }
+  if (len >= 3 && buffer[0] === 0xEF && buffer[1] === 0xBB && buffer[2] === 0xBF) {
+    // UTF-8 byte order mark detected
+    return buffer.toString('utf8', 3);
+  }
+  // Default is UTF-8 with no byte order mark
+  return buffer.toString('utf8');
+}
+
+
+/**
+ * Parse a JSON string and optionally remove comments from it first.
+ */
+export function parseJson(content: string, stripComments = true): any {
+  if (stripComments) {
+    // Simple parser to ignore strings.
+    let inString = false;
+    for (let i = 0; i < content.length; i++) {
+      if (inString) {
+        // Skip `\X` characters. Since we only care about \", we don't need to fully process valid
+        // escape sequences (e.g. unicode sequences). This algorithm could accept invalid sequences
+        // but the JSON.parse call will throw on those so the output of the whole function will
+        // still be valid.
+        switch (content[i]) {
+          case '\\': i++; break;
+          case '"': inString = false; break;
+          // Else ignore characters.
+        }
+      } else {
+        switch (content[i]) {
+          case '"': inString = true; break;
+          case '/':
+            let j = 0;
+            switch (content[i + 1]) {
+              case '/':  // Strip until end-of-line.
+                for (j = i + 2; j < content.length; j++) {
+                  if (content[j] == '\n' || content[j] == '\r') {
+                    break;
+                  }
+                }
+                break;
+              case '*':  // Strip until `*/`.
+                for (j = i + 2; j < content.length; j++) {
+                  if (content[j - 1] == '*' && content[j] == '/') {
+                    break;
+                  }
+                }
+                break;
+            }
+            if (j) {
+              // Use `j + 1` to skip the end-of-comment character.
+              content = content.substr(0, i) + content.substr(j + 1);
+              i--;
+            }
+            break;
+        }
+      }
+    }
+    // At this point we don't really care about the validity of the resulting string, since
+    // JSON.parse will validate the JSON itself.
+  }
+
+  return JSON.parse(content);
+}
+
+
+export function readJsonFile(path: string): any {
+  return parseJson(readFile(path) || '{}');
+}

packages/schematics/tooling/file-system-utility_spec.ts

@@ -0,0 +1,33 @@
+import {parseJson} from './file-system-utility';
+
+
+describe('parseJson', () => {
+  it('works', () => {
+    expect(parseJson('{ "a": 1 }')).toEqual({a: 1});
+    expect(() => parseJson('{ 1 }')).toThrow();
+  });
+
+  it('strips comments', () => {
+    expect(parseJson(`
+      // THIS IS A COMMENT
+      {
+        /* THIS IS ALSO A COMMENT */ // IGNORED BECAUSE COMMENT
+        // AGAIN, COMMENT /* THIS SHOULD NOT BE WEIRD
+        "a": "this // should not be a comment",
+        "a2": "this /* should also not be a comment",
+        /* MULTIPLE
+           LINE
+           COMMENT
+           \o/ */
+        "b" /* COMMENT */: /* YOU GUESSED IT */ 1 // COMMENT
+        , /* STILL VALID */
+        "c": 2
+      }
+    `)).toEqual({
+      a: 'this // should not be a comment',
+      a2: 'this /* should also not be a comment',
+      b: 1,
+      c: 2
+    });
+  });
+});

packages/schematics/tooling/index.ts

@@ -1,3 +1,4 @@
-export * from './export-ref';
 export * from './file-system-host';
+export * from './file-system-engine-host';
 export * from './node-module-engine-host';
+export * from './registry-engine-host';