.

Author: hansl

packages/schematics/BUILD

@@ -15,6 +15,7 @@ ts_library(
       "//packages/schematics/src/rules",
       "//packages/schematics/src/sink",
       "//packages/schematics/src/tree",
+      "//packages/schematics/tooling",
     ],
     tsconfig = "//:tsconfig.json",
     visibility = [ "//visibility:public" ],

packages/schematics/README.md

@@ -26,15 +26,94 @@ The tooling is responsible for the following tasks:
 
 1. Create the Schematic Engine, and pass in a Collection and Schematic loader.
 1. Understand and respect the Schematics metadata and dependencies between collections. Schematics can refer to dependencies, and it's the responsibility of the tool to honor those dependencies. The reference CLI uses NPM packages for its collections.
-1. Create the Options object. Options can be anything, but the schematics can specify a JSON Schema that should be respected. The reference CLI, for example, parse the arguments as a JSON object and validate it with the Schema specified by the collection.
+1. Create the Options object. Options can be anything, but the schematics can specify a JSON Schema that should be respected. The reference CLI, for example, parses the arguments as a JSON object and validate it with the Schema specified by the collection.
 1. Call the schematics with the original Tree. The tree should represent the initial state of the filesystem. The reference CLI uses the current directory for this.
 1. Create a Sink and commit the result of the schematics to the Sink. Many sinks are provided by the library; FileSystemSink and DryRunSink are examples.
 1. Output any logs propagated by the library, including debugging information.
 
 The tooling API is composed of the following pieces:
 
-## Engine
-The `SchematicEngine` is responsible for loading and constructing `Collection`s and `Schematics`'. When creating an engine, the tooling provides an `EngineHost` interface that understands how to create a `CollectionDescription` from a name, and how to create a `Schematic
+## EngineHost
+The `SchematicEngine` is responsible for loading and constructing `Collection`s and `Schematics`'. When creating an engine, the tooling provides an `EngineHost` interface that understands how to create a `CollectionDescription` from a name, how to create a `SchematicDescription` from a `CollectionDescription` and a name, as well as how to create the `Rule` factory for that Schematics. Both of which are information necessary for the `Engine` to work properly.
+
+All Description interfaces are generics that take interfaces as type parameters. Those interfaces can be used by the tooling to store additional information in the `CollectionDescription` and the `SchematicDescription`. The descriptions returned by the host are guaranteed to be the same objects when passing them as input.
+
+### CollectionDescription
+A `CollectionDescription` is the minimum amount of information that `Engine` needs to create a collection. It is currently only a `name`, which is used to validate the collection and cache it.
+
+### SchematicDescription
+A `SchematicDescription` is the minimum amount of information that `Engine` needs to create a schematic. It is currently a `name` (which is used to be cached), and a `CollectionDescription`. The collection description is asserted to be the same description as passed in. It is used later on to reference collections when schematics are created by name only.
+
+### Source from URL
+It is possible for schematics to create `Source`s from a URL. These are useful when we want to load a list of template files. There are 3 default URL protocols supported by the Engine:
+
+- `null:` returns a Tree that's invalid and will throw exceptions.
+- `empty:` returns a Tree that's empty.
+- `host:` returns a copy of the host passed to this schematic, from the context.
+
+### RuleFactory
+The other method necessary to resolve a schematics is the `RuleFactory`, a function that takes an option argument and returns a `Rule`. That factory is created from both descriptions by the host and the result will be called by the Engine when necessary. Please note that the engine cache this `RuleFactory` based on both descriptions, so if a schematic is created twice the `getSchematicRuleFactory` host function will only be called once.
+
+### Default MergeStrategy
+The `EngineHost` can have an optional `defaultMergeStrategy` to specify how the tooling wants to set the default `MergeStrategy`. This will be used if schematics don't specify a merge strategy on their own.
+
+## EngineHost Implementations
+
+### NodeModulesEngineHost
+The Schematics library provides an EngineHost that understands NPM node modules, using node modules to define collections and schematics. 
+
+This engine host use the following conventions:
+
+1. A node package needs to define a `schematics` key in its `package.json`, which points to a JSON file that contains collection metadata. This metadata is of the follpwing type:
+
+    ```typescript
+    interface NodeModuleCollectionJson {
+      name: string;
+      version?: string;
+      description: string;
+      schematics: {
+        [name: string]: {
+          factory: string;
+          description: string;
+          schema?: string;
+        }
+      };
+    }
+    ```
+
+    The name must be the same name as the NPM package.
+1. The `schematics` dictionary is used to resolve schematics information.
+    - The `factory` is a string of the form `modulePath#ExportName`. It is resolved relative to the collection JSON file, and the `ExportName` will be `default` if not specified.
+    - The `schema` is a string that points to a JSON Schema file (relative to the collection JSON file).
+    - The `description` field is a description that can be used by the tooling to show to the user.
+1. The `RuleFactory` is loaded from the `factory` field above by using `require()`. The `SchematicDescription` contains the name and more information necessary for the Host to resolve more.
+1. This EngineHost also registers some URLs:
+    - `file:` (or not specifying a protocol) supports loading a file system from the disk.
+
+# Schematics
+A schematics is defined by the `RuleFactory` and its `SchematicDescription`, which contains the name and collection.
+
+## Tree
+By definition, a schematic is a transformation between a `Tree` and another `Tree`. It receives a host `Tree`, and applies a list of actions to it, potentially returning it at the end.
+
+## Action
+A tree is transformed by staging actions, which can write over a file, create new files, rename or delete existing files.
+
+## Branching
+A tree can be branched, keeping its history, then adding actions on top of it. Two trees that are being merged will ignore their common history.
+
+## Merging
+Merging two trees results in a tree containing all actions. If two actions apply on the same path, it is automatically considered a conflict and needs to be resolved.
+
+### Conflicts
+Merge conflicts are resolved using the chosen `MergeStrategy` (with the default set by the tooling):
+
+1. `MergeStrategy.Error`. Throw an exception and stops creating the schematic.
+1. `MergeStrategy.Overwrite`. The action from the last merge argument is preferred.
+1. `MergeStrategy.ContentOnly`. Creation or Renaming the same file will throw an exception, but overwriting its content will resolve as if `MergeStrategy.Overwrite` is chosen.
+
+## Optimize
+Optimizing a tree results in the tree with a smaller staging; actions that overrules each other within the same tree are removed or simplified. The history of the tree is NOT preserved, but only the staged actions are changed.
 
 # Examples
 
@@ -52,11 +131,20 @@ export default function MySchematic(options: any) {
 }
 ```
 
-A few things from this example:
-
-1. The function receives the list of options from the tooling.
-1. It returns a [`Rule`](src/engine/interface.ts#L73), which is a transformation from a `Tree` to another `Tree`.
+## Templated Source
+An example of a simple Schematics which reads a directory and apply templates to its content and path.
 
+```typescript
+import {apply, mergeWith, template, url} from '@angular/schematics';
+
+export default function(options: any) {
+  return mergeWith([
+    apply(url('./files'), [
+      template({ utils: stringUtils, ...options })
+    ])
+  ]);
+};
+```
 
 
 # Future Work
@@ -66,4 +154,5 @@ Schematics is not done yet. Here's a list of things we are considering:
 * Smart defaults for Options. Having a JavaScript function for default values based on other default values.
 * Prompt for input options. This should only be prompted for the original schematics, dependencies to other schematics should not trigger another prompting.
 * Tasks for running tooling-specific jobs before and after a schematics has been scaffolded. Such tasks can involve initialize git, or npm install. A specific list of tasks should be provided by the tool, with unsupported tasks generating an error. 
-
+* Better URL support for more consistency. Right now tools define their own URLs without having consistency between two tools, which means that there is still some cohesion between the schematic and the tool.
+* Annotation support. Annotations are being designed right now, but they will be a type-safe way to attach metadata to a file that is updated if the file changes content. Such Annotation could tell if a file is, e.g., a test file, or binary, or the annotation could be the TypeScript AST of the file itself.

packages/schematics/src/engine/engine.ts

@@ -10,17 +10,18 @@ import {
   Collection,
   Engine,
   EngineHost,
-  ProtocolHandler,
   Schematic,
   Source,
   TypedSchematicContext
 } from './interface';
 import {SchematicImpl} from './schematic';
 import {BaseException} from '../exception/exception';
-import {empty} from '../tree/static';
+import {MergeStrategy} from '../tree/interface';
+import {NullTree} from '../tree/null';
+import {branch, empty} from '../tree/static';
 
 import {Url, parse, format} from 'url';
-import {MergeStrategy} from '../tree/interface';
+import 'rxjs/add/operator/map';
 
 
 export class UnknownUrlSourceProtocol extends BaseException {
@@ -38,56 +39,77 @@ export class UnknownSchematicException extends BaseException {
 
 
 export class SchematicEngine<CollectionT, SchematicT> implements Engine<CollectionT, SchematicT> {
-  private _protocolMap = new Map<string, ProtocolHandler>();
+  private _collectionCache = new Map<string, CollectionImpl<CollectionT, SchematicT>>();
+  private _schematicCache
+    = new Map<string, Map<string, SchematicImpl<CollectionT, SchematicT>>>();
 
   constructor(private _host: EngineHost<CollectionT, SchematicT>) {
-    // Default implementations.
-    this._protocolMap.set('null', () => {
-      return () => empty();
-    });
-    this._protocolMap.set('', (url: Url) => {
-      // Make a copy, change the protocol.
-      const fileUrl = parse(format(url));
-      fileUrl.protocol = 'file:';
-      return (context: TypedSchematicContext<CollectionT, SchematicT>) => {
-        return context.engine.createSourceFromUrl(fileUrl)(context);
-      };
-    });
   }
 
   get defaultMergeStrategy() { return this._host.defaultMergeStrategy || MergeStrategy.Default; }
 
   createCollection(name: string): Collection<CollectionT, SchematicT> {
+    let collection = this._collectionCache.get(name);
+    if (collection) {
+      return collection;
+    }
+
     const description = this._host.createCollectionDescription(name);
     if (!description) {
       throw new UnknownCollectionException(name);
     }
 
-    return new CollectionImpl<CollectionT, SchematicT>(description, this);
+    collection = new CollectionImpl<CollectionT, SchematicT>(description, this);
+    this._collectionCache.set(name, collection);
+    this._schematicCache.set(name, new Map());
+    return collection;
   }
 
   createSchematic(
       name: string,
       collection: Collection<CollectionT, SchematicT>): Schematic<CollectionT, SchematicT> {
+    const collectionImpl = this._collectionCache.get(collection.name);
+    const schematicMap = this._schematicCache.get(collection.name);
+    if (!collectionImpl || !schematicMap || collectionImpl !== collection) {
+      // This is weird, maybe the collection was created by another engine?
+      throw new UnknownCollectionException(collection.name);
+    }
+
+    let schematic = schematicMap.get(name);
+    if (schematic) {
+      return schematic;
+    }
+
     const description = this._host.createSchematicDescription(name, collection.description);
     if (!description) {
       throw new UnknownSchematicException(name, collection);
     }
     const factory = this._host.getSchematicRuleFactory(description, collection.description);
+    schematic = new SchematicImpl<CollectionT, SchematicT>(description, factory, collection, this);
 
-    return new SchematicImpl<CollectionT, SchematicT>(description, factory, collection, this);
-  }
-
-  registerUrlProtocolHandler(protocol: string, handler: ProtocolHandler) {
-    this._protocolMap.set(protocol, handler);
+    schematicMap.set(name, schematic);
+    return schematic;
   }
 
   createSourceFromUrl(url: Url): Source {
-    const protocol = (url.protocol || '').replace(/:$/, '');
-    const handler = this._protocolMap.get(protocol);
-    if (!handler) {
-      throw new UnknownUrlSourceProtocol(url.toString());
+    switch (url.protocol) {
+      case 'null:': return () => new NullTree();
+      case 'empty:': return () => empty();
+      case 'host:': return (context: TypedSchematicContext<CollectionT, SchematicT>) => {
+        return context.host.map(tree => branch(tree));
+      };
+      case '':
+        const fileUrl = parse(format(url));
+        fileUrl.protocol = 'file:';
+        return (context: TypedSchematicContext<CollectionT, SchematicT>) => {
+          return context.engine.createSourceFromUrl(fileUrl)(context);
+        };
+      default:
+        const hostSource = this._host.createSourceFromUrl(url);
+        if (!hostSource) {
+          throw new UnknownUrlSourceProtocol(url.toString());
+        }
+        return hostSource;
     }
-    return handler(url);
   }
 }

packages/schematics/src/engine/interface.ts

@@ -31,6 +31,11 @@ export type SchematicDescription<CollectionT extends {}, SchematicT extends {}>
 };
 
 
+/**
+ * The Host for the Engine. Specifically, the piece of the tooling responsible for resolving
+ * collections and schematics descriptions. The SchematicT and CollectionT type parameters contain
+ * additional metadata that you want to store while remaining type-safe.
+ */
 export interface EngineHost<CollectionT extends {}, SchematicT extends {}> {
   createCollectionDescription(name: string): CollectionDescription<CollectionT> | null;
   createSchematicDescription(
@@ -40,6 +45,7 @@ export interface EngineHost<CollectionT extends {}, SchematicT extends {}> {
   getSchematicRuleFactory<OptionT>(
       schematic: SchematicDescription<CollectionT, SchematicT>,
       collection: CollectionDescription<CollectionT>): RuleFactory<OptionT>;
+  createSourceFromUrl(url: Url): Source | null;
 
   readonly defaultMergeStrategy?: MergeStrategy;
 }
@@ -59,27 +65,16 @@ export interface Engine<CollectionT extends {}, SchematicT extends {}> {
   createSchematic(
       name: string,
       collection: Collection<CollectionT, SchematicT>): Schematic<CollectionT, SchematicT>;
-  registerUrlProtocolHandler(protocol: string, handler: ProtocolHandler): void;
   createSourceFromUrl(url: Url): Source;
 
   readonly defaultMergeStrategy: MergeStrategy;
 }
 
 
-export interface Schematic<CollectionT, SchematicT> {
-  readonly description: SchematicDescription<CollectionT, SchematicT>;
-  readonly collection: Collection<CollectionT, SchematicT>;
-
-  call<T>(options: T, host: Observable<Tree>): Observable<Tree>;
-}
-
-
-export interface ProtocolHandler {
-  (url: Url): Source;
-}
-
-
-
+/**
+ * A Collection as created by the Engine. This should be used by the tool to create schematics,
+ * or by rules to create other schematics as well.
+ */
 export interface Collection<CollectionT, SchematicT> {
   readonly name: string;
   readonly description: CollectionDescription<CollectionT>;
@@ -88,21 +83,52 @@ export interface Collection<CollectionT, SchematicT> {
 }
 
 
+/**
+ * A Schematic as created by the Engine. This should be used by the tool to execute the main
+ * schematics, or by rules to execute other schematics as well.
+ */
+export interface Schematic<CollectionT, SchematicT> {
+  readonly description: SchematicDescription<CollectionT, SchematicT>;
+  readonly collection: Collection<CollectionT, SchematicT>;
+
+  call<T>(options: T, host: Observable<Tree>): Observable<Tree>;
+}
+
+
+/**
+ * A SchematicContext. Contains information necessary for Schematics to execute some rules, for
+ * example when using another schematics, as we need the engine and collection.
+ */
 export interface TypedSchematicContext<CollectionT, SchematicT> {
   readonly engine: Engine<CollectionT, SchematicT>;
   readonly schematic: Schematic<CollectionT, SchematicT>;
   readonly host: Observable<Tree>;
   readonly strategy: MergeStrategy;
 }
+
+
+/**
+ * This is used by the Schematics implementations in order to avoid needing to have typing from
+ * the tooling. Schematics are not specific to a tool.
+ */
 export type SchematicContext = TypedSchematicContext<any, any>;
 
 
+/**
+ * A rule factory, which is normally the way schematics are implemented. Returned by the tooling
+ * after loading a schematic description.
+ */
 export type RuleFactory<T> = (options: T) => Rule;
 
+
 /**
+ * A source is a function that generates a Tree from a specific context. A rule transforms a tree
+ * into another tree from a specific context. In both cases, an Observable can be returned if
+ * the source or the rule are asynchronous. Only the last Tree generated in the observable will
+ * be used though.
+ *
  * We obfuscate the context of Source and Rule because the schematic implementation should not
  * know which types is the schematic or collection metadata, as they are both tooling specific.
  */
-export type Source = (context: TypedSchematicContext<any, any>) => Tree | Observable<Tree>;
-export type Rule = (tree: Tree,
-                    context: TypedSchematicContext<any, any>) => Tree | Observable<Tree>;
+export type Source = (context: SchematicContext) => Tree | Observable<Tree>;
+export type Rule = (tree: Tree, context: SchematicContext) => Tree | Observable<Tree>;

packages/schematics/src/exception/exception.ts

@@ -28,3 +28,8 @@ export class ContentHasMutatedException extends BaseException {
 export class InvalidUpdateRecordException extends BaseException {
   constructor() { super(`Invalid record instance.`); }
 }
+export class MergeConflictException extends BaseException {
+  constructor(path: string) {
+    super(`A merge conflicted on path "${path}".`);
+  }
+}