Skip to content

Clean up ApiGenerator #3755

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 23 commits into from
May 28, 2019
Merged

Clean up ApiGenerator #3755

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
09d0240
Update code configuration for low level client to only generate metho…
Mpdreamz Apr 24, 2019
5966447
Fixed locations to be nix friendly
Mpdreamz Apr 24, 2019
2825868
Removed alternative HTTP method named methods in the low level client
Mpdreamz Apr 24, 2019
bdf65b2
progress commit
Mpdreamz Apr 24, 2019
cdb327c
Simplified CsharpMethodsWithQueryString
Mpdreamz Apr 24, 2019
6de0d9b
progress commit
Mpdreamz Apr 24, 2019
8271368
progress commit
Mpdreamz Apr 24, 2019
9c7518d
can now use partials/include this should start cleaning up the cshtml…
Mpdreamz Apr 24, 2019
6a4dfe4
Able to introduce partial methods now
Mpdreamz Apr 28, 2019
a21333b
Add partial views
Mpdreamz Apr 28, 2019
a078b9e
RequestDescriptorBase separate view
Mpdreamz May 8, 2019
628f30c
moved descriptors to partials (initial stab)
Mpdreamz May 9, 2019
900d0c7
reindent generated code automatically
Mpdreamz May 9, 2019
cf31801
Split name and namespace
Mpdreamz May 9, 2019
12fc139
Request/Descriptors are no longer backed by CsharpMethods
Mpdreamz May 10, 2019
28d56b4
start cleaning up CsharpMethod
Mpdreamz May 16, 2019
b94fcbc
PatchMethod on overrides is gone,
Mpdreamz May 16, 2019
a248588
Cleaned up code configuration a tad
Mpdreamz May 16, 2019
27c4239
deduplication is now gone, marked aliases that were duplicate as depr…
Mpdreamz May 20, 2019
38d8a1e
In case of GET, OTHER prefer OTHER
Mpdreamz May 20, 2019
8e5114f
rebased against 7.x and reworked replace files into patch files
Mpdreamz May 20, 2019
fe48451
skip params also on requestparameters and prefer plural source_includes
Mpdreamz May 20, 2019
5bf421f
reapply fix from #3700 to make sure virtual paths as root are support…
Mpdreamz May 20, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
94 changes: 94 additions & 0 deletions src/CodeGeneration/ApiGenerator/ApiEndpointFactory.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using ApiGenerator.Domain;
using ApiGenerator.Overrides.Descriptors;
using Newtonsoft.Json.Linq;

namespace ApiGenerator
{
public static class ApiEndpointFactory
{
public static ApiEndpoint FromFile(string jsonFile)
{
var officialJsonSpec = JObject.Parse(File.ReadAllText(jsonFile));
PatchOfficialSpec(officialJsonSpec, jsonFile);
var (name, endpoint) = officialJsonSpec.ToObject<Dictionary<string, ApiEndpoint>>().First();

endpoint.FileName = Path.GetFileName(jsonFile);
endpoint.Name = name;
var tokens = name.Split(".");

endpoint.MethodName = tokens.Last();
if (tokens.Length > 1)
endpoint.Namespace = tokens[0];
//todo side effect
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Stray TODO?

endpoint.CsharpNames = new CsharpNames(name, endpoint.MethodName, endpoint.Namespace);
endpoint.Url.CsharpNames = endpoint.CsharpNames;

LoadOverridesOnEndpoint(endpoint);
PatchRequestParameters(endpoint);

EnforceRequiredOnParts(jsonFile, endpoint.Url);
return endpoint;
}

/// <summary>
/// This makes sure required is configured correctly by inspecting the paths.
/// Will emit a warning if the spec file got this wrong
/// </summary>
private static void EnforceRequiredOnParts(string jsonFile, UrlInformation url)
{
if (url.IsPartless) return;
foreach (var part in url.Parts)
{
var required = url.Paths.All(p => p.Path.Contains($"{{{part.Name}}}"));
if (part.Required != required)
ApiGenerator.Warnings.Add($"{jsonFile} has part: {part.Name} listed as {part.Required} but should be {required}");
part.Required = required;
}
}

private static void LoadOverridesOnEndpoint(ApiEndpoint endpoint)
{
var method = endpoint.CsharpNames.MethodName;
if (CodeConfiguration.ApiNameMapping.TryGetValue(endpoint.Name, out var mapsApiMethodName))
method = mapsApiMethodName;

var typeName = "ApiGenerator.Overrides.Endpoints." + method + "Overrides";
var type = GeneratorLocations.Assembly.GetType(typeName);
if (type != null && Activator.CreateInstance(type) is IEndpointOverrides overrides)
endpoint.Overrides = overrides;
}

private static void PatchRequestParameters(ApiEndpoint endpoint)
{
var newParams = ApiQueryParametersPatcher.Patch(endpoint.Name, endpoint.Url.Params, endpoint.Overrides);
endpoint.Url.Params = newParams;
}

/// <summary>
/// Finds a patch file in patches and union merges this with the official spec.
/// This allows us to check in tweaks should breaking changes occur in the spec before we catch them
/// </summary>
private static void PatchOfficialSpec(JObject original, string jsonFile)
{
var directory = Path.GetDirectoryName(jsonFile);
var patchFile = Path.Combine(directory,"..", "_Patches", Path.GetFileNameWithoutExtension(jsonFile)) + ".patch.json";
if (!File.Exists(patchFile)) return;

var patchedJson = JObject.Parse(File.ReadAllText(patchFile));

var pathsOverride = patchedJson.SelectToken("*.url.paths");

original.Merge(patchedJson, new JsonMergeSettings
{
MergeArrayHandling = MergeArrayHandling.Union
});

if (pathsOverride != null) original.SelectToken("*.url.paths").Replace(pathsOverride);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

👍 simplifies the .replace.json files...

}

}
}
140 changes: 50 additions & 90 deletions src/CodeGeneration/ApiGenerator/ApiGenerator.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,15 +4,20 @@
using System.IO;
using System.Linq;
using ApiGenerator.Domain;
using Microsoft.CodeAnalysis;
using Microsoft.CodeAnalysis.CSharp;
using Newtonsoft.Json.Linq;
using RazorLight;
using RazorLight.Generation;
using RazorLight.Razor;
using ShellProgressBar;

namespace ApiGenerator
{
public class ApiGenerator
public partial class ApiGenerator
{
private static readonly RazorLightEngine Razor = new RazorLightEngineBuilder()
.UseProject(new FileSystemRazorProject(Path.GetFullPath(GeneratorLocations.ViewFolder)))
.UseMemoryCachingProvider()
.Build();

Expand Down Expand Up @@ -52,11 +57,12 @@ public static void Generate(string downloadBranch, params string[] folders)

private static RestApiSpec CreateRestApiSpecModel(string downloadBranch, string[] folders)
{
var directories = Directory.GetDirectories(CodeConfiguration.RestSpecificationFolder, "*", SearchOption.AllDirectories)
var directories = Directory.GetDirectories(GeneratorLocations.RestSpecificationFolder, "*", SearchOption.AllDirectories)
.Where(f => folders == null || folders.Length == 0 || folders.Contains(new DirectoryInfo(f).Name))
.OrderBy(f=>new FileInfo(f).Name)
.ToList();

var endpoints = new Dictionary<string, ApiEndpoint>();
var endpoints = new SortedDictionary<string, ApiEndpoint>();
var seenFiles = new HashSet<string>();
using (var pbar = new ProgressBar(directories.Count, $"Listing {directories.Count} directories",
new ProgressBarOptions { BackgroundColor = ConsoleColor.DarkGray }))
Expand All @@ -66,7 +72,7 @@ private static RestApiSpec CreateRestApiSpecModel(string downloadBranch, string[
.Where(f => f.EndsWith(".json") && !CodeConfiguration.IgnoredApis.Contains(new FileInfo(f).Name))
.ToList()
);
var commonFile = Path.Combine(CodeConfiguration.RestSpecificationFolder, "Core", "_common.json");
var commonFile = Path.Combine(GeneratorLocations.RestSpecificationFolder, "Core", "_common.json");
if (!File.Exists(commonFile)) throw new Exception($"Expected to find {commonFile}");

RestApiSpec.CommonApiQueryParameters = CreateCommonApiQueryParameters(commonFile);
Expand All @@ -79,15 +85,12 @@ private static RestApiSpec CreateRestApiSpecModel(string downloadBranch, string[
foreach (var file in jsonFiles)
{
if (file.EndsWith("_common.json")) continue;
else if (file.EndsWith(".obsolete.json")) continue;
else if (file.EndsWith(".patch.json")) continue;
else if (file.EndsWith(".replace.json")) continue;
else
{
var endpoint = CreateApiEndpoint(file);
endpoint.Value.FileName = Path.GetFileName(file);
var endpoint = ApiEndpointFactory.FromFile(file);
seenFiles.Add(Path.GetFileNameWithoutExtension(file));
endpoints.Add(endpoint.Key, endpoint.Value);
endpoints.Add(endpoint.Name, endpoint);
}

fileProgress.Tick();
Expand All @@ -108,127 +111,84 @@ private static RestApiSpec CreateRestApiSpecModel(string downloadBranch, string[
return new RestApiSpec { Endpoints = endpoints, Commit = downloadBranch };
}

public static string PascalCase(string s)
private static SortedDictionary<string, QueryParameters> CreateCommonApiQueryParameters(string jsonFile)
{
var textInfo = new CultureInfo("en-US").TextInfo;
return textInfo.ToTitleCase(s.ToLowerInvariant()).Replace("_", string.Empty).Replace(".", string.Empty);
var json = File.ReadAllText(jsonFile);
var jobject = JObject.Parse(json);
var commonParameters = jobject.Property("params").Value.ToObject<Dictionary<string, QueryParameters>>();
return ApiQueryParametersPatcher.Patch(null, commonParameters, null, false);
}

private static KeyValuePair<string, ApiEndpoint> CreateApiEndpoint(string jsonFile)
{
var replaceFile = Path.Combine(Path.GetDirectoryName(jsonFile), Path.GetFileNameWithoutExtension(jsonFile)) + ".replace.json";
if (File.Exists(replaceFile))
{
var replaceSpec = JObject.Parse(File.ReadAllText(replaceFile));
var endpointReplaced = replaceSpec.ToObject<Dictionary<string, ApiEndpoint>>().First();
endpointReplaced.Value.RestSpecName = endpointReplaced.Key;
endpointReplaced.Value.CsharpMethodName = CreateMethodName(endpointReplaced.Key);
return endpointReplaced;
}

var officialJsonSpec = JObject.Parse(File.ReadAllText(jsonFile));
PatchOfficialSpec(officialJsonSpec, jsonFile);
var endpoint = officialJsonSpec.ToObject<Dictionary<string, ApiEndpoint>>().First();
endpoint.Value.RestSpecName = endpoint.Key;
endpoint.Value.CsharpMethodName = CreateMethodName(endpoint.Key);

PatchUrlParts(jsonFile, endpoint.Value.Url);
return endpoint;
}

private static void PatchUrlParts(string jsonFile, ApiUrl url)
private static string DoRazor(string name, string template, RestApiSpec model)
{
if (url.IsPartless) return;
foreach (var kv in url.Parts)
try
{
var required = url.ExposedApiPaths.All(p => p.Path.Contains($"{{{kv.Key}}}"));
if (kv.Value.Required != required)
Warnings.Add($"{jsonFile} has part: {kv.Key} listed as {kv.Value.Required} but should be {required}");
kv.Value.Required = required;
return Razor.CompileRenderStringAsync<RestApiSpec>(name, template, model).GetAwaiter().GetResult();
}
}

private static void PatchOfficialSpec(JObject original, string jsonFile)
{
var directory = Path.GetDirectoryName(jsonFile);
var patchFile = Path.Combine(directory,"..", "_Patches", Path.GetFileNameWithoutExtension(jsonFile)) + ".patch.json";
if (!File.Exists(patchFile)) return;

var patchedJson = JObject.Parse(File.ReadAllText(patchFile));

var pathsOverride = patchedJson.SelectToken("*.url.paths");

original.Merge(patchedJson, new JsonMergeSettings
catch (TemplateGenerationException e)
{
MergeArrayHandling = MergeArrayHandling.Union
});

if (pathsOverride != null) original.SelectToken("*.url.paths").Replace(pathsOverride);
foreach (var d in e.Diagnostics)
{
Console.WriteLine(d.GetMessage());
}
throw e;
}
}

private static Dictionary<string, ApiQueryParameters> CreateCommonApiQueryParameters(string jsonFile)
private static void WriteFormattedCsharpFile(string path, string contents)
{
var json = File.ReadAllText(jsonFile);
var jobject = JObject.Parse(json);
var commonParameters = jobject.Property("params").Value.ToObject<Dictionary<string, ApiQueryParameters>>();
return ApiQueryParametersPatcher.Patch(null, commonParameters, null, false);
var tree = CSharpSyntaxTree.ParseText(contents);
var root = tree.GetRoot().NormalizeWhitespace(indentation:"\t", "\n", elasticTrivia: false);
contents = root.ToFullString();
File.WriteAllText(path, contents);
}

private static string CreateMethodName(string apiEndpointKey) => PascalCase(apiEndpointKey);

private static string DoRazor(string name, string template, RestApiSpec model)
{
var engine = new RazorLightEngineBuilder()
.AddPrerenderCallbacks(t =>
{
}).Build();
return engine.CompileRenderAsync(name, template, model).GetAwaiter().GetResult();
}

private static void GenerateClientInterface(RestApiSpec model)
{
var targetFile = CodeConfiguration.EsNetFolder + @"IElasticLowLevelClient.Generated.cs";
var targetFile = GeneratorLocations.EsNetFolder + @"IElasticLowLevelClient.Generated.cs";
var source = DoRazor(nameof(GenerateClientInterface),
File.ReadAllText(CodeConfiguration.ViewFolder + @"IElasticLowLevelClient.Generated.cshtml"), model);
File.WriteAllText(targetFile, source);
File.ReadAllText(GeneratorLocations.ViewFolder + @"IElasticLowLevelClient.Generated.cshtml"), model);
WriteFormattedCsharpFile(targetFile, source);
}

private static void GenerateRawClient(RestApiSpec model)
{
var targetFile = CodeConfiguration.EsNetFolder + @"ElasticLowLevelClient.Generated.cs";
var targetFile = GeneratorLocations.EsNetFolder + @"ElasticLowLevelClient.Generated.cs";
var source = DoRazor(nameof(GenerateRawClient),
File.ReadAllText(CodeConfiguration.ViewFolder + @"ElasticLowLevelClient.Generated.cshtml"), model);
File.WriteAllText(targetFile, source);
File.ReadAllText(GeneratorLocations.ViewFolder + @"ElasticLowLevelClient.Generated.cshtml"), model);
WriteFormattedCsharpFile(targetFile, source);
}

private static void GenerateDescriptors(RestApiSpec model)
{
var targetFile = CodeConfiguration.NestFolder + @"_Generated\_Descriptors.Generated.cs";
var source = DoRazor(nameof(GenerateDescriptors), File.ReadAllText(CodeConfiguration.ViewFolder + @"_Descriptors.Generated.cshtml"),
var targetFile = GeneratorLocations.NestFolder + @"_Generated/_Descriptors.generated.cs";
var source = DoRazor(nameof(GenerateDescriptors), File.ReadAllText(GeneratorLocations.ViewFolder + @"_Descriptors.Generated.cshtml"),
model);
File.WriteAllText(targetFile, source);
WriteFormattedCsharpFile(targetFile, source);
}

private static void GenerateRequests(RestApiSpec model)
{
var targetFile = CodeConfiguration.NestFolder + @"_Generated\_Requests.Generated.cs";
var source = DoRazor(nameof(GenerateRequests), File.ReadAllText(CodeConfiguration.ViewFolder + @"_Requests.Generated.cshtml"), model);
File.WriteAllText(targetFile, source);
var targetFile = GeneratorLocations.NestFolder + @"_Generated/_Requests.generated.cs";
var source = DoRazor(nameof(GenerateRequests), File.ReadAllText(GeneratorLocations.ViewFolder + @"_Requests.Generated.cshtml"), model);
WriteFormattedCsharpFile(targetFile, source);
}

private static void GenerateRequestParameters(RestApiSpec model)
{
var targetFile = CodeConfiguration.EsNetFolder + @"Domain\RequestParameters\RequestParameters.Generated.cs";
var targetFile = GeneratorLocations.EsNetFolder + @"Domain/RequestParameters/RequestParameters.Generated.cs";
var source = DoRazor(nameof(GenerateRequestParameters),
File.ReadAllText(CodeConfiguration.ViewFolder + @"RequestParameters.Generated.cshtml"), model);
File.WriteAllText(targetFile, source);
File.ReadAllText(GeneratorLocations.ViewFolder + @"RequestParameters.Generated.cshtml"), model);
WriteFormattedCsharpFile(targetFile, source);
}

private static void GenerateEnums(RestApiSpec model)
{
var targetFile = CodeConfiguration.EsNetFolder + @"Domain\Enums.Generated.cs";
var source = DoRazor(nameof(GenerateEnums), File.ReadAllText(CodeConfiguration.ViewFolder + @"Enums.Generated.cshtml"), model);
File.WriteAllText(targetFile, source);
var targetFile = GeneratorLocations.EsNetFolder + @"Domain/Enums.Generated.cs";
var source = DoRazor(nameof(GenerateEnums), File.ReadAllText(GeneratorLocations.ViewFolder + @"Enums.Generated.cshtml"), model);
WriteFormattedCsharpFile(targetFile, source);
}
}
}
3 changes: 2 additions & 1 deletion src/CodeGeneration/ApiGenerator/ApiGenerator.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,11 +11,12 @@
<LangVersion>Latest</LangVersion>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.CodeAnalysis.CSharp" Version="3.1.0-beta3-final" />
<PackageReference Include="Newtonsoft.Json" Version="12.0.1" />
<PackageReference Include="ShellProgressBar" Version="4.2.0" />
<PackageReference Include="CsQuery.Core" Version="2.0.1" />
<!-- https://github.com/toddams/RazorLight/issues/172 -->
<PackageReference Include="RazorLight.Unofficial" Version="2.0.0-beta1.1" />
<PackageReference Include="RazorLight.Unofficial" Version="2.0.0-beta1.3" />
<!--<PackageReference Include="RazorLight" Version="2.0.0-beta1" />-->
</ItemGroup>
</Project>
Loading