Skip to content

[GR-34108] Extend boot module layer to include required modules #3821

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 7 commits into from
Oct 30, 2021
Merged

[GR-34108] Extend boot module layer to include required modules #3821

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
165b245
Extend boot module layer to include required modules
Sep 19, 2021
0b1dc17
Move reflection fields and nameToModule synthesizing logic into a fin…
Sep 30, 2021
93c9ed9
Support for static required modules
Oct 8, 2021
6d81689
Rename notFoundModules to missingModules
Oct 8, 2021
6255d13
Style fixes
Oct 8, 2021
d39874f
Add workaround for reflection filters
Oct 15, 2021
8d88db0
Fix several style issues
Oct 21, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
241 changes: 219 additions & 22 deletions ...tratevm/src/com.oracle.svm.hosted.jdk11/src/com/oracle/svm/hosted/ModuleLayerFeature.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
import com.oracle.svm.core.jdk11.BootModuleLayerSupport;
import com.oracle.svm.core.jdk.JDK11OrLater;
import com.oracle.svm.core.util.VMError;
import com.oracle.svm.util.ModuleSupport;
import com.oracle.svm.util.ReflectionUtil;
import org.graalvm.nativeimage.ImageSingletons;
import org.graalvm.nativeimage.Platform;
Expand All @@ -45,13 +46,20 @@
import java.lang.reflect.Constructor;
import java.lang.reflect.Field;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.NoSuchElementException;
import java.util.Optional;
import java.util.Set;
import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.Stream;

/**
* This feature:
Expand Down Expand Up @@ -86,10 +94,10 @@
@AutomaticFeature
@Platforms(Platform.HOSTED_ONLY.class)
public final class ModuleLayerFeature implements Feature {

private Field moduleNameToModuleField;
private Field moduleParentsField;
private Constructor<ModuleLayer> moduleLayerConstructor;
private Field moduleLayerNameToModuleField;
private Field moduleLayerParentsField;
private NameToModuleSynthesizer nameToModuleSynthesizer;

@Override
public boolean isInConfiguration(IsInConfigurationAccess access) {
Expand All @@ -99,17 +107,19 @@ public boolean isInConfiguration(IsInConfigurationAccess access) {
@Override
public void afterRegistration(AfterRegistrationAccess access) {
ImageSingletons.add(BootModuleLayerSupport.class, new BootModuleLayerSupport());
moduleNameToModuleField = ReflectionUtil.lookupField(ModuleLayer.class, "nameToModule");
moduleParentsField = ReflectionUtil.lookupField(ModuleLayer.class, "parents");
moduleLayerConstructor = ReflectionUtil.lookupConstructor(ModuleLayer.class, Configuration.class, List.class, Function.class);
moduleLayerNameToModuleField = ReflectionUtil.lookupField(ModuleLayer.class, "nameToModule");
moduleLayerParentsField = ReflectionUtil.lookupField(ModuleLayer.class, "parents");
nameToModuleSynthesizer = new NameToModuleSynthesizer();
}

@Override
public void beforeAnalysis(BeforeAnalysisAccess access) {
FeatureImpl.BeforeAnalysisAccessImpl accessImpl = (FeatureImpl.BeforeAnalysisAccessImpl) access;
Map<String, Module> baseModules = ModuleLayer.boot().modules()
Set<String> baseModules = ModuleLayer.boot().modules()
.stream()
.collect(Collectors.toMap(Module::getName, m -> m));
.map(Module::getName)
.collect(Collectors.toSet());
ModuleLayer runtimeBootLayer = synthesizeRuntimeBootLayer(accessImpl.imageClassLoader, baseModules);
BootModuleLayerSupport.instance().setBootLayer(runtimeBootLayer);
}
Expand All @@ -119,49 +129,73 @@ public void afterAnalysis(AfterAnalysisAccess access) {
FeatureImpl.AfterAnalysisAccessImpl accessImpl = (FeatureImpl.AfterAnalysisAccessImpl) access;
AnalysisUniverse universe = accessImpl.getUniverse();

Map<String, Module> reachableModules = universe.getTypes()
Stream<Module> analysisReachableModules = universe.getTypes()
.stream()
.filter(t -> t.isReachable() && !t.isArray())
.map(t -> t.getJavaClass().getModule())
.distinct()
.filter(m -> m.isNamed() && !m.getDescriptor().modifiers().contains(ModuleDescriptor.Modifier.SYNTHETIC))
.collect(Collectors.toMap(Module::getName, m -> m));
.distinct();

Set<String> allReachableModules = analysisReachableModules
.filter(Module::isNamed)
.filter(m -> !m.getDescriptor().modifiers().contains(ModuleDescriptor.Modifier.SYNTHETIC))
.flatMap(ModuleLayerFeature::extractRequiredModuleNames)
.collect(Collectors.toSet());

ModuleLayer runtimeBootLayer = synthesizeRuntimeBootLayer(accessImpl.imageClassLoader, reachableModules);
ModuleLayer runtimeBootLayer = synthesizeRuntimeBootLayer(accessImpl.imageClassLoader, allReachableModules);
BootModuleLayerSupport.instance().setBootLayer(runtimeBootLayer);
}

private ModuleLayer synthesizeRuntimeBootLayer(ImageClassLoader cl, Map<String, Module> reachableModules) {
/*
* Creates a stream of module names that are reachable from a given module through "requires"
*/
private static Stream<String> extractRequiredModuleNames(Module m) {
Stream<String> requiredModules = m.getDescriptor().requires().stream().map(ModuleDescriptor.Requires::name);
return Stream.concat(Stream.of(m.getName()), requiredModules);
}

private ModuleLayer synthesizeRuntimeBootLayer(ImageClassLoader cl, Set<String> reachableModules) {
Configuration cf = synthesizeRuntimeBootLayerConfiguration(cl.modulepath(), reachableModules);
try {
ModuleLayer runtimeBootLayer = moduleLayerConstructor.newInstance(cf, List.of(), null);
patchRuntimeBootLayer(runtimeBootLayer, reachableModules);
// Ensure that the lazy field ModuleLayer.modules gets set
runtimeBootLayer.modules();
Map<String, Module> nameToModule = nameToModuleSynthesizer.synthesizeNameToModule(runtimeBootLayer, cl.getClassLoader());
patchRuntimeBootLayer(runtimeBootLayer, nameToModule);
return runtimeBootLayer;
} catch (InstantiationException | IllegalAccessException | InvocationTargetException ex) {
throw VMError.shouldNotReachHere("Failed to synthesize the runtime boot module layer.", ex);
}
}

private static Configuration synthesizeRuntimeBootLayerConfiguration(List<Path> mp, Map<String, Module> reachableModules) {
private static Configuration synthesizeRuntimeBootLayerConfiguration(List<Path> mp, Set<String> reachableModules) {
ModuleFinder beforeFinder = new BootModuleLayerModuleFinder();
ModuleFinder afterFinder = ModuleFinder.of(mp.toArray(Path[]::new));
Set<String> roots = reachableModules.keySet();

try {
return Configuration.empty().resolve(beforeFinder, afterFinder, roots);
ModuleFinder composed = ModuleFinder.compose(beforeFinder, afterFinder);
List<String> missingModules = new ArrayList<>();
for (String module : reachableModules) {
Optional<ModuleReference> mref = composed.find(module);
if (mref.isEmpty()) {
missingModules.add(module);
}
}
reachableModules.removeAll(missingModules);

return Configuration.empty().resolve(beforeFinder, afterFinder, reachableModules);
} catch (FindException | ResolutionException | SecurityException ex) {
throw VMError.shouldNotReachHere("Failed to synthesize the runtime boot module layer configuration.", ex);
}
}

private void patchRuntimeBootLayer(ModuleLayer runtimeBootLayer, Map<String, Module> reachableModules) {
private void patchRuntimeBootLayer(ModuleLayer runtimeBootLayer, Map<String, Module> nameToModule) {
try {
moduleNameToModuleField.set(runtimeBootLayer, reachableModules);
moduleParentsField.set(runtimeBootLayer, List.of(ModuleLayer.empty()));
moduleLayerNameToModuleField.set(runtimeBootLayer, nameToModule);
moduleLayerParentsField.set(runtimeBootLayer, List.of(ModuleLayer.empty()));
} catch (IllegalAccessException ex) {
throw VMError.shouldNotReachHere("Failed to patch the runtime boot module layer.", ex);
}

// Ensure that the lazy modules field gets set
runtimeBootLayer.modules();
}

static class BootModuleLayerModuleFinder implements ModuleFinder {
Expand All @@ -184,4 +218,167 @@ public Set<ModuleReference> findAll() {
.collect(Collectors.toSet());
}
}

private static final class NameToModuleSynthesizer {
private final Module everyoneModule;
private final Set<Module> everyoneSet;
private final Constructor<Module> moduleConstructor;
private final Field moduleLayerField;
private final Field moduleReadsField;
private final Field moduleOpenPackagesField;
private final Field moduleExportedPackagesField;
private final Method moduleFindModuleMethod;

NameToModuleSynthesizer() {
Method classGetDeclaredMethods0Method = ReflectionUtil.lookupMethod(Class.class, "getDeclaredFields0", boolean.class);
try {
ModuleSupport.openModuleByClass(Module.class, ModuleLayerFeature.class);
Field[] moduleClassFields = (Field[]) classGetDeclaredMethods0Method.invoke(Module.class, false);

Field everyoneModuleField = findFieldByName(moduleClassFields, "EVERYONE_MODULE");
everyoneModuleField.setAccessible(true);
everyoneModule = (Module) everyoneModuleField.get(null);

moduleLayerField = findFieldByName(moduleClassFields, "layer");
moduleReadsField = findFieldByName(moduleClassFields, "reads");
moduleOpenPackagesField = findFieldByName(moduleClassFields, "openPackages");
moduleExportedPackagesField = findFieldByName(moduleClassFields, "exportedPackages");
moduleLayerField.setAccessible(true);
moduleReadsField.setAccessible(true);
moduleOpenPackagesField.setAccessible(true);
moduleExportedPackagesField.setAccessible(true);
} catch (ReflectiveOperationException | NoSuchElementException ex) {
throw VMError.shouldNotReachHere("Failed to find the value of EVERYONE_MODULE field of Module class.", ex);
}
everyoneSet = Set.of(everyoneModule);
moduleConstructor = ReflectionUtil.lookupConstructor(Module.class, ClassLoader.class, ModuleDescriptor.class);
moduleFindModuleMethod = ReflectionUtil.lookupMethod(Module.class, "findModule", String.class, Map.class, Map.class, List.class);
}

private static Field findFieldByName(Field[] fields, String name) {
return Arrays.stream(fields).filter(f -> f.getName().equals(name)).findAny().get();
}

/**
* This method creates Module instances that will populate the runtime boot module layer of
* the image. This implementation is copy-pasted from Module#defineModules(Configuration,
* Function, ModuleLayer) with few simplifications (removing multiple classloader support)
* and removal of VM state updates (otherwise we would be re-defining modules to the host
* VM).
*/
Map<String, Module> synthesizeNameToModule(ModuleLayer runtimeBootLayer, ClassLoader cl)
throws IllegalAccessException, InvocationTargetException, InstantiationException {
Configuration cf = runtimeBootLayer.configuration();

int cap = (int) (cf.modules().size() / 0.75f + 1.0f);
Map<String, Module> nameToModule = new HashMap<>(cap);

/*
* Remove mapping of modules to classloaders. Create module instances without defining
* them to the VM
*/
for (ResolvedModule resolvedModule : cf.modules()) {
ModuleReference mref = resolvedModule.reference();
ModuleDescriptor descriptor = mref.descriptor();
String name = descriptor.name();
Module m = moduleConstructor.newInstance(cl, descriptor);
moduleLayerField.set(m, runtimeBootLayer);
nameToModule.put(name, m);
}

/*
* Setup readability and exports/opens. This part is unchanged, save for field setters
* and VM update removals
*/
for (ResolvedModule resolvedModule : cf.modules()) {
ModuleReference mref = resolvedModule.reference();
ModuleDescriptor descriptor = mref.descriptor();

String mn = descriptor.name();
Module m = nameToModule.get(mn);
assert m != null;

Set<Module> reads = new HashSet<>();
for (ResolvedModule other : resolvedModule.reads()) {
Module m2 = nameToModule.get(other.name());
reads.add(m2);
}
moduleReadsField.set(m, reads);

if (!descriptor.isOpen() && !descriptor.isAutomatic()) {
if (descriptor.opens().isEmpty()) {
Map<String, Set<Module>> exportedPackages = new HashMap<>();
for (ModuleDescriptor.Exports exports : m.getDescriptor().exports()) {
String source = exports.source();
if (exports.isQualified()) {
Set<Module> targets = new HashSet<>();
for (String target : exports.targets()) {
Module m2 = nameToModule.get(target);
if (m2 != null) {
targets.add(m2);
}
}
if (!targets.isEmpty()) {
exportedPackages.put(source, targets);
}
} else {
exportedPackages.put(source, everyoneSet);
}
}
moduleExportedPackagesField.set(m, exportedPackages);
} else {
Map<String, Set<Module>> openPackages = new HashMap<>();
Map<String, Set<Module>> exportedPackages = new HashMap<>();
for (ModuleDescriptor.Opens opens : descriptor.opens()) {
String source = opens.source();
if (opens.isQualified()) {
Set<Module> targets = new HashSet<>();
for (String target : opens.targets()) {
Module m2 = (Module) moduleFindModuleMethod.invoke(null, target, Map.of(), nameToModule, runtimeBootLayer.parents());
if (m2 != null) {
targets.add(m2);
}
}
if (!targets.isEmpty()) {
openPackages.put(source, targets);
}
} else {
openPackages.put(source, everyoneSet);
}
}

for (ModuleDescriptor.Exports exports : descriptor.exports()) {
String source = exports.source();
Set<Module> openToTargets = openPackages.get(source);
if (openToTargets != null && openToTargets.contains(everyoneModule)) {
continue;
}

if (exports.isQualified()) {
Set<Module> targets = new HashSet<>();
for (String target : exports.targets()) {
Module m2 = (Module) moduleFindModuleMethod.invoke(null, target, Map.of(), nameToModule, runtimeBootLayer.parents());
if (m2 != null) {
if (openToTargets == null || !openToTargets.contains(m2)) {
targets.add(m2);
}
}
}
if (!targets.isEmpty()) {
exportedPackages.put(source, targets);
}
} else {
exportedPackages.put(source, everyoneSet);
}
}

moduleOpenPackagesField.set(m, openPackages);
moduleExportedPackagesField.set(m, exportedPackages);
}
}
}

return nameToModule;
}
}
}
5 changes: 4 additions & 1 deletion substratevm/src/com.oracle.svm.util/src/com/oracle/svm/util/ReflectionUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,10 @@
import java.lang.reflect.Method;

/**
* This class contains utility methods for commonly used reflection functionality.
* This class contains utility methods for commonly used reflection functionality. Note that lookups
* will not work on JDK 17 in cases when the field/method is filtered. See
* jdk.internal.reflect.Reflection#fieldFilterMap for more information or
* com.oracle.svm.hosted.ModuleLayerFeature for an example of a workaround in such cases.
*/
public final class ReflectionUtil {

Expand Down