Bootstrap entitlements for testing

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/LegacyYamlRestTestPluginFuncTest.groovy

@@ -21,7 +21,7 @@ import org.gradle.testkit.runner.TaskOutcome
 class LegacyYamlRestTestPluginFuncTest extends AbstractRestResourcesFuncTest {
 
     def setup() {
-        configurationCacheCompatible = true
+        configurationCacheCompatible = false // This is interfering with the entitlements logic in ElasticsearchTestBasePlugin
         buildApiRestrictionsDisabled = true
     }
 

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java

@@ -24,6 +24,7 @@
 import org.gradle.api.Project;
 import org.gradle.api.Task;
 import org.gradle.api.artifacts.Configuration;
+import org.gradle.api.artifacts.ConfigurationContainer;
 import org.gradle.api.file.FileCollection;
 import org.gradle.api.plugins.JavaPlugin;
 import org.gradle.api.provider.ProviderFactory;
@@ -173,6 +174,16 @@ public void execute(Task t) {
             // we use 'temp' relative to CWD since this is per JVM and tests are forbidden from writing to CWD
             nonInputProperties.systemProperty("java.io.tmpdir", test.getWorkingDir().toPath().resolve("temp"));
 
+            SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class);
+            SourceSet mainSourceSet = sourceSets.findByName(SourceSet.MAIN_SOURCE_SET_NAME);
+            SourceSet testSourceSet = sourceSets.findByName(SourceSet.TEST_SOURCE_SET_NAME);
+            if (mainSourceSet != null && testSourceSet != null) {
+                FileCollection mainRuntime = mainSourceSet.getRuntimeClasspath();
+                FileCollection testRuntime = testSourceSet.getRuntimeClasspath();
+                FileCollection testOnlyFiles = testRuntime.minus(mainRuntime);
+                test.doFirst(task -> test.environment("es.entitlement.testOnlyPath", testOnlyFiles.getAsPath()));
+            }
+
             test.systemProperties(getProviderFactory().systemPropertiesPrefixedBy("tests.").get());
             test.systemProperties(getProviderFactory().systemPropertiesPrefixedBy("es.").get());
 
@@ -205,27 +216,27 @@ public void execute(Task t) {
             }
 
             /*
-             *  If this project builds a shadow JAR than any unit tests should test against that artifact instead of
+             *  If this project builds a shadow JAR then any unit tests should test against that artifact instead of
              *  compiled class output and dependency jars. This better emulates the runtime environment of consumers.
              */
             project.getPluginManager().withPlugin("com.gradleup.shadow", p -> {
                 if (test.getName().equals(JavaPlugin.TEST_TASK_NAME)) {
                     // Remove output class files and any other dependencies from the test classpath, since the shadow JAR includes these
-                    SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class);
-                    FileCollection mainRuntime = sourceSets.getByName(SourceSet.MAIN_SOURCE_SET_NAME).getRuntimeClasspath();
                     // Add any "shadow" dependencies. These are dependencies that are *not* bundled into the shadow JAR
                     Configuration shadowConfig = project.getConfigurations().getByName(ShadowBasePlugin.CONFIGURATION_NAME);
                     // Add the shadow JAR artifact itself
                     FileCollection shadowJar = project.files(project.getTasks().named("shadowJar"));
-                    FileCollection testRuntime = sourceSets.getByName(SourceSet.TEST_SOURCE_SET_NAME).getRuntimeClasspath();
+                    FileCollection mainRuntime = mainSourceSet.getRuntimeClasspath();
+                    FileCollection testRuntime = testSourceSet.getRuntimeClasspath();
                     test.setClasspath(testRuntime.minus(mainRuntime).plus(shadowConfig).plus(shadowJar));
                 }
             });
         });
-        configureImmutableCollectionsPatch(project);
+        configureJavaBasePatch(project);
+        configureEntitlements(project);
     }
 
-    private void configureImmutableCollectionsPatch(Project project) {
+    private void configureJavaBasePatch(Project project) {
         String patchProject = ":test:immutable-collections-patch";
         if (project.findProject(patchProject) == null) {
             return; // build tests may not have this project, just skip
@@ -235,16 +246,74 @@ private void configureImmutableCollectionsPatch(Project project) {
             .create(configurationName, config -> config.setCanBeConsumed(false));
         var deps = project.getDependencies();
         deps.add(configurationName, deps.project(Map.of("path", patchProject, "configuration", "patch")));
+
+        // If ElasticsearchJavaBasePlugin has specified a dependency on the entitlement bridge jar,
+        // then it needs to be added to the --patch-module=java.base command line option.
+        ConfigurationContainer configurations = project.getConfigurations();
+
         project.getTasks().withType(Test.class).matching(task -> task.getName().equals("test")).configureEach(test -> {
             test.getInputs().files(patchedFileCollection);
             test.systemProperty("tests.hackImmutableCollections", "true");
+
+            Configuration bridgeJarConfig = configurations.findByName("entitlementBridgeJar");
+            String bridgeJarPart;
+            if (bridgeJarConfig == null) {
+                bridgeJarPart = "";
+            } else {
+                test.getInputs().files(bridgeJarConfig);
+                bridgeJarPart = File.pathSeparator + bridgeJarConfig.getSingleFile().getAbsolutePath();
+            }
+
             test.getJvmArgumentProviders()
                 .add(
                     () -> List.of(
-                        "--patch-module=java.base=" + patchedFileCollection.getSingleFile() + "/java.base",
+                        "--patch-module=java.base=" + patchedFileCollection.getSingleFile() + "/java.base" + bridgeJarPart,
                         "--add-opens=java.base/java.util=ALL-UNNAMED"
                     )
                 );
         });
     }
+
+    private static void configureEntitlements(Project project) {
+        Configuration agentJarConfig = project.getConfigurations().create("entitlementAgentJar");
+        Project agent = project.findProject(":libs:entitlement:agent");
+        if (agent != null) {
+            agentJarConfig.defaultDependencies(
+                deps -> { deps.add(project.getDependencies().project(Map.of("path", ":libs:entitlement:agent"))); }
+            );
+        }
+        FileCollection agentFiles = agentJarConfig;
+
+        Configuration bridgeJarConfig = project.getConfigurations().create("entitlementBridgeJar");
+        Project bridge = project.findProject(":libs:entitlement:bridge");
+        if (bridge != null) {
+            bridgeJarConfig.defaultDependencies(
+                deps -> { deps.add(project.getDependencies().project(Map.of("path", ":libs:entitlement:bridge"))); }
+            );
+        }
+        FileCollection bridgeFiles = bridgeJarConfig;
+
+        project.getTasks().withType(Test.class).configureEach(test -> {
+            // See also SystemJvmOptions.maybeAttachEntitlementAgent.
+
+            // Agent
+            if (agentFiles.isEmpty() == false) {
+                test.getInputs().files(agentFiles);
+                test.systemProperty("es.entitlement.agentJar", agentFiles.getAsPath());
+                test.systemProperty("jdk.attach.allowAttachSelf", true);
+            }
+
+            // Bridge
+            if (bridgeFiles.isEmpty() == false) {
+                String modulesContainingEntitlementInstrumentation = "java.logging,java.net.http,java.naming,jdk.net";
+                test.getInputs().files(bridgeFiles);
+                // Tests may not be modular, but the JDK still is
+                test.jvmArgs(
+                    "--add-exports=java.base/org.elasticsearch.entitlement.bridge=ALL-UNNAMED,"
+                        + modulesContainingEntitlementInstrumentation
+                );
+            }
+        });
+    }
+
 }

build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.java

@@ -184,7 +184,7 @@ private ChangelogEntry makeHighlightsEntry(int pr, boolean notable) {
     }
 
     private String getResource(String name) throws Exception {
-        return Files.readString(Paths.get(Objects.requireNonNull(this.getClass().getResource(name)).toURI()), StandardCharsets.UTF_8);
+        return Files.readString(Paths.get(Objects.requireNonNull(this.getClass().getResource(name)).toURI()), StandardCharsets.UTF_8).replace("\r", "");
     }
 
     private void writeResource(String name, String contents) throws Exception {

build-tools/src/main/java/org/elasticsearch/gradle/test/TestBuildInfoPlugin.java

@@ -18,8 +18,11 @@
 import org.gradle.api.provider.ProviderFactory;
 import org.gradle.api.tasks.SourceSet;
 import org.gradle.api.tasks.SourceSetContainer;
+import org.gradle.api.tasks.testing.Test;
 import org.gradle.language.jvm.tasks.ProcessResources;
 
+import java.util.List;
+
 import javax.inject.Inject;
 
 /**
@@ -53,5 +56,9 @@ public void apply(Project project) {
         project.getTasks().withType(ProcessResources.class).named("processResources").configure(task -> {
             task.into("META-INF", copy -> copy.from(testBuildInfoTask));
         });
+
+        project.getTasks().withType(Test.class).matching(test -> List.of("test").contains(test.getName())).configureEach(test -> {
+            test.systemProperty("es.entitlement.enableForTests", "true");
+        });
     }
 }

libs/build.gradle

@@ -45,4 +45,14 @@ configure(childProjects.values()) {
      */
     apply plugin: 'elasticsearch.build'
   }
+
+  // This is for any code potentially included in the server at runtime.
+  // Omit oddball libraries that aren't in server.
+  def nonServerLibs = ['plugin-scanner']
+  if (false == nonServerLibs.contains(project.name)) {
+    project.getTasks().withType(Test.class).matching(test -> ['test'].contains(test.name)).configureEach(test -> {
+      test.systemProperty('es.entitlement.enableForTests', 'true')
+    })
+  }
+
 }

libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyCheckerImpl.java

@@ -135,7 +135,7 @@ private void neverEntitled(Class<?> callerClass, Supplier<String> operationDescr
             Strings.format(
                 "component [%s], module [%s], class [%s], operation [%s]",
                 entitlements.componentName(),
-                PolicyCheckerImpl.getModuleName(requestingClass),
+                entitlements.moduleName(),
                 requestingClass,
                 operationDescription.get()
             ),
@@ -247,7 +247,7 @@ public void checkFileRead(Class<?> callerClass, Path path, boolean followLinks)
                 Strings.format(
                     "component [%s], module [%s], class [%s], entitlement [file], operation [read], path [%s]",
                     entitlements.componentName(),
-                    PolicyCheckerImpl.getModuleName(requestingClass),
+                    entitlements.moduleName(),
                     requestingClass,
                     realPath == null ? path : Strings.format("%s -> %s", path, realPath)
                 ),
@@ -279,7 +279,7 @@ public void checkFileWrite(Class<?> callerClass, Path path) {
                 Strings.format(
                     "component [%s], module [%s], class [%s], entitlement [file], operation [write], path [%s]",
                     entitlements.componentName(),
-                    PolicyCheckerImpl.getModuleName(requestingClass),
+                    entitlements.moduleName(),
                     requestingClass,
                     path
                 ),
@@ -383,7 +383,7 @@ public void checkWriteProperty(Class<?> callerClass, String property) {
                     () -> Strings.format(
                         "Entitled: component [%s], module [%s], class [%s], entitlement [write_system_properties], property [%s]",
                         entitlements.componentName(),
-                        PolicyCheckerImpl.getModuleName(requestingClass),
+                        entitlements.moduleName(),
                         requestingClass,
                         property
                     )
@@ -394,7 +394,7 @@ public void checkWriteProperty(Class<?> callerClass, String property) {
             Strings.format(
                 "component [%s], module [%s], class [%s], entitlement [write_system_properties], property [%s]",
                 entitlements.componentName(),
-                PolicyCheckerImpl.getModuleName(requestingClass),
+                entitlements.moduleName(),
                 requestingClass,
                 property
             ),
@@ -447,7 +447,7 @@ private void checkFlagEntitlement(
                 Strings.format(
                     "component [%s], module [%s], class [%s], entitlement [%s]",
                     classEntitlements.componentName(),
-                    PolicyCheckerImpl.getModuleName(requestingClass),
+                    classEntitlements.moduleName(),
                     requestingClass,
                     PolicyParser.buildEntitlementNameFromClass(entitlementClass)
                 ),
@@ -460,7 +460,7 @@ private void checkFlagEntitlement(
                 () -> Strings.format(
                     "Entitled: component [%s], module [%s], class [%s], entitlement [%s]",
                     classEntitlements.componentName(),
-                    PolicyCheckerImpl.getModuleName(requestingClass),
+                    classEntitlements.moduleName(),
                     requestingClass,
                     PolicyParser.buildEntitlementNameFromClass(entitlementClass)
                 )

libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java

@@ -118,8 +118,9 @@ public enum ComponentKind {
      *
      * @param componentName the plugin name or else one of the special component names like "(server)".
      */
-    record ModuleEntitlements(
+    protected record ModuleEntitlements(
         String componentName,
+        String moduleName,
         Map<Class<? extends Entitlement>, List<Entitlement>> entitlementsByType,
         FileAccessTree fileAccess,
         Logger logger
@@ -148,7 +149,13 @@ private FileAccessTree getDefaultFileAccess(Collection<Path> componentPaths) {
 
     // pkg private for testing
     ModuleEntitlements defaultEntitlements(String componentName, Collection<Path> componentPaths, String moduleName) {
-        return new ModuleEntitlements(componentName, Map.of(), getDefaultFileAccess(componentPaths), getLogger(componentName, moduleName));
+        return new ModuleEntitlements(
+            componentName,
+            moduleName,
+            Map.of(),
+            getDefaultFileAccess(componentPaths),
+            getLogger(componentName, moduleName)
+        );
     }
 
     // pkg private for testing
@@ -166,6 +173,7 @@ ModuleEntitlements policyEntitlements(
         }
         return new ModuleEntitlements(
             componentName,
+            moduleName,
             entitlements.stream().collect(groupingBy(Entitlement::getClass)),
             FileAccessTree.of(componentName, moduleName, filesEntitlement, pathLookup, componentPaths, exclusivePaths),
             getLogger(componentName, moduleName)
@@ -293,11 +301,11 @@ private static Logger getLogger(String componentName, String moduleName) {
      */
     private static final ConcurrentHashMap<String, Logger> MODULE_LOGGERS = new ConcurrentHashMap<>();
 
-    ModuleEntitlements getEntitlements(Class<?> requestingClass) {
+    protected ModuleEntitlements getEntitlements(Class<?> requestingClass) {
         return moduleEntitlementsMap.computeIfAbsent(requestingClass.getModule(), m -> computeEntitlements(requestingClass));
     }
 
-    private ModuleEntitlements computeEntitlements(Class<?> requestingClass) {
+    protected final ModuleEntitlements computeEntitlements(Class<?> requestingClass) {
         var policyScope = scopeResolver.apply(requestingClass);
         var componentName = policyScope.componentName();
         var moduleName = policyScope.moduleName();
@@ -336,8 +344,7 @@ private ModuleEntitlements computeEntitlements(Class<?> requestingClass) {
         }
     }
 
-    // pkg private for testing
-    static Collection<Path> getComponentPathsFromClass(Class<?> requestingClass) {
+    protected Collection<Path> getComponentPathsFromClass(Class<?> requestingClass) {
         var codeSource = requestingClass.getProtectionDomain().getCodeSource();
         if (codeSource == null) {
             return List.of();

libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyManagerTests.java

@@ -89,7 +89,6 @@ public void testGetEntitlements() {
         AtomicReference<PolicyScope> policyScope = new AtomicReference<>();
 
         // A common policy with a variety of entitlements to test
-        Collection<Path> thisSourcePaths = PolicyManager.getComponentPathsFromClass(getClass());
         var plugin1SourcePaths = List.of(Path.of("modules", "plugin1"));
         var policyManager = new PolicyManager(
             new Policy("server", List.of(new Scope("org.example.httpclient", List.of(new OutboundNetworkEntitlement())))),
@@ -99,6 +98,7 @@ public void testGetEntitlements() {
             Map.of("plugin1", plugin1SourcePaths),
             TEST_PATH_LOOKUP
         );
+        Collection<Path> thisSourcePaths = policyManager.getComponentPathsFromClass(getClass());
 
         // "Unspecified" below means that the module is not named in the policy
 

modules/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/AttachmentProcessorTests.java

@@ -15,6 +15,7 @@
 import org.elasticsearch.ingest.Processor;
 import org.elasticsearch.ingest.RandomDocumentPicks;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.ESTestCase.WithoutEntitlements;
 import org.junit.Before;
 
 import java.io.InputStream;
@@ -38,6 +39,7 @@
 import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
 
+@WithoutEntitlements // ES-12084
 public class AttachmentProcessorTests extends ESTestCase {
 
     private Processor processor;

modules/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/TikaDocTests.java

@@ -14,6 +14,7 @@
 import org.apache.tika.metadata.Metadata;
 import org.elasticsearch.core.PathUtils;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.ESTestCase.WithoutEntitlements;
 
 import java.nio.file.DirectoryStream;
 import java.nio.file.Files;
@@ -25,6 +26,7 @@
  * comes back and no exception.
  */
 @SuppressFileSystems("ExtrasFS") // don't try to parse extraN
+@WithoutEntitlements // ES-12084
 public class TikaDocTests extends ESTestCase {
 
     /** some test files from tika test suite, zipped up */

modules/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/TikaImplTests.java

@@ -9,7 +9,9 @@
 package org.elasticsearch.ingest.attachment;
 
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.ESTestCase.WithoutEntitlements;
 
+@WithoutEntitlements // ES-12084
 public class TikaImplTests extends ESTestCase {
 
     public void testTikaLoads() throws Exception {

modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RegisteredDomainProcessorFactoryTests.java

@@ -11,13 +11,15 @@
 
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.ESTestCase.WithoutEntitlements;
 import org.junit.Before;
 
 import java.util.HashMap;
 import java.util.Map;
 
 import static org.hamcrest.Matchers.equalTo;
 
+@WithoutEntitlements // ES-12084
 public class RegisteredDomainProcessorFactoryTests extends ESTestCase {
 
     private RegisteredDomainProcessor.Factory factory;