Merge branch 'main' into feature/apis

Author: GalLalouche

.buildkite/scripts/dra-workflow.sh

@@ -71,6 +71,7 @@ echo --- Building release artifacts
   buildReleaseArtifacts \
   exportCompressedDockerImages \
   exportDockerContexts \
+  :zipAggregation \
   :distribution:generateDependenciesReport
 
 PATH="$PATH:${JAVA_HOME}/bin" # Required by the following script

build-tools-internal/gradle/wrapper/gradle-wrapper.properties

@@ -1,7 +1,7 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionSha256Sum=d7042b3c11565c192041fc8c4703f541b888286404b4f267138c1d094d8ecdca
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.14.1-all.zip
+distributionSha256Sum=443c9c8ee2ac1ee0e11881a40f2376d79c66386264a44b24a9f8ca67e633375f
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.14.2-all.zip
 networkTimeout=10000
 validateDistributionUrl=true
 zipStoreBase=GRADLE_USER_HOME

build-tools-internal/src/main/resources/changelog-schema.json

@@ -86,6 +86,7 @@
             "Rollup",
             "SQL",
             "Search",
+            "Searchable Snapshots",
             "Security",
             "Snapshot/Restore",
             "Stats",

build-tools-internal/src/main/resources/minimumGradleVersion

@@ -1 +1 @@
-8.14.1
+8.14.2

build.gradle

@@ -68,6 +68,12 @@ nmcpAggregation {
 }
 
 tasks.named('zipAggregation').configure {
+  // put this in a place that works well with our DRA infrastructure
+  archiveFileName.unset();
+  archiveBaseName.set("elasticsearch-maven-aggregation")
+  archiveVersion.set(VersionProperties.elasticsearch)
+  destinationDirectory.set(layout.buildDirectory.dir("distributions"));
+
   dependsOn gradle.includedBuild('build-tools').task(':zipElasticPublication')
   from(zipTree(gradle.includedBuild('build-tools').task(':zipElasticPublication').resolveTask().archiveFile.get()))
 }

docs/changelog/128910.yaml

@@ -0,0 +1,5 @@
+pr: 128910
+summary: Fix `FieldAttribute` name usage in `InferNonNullAggConstraint`
+area: ES|QL
+type: bug
+issues: []

docs/changelog/128913.yml

@@ -0,0 +1,5 @@
+pr: 128913
+summary: "[apm-data] Enable 'date_detection' for all apm data streams"
+area: Data streams
+type: enhancement
+issues: []

docs/changelog/129164.yaml

@@ -0,0 +1,5 @@
+pr: 129164
+summary: Log partial failures
+area: ES|QL
+type: feature
+issues: []

docs/changelog/129245.yaml

@@ -0,0 +1,5 @@
+pr: 129245
+summary: Throttle indexing when disk IO throttling is disabled
+area: Engine
+type: enhancement
+issues: []

docs/reference/query-languages/esql/_snippets/lists/search-functions.md

@@ -2,5 +2,4 @@
 * [preview] [`MATCH`](../../functions-operators/search-functions.md#esql-match)
 * [preview] [`MATCH_PHRASE`](../../functions-operators/search-functions.md#esql-match_phrase)
 * [preview] [`QSTR`](../../functions-operators/search-functions.md#esql-qstr)
-  % * [preview] [
-  `TERM`](../../functions-operators/search-functions.md#esql-term)
+% * [preview] [`TERM`](../../functions-operators/search-functions.md#esql-term)

gradle/wrapper/gradle-wrapper.properties

@@ -1,7 +1,7 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionSha256Sum=d7042b3c11565c192041fc8c4703f541b888286404b4f267138c1d094d8ecdca
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.14.1-all.zip
+distributionSha256Sum=443c9c8ee2ac1ee0e11881a40f2376d79c66386264a44b24a9f8ca67e633375f
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.14.2-all.zip
 networkTimeout=10000
 validateDistributionUrl=true
 zipStoreBase=GRADLE_USER_HOME

muted-tests.yml

@@ -520,6 +520,9 @@ tests:
 - class: org.elasticsearch.xpack.esql.qa.single_node.GenerativeForkIT
   method: test {lookup-join.MultipleBatches*
   issue: https://github.com/elastic/elasticsearch/issues/129210
+- class: org.elasticsearch.compute.data.sort.LongTopNSetTests
+  method: testCrankyBreaker
+  issue: https://github.com/elastic/elasticsearch/issues/129257
 
 # Examples:
 #

plugins/examples/gradle/wrapper/gradle-wrapper.properties

@@ -1,7 +1,7 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionSha256Sum=d7042b3c11565c192041fc8c4703f541b888286404b4f267138c1d094d8ecdca
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.14.1-all.zip
+distributionSha256Sum=443c9c8ee2ac1ee0e11881a40f2376d79c66386264a44b24a9f8ca67e633375f
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.14.2-all.zip
 networkTimeout=10000
 validateDistributionUrl=true
 zipStoreBase=GRADLE_USER_HOME

qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/ParameterizedFullClusterRestartTestCase.java

@@ -134,7 +134,11 @@ public boolean isRunningAgainstOldCluster() {
         return requestedUpgradeStatus == OLD;
     }
 
-    public static String getOldClusterVersion() {
+    /**
+     * The version of the "old" (initial) cluster. It is an opaque string, do not even think about parsing it for version
+     * comparison. Use (test) cluster features and {@link ParameterizedFullClusterRestartTestCase#oldClusterHasFeature} instead.
+     */
+    protected static String getOldClusterVersion() {
         return System.getProperty("tests.bwc.main.version", OLD_CLUSTER_VERSION);
     }
 

qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/AbstractRollingUpgradeTestCase.java

@@ -30,10 +30,11 @@ public abstract class AbstractRollingUpgradeTestCase extends ParameterizedRollin
     private static final ElasticsearchCluster cluster = buildCluster();
 
     private static ElasticsearchCluster buildCluster() {
-        Version oldVersion = Version.fromString(OLD_CLUSTER_VERSION);
+        // Note we need to use OLD_CLUSTER_VERSION directly here, as it may contain special values (e.g. 0.0.0) the ElasticsearchCluster
+        // builder uses to lookup a particular distribution
         var cluster = ElasticsearchCluster.local()
             .distribution(DistributionType.DEFAULT)
-            .version(getOldClusterTestVersion())
+            .version(OLD_CLUSTER_VERSION)
             .nodes(NODE_NUM)
             .setting("path.repo", new Supplier<>() {
                 @Override
@@ -46,8 +47,9 @@ public String get() {
             .feature(FeatureFlag.TIME_SERIES_MODE);
 
         // Avoid triggering bogus assertion when serialized parsed mappings don't match with original mappings, because _source key is
-        // inconsistent
-        if (oldVersion.before(Version.fromString("8.18.0"))) {
+        // inconsistent. As usual, we operate under the premise that "versionless" clusters (serverless) are on the latest code and
+        // do not need this.
+        if (Version.tryParse(getOldClusterVersion()).map(v -> v.before(Version.fromString("8.18.0"))).orElse(false)) {
             cluster.jvmArg("-da:org.elasticsearch.index.mapper.DocumentMapper");
             cluster.jvmArg("-da:org.elasticsearch.index.mapper.MapperService");
         }

qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/AbstractRollingUpgradeWithSecurityTestCase.java

@@ -35,10 +35,11 @@ public abstract class AbstractRollingUpgradeWithSecurityTestCase extends Paramet
     private static final ElasticsearchCluster cluster = buildCluster();
 
     private static ElasticsearchCluster buildCluster() {
-        Version oldVersion = Version.fromString(OLD_CLUSTER_VERSION);
+        // Note we need to use OLD_CLUSTER_VERSION directly here, as it may contain special values (e.g. 0.0.0) the ElasticsearchCluster
+        // builder uses to lookup a particular distribution
         var cluster = ElasticsearchCluster.local()
             .distribution(DistributionType.DEFAULT)
-            .version(getOldClusterTestVersion())
+            .version(OLD_CLUSTER_VERSION)
             .nodes(NODE_NUM)
             .user(USER, PASS)
             .setting("xpack.security.autoconfiguration.enabled", "false")
@@ -51,8 +52,8 @@ public String get() {
             });
 
         // Avoid triggering bogus assertion when serialized parsed mappings don't match with original mappings, because _source key is
-        // inconsistent
-        if (oldVersion.before(Version.fromString("8.18.0"))) {
+        // inconsistent. Assume non-parseable versions (serverless) do not need this.
+        if (Version.tryParse(getOldClusterVersion()).map(v -> v.before(Version.fromString("8.18.0"))).orElse(false)) {
             cluster.jvmArg("-da:org.elasticsearch.index.mapper.DocumentMapper");
             cluster.jvmArg("-da:org.elasticsearch.index.mapper.MapperService");
         }

qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/DenseVectorMappingUpdateIT.java

@@ -82,7 +82,7 @@ public DenseVectorMappingUpdateIT(@Name("upgradedNodes") int upgradedNodes) {
     }
 
     public void testDenseVectorMappingUpdateOnOldCluster() throws IOException {
-        if (getOldClusterTestVersion().after(Version.V_8_7_0.toString())) {
+        if (oldClusterHasFeature("gte_v8.7.1")) {
             String indexName = "test_index";
             if (isOldCluster()) {
                 Request createIndex = new Request("PUT", "/" + indexName);

qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/FileSettingsUpgradeIT.java

@@ -14,6 +14,7 @@
 import org.elasticsearch.client.Request;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.core.SuppressForbidden;
+import org.elasticsearch.core.UpdateForV10;
 import org.elasticsearch.test.cluster.ElasticsearchCluster;
 import org.elasticsearch.test.cluster.FeatureFlag;
 import org.elasticsearch.test.cluster.local.distribution.DistributionType;
@@ -33,8 +34,12 @@
 
 public class FileSettingsUpgradeIT extends ParameterizedRollingUpgradeTestCase {
 
+    @UpdateForV10(owner = UpdateForV10.Owner.CORE_INFRA) // Remove this rule entirely
     private static final RunnableTestRuleAdapter versionLimit = new RunnableTestRuleAdapter(
-        () -> assumeTrue("Only valid when upgrading from pre-file settings", getOldClusterTestVersion().before(new Version(8, 4, 0)))
+        () -> assumeTrue(
+            "Only valid when upgrading from pre-file settings",
+            Version.tryParse(getOldClusterVersion()).map(v -> v.before(new Version(8, 4, 0))).orElse(false)
+        )
     );
 
     private static final String settingsJSON = """
@@ -52,9 +57,11 @@ public class FileSettingsUpgradeIT extends ParameterizedRollingUpgradeTestCase {
 
     private static final TemporaryFolder repoDirectory = new TemporaryFolder();
 
+    // Note we need to use OLD_CLUSTER_VERSION directly here, as it may contain special values (e.g. 0.0.0) the ElasticsearchCluster
+    // builder uses to lookup a particular distribution
     private static final ElasticsearchCluster cluster = ElasticsearchCluster.local()
         .distribution(DistributionType.DEFAULT)
-        .version(getOldClusterTestVersion())
+        .version(OLD_CLUSTER_VERSION)
         .nodes(NODE_NUM)
         .setting("path.repo", new Supplier<>() {
             @Override

qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/LogsUsageRollingUpgradeIT.java

@@ -30,7 +30,7 @@ public LogsUsageRollingUpgradeIT(@Name("upgradedNodes") int upgradedNodes) {
     }
 
     public void testUsage() throws Exception {
-        assumeTrue("logsdb.prior_logs_usage only gets set in 8.x", getOldClusterTestVersion().before("9.0.0"));
+        assumeFalse("logsdb.prior_logs_usage only gets set in 8.x", oldClusterHasFeature("gte_v9.0.0"));
         String dataStreamName = "logs-mysql-error";
         if (isOldCluster()) {
             bulkIndex(dataStreamName, 4, 256, Instant.now());

qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/ParameterizedRollingUpgradeTestCase.java

@@ -127,11 +127,6 @@ public static void resetNodes() {
         upgradeFailed = false;
     }
 
-    @Deprecated // Use the new testing framework and oldClusterHasFeature(feature) instead
-    protected static String getOldClusterVersion() {
-        return OLD_CLUSTER_VERSION;
-    }
-
     protected static boolean oldClusterHasFeature(String featureId) {
         assert oldClusterTestFeatureService != null;
         return oldClusterTestFeatureService.clusterHasFeature(featureId);
@@ -146,12 +141,20 @@ protected static IndexVersion getOldClusterIndexVersion() {
         return oldIndexVersion;
     }
 
-    protected static Version getOldClusterTestVersion() {
-        return Version.fromString(OLD_CLUSTER_VERSION);
+    /**
+     * The version of the "old" (initial) cluster. It is an opaque string, do not even think about parsing it for version
+     * comparison. Use (test) cluster features and {@link ParameterizedRollingUpgradeTestCase#oldClusterHasFeature} instead.
+     */
+    protected static String getOldClusterVersion() {
+        return System.getProperty("tests.bwc.main.version", OLD_CLUSTER_VERSION);
     }
 
-    protected static boolean isOldClusterVersion(String nodeVersion) {
-        return OLD_CLUSTER_VERSION.equals(nodeVersion);
+    protected static boolean isOldClusterVersion(String nodeVersion, String buildHash) {
+        String bwcRefSpec = System.getProperty("tests.bwc.refspec.main");
+        if (bwcRefSpec != null) {
+            return bwcRefSpec.equals(buildHash);
+        }
+        return getOldClusterVersion().equals(nodeVersion);
     }
 
     protected static boolean isOldCluster() {

qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/SnapshotBasedRecoveryIT.java

@@ -88,21 +88,22 @@ public void testSnapshotBasedRecovery() throws Exception {
                 }
 
                 String primaryNodeId = getPrimaryNodeIdOfShard(indexName, 0);
-                String primaryNodeVersion = getNodeVersion(primaryNodeId);
+                var primaryNodeVersion = getNodeVersion(primaryNodeId);
 
                 // Sometimes the primary shard ends on the upgraded node (i.e. after a rebalance)
                 // This causes issues when removing and adding replicas, since then we cannot allocate to any of the old nodes.
                 // That is an issue only for the first mixed round.
                 // In that case we exclude the upgraded node from the shard allocation and cancel the shard to force moving
                 // the primary to a node in the old version, this allows adding replicas in the first mixed round.
                 logger.info("--> Primary node in first mixed round {} / {}", primaryNodeId, primaryNodeVersion);
-                if (isOldClusterVersion(primaryNodeVersion) == false) {
+                if (isOldClusterVersion(primaryNodeVersion.version(), primaryNodeVersion.buildHash()) == false) {
                     logger.info("--> cancelling primary shard on node [{}]", primaryNodeId);
                     cancelShard(indexName, 0, primaryNodeId);
                     logger.info("--> done cancelling primary shard on node [{}]", primaryNodeId);
 
                     String currentPrimaryNodeId = getPrimaryNodeIdOfShard(indexName, 0);
-                    assertTrue(isOldClusterVersion(getNodeVersion(currentPrimaryNodeId)));
+                    var currentPrimaryNodeVersion = getNodeVersion(currentPrimaryNodeId);
+                    assertTrue(isOldClusterVersion(currentPrimaryNodeVersion.version(), currentPrimaryNodeVersion.buildHash()));
                 }
             } else {
                 logger.info("--> not in first upgrade round, removing exclusions for [{}]", indexName);
@@ -137,17 +138,24 @@ private List<String> getUpgradedNodeIds() throws IOException {
         List<String> upgradedNodes = new ArrayList<>();
         for (Map.Entry<String, Map<String, Object>> nodeInfoEntry : nodes.entrySet()) {
             String nodeVersion = extractValue(nodeInfoEntry.getValue(), "version");
-            if (isOldClusterVersion(nodeVersion) == false) {
+            String nodeBuildHash = extractValue(nodeInfoEntry.getValue(), "build_hash");
+            if (isOldClusterVersion(nodeVersion, nodeBuildHash) == false) {
                 upgradedNodes.add(nodeInfoEntry.getKey());
             }
         }
         return upgradedNodes;
     }
 
-    private String getNodeVersion(String primaryNodeId) throws IOException {
+    private record NodeVersion(String version, String buildHash) {}
+
+    private NodeVersion getNodeVersion(String primaryNodeId) throws IOException {
         Request request = new Request(HttpGet.METHOD_NAME, "_nodes/" + primaryNodeId);
         Response response = client().performRequest(request);
-        return extractValue(responseAsMap(response), "nodes." + primaryNodeId + ".version");
+        Map<String, Object> responseAsMap = responseAsMap(response);
+        return new NodeVersion(
+            extractValue(responseAsMap, "nodes." + primaryNodeId + ".version"),
+            extractValue(responseAsMap, "nodes." + primaryNodeId + ".build_hash")
+        );
     }
 
     private String getPrimaryNodeIdOfShard(String indexName, int shard) throws Exception {

qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/SourceModeRollingUpgradeIT.java

@@ -27,7 +27,7 @@ public SourceModeRollingUpgradeIT(@Name("upgradedNodes") int upgradedNodes) {
     }
 
     public void testConfigureStoredSourceBeforeIndexCreationLegacy() throws IOException {
-        assumeTrue("testing deprecation warnings and deprecation migrations", getOldClusterTestVersion().before("9.0.0"));
+        assumeFalse("testing deprecation warnings and deprecation migrations", oldClusterHasFeature("gte_v9.0.0"));
         String templateName = "logs@custom";
         if (isOldCluster()) {
             var storedSourceMapping = """
@@ -56,7 +56,7 @@ public void testConfigureStoredSourceBeforeIndexCreationLegacy() throws IOExcept
     }
 
     public void testConfigureStoredSourceWhenIndexIsCreatedLegacy() throws IOException {
-        assumeTrue("testing deprecation warnings and deprecation migrations", getOldClusterTestVersion().before("9.0.0"));
+        assumeFalse("testing deprecation warnings and deprecation migrations", oldClusterHasFeature("gte_v9.0.0"));
         String templateName = "logs@custom";
         if (isOldCluster()) {
             var storedSourceMapping = """