Merge branch 'main' into remove-snapshot-failures

elastic · Feb 3, 2025 · 0d776db · 0d776db
2 parents 3833ff4 + 92d1d31
commit 0d776db
Show file tree

Hide file tree

Showing 110 changed files with 1,745 additions and 1,101 deletions.
diff --git a/.buildkite/pipelines/periodic.template.yml b/.buildkite/pipelines/periodic.template.yml
@@ -86,6 +86,7 @@ steps:
             ES_RUNTIME_JAVA:
               - openjdk21
               - openjdk23
+              - openjdk24
             GRADLE_TASK:
               - checkPart1
               - checkPart2

diff --git a/.buildkite/pipelines/periodic.yml b/.buildkite/pipelines/periodic.yml
@@ -505,6 +505,7 @@ steps:
             ES_RUNTIME_JAVA:
               - openjdk21
               - openjdk23
+              - openjdk24
             GRADLE_TASK:
               - checkPart1
               - checkPart2

diff --git a/docs/changelog/120957.yaml b/docs/changelog/120957.yaml
@@ -0,0 +1,5 @@
+pr: 120957
+summary: Introduce `AllocationBalancingRoundSummaryService`
+area: Allocation
+type: enhancement
+issues: []
diff --git a/docs/internal/Versioning.md b/docs/internal/Versioning.md
@@ -35,19 +35,19 @@ Every change to the transport protocol is represented by a new transport version
 higher than all previous transport versions, which then becomes the highest version
 recognized by that build of Elasticsearch. The version ids are stored
 as constants in the `TransportVersions` class.
-Each id has a standard pattern `M_NNN_SS_P`, where:
+Each id has a standard pattern `M_NNN_S_PP`, where:
 * `M` is the major version
 * `NNN` is an incrementing id
-* `SS` is used in subsidiary repos amending the default transport protocol
-* `P` is used for patches and backports
+* `S` is used in subsidiary repos amending the default transport protocol
+* `PP` is used for patches and backports
 
 When you make a change to the serialization form of any object,
 you need to create a new sequential constant in `TransportVersions`,
 introduced in the same PR that adds the change, that increments
 the `NNN` component from the previous highest version,
 with other components  set to zero.
-For example, if the previous version number is `8_413_00_1`,
-the next version number should be `8_414_00_0`.
+For example, if the previous version number is `8_413_0_01`,
+the next version number should be `8_414_0_00`.
 
 Once you have defined your constant, you then need to use it
 in serialization code. If the transport version is at or above the new id,
@@ -166,7 +166,7 @@ also has that change, and knows about the patch backport ids and what they mean.
 
 Index version is a single incrementing version number for the index data format,
 metadata, and associated mappings. It is declared the same way as the
-transport version - with the pattern `M_NNN_SS_P`, for the major version, version id,
+transport version - with the pattern `M_NNN_S_PP`, for the major version, version id,
 subsidiary version id, and patch number respectively.
 
 Index version is stored in index metadata when an index is created,

diff --git a/...entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java b/...entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java
@@ -398,7 +398,9 @@ private ModuleEntitlements computeEntitlements(Class<?> requestingClass) {
         var pluginName = pluginResolver.apply(requestingClass);
         if (pluginName != null) {
             var pluginEntitlements = pluginsEntitlements.get(pluginName);
-            if (pluginEntitlements != null) {
+            if (pluginEntitlements == null) {
+                return ModuleEntitlements.NONE;
+            } else {
                 final String scopeName;
                 if (requestingModule.isNamed() == false) {
                     scopeName = ALL_UNNAMED;

diff --git a/...lement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyManagerTests.java b/...lement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyManagerTests.java
@@ -271,7 +271,7 @@ public void testAgentsEntitlements() throws IOException, ClassNotFoundException
             createEmptyTestServerPolicy(),
             List.of(new CreateClassLoaderEntitlement()),
             Map.of(),
-            c -> "test",
+            c -> c.getPackageName().startsWith(TEST_AGENTS_PACKAGE_NAME) ? null : "test",
             TEST_AGENTS_PACKAGE_NAME,
             NO_ENTITLEMENTS_MODULE
         );
@@ -357,6 +357,22 @@ public void testDuplicateFlagEntitlements() {
         );
     }
 
+    /**
+     * If the plugin resolver tells us a class is in a plugin, don't conclude that it's in an agent.
+     */
+    public void testPluginResolverOverridesAgents() {
+        var policyManager = new PolicyManager(
+            createEmptyTestServerPolicy(),
+            List.of(new CreateClassLoaderEntitlement()),
+            Map.of(),
+            c -> "test", // Insist that the class is in a plugin
+            TEST_AGENTS_PACKAGE_NAME,
+            NO_ENTITLEMENTS_MODULE
+        );
+        ModuleEntitlements notAgentsEntitlements = policyManager.getEntitlements(TestAgent.class);
+        assertThat(notAgentsEntitlements.hasEntitlement(CreateClassLoaderEntitlement.class), is(false));
+    }
+
     private static Class<?> makeClassInItsOwnModule() throws IOException, ClassNotFoundException {
         final Path home = createTempDir();
         Path jar = createMockPluginJar(home);

diff --git a/muted-tests.yml b/muted-tests.yml
@@ -143,9 +143,6 @@ tests:
 - class: org.elasticsearch.datastreams.DataStreamsClientYamlTestSuiteIT
   method: test {p0=data_stream/120_data_streams_stats/Multiple data stream}
   issue: https://github.com/elastic/elasticsearch/issues/118217
-- class: org.elasticsearch.action.search.SearchQueryThenFetchAsyncActionTests
-  method: testBottomFieldSort
-  issue: https://github.com/elastic/elasticsearch/issues/118214
 - class: org.elasticsearch.xpack.searchablesnapshots.RetrySearchIntegTests
   method: testSearcherId
   issue: https://github.com/elastic/elasticsearch/issues/118374
@@ -335,8 +332,6 @@ tests:
 - class: org.elasticsearch.upgrades.VectorSearchIT
   method: testBBQVectorSearch {upgradedNodes=0}
   issue: https://github.com/elastic/elasticsearch/issues/121253
-- class: org.elasticsearch.lucene.FullClusterRestartLuceneIndexCompatibilityIT
-  issue: https://github.com/elastic/elasticsearch/issues/121257
 - class: org.elasticsearch.upgrades.VectorSearchIT
   method: testBBQVectorSearch {upgradedNodes=1}
   issue: https://github.com/elastic/elasticsearch/issues/121271
@@ -392,8 +387,6 @@ tests:
 - class: org.elasticsearch.xpack.ml.integration.ClassificationIT
   method: testDependentVariableIsAliasToNested
   issue: https://github.com/elastic/elasticsearch/issues/121415
-- class: org.elasticsearch.datastreams.TSDBPassthroughIndexingIT
-  issue: https://github.com/elastic/elasticsearch/issues/121464
 - class: org.elasticsearch.xpack.esql.heap_attack.HeapAttackIT
   method: testLookupExplosionBigStringManyMatches
   issue: https://github.com/elastic/elasticsearch/issues/121465
@@ -409,6 +402,56 @@ tests:
 - class: org.elasticsearch.xpack.esql.heap_attack.HeapAttackIT
   method: testLookupExplosionManyMatches
   issue: https://github.com/elastic/elasticsearch/issues/121481
+- class: org.elasticsearch.xpack.security.profile.ProfileIntegTests
+  method: testGetUsersWithProfileUid
+  issue: https://github.com/elastic/elasticsearch/issues/121483
+- class: org.elasticsearch.xpack.security.CoreWithSecurityClientYamlTestSuiteIT
+  method: test {yaml=cat.aliases/10_basic/Empty cluster}
+  issue: https://github.com/elastic/elasticsearch/issues/121484
+- class: org.elasticsearch.xpack.transform.checkpoint.TransformCCSCanMatchIT
+  method: testTransformLifecycle_RangeQueryThatMatchesNoShards
+  issue: https://github.com/elastic/elasticsearch/issues/121480
+- class: org.elasticsearch.xpack.esql.action.CrossClusterAsyncQueryIT
+  method: testStopQueryLocal
+  issue: https://github.com/elastic/elasticsearch/issues/121487
+- class: org.elasticsearch.xpack.esql.action.CrossClusterAsyncQueryIT
+  method: testSuccessfulPathways
+  issue: https://github.com/elastic/elasticsearch/issues/121488
+- class: org.elasticsearch.xpack.esql.action.CrossClusterAsyncQueryIT
+  method: testAsyncQueriesWithLimit0
+  issue: https://github.com/elastic/elasticsearch/issues/121489
+- class: org.elasticsearch.xpack.security.profile.ProfileIntegTests
+  method: testSuggestProfilesWithHint
+  issue: https://github.com/elastic/elasticsearch/issues/121116
+- class: org.elasticsearch.xpack.sql.qa.single_node.JdbcDocCsvSpecIT
+  method: test {docs.testFilterToday}
+  issue: https://github.com/elastic/elasticsearch/issues/121474
+- class: org.elasticsearch.xpack.security.profile.ProfileIntegTests
+  method: testSuggestProfileWithData
+  issue: https://github.com/elastic/elasticsearch/issues/121258
+- class: org.elasticsearch.ingest.geoip.FullClusterRestartIT
+  method: testGeoIpSystemFeaturesMigration {cluster=UPGRADED}
+  issue: https://github.com/elastic/elasticsearch/issues/121115
+- class: org.elasticsearch.xpack.core.ilm.SetSingleNodeAllocateStepTests
+  method: testPerformActionSomeShardsOnlyOnNewNodesButNewNodesInvalidAttrs
+  issue: https://github.com/elastic/elasticsearch/issues/121495
+- class: org.elasticsearch.xpack.security.CoreWithSecurityClientYamlTestSuiteIT
+  method: test {yaml=cat.aliases/40_hidden/Test cat aliases output with a visible index with a hidden alias}
+  issue: https://github.com/elastic/elasticsearch/issues/121128
+- class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT
+  method: test {p0=search.vectors/42_knn_search_int4_flat/Vector similarity with filter only}
+  issue: https://github.com/elastic/elasticsearch/issues/121412
+- class: org.elasticsearch.xpack.inference.common.InferenceServiceNodeLocalRateLimitCalculatorTests
+  issue: https://github.com/elastic/elasticsearch/issues/121294
+- class: org.elasticsearch.xpack.ml.integration.ClassificationIT
+  method: testDependentVariableIsAliasToKeyword
+  issue: https://github.com/elastic/elasticsearch/issues/121492
+- class: org.elasticsearch.xpack.security.CoreWithSecurityClientYamlTestSuiteIT
+  method: test {yaml=cat.aliases/10_basic/Complex alias}
+  issue: https://github.com/elastic/elasticsearch/issues/121513
+- class: org.elasticsearch.xpack.security.CoreWithSecurityClientYamlTestSuiteIT
+  method: test {yaml=snapshot.create/10_basic/Create a snapshot for missing index}
+  issue: https://github.com/elastic/elasticsearch/issues/121536
 
 # Examples:
 #

diff --git a/...aRestTest/java/org/elasticsearch/lucene/FullClusterRestartLuceneIndexCompatibilityIT.java b/...aRestTest/java/org/elasticsearch/lucene/FullClusterRestartLuceneIndexCompatibilityIT.java
@@ -11,8 +11,6 @@
 
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.index.translog.Translog;
 import org.elasticsearch.repositories.fs.FsRepository;
 import org.elasticsearch.test.cluster.util.Version;
 
@@ -184,7 +182,6 @@ public void testClosedIndexUpgrade() throws Exception {
                 Settings.builder()
                     .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
                     .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, randomInt(2))
-                    .put(IndexSettings.INDEX_TRANSLOG_DURABILITY_SETTING.getKey(), randomFrom(Translog.Durability.values()))
                     .build()
             );
             indexDocs(index, numDocs);

diff --git a/...estTest/java/org/elasticsearch/lucene/RollingUpgradeLuceneIndexCompatibilityTestCase.java b/...estTest/java/org/elasticsearch/lucene/RollingUpgradeLuceneIndexCompatibilityTestCase.java
@@ -13,8 +13,6 @@
 import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.index.translog.Translog;
 import org.elasticsearch.repositories.fs.FsRepository;
 import org.elasticsearch.test.cluster.util.Version;
 
@@ -189,11 +187,7 @@ public void testClosedIndexUpgrade() throws Exception {
             createIndex(
                 client(),
                 index,
-                Settings.builder()
-                    .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
-                    .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
-                    .put(IndexSettings.INDEX_TRANSLOG_DURABILITY_SETTING.getKey(), randomFrom(Translog.Durability.values()))
-                    .build()
+                Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0).build()
             );
             indexDocs(index, numDocs);
             return;

diff --git a/server/src/main/java/org/elasticsearch/ReleaseVersions.java b/server/src/main/java/org/elasticsearch/ReleaseVersions.java
@@ -78,10 +78,10 @@ public static IntFunction<String> generateVersionsLookup(Class<?> versionContain
             // replace all version lists with the smallest & greatest versions
             versions.replaceAll((k, v) -> {
                 if (v.size() == 1) {
-                    return List.of(v.get(0));
+                    return List.of(v.getFirst());
                 } else {
                     v.sort(Comparator.naturalOrder());
-                    return List.of(v.get(0), v.get(v.size() - 1));
+                    return List.of(v.getFirst(), v.getLast());
                 }
             });
 
@@ -100,14 +100,14 @@ private static IntFunction<String> lookupFunction(NavigableMap<Integer, List<Ver
 
             String lowerBound, upperBound;
             if (versionRange != null) {
-                lowerBound = versionRange.get(0).toString();
-                upperBound = lastItem(versionRange).toString();
+                lowerBound = versionRange.getFirst().toString();
+                upperBound = versionRange.getLast().toString();
             } else {
                 // infer the bounds from the surrounding entries
                 var lowerRange = versions.lowerEntry(id);
                 if (lowerRange != null) {
                     // the next version is just a guess - might be a newer revision, might be a newer minor or major...
-                    lowerBound = nextVersion(lastItem(lowerRange.getValue())).toString();
+                    lowerBound = nextVersion(lowerRange.getValue().getLast()).toString();
                 } else {
                     // a really old version we don't have a record for
                     // assume it's an old version id - we can just return it directly
@@ -122,7 +122,7 @@ private static IntFunction<String> lookupFunction(NavigableMap<Integer, List<Ver
                 var upperRange = versions.higherEntry(id);
                 if (upperRange != null) {
                     // too hard to guess what version this id might be for using the next version - just use it directly
-                    upperBound = upperRange.getValue().get(0).toString();
+                    upperBound = upperRange.getValue().getFirst().toString();
                 } else {
                     // a newer version than all we know about? Can't map it...
                     upperBound = "[" + id + "]";
@@ -133,10 +133,6 @@ private static IntFunction<String> lookupFunction(NavigableMap<Integer, List<Ver
         };
     }
 
-    private static <T> T lastItem(List<T> list) {
-        return list.get(list.size() - 1);
-    }
-
     private static Version nextVersion(Version version) {
         return new Version(version.id + 100);   // +1 to revision
     }

diff --git a/server/src/main/java/org/elasticsearch/TransportVersion.java b/server/src/main/java/org/elasticsearch/TransportVersion.java
@@ -130,20 +130,20 @@ public static TransportVersion fromString(String str) {
      * When a patch version of an existing transport version is created, {@code transportVersion.isPatchFrom(patchVersion)}
      * will match any transport version at or above {@code patchVersion} that is also of the same base version.
      * <p>
-     * For example, {@code version.isPatchFrom(8_800_00_4)} will return the following for the given {@code version}:
+     * For example, {@code version.isPatchFrom(8_800_0_04)} will return the following for the given {@code version}:
      * <ul>
-     *     <li>{@code 8_799_00_0.isPatchFrom(8_800_00_4)}: {@code false}</li>
-     *     <li>{@code 8_799_00_9.isPatchFrom(8_800_00_4)}: {@code false}</li>
-     *     <li>{@code 8_800_00_0.isPatchFrom(8_800_00_4)}: {@code false}</li>
-     *     <li>{@code 8_800_00_3.isPatchFrom(8_800_00_4)}: {@code false}</li>
-     *     <li>{@code 8_800_00_4.isPatchFrom(8_800_00_4)}: {@code true}</li>
-     *     <li>{@code 8_800_00_9.isPatchFrom(8_800_00_4)}: {@code true}</li>
-     *     <li>{@code 8_800_01_0.isPatchFrom(8_800_00_4)}: {@code false}</li>
-     *     <li>{@code 8_801_00_0.isPatchFrom(8_800_00_4)}: {@code false}</li>
+     *     <li>{@code 8_799_0_00.isPatchFrom(8_800_0_04)}: {@code false}</li>
+     *     <li>{@code 8_799_0_09.isPatchFrom(8_800_0_04)}: {@code false}</li>
+     *     <li>{@code 8_800_0_00.isPatchFrom(8_800_0_04)}: {@code false}</li>
+     *     <li>{@code 8_800_0_03.isPatchFrom(8_800_0_04)}: {@code false}</li>
+     *     <li>{@code 8_800_0_04.isPatchFrom(8_800_0_04)}: {@code true}</li>
+     *     <li>{@code 8_800_0_49.isPatchFrom(8_800_0_04)}: {@code true}</li>
+     *     <li>{@code 8_800_1_00.isPatchFrom(8_800_0_04)}: {@code false}</li>
+     *     <li>{@code 8_801_0_00.isPatchFrom(8_800_0_04)}: {@code false}</li>
      * </ul>
      */
     public boolean isPatchFrom(TransportVersion version) {
-        return onOrAfter(version) && id < version.id + 10 - (version.id % 10);
+        return onOrAfter(version) && id < version.id + 100 - (version.id % 100);
     }
 
     /**