Making k and num_candidates optional for knn search

docs/changelog/101209.yaml

@@ -0,0 +1,6 @@
+pr: 101209
+summary: "Making `k` and `num_candidates` optional for knn search"
+area: Vector Search
+type: enhancement
+issues:
+ - 97533

docs/reference/query-dsl/knn-query.asciidoc

@@ -94,10 +94,10 @@ as the vector field you are searching against.
 `num_candidates`::
 +
 --
-(Required, integer) The number of nearest neighbor candidates to consider per shard.
+(Optional, integer) The number of nearest neighbor candidates to consider per shard.
 Cannot exceed 10,000. {es} collects `num_candidates` results from each shard, then
 merges them to find the top results. Increasing `num_candidates` tends to improve the
-accuracy of the final results.
+accuracy of the final results. Defaults to `Math.min(1.5 * size, 10_000)`.
 --
 
 `filter`::

docs/reference/rest-api/common-parms.asciidoc

@@ -584,14 +584,15 @@ end::knn-filter[]
 
 tag::knn-k[]
 Number of nearest neighbors to return as top hits. This value must be less than
-`num_candidates`.
+`num_candidates`. Defaults to `size`.
 end::knn-k[]
 
 tag::knn-num-candidates[]
-The number of nearest neighbor candidates to consider per shard. Cannot exceed
-10,000. {es} collects `num_candidates` results from each shard, then merges them
+The number of nearest neighbor candidates to consider per shard.
+Needs to be greater than `k`, or `size` if `k` is omitted, and cannot exceed 10,000.
+{es} collects `num_candidates` results from each shard, then merges them
 to find the top `k` results. Increasing `num_candidates` tends to improve the
-accuracy of the final `k` results.
+accuracy of the final `k` results. Defaults to `Math.min(1.5 * k, 10_000)`.
 end::knn-num-candidates[]
 
 tag::knn-query-vector[]

docs/reference/search/knn-search.asciidoc

@@ -102,22 +102,22 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=routing]
 include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=knn-filter]
 
 `knn`::
-(Required, object) 
+(Required, object)
 include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=knn]
 +
 .Properties of `knn` object
 [%collapsible%open]
 ====
 `field`::
-(Required, string) 
+(Required, string)
 include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=knn-field]
 
 `k`::
-(Required, integer) 
+(Optional, integer)
 include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=knn-k]
 
 `num_candidates`::
-(Required, integer) 
+(Optional, integer)
 include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=knn-num-candidates]
 
 `query_vector`::

docs/reference/search/search.asciidoc

@@ -505,11 +505,11 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=knn-field]
 include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=knn-filter]
 
 `k`::
-(Required, integer)
+(Optional, integer)
 include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=knn-k]
 
 `num_candidates`::
-(Required, integer)
+(Optional, integer)
 include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=knn-num-candidates]
 
 `query_vector`::

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/150_knn_search_missing_params.yml

@@ -0,0 +1,139 @@
+setup:
+  - skip:
+      version: ' - 8.12.99'
+      reason: '[k] and [num_candidates] were made optional for kNN search in 8.13.0'
+  - do:
+      indices.create:
+        index: knn_search_test_index
+        body:
+          mappings:
+            properties:
+                  vector:
+                    type: dense_vector
+                    dims: 5
+                    index: true
+                    similarity: l2_norm
+
+  - do:
+      index:
+        index: knn_search_test_index
+        id: "1"
+        body:
+          vector: [1.0, -10.5, 1.3, 0.593, 41]
+
+  - do:
+      index:
+        index: knn_search_test_index
+        id: "2"
+        body:
+          vector: [-0.5, 100.0, -13, 14.8, -156.0]
+
+  - do:
+      index:
+        index: knn_search_test_index
+        id: "3"
+        body:
+          vector: [0.5, 111.3, -13.0, 14.8, -156.0]
+
+  - do:
+      indices.refresh: {}
+
+---
+"kNN with missing k param using default size":
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: knn_search_test_index
+        body:
+          knn:
+            field: vector
+            query_vector: [-0.5, 90.0, -10, 14.8, -156.0]
+            num_candidates: 10
+
+  - match: {hits.total: 3}
+
+---
+"kNN with missing k param using provided size":
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: knn_search_test_index
+        body:
+          knn:
+            field: vector
+            query_vector: [-0.5, 90.0, -10, 14.8, -156.0]
+            num_candidates: 10
+          size: 2
+
+  - match: {hits.total: 2}
+
+---
+"kNN search with missing num_candidates param":
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: knn_search_test_index
+        body:
+          knn:
+            field: vector
+            query_vector: [-0.5, 90.0, -10, 14.8, -156.0]
+            k: 2
+
+  - match: {hits.total: 2}
+
+---
+"kNN search with missing both k and num_candidates param - default size":
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: knn_search_test_index
+        body:
+          knn:
+            field: vector
+            query_vector: [-0.5, 90.0, -10, 14.8, -156.0]
+
+  - match: {hits.total: 3}
+
+
+---
+"kNN search with missing both k and num_candidates param - provided size":
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: knn_search_test_index
+        body:
+          knn:
+            field: vector
+            query_vector: [-0.5, 90.0, -10, 14.8, -156.0]
+          size: 2
+
+  - match: {hits.total: 2}
+
+---
+"kNN search with missing k, and num_candidates < size":
+
+  - do:
+      catch: bad_request
+      search:
+        index: knn_search_test_index
+        body:
+          knn:
+            field: vector
+            query_vector: [-0.5, 90.0, -10, 14.8, -156.0]
+            num_candidates: 2
+          size: 10
+
+---
+"kNN search with missing k, default size, and invalid num_candidates":
+
+  - do:
+      catch: bad_request
+      search:
+        index: knn_search_test_index
+        body:
+          knn:
+            field: vector
+            query_vector: [-0.5, 90.0, -10, 14.8, -156.0]
+            num_candidates: 2

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/160_knn_query_missing_params.yml

@@ -0,0 +1,172 @@
+setup:
+  - skip:
+      version: ' - 8.12.99'
+      reason: '[k] and [num_candidates] were made optional for kNN query in 8.13.0'
+  - do:
+      indices.create:
+        index: knn_query_test_index
+        body:
+          mappings:
+            properties:
+                  vector:
+                    type: dense_vector
+                    dims: 3
+                    index: true
+                    similarity: l2_norm
+                  category:
+                    type: keyword
+                  nested:
+                    type: nested
+                    properties:
+                      paragraph_id:
+                        type: keyword
+                      vector:
+                        type: dense_vector
+                        dims: 5
+                        index: true
+                        similarity: l2_norm
+
+  - do:
+      index:
+        index: knn_query_test_index
+        id: "1"
+        body:
+          vector: [1.0, 1.0, 0.0]
+          category: A
+          nested:
+            - paragraph_id: 0
+              vector: [ 230.0, 300.33, -34.8988, 15.555, -200.0 ]
+            - paragraph_id: 1
+              vector: [ 240.0, 300, -3, 1, -20 ]
+
+  - do:
+      index:
+        index: knn_query_test_index
+        id: "2"
+        body:
+          vector: [1.0, 0.5, 1.0]
+          category: A
+          nested:
+            - paragraph_id: 2
+              vector: [ 0, 100.0, 0, 14.8, -156.0 ]
+
+  - do:
+      index:
+        index: knn_query_test_index
+        id: "3"
+        body:
+          vector: [-1, -1, -1]
+          category: B
+          nested:
+            - paragraph_id: 0
+              vector: [ 100, 200.0, 300, 14.8, -156.0 ]
+
+  - do:
+      indices.refresh: {}
+
+---
+"kNN query with missing num_candidates param - default size":
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: knn_query_test_index
+        body:
+          query:
+            knn:
+              field: vector
+              query_vector: [0, 0, 0]
+
+  - match: { hits.total: 3 }
+
+---
+"kNN query with missing num_candidates param - size provided":
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: knn_query_test_index
+        body:
+          query:
+            knn:
+              field: vector
+              query_vector: [1, 1, 1]
+          size: 1
+  - match: { hits.total: 2 }  # due to num_candidates defined as round(1.5 * size), so we only see 2 results
+  - length: { hits.hits: 1 }  # one result is only returned though
+
+---
+"kNN query with num_candidates less than size":
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: knn_query_test_index
+        body:
+          query:
+            knn:
+              field: vector
+              query_vector: [-1, -1, -1]
+              num_candidates: 1
+          size: 10
+
+  - match: { hits.total: 1 }
+
+
+---
+"kNN query in a bool clause - missing num_candidates":
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: knn_query_test_index
+        body:
+          query:
+            bool:
+              must:
+                - term:
+                    category: A
+                - knn:
+                    field: vector
+                    query_vector: [ 1, 1, 0]
+          size: 1
+
+  - match: { hits.total: 2 }  # due to num_candidates defined as round(1.5 * size), so we only see 2 results from cat:A
+  - length: { hits.hits: 1 }
+
+---
+"kNN search in a dis_max query - missing num_candidates":
+  - do:
+      search:
+        index: knn_query_test_index
+        body:
+          query:
+            dis_max:
+              queries:
+                - knn:
+                    field: vector
+                    query_vector: [1, 1, 0]
+                - match:
+                    category: B
+              tie_breaker: 0.8
+          size: 1
+
+  - match: { hits.total.value: 3 } # 2 knn result + 1 extra from match query
+  - length: { hits.hits: 1 }
+
+---
+"kNN search used in nested field - missing num_candidates":
+  - do:
+      search:
+        index: knn_query_test_index
+        body:
+          query:
+            nested:
+              path: nested
+              query:
+                knn:
+                  field: nested.vector
+                  query_vector: [ -0.5, 90.0, -10, 14.8, -156.0 ]
+              inner_hits: { size: 1, "fields": [ "nested.paragraph_id" ], _source: false }
+          size: 1
+
+  - match: { hits.total.value: 2 }
+  - length: { hits.hits: 1 }