Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions docs/changelog/138621.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
pr: 138621
summary: Release CHUNK function as tech preview
area: ES|QL
type: enhancement
issues: []
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,9 @@ mapped_pages:
:::{include} ../_snippets/functions/layout/byte_length.md
:::

:::{include} ../_snippets/functions/layout/chunk.md
:::

:::{include} ../_snippets/functions/layout/concat.md
:::

Expand Down

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Original file line number Diff line number Diff line change
Expand Up @@ -1665,7 +1665,7 @@ public enum Cap {
/**
* Chunk function.
*/
CHUNK_FUNCTION_V2(Build.current().isSnapshot()),
CHUNK_FUNCTION_V2(),

/**
* Support for vector similarity functtions pushdown
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -434,7 +434,8 @@ private static FunctionDefinition[][] functions() {
def(Trim.class, Trim::new, "trim"),
def(UrlEncode.class, UrlEncode::new, "url_encode"),
def(UrlEncodeComponent.class, UrlEncodeComponent::new, "url_encode_component"),
def(UrlDecode.class, UrlDecode::new, "url_decode") },
def(UrlDecode.class, UrlDecode::new, "url_decode"),
def(Chunk.class, bi(Chunk::new), "chunk") },
// date
new FunctionDefinition[] {
def(DateDiff.class, tric(DateDiff::new), "date_diff"),
Expand Down Expand Up @@ -572,8 +573,7 @@ private static FunctionDefinition[][] snapshotFunctions() {
def(L1Norm.class, L1Norm::new, "v_l1_norm"),
def(L2Norm.class, L2Norm::new, "v_l2_norm"),
def(Magnitude.class, Magnitude::new, "v_magnitude"),
def(Hamming.class, Hamming::new, "v_hamming"),
def(Chunk.class, bi(Chunk::new), "chunk") } };
def(Hamming.class, Hamming::new, "v_hamming") } };
}

public EsqlFunctionRegistry snapshotRegistry() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3293,47 +3293,45 @@ public void testSubqueryInFromWithLookupJoinOnFullTextFunction() {
}

public void testChunkFunctionInvalidInputs() {
if (EsqlCapabilities.Cap.CHUNK_FUNCTION_V2.isEnabled()) {
assertThat(
error("from test | EVAL chunks = CHUNK(body, null)", fullTextAnalyzer, VerificationException.class),
equalTo("1:27: invalid chunking_settings, found [null]")
);
assertThat(
error("from test | EVAL chunks = CHUNK(body, {\"strategy\": \"invalid\"})", fullTextAnalyzer, VerificationException.class),
equalTo("1:27: Invalid chunkingStrategy invalid")
);
assertThat(
error(
"from test | EVAL chunks = CHUNK(body, {\"strategy\": \"sentence\", \"max_chunk_size\": 5, \"sentence_overlap\": 1})",
fullTextAnalyzer,
VerificationException.class
),
equalTo(
"1:27: Validation Failed: 1: [chunking_settings] Invalid value [5.0]. "
+ "[max_chunk_size] must be a greater than or equal to [20.0];"
)
);
assertThat(
error(
"from test | EVAL chunks = CHUNK(body, {\"strategy\": \"sentence\", \"max_chunk_size\": 5, \"sentence_overlap\": 5})",
fullTextAnalyzer,
VerificationException.class
),
equalTo(
"1:27: Validation Failed: 1: [chunking_settings] Invalid value [5.0]. "
+ "[max_chunk_size] must be a greater than or equal to [20.0];2: sentence_overlap[5] must be either 0 or 1;"
)
);
assertThat(
error(
"from test | EVAL chunks = CHUNK(body, {\"strategy\": \"sentence\", \"max_chunk_size\": 20, "
+ "\"sentence_overlap\": 1, \"extra_value\": \"foo\"})",
fullTextAnalyzer,
VerificationException.class
),
equalTo("1:27: Validation Failed: 1: Sentence based chunking settings can not have the following settings: [extra_value];")
);
}
assertThat(
error("from test | EVAL chunks = CHUNK(body, null)", fullTextAnalyzer, VerificationException.class),
equalTo("1:27: invalid chunking_settings, found [null]")
);
assertThat(
error("from test | EVAL chunks = CHUNK(body, {\"strategy\": \"invalid\"})", fullTextAnalyzer, VerificationException.class),
equalTo("1:27: Invalid chunkingStrategy invalid")
);
assertThat(
error(
"from test | EVAL chunks = CHUNK(body, {\"strategy\": \"sentence\", \"max_chunk_size\": 5, \"sentence_overlap\": 1})",
fullTextAnalyzer,
VerificationException.class
),
equalTo(
"1:27: Validation Failed: 1: [chunking_settings] Invalid value [5.0]. "
+ "[max_chunk_size] must be a greater than or equal to [20.0];"
)
);
assertThat(
error(
"from test | EVAL chunks = CHUNK(body, {\"strategy\": \"sentence\", \"max_chunk_size\": 5, \"sentence_overlap\": 5})",
fullTextAnalyzer,
VerificationException.class
),
equalTo(
"1:27: Validation Failed: 1: [chunking_settings] Invalid value [5.0]. "
+ "[max_chunk_size] must be a greater than or equal to [20.0];2: sentence_overlap[5] must be either 0 or 1;"
)
);
assertThat(
error(
"from test | EVAL chunks = CHUNK(body, {\"strategy\": \"sentence\", \"max_chunk_size\": 20, "
+ "\"sentence_overlap\": 1, \"extra_value\": \"foo\"})",
fullTextAnalyzer,
VerificationException.class
),
equalTo("1:27: Validation Failed: 1: Sentence based chunking settings can not have the following settings: [extra_value];")
);
}

private void checkVectorFunctionsNullArgs(String functionInvocation) throws Exception {
Expand Down
Loading