Merge branch '8.16' into backport/8.16/pr-117157

n1v0lg · Nov 21, 2024 · 80adbaa · 80adbaa
2 parents 3de53b3 + a0d057f
commit 80adbaa
Show file tree

Hide file tree

Showing 5 changed files with 38 additions and 2 deletions.
diff --git a/docs/changelog/116358.yaml b/docs/changelog/116358.yaml
@@ -0,0 +1,5 @@
+pr: 116358
+summary: Update Deberta tokenizer
+area: Machine Learning
+type: bug
+issues: []
diff --git a/docs/changelog/117105.yaml b/docs/changelog/117105.yaml
@@ -0,0 +1,6 @@
+pr: 117105
+summary: Fix long metric deserialize & add - auto-resize needs to be set manually
+area: CCS
+type: bug
+issues:
+ - 116914
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/LongMetric.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/LongMetric.java
@@ -74,6 +74,7 @@ public static LongMetricValue fromStream(StreamInput in) throws IOException {
             try {
                 // TODO: not sure what is the good value for minBarForHighestToLowestValueRatio here?
                 Histogram dh = Histogram.decodeFromCompressedByteBuffer(bb, 1);
+                dh.setAutoResize(true);
                 return new LongMetricValue(dh);
             } catch (DataFormatException e) {
                 throw new IOException(e);

diff --git a/...src/test/java/org/elasticsearch/action/admin/cluster/stats/CCSTelemetrySnapshotTests.java b/...src/test/java/org/elasticsearch/action/admin/cluster/stats/CCSTelemetrySnapshotTests.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.action.admin.cluster.stats.CCSTelemetrySnapshot.PerClusterCCSTelemetry;
 import org.elasticsearch.action.admin.cluster.stats.LongMetric.LongMetricValue;
 import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.core.Tuple;
@@ -32,9 +33,13 @@
 public class CCSTelemetrySnapshotTests extends AbstractWireSerializingTestCase<CCSTelemetrySnapshot> {
 
     private LongMetricValue randomLongMetricValue() {
+        return randomLongMetricValueBetween(0, 1_000_000);
+    }
+
+    private LongMetricValue randomLongMetricValueBetween(int low, int high) {
         LongMetric v = new LongMetric();
         for (int i = 0; i < randomIntBetween(5, 10); i++) {
-            v.record(randomIntBetween(0, 1_000_000));
+            v.record(randomIntBetween(low, high));
         }
         return v.getValue();
     }
@@ -330,4 +335,21 @@ private String readJSONFromResource(String fileName) throws IOException {
             return new String(inputStream.readAllBytes(), StandardCharsets.UTF_8);
         }
     }
+
+    public void testRanges() throws IOException {
+        var value1 = randomLongMetricValueBetween(1_000_000, 10_000_000);
+        var count1 = value1.count();
+        var max1 = value1.max();
+        var output = new BytesStreamOutput();
+        value1.writeTo(output);
+        var value1Read = LongMetricValue.fromStream(output.bytes().streamInput());
+        var value2 = randomLongMetricValueBetween(0, 100);
+        var count2 = value2.count();
+        output = new BytesStreamOutput();
+        value2.writeTo(output);
+        var value2Read = LongMetricValue.fromStream(output.bytes().streamInput());
+        value2Read.add(value1Read);
+        assertThat(value2Read.count(), equalTo(count1 + count2));
+        assertThat(value2Read.max(), equalTo(max1));
+    }
 }
diff --git a/...l/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/UnigramTokenizer.java b/...l/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/UnigramTokenizer.java
@@ -367,8 +367,10 @@ List<DelimitedToken.Encoded> tokenize(CharSequence inputSequence, IntToIntFuncti
                         new DelimitedToken.Encoded(
                             Strings.format("<0x%02X>", bytes[i]),
                             pieces[i],
+                            // even though we are changing the number of characters in the output, we don't
+                            // need to change the offsets. The offsets refer to the input characters
                             offsetCorrection.apply(node.startsAtCharPos),
-                            offsetCorrection.apply(startsAtBytes + i)
+                            offsetCorrection.apply(endsAtChars)
                         )
                     );
                 }