New Monitor server implementation #5012
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,103 @@ | ||
| /* | ||
| * Licensed to the Apache Software Foundation (ASF) under one | ||
| * or more contributor license agreements. See the NOTICE file | ||
| * distributed with this work for additional information | ||
| * regarding copyright ownership. The ASF licenses this file | ||
| * to you under the Apache License, Version 2.0 (the | ||
| * "License"); you may not use this file except in compliance | ||
| * with the License. You may obtain a copy of the License at | ||
| * | ||
| * https://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, | ||
| * software distributed under the License is distributed on an | ||
| * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | ||
| * KIND, either express or implied. See the License for the | ||
| * specific language governing permissions and limitations | ||
| * under the License. | ||
| */ | ||
| // automatically generated by the FlatBuffers compiler, do not modify | ||
|
|
||
| package org.apache.accumulo.core.metrics.flatbuffers; | ||
|
|
||
| import com.google.flatbuffers.BaseVector; | ||
| import com.google.flatbuffers.BooleanVector; | ||
| import com.google.flatbuffers.ByteVector; | ||
| import com.google.flatbuffers.Constants; | ||
| import com.google.flatbuffers.DoubleVector; | ||
| import com.google.flatbuffers.FlatBufferBuilder; | ||
| import com.google.flatbuffers.FloatVector; | ||
| import com.google.flatbuffers.IntVector; | ||
| import com.google.flatbuffers.LongVector; | ||
| import com.google.flatbuffers.ShortVector; | ||
| import com.google.flatbuffers.StringVector; | ||
| import com.google.flatbuffers.Struct; | ||
| import com.google.flatbuffers.Table; | ||
| import com.google.flatbuffers.UnionVector; | ||
| import java.nio.ByteBuffer; | ||
| import java.nio.ByteOrder; | ||
|
|
||
| @SuppressWarnings("unused") | ||
| public final class FMetric extends Table { | ||
| public static void ValidateVersion() { Constants.FLATBUFFERS_24_3_25(); } | ||
| public static FMetric getRootAsFMetric(ByteBuffer _bb) { return getRootAsFMetric(_bb, new FMetric()); } | ||
| public static FMetric getRootAsFMetric(ByteBuffer _bb, FMetric obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } | ||
| public void __init(int _i, ByteBuffer _bb) { __reset(_i, _bb); } | ||
| public FMetric __assign(int _i, ByteBuffer _bb) { __init(_i, _bb); return this; } | ||
|
|
||
| public String name() { int o = __offset(4); return o != 0 ? __string(o + bb_pos) : null; } | ||
| public ByteBuffer nameAsByteBuffer() { return __vector_as_bytebuffer(4, 1); } | ||
| public ByteBuffer nameInByteBuffer(ByteBuffer _bb) { return __vector_in_bytebuffer(_bb, 4, 1); } | ||
| public String type() { int o = __offset(6); return o != 0 ? __string(o + bb_pos) : null; } | ||
| public ByteBuffer typeAsByteBuffer() { return __vector_as_bytebuffer(6, 1); } | ||
| public ByteBuffer typeInByteBuffer(ByteBuffer _bb) { return __vector_in_bytebuffer(_bb, 6, 1); } | ||
| public org.apache.accumulo.core.metrics.flatbuffers.FTag tags(int j) { return tags(new org.apache.accumulo.core.metrics.flatbuffers.FTag(), j); } | ||
| public org.apache.accumulo.core.metrics.flatbuffers.FTag tags(org.apache.accumulo.core.metrics.flatbuffers.FTag obj, int j) { int o = __offset(8); return o != 0 ? obj.__assign(__indirect(__vector(o) + j * 4), bb) : null; } | ||
| public int tagsLength() { int o = __offset(8); return o != 0 ? __vector_len(o) : 0; } | ||
| public org.apache.accumulo.core.metrics.flatbuffers.FTag.Vector tagsVector() { return tagsVector(new org.apache.accumulo.core.metrics.flatbuffers.FTag.Vector()); } | ||
| public org.apache.accumulo.core.metrics.flatbuffers.FTag.Vector tagsVector(org.apache.accumulo.core.metrics.flatbuffers.FTag.Vector obj) { int o = __offset(8); return o != 0 ? obj.__assign(__vector(o), 4, bb) : null; } | ||
| public double dvalue() { int o = __offset(10); return o != 0 ? bb.getDouble(o + bb_pos) : 0.0; } | ||
| public int ivalue() { int o = __offset(12); return o != 0 ? bb.getInt(o + bb_pos) : 0; } | ||
| public long lvalue() { int o = __offset(14); return o != 0 ? bb.getLong(o + bb_pos) : 0L; } | ||
|
|
||
| public static int createFMetric(FlatBufferBuilder builder, | ||
| int nameOffset, | ||
| int typeOffset, | ||
| int tagsOffset, | ||
| double dvalue, | ||
| int ivalue, | ||
| long lvalue) { | ||
| builder.startTable(6); | ||
| FMetric.addLvalue(builder, lvalue); | ||
| FMetric.addDvalue(builder, dvalue); | ||
| FMetric.addIvalue(builder, ivalue); | ||
| FMetric.addTags(builder, tagsOffset); | ||
| FMetric.addType(builder, typeOffset); | ||
| FMetric.addName(builder, nameOffset); | ||
| return FMetric.endFMetric(builder); | ||
| } | ||
|
|
||
| public static void startFMetric(FlatBufferBuilder builder) { builder.startTable(6); } | ||
| public static void addName(FlatBufferBuilder builder, int nameOffset) { builder.addOffset(0, nameOffset, 0); } | ||
| public static void addType(FlatBufferBuilder builder, int typeOffset) { builder.addOffset(1, typeOffset, 0); } | ||
| public static void addTags(FlatBufferBuilder builder, int tagsOffset) { builder.addOffset(2, tagsOffset, 0); } | ||
| public static int createTagsVector(FlatBufferBuilder builder, int[] data) { builder.startVector(4, data.length, 4); for (int i = data.length - 1; i >= 0; i--) builder.addOffset(data[i]); return builder.endVector(); } | ||
| public static void startTagsVector(FlatBufferBuilder builder, int numElems) { builder.startVector(4, numElems, 4); } | ||
| public static void addDvalue(FlatBufferBuilder builder, double dvalue) { builder.addDouble(3, dvalue, 0.0); } | ||
| public static void addIvalue(FlatBufferBuilder builder, int ivalue) { builder.addInt(4, ivalue, 0); } | ||
| public static void addLvalue(FlatBufferBuilder builder, long lvalue) { builder.addLong(5, lvalue, 0L); } | ||
| public static int endFMetric(FlatBufferBuilder builder) { | ||
| int o = builder.endTable(); | ||
| return o; | ||
| } | ||
| public static void finishFMetricBuffer(FlatBufferBuilder builder, int offset) { builder.finish(offset); } | ||
| public static void finishSizePrefixedFMetricBuffer(FlatBufferBuilder builder, int offset) { builder.finishSizePrefixed(offset); } | ||
|
|
||
| public static final class Vector extends BaseVector { | ||
| public Vector __assign(int _vector, int _element_size, ByteBuffer _bb) { __reset(_vector, _element_size, _bb); return this; } | ||
|
|
||
| public FMetric get(int j) { return get(new FMetric(), j); } | ||
| public FMetric get(FMetric obj, int j) { return obj.__assign(__indirect(__element(j), bb), bb); } | ||
| } | ||
| } | ||
|
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,79 @@ | ||
| /* | ||
| * Licensed to the Apache Software Foundation (ASF) under one | ||
| * or more contributor license agreements. See the NOTICE file | ||
| * distributed with this work for additional information | ||
| * regarding copyright ownership. The ASF licenses this file | ||
| * to you under the Apache License, Version 2.0 (the | ||
| * "License"); you may not use this file except in compliance | ||
| * with the License. You may obtain a copy of the License at | ||
| * | ||
| * https://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, | ||
| * software distributed under the License is distributed on an | ||
| * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | ||
| * KIND, either express or implied. See the License for the | ||
| * specific language governing permissions and limitations | ||
| * under the License. | ||
| */ | ||
| // automatically generated by the FlatBuffers compiler, do not modify | ||
|
|
||
| package org.apache.accumulo.core.metrics.flatbuffers; | ||
|
|
||
| import com.google.flatbuffers.BaseVector; | ||
| import com.google.flatbuffers.BooleanVector; | ||
| import com.google.flatbuffers.ByteVector; | ||
| import com.google.flatbuffers.Constants; | ||
| import com.google.flatbuffers.DoubleVector; | ||
| import com.google.flatbuffers.FlatBufferBuilder; | ||
| import com.google.flatbuffers.FloatVector; | ||
| import com.google.flatbuffers.IntVector; | ||
| import com.google.flatbuffers.LongVector; | ||
| import com.google.flatbuffers.ShortVector; | ||
| import com.google.flatbuffers.StringVector; | ||
| import com.google.flatbuffers.Struct; | ||
| import com.google.flatbuffers.Table; | ||
| import com.google.flatbuffers.UnionVector; | ||
| import java.nio.ByteBuffer; | ||
| import java.nio.ByteOrder; | ||
|
|
||
| @SuppressWarnings("unused") | ||
| public final class FTag extends Table { | ||
| public static void ValidateVersion() { Constants.FLATBUFFERS_24_3_25(); } | ||
| public static FTag getRootAsFTag(ByteBuffer _bb) { return getRootAsFTag(_bb, new FTag()); } | ||
| public static FTag getRootAsFTag(ByteBuffer _bb, FTag obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__assign(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } | ||
| public void __init(int _i, ByteBuffer _bb) { __reset(_i, _bb); } | ||
| public FTag __assign(int _i, ByteBuffer _bb) { __init(_i, _bb); return this; } | ||
|
|
||
| public String key() { int o = __offset(4); return o != 0 ? __string(o + bb_pos) : null; } | ||
| public ByteBuffer keyAsByteBuffer() { return __vector_as_bytebuffer(4, 1); } | ||
| public ByteBuffer keyInByteBuffer(ByteBuffer _bb) { return __vector_in_bytebuffer(_bb, 4, 1); } | ||
| public String value() { int o = __offset(6); return o != 0 ? __string(o + bb_pos) : null; } | ||
| public ByteBuffer valueAsByteBuffer() { return __vector_as_bytebuffer(6, 1); } | ||
| public ByteBuffer valueInByteBuffer(ByteBuffer _bb) { return __vector_in_bytebuffer(_bb, 6, 1); } | ||
|
|
||
| public static int createFTag(FlatBufferBuilder builder, | ||
| int keyOffset, | ||
| int valueOffset) { | ||
| builder.startTable(2); | ||
| FTag.addValue(builder, valueOffset); | ||
| FTag.addKey(builder, keyOffset); | ||
| return FTag.endFTag(builder); | ||
| } | ||
|
|
||
| public static void startFTag(FlatBufferBuilder builder) { builder.startTable(2); } | ||
| public static void addKey(FlatBufferBuilder builder, int keyOffset) { builder.addOffset(0, keyOffset, 0); } | ||
| public static void addValue(FlatBufferBuilder builder, int valueOffset) { builder.addOffset(1, valueOffset, 0); } | ||
| public static int endFTag(FlatBufferBuilder builder) { | ||
| int o = builder.endTable(); | ||
| return o; | ||
| } | ||
|
|
||
| public static final class Vector extends BaseVector { | ||
| public Vector __assign(int _vector, int _element_size, ByteBuffer _bb) { __reset(_vector, _element_size, _bb); return this; } | ||
|
|
||
| public FTag get(int j) { return get(new FTag(), j); } | ||
| public FTag get(FTag obj, int j) { return obj.__assign(__indirect(__element(j), bb), bb); } | ||
| } | ||
| } | ||
|
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,35 @@ | ||
| /* | ||
| * Licensed to the Apache Software Foundation (ASF) under one | ||
| * or more contributor license agreements. See the NOTICE file | ||
| * distributed with this work for additional information | ||
| * regarding copyright ownership. The ASF licenses this file | ||
| * to you under the Apache License, Version 2.0 (the | ||
| * "License"); you may not use this file except in compliance | ||
| * with the License. You may obtain a copy of the License at | ||
| * | ||
| * https://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, | ||
| * software distributed under the License is distributed on an | ||
| * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | ||
| * KIND, either express or implied. See the License for the | ||
| * specific language governing permissions and limitations | ||
| * under the License. | ||
| */ | ||
| namespace org.apache.accumulo.core.metrics.flatbuffers; | ||
|
|
||
| table FTag { | ||
| key:string; | ||
| value:string; | ||
| } | ||
|
|
||
| table FMetric { | ||
| name:string; | ||
| type:string; | ||
| tags:[FTag]; | ||
| dvalue:double; | ||
| ivalue:int; | ||
| lvalue:long; | ||
| } | ||
|
|
||
| root_type FMetric; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This is a pretty low-quality Maven plugin that hasn't been touched in 4 years. It doesn't publish generated maven plugin documentation to know what to configure, and it installs software (and compiles it) into the user's home directory, instead of following Maven best practices to use the target directory (or at least /tmp).
I am reluctant to buy in to using this plugin for these reasons, and am wondering if the goal of the plugin could be done with flatc on the command-line like we do with thrift, or if it could be achieved without using flatbuffers at all.
Also, why not just use Thrift for the internal type? It seems unnecessary to bring in another library that is essentially serving the same purpose, especially since you're also adding generated thrift types that do serialization in the same PR.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
The reason I'm using Flatbuffers vs Thrift is for the reason I mentioned in my initial comment - memory and time efficiency. IIRC, and I'm on my phone so I can't easily check, Thrift fully deserializes the the binary sent over the wire. Flatbuffers does not, it deserializes what you need when you need it. Deserializing all of the metrics from the servers would create a lot of objects which may potentially go unused if nobody is using the UI.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
You're probably right about how thrift deserializes. It's a safe assumption that if you've made an RPC call, you want to make use of the response, so that's probably what thrift does.
We've also used JSON in many places, which has similar benefits. I wonder what the difference in performance would be if we just used Gson or Jackson to handle the serialization of the objects, rather than introduce another dependency and workflow into the build. It'd be nice to be somewhat consistent with how we do serialization, and it would be nice to avoid the complexity of another IDL build workflow and dependency, especially if the plugin that makes the workflow easier isn't well maintained.
The way the monitor currently avoids unnecessary work is by only pulling stuff from the Manager when needed to serve content to a user request. It rate limits the frequency of the pulling to prevent excessive calls to the Manager. I don't know if it would eliminate the entire benefits of flatbuffers, but I'm thinking we should probably do the same here, because there's no reason for the monitor to constantly poll the whole cluster when nobody is querying any pages. If that alone is enough to eliminate the unnecessary deserialization when nobody's asking for it, then it might make things dramatically simpler.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
You had mentioned the flatbuffers-maven-plugin in an earlier comment and I didn't address it. I have no issue with removing it and using the same mechanism that we use for generating the Thrift code. I added the plugin because it was the easiest way to solve the problem and continue moving forward on this idea.