From d0b136069a76c7e11f92d5c2fca50a9810d5b6bb Mon Sep 17 00:00:00 2001 From: Jack Smith Date: Thu, 5 Nov 2020 09:50:58 -0500 Subject: [PATCH 01/11] Checkout ChunkResolver from eager-execution-a branch --- .../hubspot/jinjava/util/ChunkResolver.java | 380 ++++++++++++++++++ .../jinjava/util/ChunkResolverTest.java | 254 ++++++++++++ 2 files changed, 634 insertions(+) create mode 100644 src/main/java/com/hubspot/jinjava/util/ChunkResolver.java create mode 100644 src/test/java/com/hubspot/jinjava/util/ChunkResolverTest.java diff --git a/src/main/java/com/hubspot/jinjava/util/ChunkResolver.java b/src/main/java/com/hubspot/jinjava/util/ChunkResolver.java new file mode 100644 index 000000000..4957d1acd --- /dev/null +++ b/src/main/java/com/hubspot/jinjava/util/ChunkResolver.java @@ -0,0 +1,380 @@ +package com.hubspot.jinjava.util; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.module.SimpleModule; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.hubspot.jinjava.interpret.DeferredValueException; +import com.hubspot.jinjava.interpret.JinjavaInterpreter; +import com.hubspot.jinjava.interpret.UnknownTokenException; +import com.hubspot.jinjava.objects.date.JsonPyishDateSerializer; +import com.hubspot.jinjava.objects.date.PyishDate; +import com.hubspot.jinjava.tree.parse.Token; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import org.apache.commons.lang3.StringUtils; + +/** + * This class takes a string and resolves it in chunks. This allows for + * strings with deferred values within them to be partially resolved, as much + * as they can be with a deferred value. + * E.g with foo=3, bar=2: + * "range(0,foo)[-1] + deferred/bar" -> "2 + deferred/2" + * This class is not thread-safe. Do not reuse between threads. + */ +public class ChunkResolver { + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper() + .registerModule( + new SimpleModule().addSerializer(PyishDate.class, new JsonPyishDateSerializer()) + ); + + private static final Set RESERVED_KEYWORDS = ImmutableSet.of( + "and", + "block", + "cycle", + "elif", + "else", + "endblock", + "endfilter", + "endfor", + "endif", + "endmacro", + "endraw", + "endtrans", + "extends", + "filter", + "for", + "if", + "in", + "include", + "is", + "macro", + "not", + "or", + "pluralize", + "print", + "raw", + "recursive", + "set", + "trans", + "call", + "endcall", + "__macros__" + ); + + // ( -> ) + // { -> } + // [ -> ] + private static final Map CHUNK_LEVEL_MARKER_MAP = ImmutableMap.of( + '(', + ')', + '{', + '}', + '[', + ']' + ); + + private final char[] value; + private final int length; + private final Token token; + private final JinjavaInterpreter interpreter; + private final Set deferredWords; + + private boolean useMiniChunks = true; + private int nextPos = 0; + private char prevChar = 0; + private boolean inQuote = false; + private char quoteChar = 0; + + public ChunkResolver(String s, Token token, JinjavaInterpreter interpreter) { + value = s.toCharArray(); + length = value.length; + this.token = token; + this.interpreter = interpreter; + deferredWords = new HashSet<>(); + } + + /** + * use Comma as token/mini chunk split or not true use it; false don't use it. + * + * @param onOrOff + * flag to indicate whether or not to split on commas + * @return this instance for method chaining + */ + public ChunkResolver useMiniChunks(boolean onOrOff) { + useMiniChunks = onOrOff; + return this; + } + + /** + * @return Any deferred words that were encountered. + */ + public Set getDeferredWords() { + return deferredWords; + } + + /** + * Chunkify and resolve variables and expressions within the string. + * Tokens are resolved within "chunks" where a chunk is surrounded by a markers + * of {}, [], (). The contents inside of a chunk are split by whitespace + * and/or comma, and these "tokens" resolved individually. + * + * The main chunk itself does not get resolved. + * e.g. + * `false || (foo), 'bar'` -> `true, 'bar'` + * `[(foo == bar), deferred, bar]` -> `[true,deferred,'hello']` + * @return String with chunk layers within it being partially or fully resolved. + */ + public String resolveChunks() { + nextPos = 0; + boolean isHideInterpreterErrorsStart = interpreter + .getContext() + .isHideInterpreterErrors(); + try { + interpreter.getContext().setHideInterpreterErrors(true); + return String.join("", getChunk(null)); + } finally { + interpreter.getContext().setHideInterpreterErrors(isHideInterpreterErrorsStart); + } + } + + /** + * Chunkify and resolve variables and expressions within the string. + * Rather than concatenating the chunks, they are split by mini-chunks, + * with the comma splitter ommitted from the list of results. + * Therefore an expression of "1, 1 + 1, 1 + range(deferred)" becomes a List of ["1", "2", "1 + range(deferred)"]. + * Tokens are resolved within "chunks" where a chunk is surrounded by a markers + * of {}, [], (). The contents inside of a chunk are split by whitespace + * and/or comma, and these "tokens" resolved individually. + * + * The main chunk itself does not get resolved. + * e.g. + * `false || (foo), 'bar'` -> `true, 'bar'` + * `[(foo == bar), deferred, bar]` -> `[true,deferred,'hello']` + * @return String with chunk layers within it being partially or fully resolved. + */ + public List splitChunks() { + nextPos = 0; + boolean isHideInterpreterErrorsStart = interpreter + .getContext() + .isHideInterpreterErrors(); + try { + interpreter.getContext().setHideInterpreterErrors(true); + List miniChunks = getChunk(null); + if (useMiniChunks) { + return miniChunks + .stream() + .filter(s -> s.length() > 1 || !isMiniChunkSplitter(s.charAt(0))) + .collect(Collectors.toList()); + } else { + return Collections.singletonList(String.join("", miniChunks)); + } + } finally { + interpreter.getContext().setHideInterpreterErrors(isHideInterpreterErrorsStart); + } + } + + /** + * e.g. `[0, foo + bar]`: + * `0, foo + bar` is a chunk + * `0` and `foo + bar` are mini chunks + * `0`, `,`, ` `, `foo`, ` `, `+`, ` `, and `bar` are the tokens + * @param chunkLevelMarker the marker `(`, `[`, `{` that started this chunk + * @return the resolved chunk + */ + private List getChunk(Character chunkLevelMarker) { + List chunks = new ArrayList<>(); + // Mini chunks are split by commas. + StringBuilder miniChunkBuilder = new StringBuilder(); + StringBuilder tokenBuilder = new StringBuilder(); + while (nextPos < length) { + char c = value[nextPos++]; + if (inQuote) { + if (c == quoteChar && prevChar != '\\') { + inQuote = false; + } + } else if ((c == '\'' || c == '"') && prevChar != '\\') { + inQuote = true; + quoteChar = c; + } else if ( + chunkLevelMarker != null && CHUNK_LEVEL_MARKER_MAP.get(chunkLevelMarker) == c + ) { + prevChar = c; + break; + } else if (CHUNK_LEVEL_MARKER_MAP.containsKey(c)) { + prevChar = c; + tokenBuilder.append(c); + tokenBuilder.append(resolveChunk(String.join("", getChunk(c)))); + tokenBuilder.append(prevChar); + continue; + } else if (isTokenSplitter(c)) { + prevChar = c; + + miniChunkBuilder.append(resolveToken(tokenBuilder.toString())); + tokenBuilder = new StringBuilder(); + if (isMiniChunkSplitter(c)) { + chunks.add(resolveChunk(miniChunkBuilder.toString())); + chunks.add(String.valueOf(c)); + miniChunkBuilder = new StringBuilder(); + } else { + miniChunkBuilder.append(c); + } + continue; + } + prevChar = c; + tokenBuilder.append(c); + } + miniChunkBuilder.append(resolveToken(tokenBuilder.toString())); + chunks.add(resolveChunk(miniChunkBuilder.toString())); + return chunks; + } + + private boolean isTokenSplitter(char c) { + return (!Character.isLetterOrDigit(c) && c != '_' && c != '.'); + } + + private boolean isMiniChunkSplitter(char c) { + return useMiniChunks && c == ','; + } + + private String resolveToken(String token) { + if (StringUtils.isBlank(token)) { + return ""; + } + try { + String resolvedToken; + if (WhitespaceUtils.isQuoted(token) || RESERVED_KEYWORDS.contains(token)) { + resolvedToken = token; + } else { + Object val = interpreter.retraceVariable( + token, + this.token.getLineNumber(), + this.token.getStartPosition() + ); + if (val == null) { + try { + val = interpreter.resolveELExpression(token, this.token.getLineNumber()); + } catch (UnknownTokenException e) { + // val is still null + } + } + if (val == null) { + resolvedToken = token; + } else { + resolvedToken = getValueAsJinjavaString(val); + } + } + return resolvedToken.trim(); + } catch (DeferredValueException | JsonProcessingException e) { + deferredWords.addAll(findDeferredWords(token)); + return token.trim(); + } + } + + // Try resolving the chunk/mini chunk as an ELExpression + private String resolveChunk(String chunk) { + if (StringUtils.isBlank(chunk)) { + return ""; + } else if (RESERVED_KEYWORDS.contains(chunk)) { + return chunk; + } + try { + String resolvedChunk; + Object val = interpreter.resolveELExpression(chunk, token.getLineNumber()); + if (val == null) { + resolvedChunk = chunk; + } else { + resolvedChunk = getValueAsJinjavaString(val); + } + return resolvedChunk.trim(); + } catch (Exception e) { + deferredWords.addAll(findDeferredWords(chunk)); + return chunk.trim(); + } + } + + public static String getValueAsJinjavaString(Object val) + throws JsonProcessingException { + return OBJECT_MAPPER + .writeValueAsString(val) + .replaceAll("(? findDeferredWords(String chunk) { + Set words = new HashSet<>(); + char[] value = chunk.toCharArray(); + int prevQuotePos = 0; + int curPos = 0; + char c; + char prevChar = 0; + boolean inQuote = false; + char quoteChar = 0; + while (curPos < chunk.length()) { + c = value[curPos]; + if (inQuote) { + if (c == quoteChar && prevChar != '\\') { + inQuote = false; + prevQuotePos = curPos; + } + } else if ((c == '\'' || c == '"') && prevChar != '\\') { + inQuote = true; + quoteChar = c; + words.addAll(findDeferredWordsInSubstring(chunk, prevQuotePos, curPos)); + } + prevChar = c; + curPos++; + } + words.addAll(findDeferredWordsInSubstring(chunk, prevQuotePos, curPos)); + return words; + } + + // Knowing that there are no quotes between start and end, + // split up the words in `chunk` and return whichever ones can't be resolved. + private Set findDeferredWordsInSubstring(String chunk, int start, int end) { + return Arrays + .stream(chunk.substring(start, end).split("[^\\w.]")) + .filter(StringUtils::isNotBlank) + .filter(w -> shouldBeEvaluated(w, token, interpreter)) + .collect(Collectors.toSet()); + } + + public static boolean shouldBeEvaluated( + String w, + Token token, + JinjavaInterpreter interpreter + ) { + try { + if (RESERVED_KEYWORDS.contains(w)) { + return false; + } + try { + Object val = interpreter.retraceVariable( + w, + token.getLineNumber(), + token.getStartPosition() + ); + if (val != null) { + // It's a variable that must now be deferred + return true; + } + } catch (UnknownTokenException e) { + // val is still null + } + // don't defer numbers, values such as true/false, etc. + return interpreter.resolveELExpression(w, token.getLineNumber()) == null; + } catch (DeferredValueException e) { + return true; + } + } +} diff --git a/src/test/java/com/hubspot/jinjava/util/ChunkResolverTest.java b/src/test/java/com/hubspot/jinjava/util/ChunkResolverTest.java new file mode 100644 index 000000000..327d6b4ee --- /dev/null +++ b/src/test/java/com/hubspot/jinjava/util/ChunkResolverTest.java @@ -0,0 +1,254 @@ +/********************************************************************** + Copyright (c) 2014 HubSpot Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + **********************************************************************/ +package com.hubspot.jinjava.util; + +import static org.assertj.core.api.Assertions.assertThat; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.hubspot.jinjava.Jinjava; +import com.hubspot.jinjava.interpret.Context; +import com.hubspot.jinjava.interpret.DeferredValue; +import com.hubspot.jinjava.interpret.JinjavaInterpreter; +import com.hubspot.jinjava.tree.parse.DefaultTokenScannerSymbols; +import com.hubspot.jinjava.tree.parse.TagToken; +import com.hubspot.jinjava.tree.parse.TokenScannerSymbols; +import java.util.List; +import java.util.Map; +import org.junit.After; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; + +public class ChunkResolverTest { + private static final TokenScannerSymbols SYMBOLS = new DefaultTokenScannerSymbols(); + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + private JinjavaInterpreter interpreter; + private TagToken tagToken; + private Context context; + + @Before + public void setUp() { + interpreter = new JinjavaInterpreter(new Jinjava().newInterpreter()); + context = interpreter.getContext(); + context.put("deferred", DeferredValue.instance()); + tagToken = new TagToken("{% foo %}", 1, 2, SYMBOLS); + JinjavaInterpreter.pushCurrent(interpreter); + } + + @After + public void cleanup() { + JinjavaInterpreter.popCurrent(); + } + + private ChunkResolver makeChunkResolver(String string) { + return new ChunkResolver(string, tagToken, interpreter).useMiniChunks(true); + } + + @Test + public void itResolvesDeferredBoolean() { + context.put("foo", "foo_val"); + ChunkResolver chunkResolver = makeChunkResolver("(111 == 112) || (foo == deferred)"); + String partiallyResolved = chunkResolver.resolveChunks(); + assertThat(partiallyResolved).isEqualTo("false || ('foo_val' == deferred)"); + assertThat(chunkResolver.getDeferredWords()).containsExactly("deferred"); + + context.put("deferred", "foo_val"); + assertThat(makeChunkResolver(partiallyResolved).resolveChunks()).isEqualTo("true"); + assertThat(interpreter.resolveELExpression(partiallyResolved, 1)).isEqualTo(true); + } + + @Test + public void itResolvesDeferredList() { + context.put("foo", "foo_val"); + context.put("bar", "bar_val"); + ChunkResolver chunkResolver = makeChunkResolver("[foo == bar, deferred, bar]"); + assertThat(chunkResolver.resolveChunks()).isEqualTo("[false,deferred,'bar_val']"); + assertThat(chunkResolver.getDeferredWords()).containsExactlyInAnyOrder("deferred"); + context.put("bar", "foo_val"); + assertThat(chunkResolver.resolveChunks()).isEqualTo("[true,deferred,'foo_val']"); + } + + @Test + public void itResolvesSimpleBoolean() { + context.put("foo", true); + ChunkResolver chunkResolver = makeChunkResolver("false || (foo), 'bar'"); + String partiallyResolved = chunkResolver.resolveChunks(); + assertThat(partiallyResolved).isEqualTo("true,'bar'"); + assertThat(chunkResolver.getDeferredWords()).isEmpty(); + } + + @Test + @SuppressWarnings("unchecked") + public void itResolvesRange() { + ChunkResolver chunkResolver = makeChunkResolver("range(0,2)"); + String partiallyResolved = chunkResolver.resolveChunks(); + assertThat(partiallyResolved).isEqualTo("[0,1]"); + assertThat(chunkResolver.getDeferredWords()).isEmpty(); + // I don't know why this is a list of longs? + assertThat((List) interpreter.resolveELExpression(partiallyResolved, 1)) + .contains(0L, 1L); + } + + @Test + @SuppressWarnings("unchecked") + public void itResolvesDeferredRange() throws Exception { + List expectedList = ImmutableList.of(1, 2, 3); + context.put("foo", 1); + context.put("bar", 3); + ChunkResolver chunkResolver = makeChunkResolver("range(deferred, foo + bar)"); + String partiallyResolved = chunkResolver.resolveChunks(); + assertThat(partiallyResolved).isEqualTo("range(deferred,4)"); + assertThat(chunkResolver.getDeferredWords()) + .containsExactlyInAnyOrder("deferred", "range"); + + context.put("deferred", 1); + assertThat(makeChunkResolver(partiallyResolved).resolveChunks()) + .isEqualTo(OBJECT_MAPPER.writeValueAsString(expectedList)); + // But this is a list of integers + assertThat((List) interpreter.resolveELExpression(partiallyResolved, 1)) + .isEqualTo(expectedList); + } + + @Test + @SuppressWarnings("unchecked") + public void itResolvesDictionary() { + Map dict = ImmutableMap.of("foo", "one", "bar", 2L); + context.put("the_dictionary", dict); + + ChunkResolver chunkResolver = makeChunkResolver("[the_dictionary, 1]"); + String partiallyResolved = chunkResolver.resolveChunks(); + assertThat(chunkResolver.getDeferredWords()).isEmpty(); + assertThat(interpreter.resolveELExpression(partiallyResolved, 1)) + .isEqualTo(ImmutableList.of(dict, 1L)); + } + + @Test + public void itResolvesNested() { + context.put("foo", 1); + context.put("bar", 3); + ChunkResolver chunkResolver = makeChunkResolver( + "[foo, range(deferred, bar), range(foo, bar)][0:2]" + ); + String partiallyResolved = chunkResolver.resolveChunks(); + assertThat(partiallyResolved).isEqualTo("[1,range(deferred,3),[1,2]][0:2]"); + assertThat(chunkResolver.getDeferredWords()) + .containsExactlyInAnyOrder("deferred", "range"); + + context.put("deferred", 2); + assertThat(makeChunkResolver(partiallyResolved).resolveChunks()).isEqualTo("[1,[2]]"); + assertThat(interpreter.resolveELExpression(partiallyResolved, 1)) + .isEqualTo(ImmutableList.of(1L, ImmutableList.of(2))); + } + + @Test + public void itSplitsOnNonWords() { + context.put("foo", 1); + context.put("bar", 4); + ChunkResolver chunkResolver = makeChunkResolver("range(0,foo) + -deferred/bar"); + String partiallyResolved = chunkResolver.resolveChunks(); + assertThat(partiallyResolved).isEqualTo("[0] + -deferred/4"); + assertThat(chunkResolver.getDeferredWords()).containsExactly("deferred"); + + context.put("deferred", 2); + assertThat(makeChunkResolver(partiallyResolved).resolveChunks()) + .isEqualTo("[0,-0.5]"); + assertThat(interpreter.resolveELExpression(partiallyResolved, 1)) + .isEqualTo(ImmutableList.of(0L, -0.5)); + } + + @Test + public void itSplitsAndIndexesOnNonWords() { + context.put("foo", 3); + context.put("bar", 4); + ChunkResolver chunkResolver = makeChunkResolver("range(-2,foo)[-1] + -deferred/bar"); + String partiallyResolved = chunkResolver.resolveChunks(); + assertThat(partiallyResolved).isEqualTo("2 + -deferred/4"); + assertThat(chunkResolver.getDeferredWords()).containsExactly("deferred"); + + context.put("deferred", 2); + assertThat(makeChunkResolver(partiallyResolved).resolveChunks()).isEqualTo("1.5"); + assertThat(interpreter.resolveELExpression(partiallyResolved, 1)).isEqualTo(1.5); + } + + @Test + @Ignore + // TODO support order of operations + public void itSupportsOrderOfOperations() { + ChunkResolver chunkResolver = makeChunkResolver("[0,1]|reverse + deferred"); + String partiallyResolved = chunkResolver.resolveChunks(); + assertThat(partiallyResolved).isEqualTo("[1,0] + deferred"); + assertThat(chunkResolver.getDeferredWords()).containsExactly("deferred"); + + context.put("deferred", 2); + assertThat(makeChunkResolver(partiallyResolved).resolveChunks()).isEqualTo("[1,0,2]"); + assertThat(interpreter.resolveELExpression(partiallyResolved, 1)) + .isEqualTo(ImmutableList.of(1L, 0L, 2L)); + } + + @Test + public void itCatchesDeferredVariables() { + ChunkResolver chunkResolver = makeChunkResolver("range(0, deferred)"); + String partiallyResolved = chunkResolver.resolveChunks(); + assertThat(partiallyResolved).isEqualTo("range(0,deferred)"); + // Since the range function is deferred, it is added to deferredWords. + assertThat(chunkResolver.getDeferredWords()) + .containsExactlyInAnyOrder("range", "deferred"); + } + + @Test + public void itSplitsChunks() { + ChunkResolver chunkResolver = makeChunkResolver("1, 1 + 1, 1 + 2"); + List miniChunks = chunkResolver.splitChunks(); + assertThat(miniChunks).containsExactly("1", "2", "3"); + assertThat(chunkResolver.getDeferredWords()).isEmpty(); + } + + @Test + public void itProperlySplitsMultiLevelChunks() { + ChunkResolver chunkResolver = makeChunkResolver( + "[5,7], 1 + 1, 1 + range(0 + 1, deferred)" + ); + List miniChunks = chunkResolver.splitChunks(); + assertThat(miniChunks).containsExactly("[5,7]", "2", "1 + range(1,deferred)"); + assertThat(chunkResolver.getDeferredWords()) + .containsExactlyInAnyOrder("range", "deferred"); + } + + @Test + public void itRespectsNoMiniChunksFlag() { + context.put("foo", 9); + ChunkResolver chunkResolver = makeChunkResolver("foo, 1 + 1, 1 + 2") + .useMiniChunks(false); + List miniChunks = chunkResolver.splitChunks(); + assertThat(miniChunks).containsExactly("9, 1 + 1, 1 + 2"); + assertThat(chunkResolver.getDeferredWords()).isEmpty(); + } + + @Test + public void itDoesntDeferReservedWords() { + context.put("foo", 0); + ChunkResolver chunkResolver = makeChunkResolver( + "[(foo > 1) or deferred, deferred].append(1)" + ); + String partiallyResolved = chunkResolver.resolveChunks(); + assertThat(partiallyResolved).isEqualTo("[false or deferred,deferred].append(1)"); + assertThat(chunkResolver.getDeferredWords()).doesNotContain("false", "or"); + assertThat(chunkResolver.getDeferredWords()).contains("deferred", ".append"); + } +} From 09803ac538070002e4ab656c6b6095871cd6b47c Mon Sep 17 00:00:00 2001 From: Jack Smith Date: Thu, 5 Nov 2020 10:40:55 -0500 Subject: [PATCH 02/11] Add pyish date serializer to json --- .../objects/date/JsonPyishDateSerializer.java | 24 +++++++++++++++++++ .../jinjava/objects/date/PyishDate.java | 3 ++- 2 files changed, 26 insertions(+), 1 deletion(-) create mode 100644 src/main/java/com/hubspot/jinjava/objects/date/JsonPyishDateSerializer.java diff --git a/src/main/java/com/hubspot/jinjava/objects/date/JsonPyishDateSerializer.java b/src/main/java/com/hubspot/jinjava/objects/date/JsonPyishDateSerializer.java new file mode 100644 index 000000000..555913314 --- /dev/null +++ b/src/main/java/com/hubspot/jinjava/objects/date/JsonPyishDateSerializer.java @@ -0,0 +1,24 @@ +package com.hubspot.jinjava.objects.date; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.JsonSerializer; +import com.fasterxml.jackson.databind.SerializerProvider; +import java.io.IOException; +import java.time.format.DateTimeFormatter; + +public class JsonPyishDateSerializer extends JsonSerializer { + + @Override + public void serialize( + PyishDate pyishDate, + JsonGenerator jsonGenerator, + SerializerProvider serializerProvider + ) + throws IOException { + jsonGenerator.writeString( + DateTimeFormatter + .ofPattern(PyishDate.PYISH_DATE_FORMAT) + .format(pyishDate.toDateTime()) + ); + } +} diff --git a/src/main/java/com/hubspot/jinjava/objects/date/PyishDate.java b/src/main/java/com/hubspot/jinjava/objects/date/PyishDate.java index 2077811ca..786614797 100644 --- a/src/main/java/com/hubspot/jinjava/objects/date/PyishDate.java +++ b/src/main/java/com/hubspot/jinjava/objects/date/PyishDate.java @@ -19,6 +19,7 @@ */ public final class PyishDate extends Date implements Serializable, PyWrapper { private static final long serialVersionUID = 1L; + public static final String PYISH_DATE_FORMAT = "yyyy-MM-dd HH:mm:ss"; private final ZonedDateTime date; @@ -102,7 +103,7 @@ public ZonedDateTime toDateTime() { @Override public String toString() { - return strftime("yyyy-MM-dd HH:mm:ss"); + return strftime(PYISH_DATE_FORMAT); } @Override From ac5a8a1b54d08e49cfc6e507f4b4d4325481311b Mon Sep 17 00:00:00 2001 From: Jack Smith Date: Thu, 5 Nov 2020 10:44:09 -0500 Subject: [PATCH 03/11] Add hideInterpreterErrors flag --- src/main/java/com/hubspot/jinjava/interpret/Context.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/main/java/com/hubspot/jinjava/interpret/Context.java b/src/main/java/com/hubspot/jinjava/interpret/Context.java index 0afa5d4fb..4cef6b493 100644 --- a/src/main/java/com/hubspot/jinjava/interpret/Context.java +++ b/src/main/java/com/hubspot/jinjava/interpret/Context.java @@ -93,6 +93,7 @@ public enum Library { private final Stack renderStack = new Stack<>(); private boolean validationMode = false; + private boolean isHideInterpreterErrors = false; public Context() { this(null, null, null); @@ -526,4 +527,12 @@ public void addDependencies(SetMultimap dependencies) { public SetMultimap getDependencies() { return this.dependencies; } + + public boolean isHideInterpreterErrors() { + return isHideInterpreterErrors; + } + + public void setHideInterpreterErrors(boolean hideInterpreterErrors) { + isHideInterpreterErrors = hideInterpreterErrors; + } } From ccd1501a7791fd0d841594f8b9cedf5a94aa2f48 Mon Sep 17 00:00:00 2001 From: Jack Smith Date: Thu, 5 Nov 2020 10:44:19 -0500 Subject: [PATCH 04/11] Add dict resolve test --- .../com/hubspot/jinjava/util/ChunkResolverTest.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/test/java/com/hubspot/jinjava/util/ChunkResolverTest.java b/src/test/java/com/hubspot/jinjava/util/ChunkResolverTest.java index 327d6b4ee..f181efa9a 100644 --- a/src/test/java/com/hubspot/jinjava/util/ChunkResolverTest.java +++ b/src/test/java/com/hubspot/jinjava/util/ChunkResolverTest.java @@ -24,6 +24,7 @@ import com.hubspot.jinjava.interpret.Context; import com.hubspot.jinjava.interpret.DeferredValue; import com.hubspot.jinjava.interpret.JinjavaInterpreter; +import com.hubspot.jinjava.objects.collections.PyMap; import com.hubspot.jinjava.tree.parse.DefaultTokenScannerSymbols; import com.hubspot.jinjava.tree.parse.TagToken; import com.hubspot.jinjava.tree.parse.TokenScannerSymbols; @@ -251,4 +252,14 @@ public void itDoesntDeferReservedWords() { assertThat(chunkResolver.getDeferredWords()).doesNotContain("false", "or"); assertThat(chunkResolver.getDeferredWords()).contains("deferred", ".append"); } + + @Test + public void itEvaluatesDict() { + context.put("foo", new PyMap(ImmutableMap.of("bar", 99))); + ChunkResolver chunkResolver = makeChunkResolver("foo.bar == deferred.bar"); + String partiallyResolved = chunkResolver.resolveChunks(); + assertThat(partiallyResolved).isEqualTo("99 == deferred.bar"); + assertThat(chunkResolver.getDeferredWords()) + .containsExactlyInAnyOrder("deferred.bar"); + } } From c309fdb51e441575e910c47d9c03e2ac12f6a68f Mon Sep 17 00:00:00 2001 From: Jack Smith Date: Thu, 5 Nov 2020 13:10:27 -0500 Subject: [PATCH 05/11] Remove unnecessary useMiniChunks flag --- .../hubspot/jinjava/util/ChunkResolver.java | 28 ++++--------------- .../jinjava/util/ChunkResolverTest.java | 12 +------- 2 files changed, 6 insertions(+), 34 deletions(-) diff --git a/src/main/java/com/hubspot/jinjava/util/ChunkResolver.java b/src/main/java/com/hubspot/jinjava/util/ChunkResolver.java index 4957d1acd..444a725fd 100644 --- a/src/main/java/com/hubspot/jinjava/util/ChunkResolver.java +++ b/src/main/java/com/hubspot/jinjava/util/ChunkResolver.java @@ -13,7 +13,6 @@ import com.hubspot.jinjava.tree.parse.Token; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -87,7 +86,6 @@ public class ChunkResolver { private final JinjavaInterpreter interpreter; private final Set deferredWords; - private boolean useMiniChunks = true; private int nextPos = 0; private char prevChar = 0; private boolean inQuote = false; @@ -101,18 +99,6 @@ public ChunkResolver(String s, Token token, JinjavaInterpreter interpreter) { deferredWords = new HashSet<>(); } - /** - * use Comma as token/mini chunk split or not true use it; false don't use it. - * - * @param onOrOff - * flag to indicate whether or not to split on commas - * @return this instance for method chaining - */ - public ChunkResolver useMiniChunks(boolean onOrOff) { - useMiniChunks = onOrOff; - return this; - } - /** * @return Any deferred words that were encountered. */ @@ -168,14 +154,10 @@ public List splitChunks() { try { interpreter.getContext().setHideInterpreterErrors(true); List miniChunks = getChunk(null); - if (useMiniChunks) { - return miniChunks - .stream() - .filter(s -> s.length() > 1 || !isMiniChunkSplitter(s.charAt(0))) - .collect(Collectors.toList()); - } else { - return Collections.singletonList(String.join("", miniChunks)); - } + return miniChunks + .stream() + .filter(s -> s.length() > 1 || !isMiniChunkSplitter(s.charAt(0))) + .collect(Collectors.toList()); } finally { interpreter.getContext().setHideInterpreterErrors(isHideInterpreterErrorsStart); } @@ -241,7 +223,7 @@ private boolean isTokenSplitter(char c) { } private boolean isMiniChunkSplitter(char c) { - return useMiniChunks && c == ','; + return c == ','; } private String resolveToken(String token) { diff --git a/src/test/java/com/hubspot/jinjava/util/ChunkResolverTest.java b/src/test/java/com/hubspot/jinjava/util/ChunkResolverTest.java index f181efa9a..403d5ec4c 100644 --- a/src/test/java/com/hubspot/jinjava/util/ChunkResolverTest.java +++ b/src/test/java/com/hubspot/jinjava/util/ChunkResolverTest.java @@ -58,7 +58,7 @@ public void cleanup() { } private ChunkResolver makeChunkResolver(String string) { - return new ChunkResolver(string, tagToken, interpreter).useMiniChunks(true); + return new ChunkResolver(string, tagToken, interpreter); } @Test @@ -231,16 +231,6 @@ public void itProperlySplitsMultiLevelChunks() { .containsExactlyInAnyOrder("range", "deferred"); } - @Test - public void itRespectsNoMiniChunksFlag() { - context.put("foo", 9); - ChunkResolver chunkResolver = makeChunkResolver("foo, 1 + 1, 1 + 2") - .useMiniChunks(false); - List miniChunks = chunkResolver.splitChunks(); - assertThat(miniChunks).containsExactly("9, 1 + 1, 1 + 2"); - assertThat(chunkResolver.getDeferredWords()).isEmpty(); - } - @Test public void itDoesntDeferReservedWords() { context.put("foo", 0); From 560961dfada0a6c5cabf82506c2e9236cb93faa0 Mon Sep 17 00:00:00 2001 From: Jack Smith Date: Thu, 5 Nov 2020 13:20:49 -0500 Subject: [PATCH 06/11] Add date serialization test --- .../hubspot/jinjava/util/ChunkResolverTest.java | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/test/java/com/hubspot/jinjava/util/ChunkResolverTest.java b/src/test/java/com/hubspot/jinjava/util/ChunkResolverTest.java index 403d5ec4c..27d431265 100644 --- a/src/test/java/com/hubspot/jinjava/util/ChunkResolverTest.java +++ b/src/test/java/com/hubspot/jinjava/util/ChunkResolverTest.java @@ -25,9 +25,13 @@ import com.hubspot.jinjava.interpret.DeferredValue; import com.hubspot.jinjava.interpret.JinjavaInterpreter; import com.hubspot.jinjava.objects.collections.PyMap; +import com.hubspot.jinjava.objects.date.PyishDate; import com.hubspot.jinjava.tree.parse.DefaultTokenScannerSymbols; import com.hubspot.jinjava.tree.parse.TagToken; import com.hubspot.jinjava.tree.parse.TokenScannerSymbols; +import java.time.Instant; +import java.time.ZoneId; +import java.time.ZonedDateTime; import java.util.List; import java.util.Map; import org.junit.After; @@ -252,4 +256,15 @@ public void itEvaluatesDict() { assertThat(chunkResolver.getDeferredWords()) .containsExactlyInAnyOrder("deferred.bar"); } + + @Test + public void itSerializesDateProperly() { + PyishDate date = new PyishDate( + ZonedDateTime.ofInstant(Instant.ofEpochMilli(1234567890L), ZoneId.systemDefault()) + ); + context.put("date", date); + ChunkResolver chunkResolver = makeChunkResolver("date"); + assertThat(WhitespaceUtils.unquote(chunkResolver.resolveChunks())) + .isEqualTo(date.toString()); + } } From 50063a2af4499e41e92567427baac84cdaec8185 Mon Sep 17 00:00:00 2001 From: Jack Smith Date: Thu, 5 Nov 2020 13:23:02 -0500 Subject: [PATCH 07/11] Remove incorrectly dated license --- .../hubspot/jinjava/util/ChunkResolverTest.java | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/src/test/java/com/hubspot/jinjava/util/ChunkResolverTest.java b/src/test/java/com/hubspot/jinjava/util/ChunkResolverTest.java index 27d431265..ec60cd09c 100644 --- a/src/test/java/com/hubspot/jinjava/util/ChunkResolverTest.java +++ b/src/test/java/com/hubspot/jinjava/util/ChunkResolverTest.java @@ -1,18 +1,3 @@ -/********************************************************************** - Copyright (c) 2014 HubSpot Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - **********************************************************************/ package com.hubspot.jinjava.util; import static org.assertj.core.api.Assertions.assertThat; From 3d50717be34f8879ad0ad78c7fbc65d015d6855f Mon Sep 17 00:00:00 2001 From: Jack Smith Date: Thu, 5 Nov 2020 13:27:41 -0500 Subject: [PATCH 08/11] Update comment for splitChunks() --- .../java/com/hubspot/jinjava/util/ChunkResolver.java | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/src/main/java/com/hubspot/jinjava/util/ChunkResolver.java b/src/main/java/com/hubspot/jinjava/util/ChunkResolver.java index 444a725fd..dd1685e29 100644 --- a/src/main/java/com/hubspot/jinjava/util/ChunkResolver.java +++ b/src/main/java/com/hubspot/jinjava/util/ChunkResolver.java @@ -136,15 +136,8 @@ public String resolveChunks() { * Rather than concatenating the chunks, they are split by mini-chunks, * with the comma splitter ommitted from the list of results. * Therefore an expression of "1, 1 + 1, 1 + range(deferred)" becomes a List of ["1", "2", "1 + range(deferred)"]. - * Tokens are resolved within "chunks" where a chunk is surrounded by a markers - * of {}, [], (). The contents inside of a chunk are split by whitespace - * and/or comma, and these "tokens" resolved individually. * - * The main chunk itself does not get resolved. - * e.g. - * `false || (foo), 'bar'` -> `true, 'bar'` - * `[(foo == bar), deferred, bar]` -> `[true,deferred,'hello']` - * @return String with chunk layers within it being partially or fully resolved. + * @return List of the expression chunk which is split into mini-chunks. */ public List splitChunks() { nextPos = 0; From 83a4ab649314e1000bc85560f61449c47ee4a2c3 Mon Sep 17 00:00:00 2001 From: Jack Smith Date: Fri, 6 Nov 2020 13:37:53 -0500 Subject: [PATCH 09/11] Comment regex replaceAll --- src/main/java/com/hubspot/jinjava/util/ChunkResolver.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/java/com/hubspot/jinjava/util/ChunkResolver.java b/src/main/java/com/hubspot/jinjava/util/ChunkResolver.java index dd1685e29..2cd9f13e6 100644 --- a/src/main/java/com/hubspot/jinjava/util/ChunkResolver.java +++ b/src/main/java/com/hubspot/jinjava/util/ChunkResolver.java @@ -279,8 +279,12 @@ public static String getValueAsJinjavaString(Object val) throws JsonProcessingException { return OBJECT_MAPPER .writeValueAsString(val) + // Replace `\n` with a newline character .replaceAll("(? `"foo"` rather than `\"foo\"` .replaceAll("(? Date: Fri, 13 Nov 2020 16:20:14 -0500 Subject: [PATCH 10/11] Remove unnecessary reserved words --- .../hubspot/jinjava/util/ChunkResolver.java | 21 ------------------- 1 file changed, 21 deletions(-) diff --git a/src/main/java/com/hubspot/jinjava/util/ChunkResolver.java b/src/main/java/com/hubspot/jinjava/util/ChunkResolver.java index 2cd9f13e6..f5688ed42 100644 --- a/src/main/java/com/hubspot/jinjava/util/ChunkResolver.java +++ b/src/main/java/com/hubspot/jinjava/util/ChunkResolver.java @@ -36,35 +36,14 @@ public class ChunkResolver { private static final Set RESERVED_KEYWORDS = ImmutableSet.of( "and", - "block", - "cycle", - "elif", - "else", - "endblock", - "endfilter", - "endfor", - "endif", - "endmacro", - "endraw", - "endtrans", - "extends", "filter", - "for", - "if", "in", - "include", "is", - "macro", "not", "or", "pluralize", - "print", - "raw", "recursive", - "set", "trans", - "call", - "endcall", "__macros__" ); From ad4b922a6087fb27bf1e0a5260906296d1f4a528 Mon Sep 17 00:00:00 2001 From: Jack Smith Date: Tue, 17 Nov 2020 12:08:33 -0500 Subject: [PATCH 11/11] Change isHide to getHide --- src/main/java/com/hubspot/jinjava/interpret/Context.java | 8 ++++---- src/main/java/com/hubspot/jinjava/util/ChunkResolver.java | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/hubspot/jinjava/interpret/Context.java b/src/main/java/com/hubspot/jinjava/interpret/Context.java index 4cef6b493..2fad8dc9d 100644 --- a/src/main/java/com/hubspot/jinjava/interpret/Context.java +++ b/src/main/java/com/hubspot/jinjava/interpret/Context.java @@ -93,7 +93,7 @@ public enum Library { private final Stack renderStack = new Stack<>(); private boolean validationMode = false; - private boolean isHideInterpreterErrors = false; + private boolean hideInterpreterErrors = false; public Context() { this(null, null, null); @@ -528,11 +528,11 @@ public SetMultimap getDependencies() { return this.dependencies; } - public boolean isHideInterpreterErrors() { - return isHideInterpreterErrors; + public boolean getHideInterpreterErrors() { + return hideInterpreterErrors; } public void setHideInterpreterErrors(boolean hideInterpreterErrors) { - isHideInterpreterErrors = hideInterpreterErrors; + this.hideInterpreterErrors = hideInterpreterErrors; } } diff --git a/src/main/java/com/hubspot/jinjava/util/ChunkResolver.java b/src/main/java/com/hubspot/jinjava/util/ChunkResolver.java index f5688ed42..1dff439eb 100644 --- a/src/main/java/com/hubspot/jinjava/util/ChunkResolver.java +++ b/src/main/java/com/hubspot/jinjava/util/ChunkResolver.java @@ -101,7 +101,7 @@ public String resolveChunks() { nextPos = 0; boolean isHideInterpreterErrorsStart = interpreter .getContext() - .isHideInterpreterErrors(); + .getHideInterpreterErrors(); try { interpreter.getContext().setHideInterpreterErrors(true); return String.join("", getChunk(null)); @@ -122,7 +122,7 @@ public List splitChunks() { nextPos = 0; boolean isHideInterpreterErrorsStart = interpreter .getContext() - .isHideInterpreterErrors(); + .getHideInterpreterErrors(); try { interpreter.getContext().setHideInterpreterErrors(true); List miniChunks = getChunk(null);