From 8e72f8cca3c5cfa5abfa3f1d77b68f456e715568 Mon Sep 17 00:00:00 2001 From: "Mark S. Miller" Date: Thu, 27 Apr 2023 17:41:20 -0700 Subject: [PATCH] feat(patterns): pattern-based compression --- packages/patterns/index.js | 1 + packages/patterns/src/patterns/compress.js | 292 ++++++++++++ .../patterns/src/patterns/patternMatchers.js | 444 +++++++++++++++++- packages/patterns/src/patterns/types.js | 23 +- packages/patterns/src/types.js | 35 ++ packages/patterns/test/test-compress.js | 273 +++++++++++ 6 files changed, 1053 insertions(+), 15 deletions(-) create mode 100644 packages/patterns/src/patterns/compress.js create mode 100644 packages/patterns/test/test-compress.js diff --git a/packages/patterns/index.js b/packages/patterns/index.js index b32e6ea809..d93b0483a2 100644 --- a/packages/patterns/index.js +++ b/packages/patterns/index.js @@ -73,6 +73,7 @@ export { getInterfaceGuardPayload, getInterfaceMethodKeys, } from './src/patterns/getGuardPayloads.js'; +export { mustCompress, mustDecompress } from './src/patterns/compress.js'; // eslint-disable-next-line import/export export * from './src/types.js'; diff --git a/packages/patterns/src/patterns/compress.js b/packages/patterns/src/patterns/compress.js new file mode 100644 index 0000000000..4c07f8237f --- /dev/null +++ b/packages/patterns/src/patterns/compress.js @@ -0,0 +1,292 @@ +// @ts-nocheck So many errors that the suppressions hamper readability. +// TODO fix and then turn at-ts-check back on +import { + assertChecker, + makeTagged, + passStyleOf, + recordNames, + recordValues, +} from '@endo/marshal'; +import { + kindOf, + assertPattern, + maybeMatchHelper, + matches, + checkMatches, + mustMatch, +} from './patternMatchers.js'; +import { isKey } from '../keys/checkKey.js'; +import { keyEQ } from '../keys/compareKeys.js'; + +/** @import {Compress, Decompress, MustCompress, MustDecompress} from '../types.js' */ + +const { fromEntries } = Object; +const { Fail, quote: q } = assert; + +const isNonCompressingMatcher = pattern => { + const patternKind = kindOf(pattern); + if (patternKind === undefined) { + return false; + } + const matchHelper = maybeMatchHelper(patternKind); + return matchHelper && matchHelper.compress === undefined; +}; + +/** + * When, for example, all the specimens in a given store match a + * specific pattern, then each of those specimens must contain the same + * literal superstructure as their one shared pattern. Therefore, storing + * that literal superstructure would be redumdant. If `specimen` does + * match `pattern`, then `compress(specimen, pattern)` will return a bindings + * array which is hopefully more compact than `specimen` as a whole, but + * carries all the information from specimen that cannot be derived just + * from knowledge that it matches this `pattern`. + * + * @type {Compress} + */ +const compress = (specimen, pattern) => { + if (isNonCompressingMatcher(pattern)) { + if (matches(specimen, pattern)) { + return harden({ compressed: specimen }); + } + return undefined; + } + + // Not yet frozen! Used to accumulate bindings + const bindings = []; + const emitBinding = binding => { + bindings.push(binding); + }; + harden(emitBinding); + + /** + * @param {Passable} innerSpecimen + * @param {Pattern} innerPattern + * @returns {boolean} + */ + const compressRecur = (innerSpecimen, innerPattern) => { + assertPattern(innerPattern); + if (isKey(innerPattern)) { + return keyEQ(innerSpecimen, innerPattern); + } + const patternKind = kindOf(innerPattern); + const specimenKind = kindOf(innerSpecimen); + switch (patternKind) { + case undefined: { + return false; + } + case 'copyArray': { + if ( + specimenKind !== 'copyArray' || + innerSpecimen.length !== innerPattern.length + ) { + return false; + } + return innerPattern.every((p, i) => compressRecur(innerSpecimen[i], p)); + } + case 'copyRecord': { + if (specimenKind !== 'copyRecord') { + return false; + } + const specimenNames = recordNames(innerSpecimen); + const pattNames = recordNames(innerPattern); + + if (specimenNames.length !== pattNames.length) { + return false; + } + const specimenValues = recordValues(innerSpecimen, specimenNames); + const pattValues = recordValues(innerPattern, pattNames); + + return pattNames.every( + (name, i) => + specimenNames[i] === name && + compressRecur(specimenValues[i], pattValues[i]), + ); + } + case 'copyMap': { + if (specimenKind !== 'copyMap') { + return false; + } + const { + payload: { keys: pattKeys, values: valuePatts }, + } = innerPattern; + const { + payload: { keys: specimenKeys, values: specimenValues }, + } = innerSpecimen; + // TODO BUG: this assumes that the keys appear in the + // same order, so we can compare values in that order. + // However, we're only guaranteed that they appear in + // the same rankOrder. Thus we must search one of these + // in the other's rankOrder. + if (!keyEQ(specimenKeys, pattKeys)) { + return false; + } + return compressRecur(specimenValues, valuePatts); + } + default: + { + const matchHelper = maybeMatchHelper(patternKind); + if (matchHelper) { + if (matchHelper.compress) { + const subCompressedRecord = matchHelper.compress( + innerSpecimen, + innerPattern.payload, + compress, + ); + if (subCompressedRecord === undefined) { + return false; + } else { + emitBinding(subCompressedRecord.compressed); + return true; + } + } else if (matches(innerSpecimen, innerPattern)) { + assert(isNonCompressingMatcher(innerPattern)); + emitBinding(innerSpecimen); + return true; + } else { + return false; + } + } + } + throw Fail`unrecognized kind: ${q(patternKind)}`; + } + }; + + if (compressRecur(specimen, pattern)) { + return harden({ compressed: bindings }); + } else { + return undefined; + } +}; +harden(compress); + +/** + * `mustCompress` is to `compress` approximately as `mustMatch` is to `matches`. + * Where `compress` indicates pattern match failure by returning `undefined`, + * `mustCompress` indicates pattern match failure by throwing an error + * with a good pattern-match-failure diagnostic. Thus, like `mustMatch`, + * `mustCompress` has an additional optional `label` parameter to be used on + * the outside of that diagnostic if needed. If `mustCompress` does return + * normally, then the pattern match succeeded and `mustCompress` returns a + * valid compressed value. + * + * @type {MustCompress} + */ +export const mustCompress = (specimen, pattern, label = undefined) => { + const compressedRecord = compress(specimen, pattern); + if (compressedRecord !== undefined) { + return compressedRecord.compressed; + } + // `compress` is validating, so we don't need to redo all of `mustMatch`. + // We use it only to generate the error. + // Should only throw + checkMatches(specimen, pattern, assertChecker, label); + throw Fail`internal: ${label}: inconsistent pattern match: ${q(pattern)}`; +}; +harden(mustCompress); + +/** + * `decompress` reverses the compression performed by `compress` + * or `mustCompress`, in order to recover the equivalent + * of the original specimen from the `bindings` array and the `pattern`. + * + * @type {Decompress} + */ +const decompress = (compressed, pattern) => { + if (isNonCompressingMatcher(pattern)) { + return compressed; + } + + assert(Array.isArray(compressed)); + passStyleOf(compressed) === 'copyArray' || + Fail`Pattern ${pattern} expected bindings array: ${compressed}`; + let i = 0; + const takeBinding = () => { + i < compressed.length || + Fail`Pattern ${q(pattern)} expects more than ${q( + compressed.length, + )} bindings: ${compressed}`; + const binding = compressed[i]; + i += 1; + return binding; + }; + harden(takeBinding); + + const decompressRecur = innerPattern => { + assertPattern(innerPattern); + if (isKey(innerPattern)) { + return innerPattern; + } + const patternKind = kindOf(innerPattern); + switch (patternKind) { + case undefined: { + throw Fail`decompress expected a pattern: ${q(innerPattern)}`; + } + case 'copyArray': { + return harden(innerPattern.map(p => decompressRecur(p))); + } + case 'copyRecord': { + const pattNames = recordNames(innerPattern); + const pattValues = recordValues(innerPattern, pattNames); + const entries = pattNames.map((name, j) => [ + name, + decompressRecur(pattValues[j]), + ]); + // Reverse so printed form looks less surprising, + // with ascenting rather than descending property names. + return harden(fromEntries(entries.reverse())); + } + case 'copyMap': { + const { + payload: { keys: pattKeys, values: valuePatts }, + } = innerPattern; + return makeTagged( + 'copyMap', + harden({ + keys: pattKeys, + values: valuePatts.map(p => decompressRecur(p)), + }), + ); + } + default: + { + const matchHelper = maybeMatchHelper(patternKind); + if (matchHelper) { + if (matchHelper.decompress) { + const subCompressed = takeBinding(); + return matchHelper.decompress( + subCompressed, + innerPattern.payload, + decompress, + ); + } else { + assert(isNonCompressingMatcher(innerPattern)); + return takeBinding(); + } + } + } + throw Fail`unrecognized pattern kind: ${q(patternKind)} ${q( + innerPattern, + )}`; + } + }; + + return decompressRecur(pattern); +}; +harden(decompress); + +/** + * `decompress` reverses the compression performed by `compress` + * or `mustCompress`, in order to recover the equivalent + * of the original specimen from `compressed` and `pattern`. + * + * @type {MustDecompress} + */ +export const mustDecompress = (compressed, pattern, label = undefined) => { + const value = decompress(compressed, pattern); + // `decompress` does some checking, but is not validating, so we + // need to do the full `mustMatch` here to validate as well as to generate + // the error if invalid. + mustMatch(value, pattern, label); + return value; +}; diff --git a/packages/patterns/src/patterns/patternMatchers.js b/packages/patterns/src/patterns/patternMatchers.js index 0aa27ec4e4..99ce409ee5 100644 --- a/packages/patterns/src/patterns/patternMatchers.js +++ b/packages/patterns/src/patterns/patternMatchers.js @@ -33,17 +33,25 @@ import { checkCopyBag, getCopyMapEntryArray, makeCopyMap, + makeCopySet, + makeCopyBag, } from '../keys/checkKey.js'; import { generateCollectionPairEntries } from '../keys/keycollection-operators.js'; /** * @import {Checker, CopyRecord, CopyTagged, Passable} from '@endo/pass-style' - * @import {ArgGuard, AwaitArgGuard, CheckPattern, GetRankCover, InterfaceGuard, MatcherNamespace, MethodGuard, MethodGuardMaker, Pattern, RawGuard, SyncValueGuard, Kind, Limits, AllLimits, Key, DefaultGuardType} from '../types.js' + * @import {ArgGuard, AwaitArgGuard, CheckPattern, GetRankCover, InterfaceGuard, MatcherNamespace, MethodGuard, MethodGuardMaker, Pattern, RawGuard, SyncValueGuard, Kind, Compress, Decompress, Limits, AllLimits, Key, DefaultGuardType} from '../types.js' * @import {MatchHelper, PatternKit} from './types.js' */ const { entries, values } = Object; -const { ownKeys } = Reflect; +const { ownKeys, apply } = Reflect; + +// TODO simplify once we can assume Object.hasOwn everywhere. This probably +// means, when we stop supporting Node 14. +const { hasOwnProperty } = Object.prototype; +const hasOwn = + Object.hasOwn || ((obj, name) => apply(hasOwnProperty, obj, [name])); /** @type {WeakSet} */ const patternMemo = new WeakSet(); @@ -760,6 +768,44 @@ const makePatternKit = () => { ); }; + /** + * @param {Passable[]} array + * @param {Pattern} patt + * @param {Compress} compress + * @returns {Passable[] | undefined} + */ + const arrayCompressMatchPattern = (array, patt, compress) => { + if (isKind(patt, 'match:any')) { + return array; + } + const bindings = []; + for (const el of array) { + const subCompressedRecord = compress(el, patt); + if (subCompressedRecord) { + bindings.push(subCompressedRecord.compressed); + } else { + return undefined; + } + } + return harden(bindings); + }; + + /** + * @param {Passable} compressed + * @param {Pattern} patt + * @param {Decompress} decompress + * @returns {Passable[]} + */ + const arrayDecompressMatchPattern = (compressed, patt, decompress) => { + if (!Array.isArray(compressed)) { + throw Fail`Compressed array must be an array: ${compressed}`; + } + if (isKind(patt, 'match:any')) { + return compressed; + } + return harden(compressed.map(subBindings => decompress(subBindings, patt))); + }; + // /////////////////////// Match Helpers ///////////////////////////////////// /** @type {MatchHelper} */ @@ -777,12 +823,31 @@ const makePatternKit = () => { /** @type {MatchHelper} */ const matchAndHelper = harden({ - tag: 'match:and', + tag: 'match:and:1', checkMatches: (specimen, patts, check) => { return patts.every(patt => checkMatches(specimen, patt, check)); }, + // Compress only according to the last conjunct + compress: (specimen, patts, compress) => { + const { length } = patts; + // We know there are at least two patts + const lastPatt = patts[length - 1]; + const allButLast = patts.slice(0, length - 1); + if ( + !allButLast.every(patt => checkMatches(specimen, patt, identChecker)) + ) { + return undefined; + } + return compress(specimen, lastPatt); + }, + + decompress: (compressed, patts, decompress) => { + const lastPatt = patts[patts.length - 1]; + return decompress(compressed, lastPatt); + }, + checkIsWellFormed: (allegedPatts, check) => { const checkIt = patt => checkPattern(patt, check); return ( @@ -804,7 +869,7 @@ const makePatternKit = () => { /** @type {MatchHelper} */ const matchOrHelper = harden({ - tag: 'match:or', + tag: 'match:or:1', checkMatches: (specimen, patts, check) => { const { length } = patts; @@ -829,6 +894,31 @@ const makePatternKit = () => { return check(false, X`${specimen} - Must match one of ${q(patts)}`); }, + // Compress to an array pair of the index of the + // first disjunct that succeeded, and the compressed according to + // that disjunct. + compress: (specimen, patts, compress) => { + assert(Array.isArray(patts)); // redundant. Just for type checker. + const { length } = patts; + if (length === 0) { + return undefined; + } + for (let i = 0; i < length; i += 1) { + const subCompressedRecord = compress(specimen, patts[i]); + if (subCompressedRecord !== undefined) { + return harden({ compressed: [i, subCompressedRecord.compressed] }); + } + } + return undefined; + }, + + decompress: (compressed, patts, decompress) => { + (Array.isArray(compressed) && compressed.length === 2) || + Fail`Or compression must be a case index and a compression by that case: ${compressed}`; + const [i, subCompressed] = compressed; + return decompress(harden(subCompressed), patts[i]); + }, + checkIsWellFormed: matchAndHelper.checkIsWellFormed, getRankCover: (patts, encodePassable) => @@ -926,7 +1016,7 @@ const makePatternKit = () => { /** @type {MatchHelper} */ const matchTaggedHelper = harden({ - tag: `match:tagged`, + tag: `match:tagged:1`, checkMatches: (specimen, [tagPatt, payloadPatt], check) => { if (passStyleOf(specimen) !== 'tagged') { @@ -943,6 +1033,35 @@ const makePatternKit = () => { ); }, + compress: (specimen, [tagPatt, payloadPatt], compress) => { + if (passStyleOf(specimen) === 'tagged') { + const compressedTagRecord = compress(getTag(specimen), tagPatt); + if (!compressedTagRecord) { + return undefined; + } + const compressedPayloadRecord = compress(specimen.payload, payloadPatt); + if (!compressedPayloadRecord) { + return undefined; + } + return harden({ + compressed: [ + compressedTagRecord.compressed, + compressedPayloadRecord.compressed, + ], + }); + } + return undefined; + }, + + decompress: (compressed, [tagPatt, payloadPatt], decompress) => { + (Array.isArray(compressed) && compressed.length === 2) || + Fail`tagged compression must be a pair ${compressed}`; + const [compressedTag, compressedPayload] = compressed; + const tag = decompress(compressedTag, tagPatt); + const payload = decompress(compressedPayload, payloadPatt); + return makeTagged(tag, payload); + }, + checkIsWellFormed: (payload, check) => checkMatches( payload, @@ -1238,7 +1357,7 @@ const makePatternKit = () => { /** @type {MatchHelper} */ const matchArrayOfHelper = harden({ - tag: `match:arrayOf`, + tag: `match:arrayOf:1`, checkMatches: (specimen, [subPatt, limits = undefined], check) => { const { arrayLengthLimit } = limit(limits); @@ -1254,6 +1373,29 @@ const makePatternKit = () => { ); }, + // Compress to an array of corresponding bindings arrays + compress: (specimen, [subPatt, limits = undefined], compress) => { + const { arrayLengthLimit } = limit(limits); + if ( + isKind(specimen, 'copyArray') && + Array.isArray(specimen) && // redundant. just for type checker. + specimen.length <= arrayLengthLimit + ) { + const compressed = arrayCompressMatchPattern( + specimen, + subPatt, + compress, + ); + if (compressed) { + return harden({ compressed }); + } + } + return undefined; + }, + + decompress: (compressed, [subPatt, _limits = undefined], decompress) => + arrayDecompressMatchPattern(compressed, subPatt, decompress), + checkIsWellFormed: (payload, check) => checkIsWellFormedWithLimit( payload, @@ -1267,7 +1409,7 @@ const makePatternKit = () => { /** @type {MatchHelper} */ const matchSetOfHelper = harden({ - tag: `match:setOf`, + tag: `match:setOf:1`, checkMatches: (specimen, [keyPatt, limits = undefined], check) => { const { numSetElementsLimit } = limit(limits); @@ -1283,6 +1425,28 @@ const makePatternKit = () => { ); }, + // Compress to an array of corresponding bindings arrays + compress: (specimen, [keyPatt, limits = undefined], compress) => { + const { numSetElementsLimit } = limit(limits); + if ( + isKind(specimen, 'copySet') && + /** @type {Array} */ (specimen.payload).length <= numSetElementsLimit + ) { + const compressed = arrayCompressMatchPattern( + specimen.payload, + keyPatt, + compress, + ); + if (compressed) { + return harden({ compressed }); + } + } + return undefined; + }, + + decompress: (compressed, [keyPatt, _limits = undefined], decompress) => + makeCopySet(arrayDecompressMatchPattern(compressed, keyPatt, decompress)), + checkIsWellFormed: (payload, check) => checkIsWellFormedWithLimit( payload, @@ -1296,7 +1460,7 @@ const makePatternKit = () => { /** @type {MatchHelper} */ const matchBagOfHelper = harden({ - tag: `match:bagOf`, + tag: `match:bagOf:1`, checkMatches: ( specimen, @@ -1326,6 +1490,46 @@ const makePatternKit = () => { ); }, + // Compress to an array of corresponding bindings arrays + compress: ( + specimen, + [keyPatt, countPatt, limits = undefined], + compress, + ) => { + const { numUniqueBagElementsLimit, decimalDigitsLimit } = limit(limits); + if ( + isKind(specimen, 'copyBag') && + /** @type {Array} */ (specimen.payload).length <= + numUniqueBagElementsLimit && + specimen.payload.every(([_key, count]) => + checkDecimalDigitsLimit(count, decimalDigitsLimit, identChecker), + ) + ) { + const compressed = arrayCompressMatchPattern( + specimen.payload, + harden([keyPatt, countPatt]), + compress, + ); + if (compressed) { + return harden({ compressed }); + } + } + return undefined; + }, + + decompress: ( + compressed, + [keyPatt, countPatt, _limits = undefined], + decompress, + ) => + makeCopyBag( + arrayDecompressMatchPattern( + compressed, + harden([keyPatt, countPatt]), + decompress, + ), + ), + checkIsWellFormed: (payload, check) => checkIsWellFormedWithLimit( payload, @@ -1339,7 +1543,7 @@ const makePatternKit = () => { /** @type {MatchHelper} */ const matchMapOfHelper = harden({ - tag: `match:mapOf`, + tag: `match:mapOf:1`, checkMatches: ( specimen, @@ -1371,6 +1575,65 @@ const makePatternKit = () => { ); }, + // Compress to a pair of bindings arrays, one for the keys + // and a matching one for the values. + compress: ( + specimen, + [keyPatt, valuePatt, limits = undefined], + compress, + ) => { + const { numMapEntriesLimit } = limit(limits); + if ( + isKind(specimen, 'copyMap') && + /** @type {Array} */ (specimen.payload.keys).length <= + numMapEntriesLimit + ) { + const compressedKeys = arrayCompressMatchPattern( + specimen.payload.keys, + keyPatt, + compress, + ); + if (compressedKeys) { + const compressedValues = arrayCompressMatchPattern( + specimen.payload.values, + valuePatt, + compress, + ); + if (compressedValues) { + return harden({ + compressed: [compressedKeys, compressedValues], + }); + } + } + } + return undefined; + }, + + decompress: ( + compressed, + [keyPatt, valuePatt, _limits = undefined], + decompress, + ) => { + (Array.isArray(compressed) && compressed.length === 2) || + Fail`Compressed map should be a pair of compressed keys and compressed values ${compressed}`; + const [compressedKeys, compressedvalues] = compressed; + return makeTagged( + 'copyMap', + harden({ + keys: arrayDecompressMatchPattern( + compressedKeys, + keyPatt, + decompress, + ), + values: arrayDecompressMatchPattern( + compressedvalues, + valuePatt, + decompress, + ), + }), + ); + }, + checkIsWellFormed: (payload, check) => checkIsWellFormedWithLimit( payload, @@ -1418,7 +1681,7 @@ const makePatternKit = () => { /** @type {MatchHelper} */ const matchSplitArrayHelper = harden({ - tag: `match:splitArray`, + tag: `match:splitArray:1`, checkMatches: ( specimen, @@ -1455,6 +1718,70 @@ const makePatternKit = () => { ); }, + compress: ( + specimen, + [requiredPatt, optionalPatt = [], restPatt = MM.any()], + compress, + ) => { + if (!checkKind(specimen, 'copyArray', identChecker)) { + return undefined; + } + const { requiredSpecimen, optionalSpecimen, restSpecimen } = + splitArrayParts(specimen, requiredPatt, optionalPatt); + const partialPatt = adaptArrayPattern( + optionalPatt, + optionalSpecimen.length, + ); + const compressedRequired = compress(requiredSpecimen, requiredPatt); + if (!compressedRequired) { + return undefined; + } + const compressedPartial = []; + for (const [i, p] of entries(partialPatt)) { + const compressedField = compress(optionalSpecimen[i], p); + if (!compressedField) { + // imperative loop so can escape early + return undefined; + } + compressedPartial.push(compressedField.compressed[0]); + } + const compressedRest = compress(restSpecimen, restPatt); + if (!compressedRest) { + return undefined; + } + return harden({ + compressed: [ + compressedRequired.compressed, + compressedPartial, + compressedRest.compressed, + ], + }); + }, + + decompress: ( + compressed, + [requiredPatt, optionalPatt = [], restPatt = MM.any()], + decompress, + ) => { + (Array.isArray(compressed) && compressed.length === 3) || + Fail`splitArray compression must be a triple ${compressed}`; + const [compressRequired, compressPartial, compressedRest] = compressed; + const partialPatt = adaptArrayPattern( + optionalPatt, + compressPartial.length, + ); + const requiredParts = decompress(compressRequired, requiredPatt); + // const optionalParts = decompress(compressPartial, partialPatt); + const optionalParts = []; + for (const [i, p] of entries(partialPatt)) { + // imperative loop just for similarity to compression code + const optionalField = decompress(harden([compressPartial[i]]), p); + optionalParts.push(optionalField); + } + const restParts = decompress(compressedRest, restPatt); + return harden([...requiredParts, ...optionalParts, ...restParts]); + }, + /** * @param {Array} splitArray * @param {Checker} check @@ -1539,7 +1866,7 @@ const makePatternKit = () => { /** @type {MatchHelper} */ const matchSplitRecordHelper = harden({ - tag: `match:splitRecord`, + tag: `match:splitRecord:1`, checkMatches: ( specimen, @@ -1568,6 +1895,87 @@ const makePatternKit = () => { ); }, + compress: ( + specimen, + [requiredPatt, optionalPatt = {}, restPatt = MM.any()], + compress, + ) => { + if (!checkKind(specimen, 'copyRecord', identChecker)) { + return undefined; + } + const { requiredSpecimen, optionalSpecimen, restSpecimen } = + splitRecordParts(specimen, requiredPatt, optionalPatt); + const partialPatt = adaptRecordPattern(optionalPatt); + + const compressedRequired = compress(requiredSpecimen, requiredPatt); + if (!compressedRequired) { + return undefined; + } + const optionalNames = recordNames(partialPatt); + const compressedPartial = []; + for (const name of optionalNames) { + if (hasOwn(optionalSpecimen, name)) { + const compressedField = compress( + optionalSpecimen[name], + partialPatt[name], + ); + if (!compressedField) { + return undefined; + } + compressedPartial.push(compressedField.compressed[0]); + } else { + compressedPartial.push(null); + } + } + const compressedRest = compress(restSpecimen, restPatt); + if (!compressedRest) { + return undefined; + } + return harden({ + compressed: [ + compressedRequired.compressed, + compressedPartial, + compressedRest.compressed, + ], + }); + }, + + decompress: ( + compressed, + [requiredPatt, optionalPatt = {}, restPatt = MM.any()], + decompress, + ) => { + (Array.isArray(compressed) && compressed.length === 3) || + Fail`splitRecord compression must be a triple ${compressed}`; + const [compressedRequired, compressedPartial, compressedRest] = + compressed; + const partialPatt = adaptRecordPattern(optionalPatt); + const requiredEntries = entries( + decompress(compressedRequired, requiredPatt), + ); + const optionalNames = recordNames(partialPatt); + compressedPartial.length === optionalNames.length || + Fail`compression or patterns must preserve cardinality: ${compressedPartial}`; + /** @type {[string, Passable][]} */ + const optionalEntries = []; + for (const [i, name] of entries(optionalNames)) { + const p = partialPatt[name]; + const c = compressedPartial[i]; + if (c !== null) { + const u = decompress(harden([c]), p); + optionalEntries.push([name, u]); + } + } + const restEntries = entries(decompress(compressedRest, restPatt)); + + const allEntries = [ + ...requiredEntries, + ...optionalEntries, + ...restEntries, + ]; + return fromUniqueEntries(allEntries); + }, + /** * @param {Array} splitArray * @param {Checker} check @@ -1642,13 +2050,21 @@ const makePatternKit = () => { const helpersByMatchTag = {}; for (const helper of helpers) { - const { tag } = helper; + const { tag, compress, decompress, ...rest } = helper; if (!matchHelperTagRE.test(tag)) { throw Fail`malformed matcher tag ${q(tag)}`; } const subTag = getMatchSubTag(tag); - subTag === undefined || - Fail`Should not be any subtags before compression ${q(tag)}`; + if (subTag === undefined) { + (compress === undefined && decompress === undefined) || + Fail`internal: compressing helper must have compression version ${q( + tag, + )}`; + } else { + (typeof compress === 'function' && typeof decompress === 'function') || + Fail`internal: expected compression methods ${q(tag)})`; + helpersByMatchTag[subTag] = { tag: subTag, ...rest }; + } helpersByMatchTag[tag] = helper; } return harden(helpersByMatchTag); diff --git a/packages/patterns/src/patterns/types.js b/packages/patterns/src/patterns/types.js index 77e835afd5..fdc6037633 100644 --- a/packages/patterns/src/patterns/types.js +++ b/packages/patterns/src/patterns/types.js @@ -4,7 +4,7 @@ export {}; /** * @import {Passable, Checker} from '@endo/pass-style' - * @import {MatcherNamespace, Pattern, GetRankCover, Kind} from '../types.js' + * @import {MatcherNamespace, Pattern, GetRankCover, Compress, Decompress, CompressedRecord, Kind} from '../types.js' */ /** @@ -28,6 +28,27 @@ export {}; * Assuming validity of `matcherPayload` as the payload of a Matcher corresponding * with this MatchHelper, reports whether `specimen` is matched by that Matcher. * + * @property {(specimen: Passable, + * matcherPayload: Passable, + * compress: Compress + * ) => (CompressedRecord | undefined)} [compress] + * Assuming a valid Matcher of this type with `matcherPayload` as its + * payload, if this specimen matches this matcher, then return a + * CompressedRecord that represents this specimen, + * perhaps more compactly, given the knowledge that it matches this matcher. + * If the specimen does not match the matcher, return undefined. + * If this matcher has a `compress` method, then it must have a matching + * `decompress` method. + * + * @property {(compressed: Passable, + * matcherPayload: Passable, + * decompress: Decompress + * ) => Passable} [decompress] + * If `compressed` is the result of a successful `compress` with this matcher, + * then `decompress` must return a Passable equivalent to the original specimen. + * If this matcher has an `decompress` method, then it must have a matching + * `compress` method. + * * @property {GetRankCover} getRankCover * Assumes this is the payload of a CopyTagged with the corresponding * matchTag. Return a RankCover to bound from below and above, diff --git a/packages/patterns/src/types.js b/packages/patterns/src/types.js index cbb4b6fa19..c84a0a5128 100644 --- a/packages/patterns/src/types.js +++ b/packages/patterns/src/types.js @@ -236,6 +236,41 @@ export {}; * that the @endo/patterns level associates with that kind. */ +/** + * @typedef {object} CompressedRecord + * @property {Passable} compressed + */ + +/** + * @callback Compress + * @param {Passable} specimen + * @param {Pattern} pattern + * @returns {CompressedRecord | undefined} + */ + +/** + * @callback MustCompress + * @param {Passable} specimen + * @param {Pattern} pattern + * @param {string|number} [label] + * @returns {Passable} + */ + +/** + * @callback Decompress + * @param {Passable} compressed + * @param {Pattern} pattern + * @returns {Passable} + */ + +/** + * @callback MustDecompress + * @param {Passable} compressed + * @param {Pattern} pattern + * @param {string|number} [label] + * @returns {Passable} + */ + /** * @typedef {object} PatternMatchers * diff --git a/packages/patterns/test/test-compress.js b/packages/patterns/test/test-compress.js new file mode 100644 index 0000000000..d745428f8c --- /dev/null +++ b/packages/patterns/test/test-compress.js @@ -0,0 +1,273 @@ +// @ts-nocheck So many errors that the suppressions hamper readability. +// TODO fix and then turn at-ts-check back on +import test from '@endo/ses-ava/prepare-endo.js'; + +import { Far, makeTagged, makeMarshal } from '@endo/marshal'; +import { + makeCopyBagFromElements, + makeCopyMap, + makeCopySet, +} from '../src/keys/checkKey.js'; +import { mustCompress, mustDecompress } from '../src/patterns/compress.js'; +import { M } from '../src/patterns/patternMatchers.js'; + +const runTests = testTriple => { + const brand = Far('simoleans', {}); + const moolaBrand = Far('moola', {}); + const timer = Far('timer', {}); + + testTriple({ brand, value: 37n }, M.any(), { brand, value: 37n }); + testTriple({ brand, value: 37n }, { brand, value: M.bigint() }, [37n]); + testTriple( + { brand, value: 37n }, + { brand: M.remotable(), value: M.bigint() }, + [37n, brand], + ); + testTriple( + { brand, value: 37n }, + { brand: M.bigint(), value: M.bigint() }, + undefined, + 'test mustCompress: brand: remotable "[Alleged: simoleans]" - Must be a bigint', + ); + testTriple({ brand, value: 37n }, M.recordOf(M.string(), M.scalar()), { + brand, + value: 37n, + }); + testTriple( + [{ foo: 'a' }, { foo: 'b' }, { foo: 'c' }], + M.arrayOf(harden({ foo: M.string() })), + [[['a'], ['b'], ['c']]], + ); + testTriple( + [{ foo: 'a' }, { foo: 'b' }, { foo: 'c' }], + // Test that without the compression version tag, there is no + // non -default compression or decompression + makeTagged('match:arrayOf', harden([{ foo: M.string() }])), + [{ foo: 'a' }, { foo: 'b' }, { foo: 'c' }], + ); + testTriple( + makeCopySet([{ foo: 'a' }, { foo: 'b' }, { foo: 'c' }]), + M.setOf(harden({ foo: M.string() })), + [[['c'], ['b'], ['a']]], + ); + testTriple( + makeCopyBagFromElements([{ foo: 'a' }, { foo: 'a' }, { foo: 'c' }]), + M.bagOf(harden({ foo: M.string() })), + [ + [ + ['c', 1n], + ['a', 2n], + ], + ], + ); + testTriple( + makeCopyBagFromElements([{ foo: 'a' }, { foo: 'a' }, { foo: 'c' }]), + M.bagOf(harden({ foo: M.string() }), 1n), + undefined, + 'test mustCompress: bag counts[1]: "[2n]" - Must be: "[1n]"', + ); + testTriple( + makeCopyBagFromElements([{ foo: 'a' }, { foo: 'b' }, { foo: 'c' }]), + M.bagOf(harden({ foo: M.string() }), 1n), + [[['c'], ['b'], ['a']]], + ); + testTriple( + makeCopyMap([ + [{ foo: 'a' }, { bar: 1 }], + [{ foo: 'b' }, { bar: 2 }], + [{ foo: 'c' }, { bar: 3 }], + ]), + M.mapOf(harden({ foo: M.string() }), harden({ bar: M.number() })), + [ + [ + [['c'], ['b'], ['a']], + [[3], [2], [1]], + ], + ], + ); + testTriple( + makeCopyMap([ + [{ foo: 'c' }, { bar: 3 }], + [{ foo: 'b' }, { bar: 2 }], + [{ foo: 'a' }, { bar: 1 }], + ]), + // TODO Add a test case where the keys are in the same rankOrder but not + // the same order. + makeCopyMap([ + [{ foo: 'c' }, M.any()], + // @ts-expect-error The array need not be generic + [{ foo: 'b' }, { bar: M.number() }], + [{ foo: 'a' }, { bar: 1 }], + ]), + [{ bar: 3 }, 2], + ); + testTriple( + { + want: { Winnings: { brand: moolaBrand, value: ['x', 'y'] } }, + give: { Bid: { brand, value: 37n } }, + exit: { afterDeadline: { deadline: 11n, timer } }, + }, + { + want: { Winnings: { brand: moolaBrand, value: M.array() } }, + give: { Bid: { brand, value: M.nat() } }, + exit: { afterDeadline: { deadline: M.gte(10n), timer } }, + }, + [['x', 'y'], 37n, 11n], + ); + testTriple( + { + want: { + Winnings: { + brand: moolaBrand, + value: makeCopyBagFromElements([ + { foo: 'a' }, + { foo: 'b' }, + { foo: 'c' }, + ]), + }, + }, + give: { Bid: { brand, value: 37n } }, + exit: { afterDeadline: { deadline: 11n, timer } }, + }, + { + want: { + Winnings: { + brand: moolaBrand, + value: M.bagOf(harden({ foo: M.string() }), 1n), + }, + }, + give: { Bid: { brand, value: M.nat() } }, + exit: { afterDeadline: { deadline: M.gte(10n), timer } }, + }, + [[['c'], ['b'], ['a']], 37n, 11n], + ); + testTriple( + 'orange', + M.or('red', 'orange', 'yellow', 'green', 'blue', 'indigo', 'violet'), + [[1, []]], + ); + testTriple( + { x: 3, y: 5 }, + M.or(harden({ x: M.number(), y: M.number() }), M.bigint(), M.record()), + [[0, [5, 3]]], + ); + testTriple( + [5n], + M.or(harden({ x: M.number(), y: M.number() }), [M.bigint()], M.record()), + [[1, [5n]]], + ); + testTriple( + { x: 3, y: 5, z: 9 }, + M.or(harden({ x: M.number(), y: M.number() }), M.bigint(), M.record()), + [[2, { x: 3, y: 5, z: 9 }]], + ); + testTriple( + { + brand, + value: [{ bar: 2 }, { bar: 1 }], + }, + { + brand, + value: M.arrayOf(M.and(M.key(), { bar: M.number() })), + }, + [[[[2]], [[1]]]], + ); + testTriple( + ['a', 'b', 'c', 'd', 'e'], + M.splitArray(['a', M.string()], [M.any()], M.any()), + [[['b'], [[1, 'c']], ['d', 'e']]], + ); + testTriple( + ['a', 'b', undefined, 'd'], + M.splitArray(['a', M.string()], ['c', 'd', 'e'], M.any()), + [ + [ + ['b'], + [ + [0, []], + [1, []], + ], + [], + ], + ], + ); + testTriple( + { a: 1, b: 2, c: undefined, d: 4, e: 5 }, + M.splitRecord({ a: 1, b: M.number() }, { c: M.any(), d: 4, f: 6 }, M.any()), + [[[2], [null, [1, []], [0, []]], { e: 5 }]], + ); + testTriple( + makeTagged('foo', { a: 1, b: 2 }), + M.tagged('foo', harden({ a: 1, b: M.number() })), + [[[], [2]]], + ); +}; + +test('compression', t => { + const testCompress = (specimen, pattern, compressed, message = undefined) => { + if (!message) { + t.deepEqual( + mustCompress(harden(specimen), harden(pattern)), + harden(compressed), + ); + } + }; + runTests(testCompress); +}); + +test('test mustCompress', t => { + const testCompress = (specimen, pattern, compressed, message = undefined) => { + if (message === undefined) { + t.deepEqual( + mustCompress(harden(specimen), harden(pattern), 'test mustCompress'), + harden(compressed), + ); + } else { + t.throws( + () => + mustCompress(harden(specimen), harden(pattern), 'test mustCompress'), + { message }, + ); + } + }; + runTests(testCompress); +}); + +test('decompression', t => { + const testDecompress = ( + specimen, + pattern, + compressed, + message = undefined, + ) => { + if (message === undefined) { + t.deepEqual( + mustDecompress(harden(compressed), harden(pattern)), + harden(specimen), + ); + } + }; + runTests(testDecompress); +}); + +test('demo compression ratio', t => { + const { toCapData } = makeMarshal(() => 's', undefined, { + serializeBodyFormat: 'smallcaps', + }); + + const testCompress = (specimen, pattern, compressed, message = undefined) => { + harden(specimen); + harden(pattern); + harden(compressed); + if (message === undefined) { + const { body: big } = toCapData(specimen); + const { body: small } = toCapData(compressed); + const ratio = small.length / big.length; + console.log('\n', big, '\n', small, '\n', ratio); + const { body: patt } = toCapData(pattern); + console.log('Pattern: ', patt); + t.assert(ratio <= 2.0); + } + }; + runTests(testCompress); +});