Skip to content

Commit

Permalink
Eslint fix and manual fixes in tests
Browse files Browse the repository at this point in the history
  • Loading branch information
wilwade committed Jul 1, 2024
1 parent 0e5d8c6 commit 91ae9ac
Show file tree
Hide file tree
Showing 31 changed files with 525 additions and 533 deletions.
16 changes: 8 additions & 8 deletions lib/bloom/sbbf.ts
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ import { Block } from '../declare';
* .init()
*/
class SplitBlockBloomFilter {
private static readonly salt: Array<number> = [
private static readonly salt: number[] = [
0x47b6137b, 0x44974d91, 0x8824ad5b, 0xa2b7289d, 0x705495c7, 0x2df1424b, 0x9efc4947, 0x5c6bfb31,
];

Expand Down Expand Up @@ -73,19 +73,19 @@ class SplitBlockBloomFilter {
* from the provided Buffer
* @param buffer a NodeJs Buffer containing bloom filter data for a row group.
*/
static from(buffer: Buffer, rowCount?: number): SplitBlockBloomFilter {
static from(buffer: Buffer, _rowCount?: number): SplitBlockBloomFilter {
if (buffer.length === 0) {
throw new Error('buffer is empty');
}
const chunkSize = SplitBlockBloomFilter.WORDS_PER_BLOCK;
const uint32sFromBuf = new Uint32Array(buffer.buffer);
let result: Array<Block> = [];
const result: Block[] = [];
const length = uint32sFromBuf.length;

for (let index = 0; index < length; index += chunkSize) {
result.push(uint32sFromBuf.subarray(index, index + chunkSize));
}
let sb = new SplitBlockBloomFilter();
const sb = new SplitBlockBloomFilter();
sb.splitBlockFilter = result;
sb.numBlocks = result.length;
// these will not be knowable when reading
Expand Down Expand Up @@ -151,7 +151,7 @@ class SplitBlockBloomFilter {
* @return mask Block
*/
static mask(hashValue: Long): Block {
let result: Block = SplitBlockBloomFilter.initBlock();
const result: Block = SplitBlockBloomFilter.initBlock();
for (let i = 0; i < result.length; i++) {
const y = hashValue.getLowBitsUnsigned() * SplitBlockBloomFilter.salt[i];
result[i] = result[i] | (1 << (y >>> 27));
Expand Down Expand Up @@ -210,9 +210,9 @@ class SplitBlockBloomFilter {
* Instance
*/

private splitBlockFilter: Array<Block> = [];
private splitBlockFilter: Block[] = [];
private desiredFalsePositiveRate: number = SplitBlockBloomFilter.DEFAULT_FALSE_POSITIVE_RATE;
private numBlocks: number = 0;
private numBlocks = 0;
private numDistinctValues: number = SplitBlockBloomFilter.DEFAULT_DISTINCT_VALUES;
private hashStrategy = new parquet_thrift.BloomFilterHash(new parquet_thrift.XxHash());
private hasher = new XxHasher();
Expand All @@ -230,7 +230,7 @@ class SplitBlockBloomFilter {
getNumFilterBlocks(): number {
return this.splitBlockFilter.length;
}
getFilter(): Array<Block> {
getFilter(): Block[] {
return this.splitBlockFilter;
}

Expand Down
20 changes: 9 additions & 11 deletions lib/bloomFilterIO/bloomFilterReader.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,17 +4,17 @@ import sbbf from '../bloom/sbbf';
import { ParquetEnvelopeReader } from '../reader';
import { ColumnChunkData } from '../declare';

const filterColumnChunksWithBloomFilters = (columnChunkDataCollection: Array<ColumnChunkData>) => {
const filterColumnChunksWithBloomFilters = (columnChunkDataCollection: ColumnChunkData[]) => {
return columnChunkDataCollection.filter((columnChunk) => {
return columnChunk.column.meta_data?.bloom_filter_offset;
});
};

type bloomFilterOffsetData = {
interface bloomFilterOffsetData {
columnName: string;
offsetBytes: number;
rowGroupIndex: number;
};
}

const toInteger = (buffer: Buffer) => {
const integer = parseInt(buffer.toString('hex'), 16);
Expand All @@ -26,9 +26,7 @@ const toInteger = (buffer: Buffer) => {
return integer;
};

export const parseBloomFilterOffsets = (
ColumnChunkDataCollection: Array<ColumnChunkData>
): Array<bloomFilterOffsetData> => {
export const parseBloomFilterOffsets = (ColumnChunkDataCollection: ColumnChunkData[]): bloomFilterOffsetData[] => {
return ColumnChunkDataCollection.map(({ rowGroupIndex, column }) => {
const { bloom_filter_offset: bloomOffset, path_in_schema: pathInSchema } = column.meta_data || {};

Expand Down Expand Up @@ -83,25 +81,25 @@ const readFilterData = async (
};

const readFilterDataFrom = (
offsets: Array<number>,
offsets: number[],
envelopeReader: InstanceType<typeof ParquetEnvelopeReader>
): Promise<Array<Buffer>> => {
): Promise<Buffer[]> => {
return Promise.all(offsets.map((offset) => readFilterData(offset, envelopeReader)));
};

export const siftAllByteOffsets = (columnChunkDataCollection: Array<ColumnChunkData>): Array<bloomFilterOffsetData> => {
export const siftAllByteOffsets = (columnChunkDataCollection: ColumnChunkData[]): bloomFilterOffsetData[] => {
return parseBloomFilterOffsets(filterColumnChunksWithBloomFilters(columnChunkDataCollection));
};

export const getBloomFiltersFor = async (
paths: Array<string>,
paths: string[],
envelopeReader: InstanceType<typeof ParquetEnvelopeReader>
) => {
const columnChunkDataCollection = envelopeReader.getAllColumnChunkDataFor(paths);
const bloomFilterOffsetData = siftAllByteOffsets(columnChunkDataCollection);
const offsetByteValues = bloomFilterOffsetData.map(({ offsetBytes }) => offsetBytes);

const filterBlocksBuffers: Array<Buffer> = await readFilterDataFrom(offsetByteValues, envelopeReader);
const filterBlocksBuffers: Buffer[] = await readFilterDataFrom(offsetByteValues, envelopeReader);

return filterBlocksBuffers.map((buffer, index) => {
const { columnName, rowGroupIndex } = bloomFilterOffsetData[index];
Expand Down
12 changes: 6 additions & 6 deletions lib/bloomFilterIO/bloomFilterWriter.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,12 @@ import SplitBlockBloomFilter from '../bloom/sbbf';
import { Block } from '../declare';
import Int64 from 'node-int64';

export type createSBBFParams = {
export interface createSBBFParams {
numFilterBytes?: number;
falsePositiveRate?: number;
numDistinct?: number;
column?: any;

Check warning on line 12 in lib/bloomFilterIO/bloomFilterWriter.ts

View workflow job for this annotation

GitHub Actions / test

Unexpected any. Specify a different type
};
}

export const createSBBF = (params: createSBBFParams): SplitBlockBloomFilter => {
const { numFilterBytes, falsePositiveRate, numDistinct } = params;
Expand All @@ -30,7 +30,7 @@ export const createSBBF = (params: createSBBFParams): SplitBlockBloomFilter => {
return bloomFilter.init();
};

const serializeFilterBlocks = (blocks: Array<Block>): Buffer =>
const serializeFilterBlocks = (blocks: Block[]): Buffer =>
Buffer.concat(blocks.map((block) => Buffer.from(block.buffer)));

const buildFilterHeader = (numBytes: number) => {
Expand All @@ -49,10 +49,10 @@ export const serializeFilterHeaders = (numberOfBytes: number) => {
return parquet_util.serializeThrift(bloomFilterHeader);
};

type serializeFilterDataParams = {
filterBlocks: Array<Block>;
interface serializeFilterDataParams {
filterBlocks: Block[];
filterByteSize: number;
};
}

export const serializeFilterData = (params: serializeFilterDataParams) => {
const serializedFilterBlocks = serializeFilterBlocks(params.filterBlocks);
Expand Down
4 changes: 2 additions & 2 deletions lib/bufferReader.ts
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ export default class BufferReader {
maxLength: number;
queueWait: number;
scheduled?: boolean;
queue: Array<BufferReaderQueueRow>;
queue: BufferReaderQueueRow[];
envelopeReader: ParquetEnvelopeReader;

constructor(envelopeReader: ParquetEnvelopeReader, options: BufferReaderOptions) {
Expand Down Expand Up @@ -56,7 +56,7 @@ export default class BufferReader {
this.queue = [];
queue.sort((a, b) => a.offset - b.offset);

var subqueue: Array<BufferReaderQueueRow> = [];
let subqueue: BufferReaderQueueRow[] = [];

const readSubqueue = async () => {
if (!subqueue.length) {
Expand Down
Loading

0 comments on commit 91ae9ac

Please sign in to comment.