Index import possibly broken in 0.7.21? (could be user error)

[jasonpolites]

Hi folks,

I'm trying to figure out a reasonable pattern for exporting to the file system. After looking at the various posts, I implemented a fairly basic file system model, referencing the instructions in the readme.

Here's my save function:

function saveIndexSync(index, dir) {
  if (!fs.existsSync(dir)) {
    fs.mkdirSync(dir);
  }
  
  index.export((key, data) => {
    if(data) {
      let file = path.join(dir, key);
      fs.writeFileSync(file, data);
    } else {
      console.error(`Data for key ${key} is empty or undefined`);
    }
  });
}

This seems to work, and I get a bunch of files on the file system. When I try to import() the index however, I get an error.

Here's my load function:

function loadIndexSync(index, dir) {
  if (fs.existsSync(dir)){
    fs.readdir(dir, (err, files) => {
      if(!err) {
        files.forEach(filename => {
          let file = path.join(dir, filename);
          let data = fs.readFileSync(file);
          let val = JSON.parse(data);
          index.import(filename, val);
        });
      } else {
        throw err;
      }
    });
  }
}

Executing the load results in the following error:

/node_modules/flexsearch/dist/flexsearch.bundle.js:33
t.import=function(a,b){if(b)switch(x(b)&&(b=JSON.parse(b)),a){case "tag":this.l=b;break;case "reg":this.m=!1;this.register=b;for(let d=0,e;d<this.h.length;d++)e=this.index[this.h[d]],e.register=b,e.m=!1;break;case "store":this.store=b;break;default:a=a.split(".");const c=a[0];a=a[1];c&&a&&this.index[c].import(a,b)}};la(Q.prototype);var Ba={encode:Aa,F:!1,G:""};const Ca=[F("[\u00e0\u00e1\u00e2\u00e3\u00e4\u00e5]"),"a",F("[\u00e8\u00e9\u00ea\u00eb]"),"e",F("[\u00ec\u00ed\u00ee\u00ef]"),"i",F("[\u00f2\u00f3\u00f4\u00f5\u00f6\u0151]"),"o",F("[\u00f9\u00fa\u00fb\u00fc\u0171]"),"u",F("[\u00fd\u0177\u00ff]"),"y",F("\u00f1"),"n",F("[\u00e7c]"),"k",F("\u00df"),"s",F(" & ")," and "];function Aa(a){var b=a;b.normalize&&(b=b.normalize("NFD").replace(ea,""));return ca.call(this,b.toLowerCase(),!a.normalize&&Ca)};var Ea={encode:Da,F:!1,G:"strict"};const Fa=/[^a-z0-9]+/,Ga={b:"p",v:"f",w:"f",z:"s",x:"s","\u00df":"s",d:"t",n:"m",c:"k",g:"k",j:"k",q:"k",i:"e",y:"e",u:"o"};function Da(a){a=Aa.call(this,a).join(" ");const b=[];if(a){const c=a.split(Fa),d=c.length;for(let e=0,f,g=0;e<d;e++)if((a=c[e])&&(!this.filter||!this.filter[a])){f=a[0];let h=Ga[f]||f,k=h;for(let m=1;m<a.length;m++){f=a[m];const n=Ga[f]||f;n&&n!==k&&(h+=n,k=n)}b[g++]=h}}return b};var Ia={encode:Ha,F:!1,G:""};const Ja=[F("ae"),"a",F("oe"),"o",F("sh"),"s",F("th"),"t",F("ph"),"f",F("pf"),"f",F("(?![aeo])h(?![aeo])"),"",F("(?!^[aeo])h(?!^[aeo])"),""];function Ha(a,b){a&&(a=Da.call(this,a).join(" "),2<a.length&&(a=E(a,Ja)),b||(1<a.length&&(a=ha(a)),a&&(a=a.split(" "))));return a};var La={encode:Ka,F:!1,G:""};const Ma=F("(?!\\b)[aeo]");function Ka(a){a&&(a=Ha.call(this,a,!0),1<a.length&&(a=a.replace(Ma,"")),1<a.length&&(a=ha(a)),a&&(a=a.split(" ")));return a};G["latin:default"]=ja;G["latin:simple"]=Ba;G["latin:balance"]=Ea;G["latin:advanced"]=Ia;G["latin:extra"]=La;const W=self;let Y;const Z={Index:K,Document:Q,Worker:O,registerCharset:function(a,b){G[a]=b},registerLanguage:function(a,b){ka[a]=b}};(Y=W.define)&&Y.amd?Y([],function(){return Z}):W.exports?W.exports=Z:W.FlexSearch=Z;}(this));
                                                                                                                                                                                                                                                                                                                ^

TypeError: Cannot read properties of undefined (reading 'import')
    at Q.t.import (/<REDACTED>/node_modules/flexsearch/dist/flexsearch.bundle.js:33:305)
    at /<REDACTED>/search.js:55:23
    at Array.forEach (<anonymous>)
    at /<REDACTED>/search.js:51:15
    at FSReqCallback.oncomplete (node:fs:188:23)

The full code for reproduction is below:

const { Document } = require("flexsearch");
const fs = require('fs');
const path = require('path');
const savePath = 'indexstore';

function createIndex() {
  return new Document({
    document: {
        id: "id",
        index: [
            "from",
            "to",
            "subject",
            "body",
            "from",
            "hasAttachments",
            "attachments[]:name",
            "attachments[]:contentType",
            "attachments[]:path"
        ],
        store: [
          "from",
          "to",
          "subject",
          "hasAttachments"
        ]
    }
  });

}

function saveIndexSync(index, dir) {
  if (!fs.existsSync(dir)) {
    fs.mkdirSync(dir);
  }
  
  index.export((key, data) => {
    if(data) {
      let file = path.join(dir, key);
      fs.writeFileSync(file, data);
    } else {
      console.error(`Data for key ${key} is empty or undefined`);
    }
  });
}

function loadIndexSync(index, dir) {
  if (fs.existsSync(dir)){
    fs.readdir(dir, (err, files) => {
      if(!err) {
        files.forEach(filename => {
          let file = path.join(dir, filename);
          let data = fs.readFileSync(file);
          let val = JSON.parse(data);
          index.import(filename, val);
        });
      } else {
        throw err;
      }
    });
  }
}

/**
 * Collapses the search results into single rows, ranked by the number of 
 * fields that matched the search term
 */
function rank(resultSet) {
  let resultMap = new Map();
  resultSet.forEach(result => {
    let docs = result.result;
    docs.forEach(doc => {
      let currentVal = resultMap.get(doc.id);
      if(!currentVal) {
        currentVal = {
          score: 1,
          doc: doc.doc
        }
        resultMap.set(doc.id, currentVal);
      } else {
        // More fields matching means a higher score/rank
        currentVal.score++
      }
    });
  });

  // Sort the map based on the score property
  return new Map([...resultMap].sort((a, b) => {
    return b[1].score - a[1].score;
  }))
}

/**
 * ################################################
 * Sample App
 * ################################################
 */


const doc1 = {
  "id" : "tempId1",
  "from" : "[email protected]",
  "to" : "[email protected]",
  "subject" : "Hello fox",
  "body" : "The quick brown fox",
  "hasAttachments": true,
  "attachments" : [
    {
      "name" : "foo.pdf",
      "contentType" : "text/pdf",
      "path" : "/foo/bar/foo.pdf"
    },
    {
      "name" : "bar.pdf",
      "contentType" : "text/pdf",
      "path" : "/foo/bar/bar.pdf"      
    }
  ]
}

const doc2 = {
  "id" : "tempId2",
  "from" : "[email protected]",
  "to" : "[email protected]",
  "subject" : "Hello dog",
  "body" : "Jumps over the lazy dog",
  "hasAttachments": true,
  "attachments" : [
    {
      "name" : "foo.jpg",
      "contentType" : "image/jpg",
      "path" : "/foo/bar/foo.jpg"
    }
  ]
}

const index = createIndex()

index.add(doc1);
index.add(doc2);

let result = index.search(`foo`,  { enrich: true });
let sorted = rank (result);

for (const r of sorted) {
  console.log(JSON.stringify(r));
}

saveIndexSync(index, savePath);

const loadedIndex = createIndex();
loadIndexSync(loadedIndex, savePath);

[spidgorny]

I've tried all version 0.7.0 until 0.7.21 - all have the same issue.
Interesting that index.export() function works fine

[ts-thomas]

This is now fixed in v0.7.23
Further improvements to provide Promise.all() compatible export ist coming in next version.