Skip to content

Commit

Permalink
fix!: remove redundant types directory
Browse files Browse the repository at this point in the history
  • Loading branch information
wldeh committed Mar 9, 2024
1 parent dc5e2dd commit dcea502
Show file tree
Hide file tree
Showing 23 changed files with 574 additions and 275 deletions.
366 changes: 331 additions & 35 deletions package-lock.json

Large diffs are not rendered by default.

24 changes: 12 additions & 12 deletions src/data/download/downloadLinkGetter.ts
Original file line number Diff line number Diff line change
@@ -1,23 +1,23 @@
import cheerio from 'cheerio'
import fetch from 'node-fetch'
import cheerio from 'cheerio';
import fetch from 'node-fetch';

const BASE_URL = 'https://app.thedigitalbiblelibrary.org'
const BASE_URL = 'https://app.thedigitalbiblelibrary.org';

export default async function getDownloadLink(url: string): Promise<string> {
try {
const initialResponse = await fetch(url)
const initialData = await initialResponse.text()
const $1 = cheerio.load(initialData)
const initialResponse = await fetch(url);
const initialData = await initialResponse.text();
const $1 = cheerio.load(initialData);
const downloadResponse = await fetch(
BASE_URL + $1('.list-group-item > a').attr('href')
)
);

const downloadData = await downloadResponse.text()
const $2 = cheerio.load(downloadData)
const href: string = BASE_URL + $2('#download_button').attr('href')
const downloadData = await downloadResponse.text();
const $2 = cheerio.load(downloadData);
const href: string = BASE_URL + $2('#download_button').attr('href');

return href
return href;
} catch (error) {
throw new Error(`getDownloadLink failed: ${error.message}`)
throw new Error(`getDownloadLink failed: ${error.message}`);
}
}
14 changes: 7 additions & 7 deletions src/data/download/downloader.ts
Original file line number Diff line number Diff line change
@@ -1,16 +1,16 @@
import fs from 'fs'
import fetch, { Blob, Response } from 'node-fetch'
import fs from 'fs';
import fetch, { Blob, Response } from 'node-fetch';

export default async function downloadZip(
url: string,
downloadPath: string
): Promise<void> {
try {
const response: Response = await fetch(url)
const blob: Blob = await response.blob()
const buffer = Buffer.from(await (blob as any).arrayBuffer())
await fs.writeFileSync(downloadPath, buffer)
const response: Response = await fetch(url);
const blob: Blob = await response.blob();
const buffer = Buffer.from(await (blob as any).arrayBuffer());
await fs.writeFileSync(downloadPath, buffer);
} catch (error) {
throw new Error(`download failed: ${error.message}`)
throw new Error(`download failed: ${error.message}`);
}
}
8 changes: 4 additions & 4 deletions src/data/download/index.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import getDownloadLink from './downloadLinkGetter'
import downloadZip from './downloader'
import getDownloadLink from './downloadLinkGetter';
import downloadZip from './downloader';

export default class Download {
static downloadZip = downloadZip
static getDownloadLink = getDownloadLink
static downloadZip = downloadZip;
static getDownloadLink = getDownloadLink;
}
16 changes: 8 additions & 8 deletions src/data/files/delete.ts
Original file line number Diff line number Diff line change
@@ -1,17 +1,17 @@
import fs from 'fs'
import fs from 'fs';

const deleteFolder = (path: string) => {
if (fs.existsSync(path)) {
fs.readdirSync(path).forEach((file) => {
const curPath = path + '/' + file
const curPath = path + '/' + file;
if (fs.lstatSync(curPath).isDirectory()) {
deleteFolder(curPath)
deleteFolder(curPath);
} else {
fs.unlinkSync(curPath)
fs.unlinkSync(curPath);
}
})
fs.rmdirSync(path)
});
fs.rmdirSync(path);
}
}
};

export default deleteFolder
export default deleteFolder;
8 changes: 4 additions & 4 deletions src/data/files/index.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import deleteFolder from './delete'
import unzip from './unzipper'
import deleteFolder from './delete';
import unzip from './unzipper';

export default class Files {
static unzip = unzip
static deleteFolder = deleteFolder
static unzip = unzip;
static deleteFolder = deleteFolder;
}
10 changes: 5 additions & 5 deletions src/data/files/unzipper.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import * as unzipFile from 'unzip-stream'
import fs from 'fs'
import * as unzipFile from 'unzip-stream';
import fs from 'fs';

export default async function unzip(
outPath: string,
Expand All @@ -10,9 +10,9 @@ export default async function unzip(
fs.createReadStream(downloadPath)
.pipe(unzipFile.Extract({ path: outPath }))
.on('error', reject)
.on('finish', resolve)
})
.on('finish', resolve);
});
} catch (error) {
throw new Error(`unzip failed: ${error.message}`)
throw new Error(`unzip failed: ${error.message}`);
}
}
18 changes: 9 additions & 9 deletions src/data/importer/folderImporter.ts
Original file line number Diff line number Diff line change
@@ -1,17 +1,17 @@
import fs from 'fs'
import fs from 'fs';

import Download from '../download'
import Files from '../files'
import Download from '../download';
import Files from '../files';

export default async function importFolder(url: string, outPath: string) {
try {
const downloadPath = `./${Math.random().toString(36).substring(2)}.zip`
const downloadLink: string = await Download.getDownloadLink(url)
const downloadPath = `./${Math.random().toString(36).substring(2)}.zip`;
const downloadLink: string = await Download.getDownloadLink(url);

await Download.downloadZip(downloadLink, downloadPath)
await Files.unzip(outPath, downloadPath)
await fs.promises.unlink(downloadPath)
await Download.downloadZip(downloadLink, downloadPath);
await Files.unzip(outPath, downloadPath);
await fs.promises.unlink(downloadPath);
} catch (error) {
throw new Error(`importFolder failed: ${error.message}`)
throw new Error(`importFolder failed: ${error.message}`);
}
}
4 changes: 2 additions & 2 deletions src/data/importer/index.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import importFolder from './folderImporter'
import importFolder from './folderImporter';

export default class Importer {
static importFolder = importFolder
static importFolder = importFolder;
}
4 changes: 2 additions & 2 deletions src/index.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
import main from './main'
import main from './main';

main()
main();
44 changes: 22 additions & 22 deletions src/main.ts
Original file line number Diff line number Diff line change
@@ -1,44 +1,44 @@
import fetch, { Response } from 'node-fetch'
import fetch, { Response } from 'node-fetch';

import Importer from './data/importer'
import Content from './processing/content'
import Directory from './processing/directory'
import Importer from './data/importer';
import Content from './processing/content';
import Directory from './processing/directory';

const url =
'https://app.thedigitalbiblelibrary.org/entries/_public_domain_entries_tabledata.json'
'https://app.thedigitalbiblelibrary.org/entries/_public_domain_entries_tabledata.json';

export default async function main(): Promise<void> {
try {
let count = 0
const response: Response = await fetch(url)
const initialData = await response.json()
const array = initialData.aaData
let count = 0;
const response: Response = await fetch(url);
const initialData = await response.json();
const array = initialData.aaData;

for await (const item of array) {
count++
console.log(`(${count + '/' + array.length}) Setting up: ${item[4]}`)
count++;
console.log(`(${count + '/' + array.length}) Setting up: ${item[4]}`);
await setupBible(
`https://app.thedigitalbiblelibrary.org/entry?id=${item[0]}`
)
);
}
} catch (error) {
throw new Error(`test failed: ${error.message}`)
throw new Error(`test failed: ${error.message}`);
}
}

async function setupBible(url: string): Promise<void> {
try {
const startTime = performance.now()
const outPath = `./${Math.random().toString(36).substring(2)}`
const startTime = performance.now();
const outPath = `./${Math.random().toString(36).substring(2)}`;

await Importer.importFolder(url, outPath)
await Directory.createDirs(outPath)
await Content.populate(outPath)
await Importer.importFolder(url, outPath);
await Directory.createDirs(outPath);
await Content.populate(outPath);

const endTime = performance.now()
const timeTaken = endTime - startTime
console.log(`Finished In ${(timeTaken / 1000).toFixed(2)} seconds @ ${new Date().toLocaleString('en-US', {timeZone: 'America/Los_Angeles'})}}`)
const endTime = performance.now();
const timeTaken = endTime - startTime;
console.log(`Finished In ${(timeTaken / 1000).toFixed(2)} seconds @ ${new Date().toLocaleString('en-US', {timeZone: 'America/Los_Angeles'})}}`);
} catch (error) {
console.error(`Error setting up Bible: ${error.message}`)
console.error(`Error setting up Bible: ${error.message}`);
}
}
56 changes: 28 additions & 28 deletions src/processing/content/contentGetter.ts
Original file line number Diff line number Diff line change
@@ -1,25 +1,25 @@
import * as global from 'src/types/globalTypes'
import * as types from 'src/types/processingTypes'
import cheerio from 'cheerio'
import fs from 'fs'
import * as global from 'src/types';
import * as types from 'src/types';
import cheerio from 'cheerio';
import fs from 'fs';

import usxParser from '../usxParser'
import usxParser from '../usxParser';

export async function booksInfo(outPath: string) {
const infoFile = fs.readFileSync(outPath + '/metadata.xml')
const $ = cheerio.load(infoFile)
const data = fs.readFileSync(outPath + '/release/versification.vrs')
const infoFile = fs.readFileSync(outPath + '/metadata.xml');
const $ = cheerio.load(infoFile);
const data = fs.readFileSync(outPath + '/release/versification.vrs');

const result = []
const result = [];

const lines = data.toString().replace(/\r\n/g, '\n').split('\n')
const lines = data.toString().replace(/\r\n/g, '\n').split('\n');

for (var i = 0; i < lines.length; i++) {
if (lines[i].includes('=') || lines[i].includes('#')) continue
const parts = lines[i].split(' ')
if (lines[i].includes('=') || lines[i].includes('#')) {continue;}
const parts = lines[i].split(' ');

for (var x = 0; x < parts.length; x++) {
if (parts[i] !== undefined) if (!parts[i].includes(':')) continue
if (parts[i] !== undefined) {if (!parts[i].includes(':')) {continue;}}

result.push({
name: $(
Expand All @@ -32,17 +32,17 @@ export async function booksInfo(outPath: string) {
.first()
.text(),
chapter: parts[x].split(':')[0],
verses: parts[x].split(':')[1]
})
verses: parts[x].split(':')[1],
});
}
}

return result.filter((a) => a.name !== '' && a.verses)
return result.filter((a) => a.name !== '' && a.verses);
}

export async function getInfo(outPath: string): Promise<global.versionInfo> {
const infoFile = await fs.promises.readFile(outPath + '/metadata.xml')
const $ = cheerio.load(infoFile)
const infoFile = await fs.promises.readFile(outPath + '/metadata.xml');
const $ = cheerio.load(infoFile);
return {
id: `${$('ldml').text() || $('language > iso').text()}-${
$('abbreviationLocal').first().text().toLowerCase() ||
Expand All @@ -54,11 +54,11 @@ export async function getInfo(outPath: string): Promise<global.versionInfo> {
language: {
name: $('language > name').text(),
code: $('language > iso').text(),
level: $('audience').text()
level: $('audience').text(),
},
country: {
name: $('country > name').text(),
code: $('country > iso').text()
code: $('country > iso').text(),
},
numeralSystem: $('numerals').text(),
script: $('script').text(),
Expand All @@ -67,14 +67,14 @@ export async function getInfo(outPath: string): Promise<global.versionInfo> {
'copyright > fullStatement > statementContent > p > strong'
).text(),
localVersionName: $('nameLocal').first().text(),
localVersionAbbreviation: $('abbreviationLocal').first().text()
}
localVersionAbbreviation: $('abbreviationLocal').first().text(),
};
}

export async function getContent(outPath: string) {
const arr = await booksInfo(outPath)
const usx = await usxParser.parseUSX(outPath)
const array = []
const arr = await booksInfo(outPath);
const usx = await usxParser.parseUSX(outPath);
const array = [];

for (var i = 0; i < arr.length; i++) {
array.push({
Expand All @@ -90,8 +90,8 @@ export async function getContent(outPath: string) {
.replace(/first/i, '1')
.replace(/second/i, '2')
.replace(/third/i, '3') && a.chapter == arr[i].chapter
)
})
),
});
}
return array.filter((a) => a.verses.length > 0)
return array.filter((a) => a.verses.length > 0);
}
Loading

0 comments on commit dcea502

Please sign in to comment.