-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
fix!: remove redundant types directory
- Loading branch information
Showing
23 changed files
with
574 additions
and
275 deletions.
There are no files selected for viewing
Large diffs are not rendered by default.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,23 +1,23 @@ | ||
import cheerio from 'cheerio' | ||
import fetch from 'node-fetch' | ||
import cheerio from 'cheerio'; | ||
import fetch from 'node-fetch'; | ||
|
||
const BASE_URL = 'https://app.thedigitalbiblelibrary.org' | ||
const BASE_URL = 'https://app.thedigitalbiblelibrary.org'; | ||
|
||
export default async function getDownloadLink(url: string): Promise<string> { | ||
try { | ||
const initialResponse = await fetch(url) | ||
const initialData = await initialResponse.text() | ||
const $1 = cheerio.load(initialData) | ||
const initialResponse = await fetch(url); | ||
const initialData = await initialResponse.text(); | ||
const $1 = cheerio.load(initialData); | ||
const downloadResponse = await fetch( | ||
BASE_URL + $1('.list-group-item > a').attr('href') | ||
) | ||
); | ||
|
||
const downloadData = await downloadResponse.text() | ||
const $2 = cheerio.load(downloadData) | ||
const href: string = BASE_URL + $2('#download_button').attr('href') | ||
const downloadData = await downloadResponse.text(); | ||
const $2 = cheerio.load(downloadData); | ||
const href: string = BASE_URL + $2('#download_button').attr('href'); | ||
|
||
return href | ||
return href; | ||
} catch (error) { | ||
throw new Error(`getDownloadLink failed: ${error.message}`) | ||
throw new Error(`getDownloadLink failed: ${error.message}`); | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,16 +1,16 @@ | ||
import fs from 'fs' | ||
import fetch, { Blob, Response } from 'node-fetch' | ||
import fs from 'fs'; | ||
import fetch, { Blob, Response } from 'node-fetch'; | ||
|
||
export default async function downloadZip( | ||
url: string, | ||
downloadPath: string | ||
): Promise<void> { | ||
try { | ||
const response: Response = await fetch(url) | ||
const blob: Blob = await response.blob() | ||
const buffer = Buffer.from(await (blob as any).arrayBuffer()) | ||
await fs.writeFileSync(downloadPath, buffer) | ||
const response: Response = await fetch(url); | ||
const blob: Blob = await response.blob(); | ||
const buffer = Buffer.from(await (blob as any).arrayBuffer()); | ||
await fs.writeFileSync(downloadPath, buffer); | ||
} catch (error) { | ||
throw new Error(`download failed: ${error.message}`) | ||
throw new Error(`download failed: ${error.message}`); | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,7 +1,7 @@ | ||
import getDownloadLink from './downloadLinkGetter' | ||
import downloadZip from './downloader' | ||
import getDownloadLink from './downloadLinkGetter'; | ||
import downloadZip from './downloader'; | ||
|
||
export default class Download { | ||
static downloadZip = downloadZip | ||
static getDownloadLink = getDownloadLink | ||
static downloadZip = downloadZip; | ||
static getDownloadLink = getDownloadLink; | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,17 +1,17 @@ | ||
import fs from 'fs' | ||
import fs from 'fs'; | ||
|
||
const deleteFolder = (path: string) => { | ||
if (fs.existsSync(path)) { | ||
fs.readdirSync(path).forEach((file) => { | ||
const curPath = path + '/' + file | ||
const curPath = path + '/' + file; | ||
if (fs.lstatSync(curPath).isDirectory()) { | ||
deleteFolder(curPath) | ||
deleteFolder(curPath); | ||
} else { | ||
fs.unlinkSync(curPath) | ||
fs.unlinkSync(curPath); | ||
} | ||
}) | ||
fs.rmdirSync(path) | ||
}); | ||
fs.rmdirSync(path); | ||
} | ||
} | ||
}; | ||
|
||
export default deleteFolder | ||
export default deleteFolder; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,7 +1,7 @@ | ||
import deleteFolder from './delete' | ||
import unzip from './unzipper' | ||
import deleteFolder from './delete'; | ||
import unzip from './unzipper'; | ||
|
||
export default class Files { | ||
static unzip = unzip | ||
static deleteFolder = deleteFolder | ||
static unzip = unzip; | ||
static deleteFolder = deleteFolder; | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,17 +1,17 @@ | ||
import fs from 'fs' | ||
import fs from 'fs'; | ||
|
||
import Download from '../download' | ||
import Files from '../files' | ||
import Download from '../download'; | ||
import Files from '../files'; | ||
|
||
export default async function importFolder(url: string, outPath: string) { | ||
try { | ||
const downloadPath = `./${Math.random().toString(36).substring(2)}.zip` | ||
const downloadLink: string = await Download.getDownloadLink(url) | ||
const downloadPath = `./${Math.random().toString(36).substring(2)}.zip`; | ||
const downloadLink: string = await Download.getDownloadLink(url); | ||
|
||
await Download.downloadZip(downloadLink, downloadPath) | ||
await Files.unzip(outPath, downloadPath) | ||
await fs.promises.unlink(downloadPath) | ||
await Download.downloadZip(downloadLink, downloadPath); | ||
await Files.unzip(outPath, downloadPath); | ||
await fs.promises.unlink(downloadPath); | ||
} catch (error) { | ||
throw new Error(`importFolder failed: ${error.message}`) | ||
throw new Error(`importFolder failed: ${error.message}`); | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,5 +1,5 @@ | ||
import importFolder from './folderImporter' | ||
import importFolder from './folderImporter'; | ||
|
||
export default class Importer { | ||
static importFolder = importFolder | ||
static importFolder = importFolder; | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,3 +1,3 @@ | ||
import main from './main' | ||
import main from './main'; | ||
|
||
main() | ||
main(); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,44 +1,44 @@ | ||
import fetch, { Response } from 'node-fetch' | ||
import fetch, { Response } from 'node-fetch'; | ||
|
||
import Importer from './data/importer' | ||
import Content from './processing/content' | ||
import Directory from './processing/directory' | ||
import Importer from './data/importer'; | ||
import Content from './processing/content'; | ||
import Directory from './processing/directory'; | ||
|
||
const url = | ||
'https://app.thedigitalbiblelibrary.org/entries/_public_domain_entries_tabledata.json' | ||
'https://app.thedigitalbiblelibrary.org/entries/_public_domain_entries_tabledata.json'; | ||
|
||
export default async function main(): Promise<void> { | ||
try { | ||
let count = 0 | ||
const response: Response = await fetch(url) | ||
const initialData = await response.json() | ||
const array = initialData.aaData | ||
let count = 0; | ||
const response: Response = await fetch(url); | ||
const initialData = await response.json(); | ||
const array = initialData.aaData; | ||
|
||
for await (const item of array) { | ||
count++ | ||
console.log(`(${count + '/' + array.length}) Setting up: ${item[4]}`) | ||
count++; | ||
console.log(`(${count + '/' + array.length}) Setting up: ${item[4]}`); | ||
await setupBible( | ||
`https://app.thedigitalbiblelibrary.org/entry?id=${item[0]}` | ||
) | ||
); | ||
} | ||
} catch (error) { | ||
throw new Error(`test failed: ${error.message}`) | ||
throw new Error(`test failed: ${error.message}`); | ||
} | ||
} | ||
|
||
async function setupBible(url: string): Promise<void> { | ||
try { | ||
const startTime = performance.now() | ||
const outPath = `./${Math.random().toString(36).substring(2)}` | ||
const startTime = performance.now(); | ||
const outPath = `./${Math.random().toString(36).substring(2)}`; | ||
|
||
await Importer.importFolder(url, outPath) | ||
await Directory.createDirs(outPath) | ||
await Content.populate(outPath) | ||
await Importer.importFolder(url, outPath); | ||
await Directory.createDirs(outPath); | ||
await Content.populate(outPath); | ||
|
||
const endTime = performance.now() | ||
const timeTaken = endTime - startTime | ||
console.log(`Finished In ${(timeTaken / 1000).toFixed(2)} seconds @ ${new Date().toLocaleString('en-US', {timeZone: 'America/Los_Angeles'})}}`) | ||
const endTime = performance.now(); | ||
const timeTaken = endTime - startTime; | ||
console.log(`Finished In ${(timeTaken / 1000).toFixed(2)} seconds @ ${new Date().toLocaleString('en-US', {timeZone: 'America/Los_Angeles'})}}`); | ||
} catch (error) { | ||
console.error(`Error setting up Bible: ${error.message}`) | ||
console.error(`Error setting up Bible: ${error.message}`); | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.