upgrade packages

.eslintrc.json

@@ -1,20 +1,19 @@
 {
-    "env": {
-        "es2021": true,
-        "node": true
-    },
-    "extends": [
-        "plugin:@typescript-eslint/recommended",
-        "plugin:prettier/recommended"
-    ],
-    "parser": "@typescript-eslint/parser",
-    "parserOptions": {
-        "ecmaVersion": 12,
-        "sourceType": "module"
-    },
-    "plugins": [
-        "@typescript-eslint"
-    ],
-    "rules": {
-    }
+  "env": {
+    "es2021": true,
+    "node": true
+  },
+  "extends": [
+    "plugin:@typescript-eslint/recommended"
+  ],
+  "parser": "@typescript-eslint/parser",
+  "parserOptions": {
+    "ecmaVersion": 12,
+    "sourceType": "module"
+  },
+  "plugins": [
+    "@typescript-eslint"
+  ],
+  "rules": {
+  }
 }

package.json

@@ -38,19 +38,16 @@
     "puppeteer": "^19.8.2"
   },
   "devDependencies": {
-    "@tsconfig/node12": "^1.0.11",
     "@types/node": "^20.6.0",
     "@types/pdf-parse": "^1.1.1",
     "@types/tap": "^15.0.8",
     "@typescript-eslint/eslint-plugin": "^5.62.0",
     "@typescript-eslint/parser": "^5.62.0",
     "eslint": "^8.49.0",
-    "eslint-config-prettier": "^8.8.0",
     "eslint-plugin-import": "^2.27.5",
     "eslint-plugin-prettier": "^5.0.0",
     "fastify-cli": "^5.7.1",
     "pdf-parse": "^1.1.1",
-    "prettier": "^3.0.3",
     "tap": "^16.3.8",
     "ts-node": "^10.9.1",
     "typescript": "^5.1.6"

src/plugins/hc-pages.ts

@@ -0,0 +1,211 @@
+import puppeteer from 'puppeteer'
+import { BrowserLaunchArgumentOptions, Page, Browser, Viewport } from 'puppeteer'
+import fp from 'fastify-plugin'
+import { FastifyInstance } from 'fastify'
+
+declare module 'fastify' {
+  interface FastifyInstance {
+    runOnPage<T>(callback: RunOnPageCallback<T>): Promise<T>
+    destroyPages(): Promise<void>
+  }
+}
+
+export const plugin = async (
+  fastify: FastifyInstance,
+  options: HcPagesOptions,
+): Promise<void> => {
+  const { pagesNum, pageOptions, launchOptions } = options
+  const hcPages = await HCPages.init(pagesNum, pageOptions, launchOptions)
+  console.log('hcPages', hcPages)
+  fastify.decorate(
+    'runOnPage',
+    async (callback: RunOnPageCallback<unknown>) => {
+      return await hcPages.runOnPage(callback)
+    }
+  )
+  fastify.decorate('destroyPages', async () => {
+    await hcPages.destroy()
+  })
+  fastify.addHook('onClose', async (instance, done) => {
+    await instance.destroyPages()
+    done()
+  })
+}
+
+export const hcPages = fp(plugin, {
+  fastify: '^4.0.0',
+  name: 'hc-pages-plugin',
+})
+
+export default hcPages
+
+export interface HcPagesOptions {
+  pagesNum?: number
+  pageOptions?: Partial<PageOptions>
+  launchOptions?: BrowserLaunchArgumentOptions
+}
+
+export interface PageOptions {
+  userAgent: string
+  pageTimeoutMilliseconds: number
+  emulateMediaTypeScreenEnabled: boolean
+  acceptLanguage: string
+  viewport?: Viewport
+}
+
+export type RunOnPageCallback<T> = (page: Page) => Promise<T>
+
+const defaultPagesNum = 3
+
+const defaultPageOptions: PageOptions = {
+  userAgent: '',
+  pageTimeoutMilliseconds: 10000,
+  emulateMediaTypeScreenEnabled: false,
+  acceptLanguage: '',
+}
+
+const defaultLaunchOptions: BrowserLaunchArgumentOptions = {
+  args: [
+    '--no-sandbox',
+    '--disable-setuid-sandbox',
+    '--disable-gpu',
+    '--disable-dev-shm-usage',
+  ],
+  headless: true
+}
+
+export class HCPages {
+  private pagesNum: number
+  private pages: Page[]
+  private readyPages: Page[]
+  private currentPromises: Promise<unknown>[]
+  private pageOptions: PageOptions
+  private browser: Browser
+
+  constructor(
+    browser: Browser,
+    pagesNum: number,
+    pageOptions = {} as Partial<PageOptions> | undefined
+  ) {
+    this.pagesNum = pagesNum
+    this.pageOptions = { ...defaultPageOptions, ...pageOptions }
+    this.browser = browser
+    this.pages = []
+    this.readyPages = []
+    this.currentPromises = []
+  }
+
+  public static init = async (
+    pagesNum = defaultPagesNum,
+    pageOptions: Partial<PageOptions> | undefined = undefined,
+    launchOptions?: BrowserLaunchArgumentOptions
+  ): Promise<HCPages> => {
+    const mergedLaunchOptions = { ...defaultLaunchOptions, ...launchOptions }
+    console.debug('launchOptions', mergedLaunchOptions)
+    const browser = await puppeteer.launch(mergedLaunchOptions)
+    console.debug(`browser.verison is ${await browser.version()}`)
+    const hcPages = new HCPages(browser, pagesNum, pageOptions)
+    hcPages.pages = await hcPages.createPages()
+    hcPages.readyPages = hcPages.pages
+    return hcPages
+  }
+
+  public async destroy(): Promise<void> {
+    await this.closePages()
+    await this.closeBrowser()
+  }
+
+  private async runCallback<T>(
+    page: Page,
+    callback: RunOnPageCallback<T>
+  ): Promise<T> {
+    try {
+      const result = await callback(page)
+      return result
+    } catch (e) {
+      console.error(e)
+      throw e
+    } finally {
+      this.readyPages.push(page)
+    }
+  }
+
+  public async runOnPage<T>(callback: RunOnPageCallback<T>): Promise<T> {
+    let page = this.readyPages.pop()
+    while (!page) {
+      await Promise.race(this.currentPromises)
+      page = this.readyPages.pop()
+    }
+
+    const promise = this.runCallback(page, callback)
+    this.currentPromises.push(promise)
+    try {
+      return await promise
+    } catch (e) {
+      console.error(e)
+      throw e
+    } finally {
+      this.currentPromises.splice(this.currentPromises.indexOf(promise), 1)
+    }
+  }
+
+  private async createPage(): Promise<Page> {
+    const page = await this.browser.newPage()
+    await this.applyPageConfigs(page)
+    return page
+  }
+
+  private async createPages(): Promise<Page[]> {
+    const pages = []
+    for (let i = 0; i < this.pagesNum; i++) {
+      const page = await this.createPage()
+      console.log(`page number ${i} is created`)
+      pages.push(page)
+    }
+    return pages
+  }
+
+  private async applyPageConfigs(page: Page): Promise<void> {
+    const {
+      pageTimeoutMilliseconds,
+      userAgent,
+      emulateMediaTypeScreenEnabled,
+      acceptLanguage,
+      viewport,
+    } = this.pageOptions
+    if (pageTimeoutMilliseconds) {
+      page.setDefaultTimeout(pageTimeoutMilliseconds)
+      console.log(`defaultTimeout set ${pageTimeoutMilliseconds}`)
+    }
+    if (viewport) {
+      await page.setViewport(viewport)
+      console.log(`viewport set ${JSON.stringify(page.viewport())}`)
+    }
+    if (userAgent) {
+      await page.setUserAgent(userAgent)
+      console.log(`user agent set ${userAgent}`)
+    }
+    if (emulateMediaTypeScreenEnabled) {
+      await page.emulateMediaType('screen')
+      console.log('emulateMediaType screen')
+    }
+    if (acceptLanguage) {
+      await page.setExtraHTTPHeaders({
+        'Accept-Language': acceptLanguage,
+      })
+      console.log(`Accept-Language set: ${acceptLanguage}`)
+    }
+  }
+
+  private async closePages(): Promise<void> {
+    for (let i = 0; i < this.pagesNum; i++) {
+      await this.pages[i].close()
+      console.log(`page number ${i} is closed`)
+    }
+  }
+
+  private async closeBrowser(): Promise<void> {
+    await this.browser.close()
+    console.log('browser is closed')
+  }
+}

yarn.lock

@@ -486,7 +486,7 @@
   resolved "https://registry.yarnpkg.com/@tsconfig/node10/-/node10-1.0.9.tgz#df4907fc07a886922637b15e02d4cebc4c0021b2"
   integrity sha512-jNsYVVxU8v5g43Erja32laIDHXeoNvFEpX33OK4d6hljo3jDhCBDhx5dhCCTMWUojscpAagGiRkBKxpdl9fxqA==
 
-"@tsconfig/node12@^1.0.11", "@tsconfig/node12@^1.0.7":
+"@tsconfig/node12@^1.0.7":
   version "1.0.11"
   resolved "https://registry.yarnpkg.com/@tsconfig/node12/-/node12-1.0.11.tgz#ee3def1f27d9ed66dac6e46a295cffb0152e058d"
   integrity sha512-cqefuRsh12pWyGsIoBKJA9luFu3mRxCA+ORZvA4ktLSzIuCUtWVxGIuXigEwO5/ywWFMZ2QEGKWvkZG1zDMTag==
@@ -1498,11 +1498,6 @@ escape-string-regexp@^4.0.0:
   resolved "https://registry.yarnpkg.com/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz#14ba83a5d373e3d311e5afca29cf5bfad965bf34"
   integrity sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==
 
-eslint-config-prettier@^8.8.0:
-  version "8.8.0"
-  resolved "https://registry.yarnpkg.com/eslint-config-prettier/-/eslint-config-prettier-8.8.0.tgz#bfda738d412adc917fd7b038857110efe98c9348"
-  integrity sha512-wLbQiFre3tdGgpDv67NQKnJuTlcUVYHas3k+DZCc2U2BadthoEY4B7hLPvAxaqdyOGCzuLfii2fqGph10va7oA==
-
 eslint-import-resolver-node@^0.3.7:
   version "0.3.7"
   resolved "https://registry.yarnpkg.com/eslint-import-resolver-node/-/eslint-import-resolver-node-0.3.7.tgz#83b375187d412324a1963d84fa664377a23eb4d7"
@@ -3272,11 +3267,6 @@ prettier-linter-helpers@^1.0.0:
   dependencies:
     fast-diff "^1.1.2"
 
-prettier@^3.0.3:
-  version "3.0.3"
-  resolved "https://registry.yarnpkg.com/prettier/-/prettier-3.0.3.tgz#432a51f7ba422d1469096c0fdc28e235db8f9643"
-  integrity sha512-L/4pUDMxcNa8R/EthV08Zt42WBO4h1rarVtK0K+QJG0X187OLo7l699jWw0GKuwzkPQ//jMFA/8Xm6Fh3J/DAg==
-
 process-on-spawn@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/process-on-spawn/-/process-on-spawn-1.0.0.tgz#95b05a23073d30a17acfdc92a440efd2baefdc93"