From ddc15f9289f1e0d757adf9f856755edcae98680d Mon Sep 17 00:00:00 2001 From: sinkaroid Date: Wed, 19 Apr 2023 21:24:12 +0700 Subject: [PATCH 1/6] broken slash will broke it --- src/LustPress.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/LustPress.ts b/src/LustPress.ts index 49168e6..8720270 100644 --- a/src/LustPress.ts +++ b/src/LustPress.ts @@ -40,6 +40,7 @@ class LustPress { return res.body; } else { console.log("Fetching from source"); + url = url.replace(/\/\//g, "/"); const res = await p({ url: url, "headers": { From cef68fb306825521a22f6f20c80d115032c56a65 Mon Sep 17 00:00:00 2001 From: sinkaroid Date: Wed, 19 Apr 2023 21:28:25 +0700 Subject: [PATCH 2/6] xhamster fix embed assets --- src/scraper/xhamster/xhamsterSearchController.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/scraper/xhamster/xhamsterSearchController.ts b/src/scraper/xhamster/xhamsterSearchController.ts index 7bb7eeb..07608c1 100644 --- a/src/scraper/xhamster/xhamsterSearchController.ts +++ b/src/scraper/xhamster/xhamsterSearchController.ts @@ -1,5 +1,6 @@ import { load } from "cheerio"; import LustPress from "../../LustPress"; +import c from "../../utils/options"; import { ISearchVideoData } from "../../interfaces"; const lust = new LustPress(); @@ -33,7 +34,7 @@ export async function scrapeContent(url: string) { image: $(el).find("img").attr("src"), duration: duration[i], views: views[i], - video: $(el).attr("data-previewvideo"), + video: `${c.XHAMSTER}/embed/${link?.split("-").pop()}` }; }).get(); } From 46c3f42838bf058c88f554b7c88200a4a06d3bf1 Mon Sep 17 00:00:00 2001 From: sinkaroid Date: Wed, 19 Apr 2023 21:29:10 +0700 Subject: [PATCH 3/6] xnxx fix nullable assets --- src/scraper/xnxx/xnxxGetController.ts | 6 ++++-- src/scraper/xnxx/xnxxGetRelatedController.ts | 5 +++-- src/scraper/xnxx/xnxxSearchController.ts | 6 +++--- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src/scraper/xnxx/xnxxGetController.ts b/src/scraper/xnxx/xnxxGetController.ts index 3683047..6654335 100644 --- a/src/scraper/xnxx/xnxxGetController.ts +++ b/src/scraper/xnxx/xnxxGetController.ts @@ -6,7 +6,6 @@ const lust = new LustPress(); export async function scrapeContent(url: string) { try { - console.log(url); const resolve = await lust.fetchBody(url); const $ = load(resolve); @@ -27,6 +26,7 @@ export async function scrapeContent(url: string) { thumbnail: string; bigimg: string; video: string; + embed: string; constructor() { const thumb = $("script") .map((i, el) => { @@ -69,6 +69,8 @@ export async function scrapeContent(url: string) { .map((i, el) => { return $(el).text(); }).get(); + this.embed = $("input#copy-video-embed").attr("value") || "None"; + this.embed = this.embed.split("iframe")[1].split(" ")[1].replace(/src=/g, "").replace(/"/g, "") || "None"; } } @@ -91,7 +93,7 @@ export async function scrapeContent(url: string) { tags: x.tags.filter((el) => el !== "Edit tags and models") }, source: x.link, - assets: lust.removeAllSingleQuoteOnArray([x.thumbnail, x.bigimg, x.video]) + assets: lust.removeAllSingleQuoteOnArray([x.embed, x.thumbnail, x.bigimg, x.video]) }; return data; diff --git a/src/scraper/xnxx/xnxxGetRelatedController.ts b/src/scraper/xnxx/xnxxGetRelatedController.ts index 8ffec25..cf8a1a2 100644 --- a/src/scraper/xnxx/xnxxGetRelatedController.ts +++ b/src/scraper/xnxx/xnxxGetRelatedController.ts @@ -22,15 +22,16 @@ export async function scrapeContent(url: string) { //stop and replace everything after the last ]; const badJson = video_related?.split("];")[0] + "]"; const actualResult = JSON.parse(String(badJson)); + //console.log(actualResult); const result = actualResult.map((el: any) => { return { link: `${c.XNXX}${el.u}`, - id: el.u, + id: el.u.slice(1, -1), title: el.t, image: el.i, duration: el.d, views: `${el.n}, ${el.r}`, - video: null + video: `${c.XNXX}/embedframe/${el.id}` }; }); return result; diff --git a/src/scraper/xnxx/xnxxSearchController.ts b/src/scraper/xnxx/xnxxSearchController.ts index 0ef0f08..065d22b 100644 --- a/src/scraper/xnxx/xnxxSearchController.ts +++ b/src/scraper/xnxx/xnxxSearchController.ts @@ -17,7 +17,8 @@ export async function scrapeContent(url: string) { .map((i, el) => { return { link: `${c.XNXX}${$(el).find("a").attr("href")}`, - id: $(el).find("a").attr("href"), + // remove first "/" and last "/" + id: $(el).find("a").attr("href")?.slice(1, -1), title: $(el).find("div.thumb-under").text().split("\n") .map((el) => el.trim()).filter((el) => el !== "")[0], image: $(el).find("img").attr("data-src"), @@ -25,8 +26,7 @@ export async function scrapeContent(url: string) { .map((el) => el.trim()).filter((el) => el !== "")[2], rating: $(el).find("div.thumb-under").text().split("\n") .map((el) => el.trim()).filter((el) => el !== "")[1], - video: null - + video: `${c.XNXX}/embedframe/${$(el).find("img").attr("data-videoid")}` }; }).get(); } From ce9901ca4f52bd9b8f707508d4a7f43c8c7e90ef Mon Sep 17 00:00:00 2001 From: sinkaroid Date: Wed, 19 Apr 2023 21:29:31 +0700 Subject: [PATCH 4/6] xvideos fix nullable assets --- src/scraper/xvideos/xvideosGetController.ts | 6 ++++-- src/scraper/xvideos/xvideosGetRelatedController.ts | 5 ++--- src/scraper/xvideos/xvideosSearchController.ts | 3 +-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/scraper/xvideos/xvideosGetController.ts b/src/scraper/xvideos/xvideosGetController.ts index 5a81088..6046994 100644 --- a/src/scraper/xvideos/xvideosGetController.ts +++ b/src/scraper/xvideos/xvideosGetController.ts @@ -25,6 +25,7 @@ export async function scrapeContent(url: string) { models: string[]; thumbnail: string; bigimg: string; + embed: string; constructor() { this.link = $("meta[property='og:url']").attr("content") || "None"; this.id = this.link.split("/")[3] + "/" + this.link.split("/")[4] || "None"; @@ -60,7 +61,8 @@ export async function scrapeContent(url: string) { } ).get(); this.models = this.models.map((el) => el.split("/")[2]); - + this.embed = $("input#copy-video-embed").attr("value") || "None"; + this.embed = this.embed.split("iframe")[1].split(" ")[1].replace(/src=/g, "").replace(/"/g, "") || "None"; } } @@ -81,7 +83,7 @@ export async function scrapeContent(url: string) { tags: xv.tags, }, source: xv.link, - assets: lust.removeAllSingleQuoteOnArray([xv.thumbnail, xv.bigimg, xv.video]) + assets: lust.removeAllSingleQuoteOnArray([xv.embed, xv.thumbnail, xv.bigimg, xv.video]) }; return data; diff --git a/src/scraper/xvideos/xvideosGetRelatedController.ts b/src/scraper/xvideos/xvideosGetRelatedController.ts index 5ec3243..43395ba 100644 --- a/src/scraper/xvideos/xvideosGetRelatedController.ts +++ b/src/scraper/xvideos/xvideosGetRelatedController.ts @@ -9,7 +9,6 @@ export async function scrapeContent(url: string) { try { const res = await lust.fetchBody(url); const $ = load(res); - console.log(url); class XvideosSearch { search: object[]; @@ -24,12 +23,12 @@ export async function scrapeContent(url: string) { const result = actualResult.map((el: any) => { return { link: `${c.XVIDEOS}${el.u}`, - id: el.u, + id: el.u.slice(1, -1), title: el.t, image: el.i, duration: el.d, views: `${el.n}, ${el.r}`, - video: null + video: `${c.XVIDEOS}/embedframe/${el.id}` }; }); return result; diff --git a/src/scraper/xvideos/xvideosSearchController.ts b/src/scraper/xvideos/xvideosSearchController.ts index b50a5af..366ffb2 100644 --- a/src/scraper/xvideos/xvideosSearchController.ts +++ b/src/scraper/xvideos/xvideosSearchController.ts @@ -7,7 +7,6 @@ const lust = new LustPress(); export async function scrapeContent(url: string) { try { - console.log(url); const res = await lust.fetchBody(url); const $ = load(res); @@ -36,7 +35,7 @@ export async function scrapeContent(url: string) { ? "" : data[i].duration || "None", rating: null, - video: null + video: `${c.XVIDEOS}/embedframe/${$(el).find("img").attr("data-videoid")}` }; }).get(); From b4d0a47991d2135a45cbd06bd978956cd4217f4b Mon Sep 17 00:00:00 2001 From: sinkaroid Date: Wed, 19 Apr 2023 21:30:49 +0700 Subject: [PATCH 5/6] it should be none instead null type --- src/scraper/youporn/youpornGetController.ts | 4 ++-- src/scraper/youporn/youpornSearchController.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/scraper/youporn/youpornGetController.ts b/src/scraper/youporn/youpornGetController.ts index f458712..6518a7d 100644 --- a/src/scraper/youporn/youpornGetController.ts +++ b/src/scraper/youporn/youpornGetController.ts @@ -19,7 +19,7 @@ export async function scrapeContent(url: string) { rating: string; publish: string; upVote: string; - downVote: null; + downVote: string; video: string; tags: string[]; models: string[]; @@ -33,7 +33,7 @@ export async function scrapeContent(url: string) { this.rating = $("div.feature").find("span").text().replace(/[^0-9.,%]/g, "") || "0"; this.publish = $("div.video-uploaded").find("span").text() || "None"; this.upVote = this.views; - this.downVote = null; + this.downVote = "None"; this.video = `https://www.youporn.com/embed/${this.id}`; this.tags = $("a[data-espnode='category_tag'], a[data-espnode='porntag_tag']") .map((i, el) => { diff --git a/src/scraper/youporn/youpornSearchController.ts b/src/scraper/youporn/youpornSearchController.ts index d765b40..2382fe3 100644 --- a/src/scraper/youporn/youpornSearchController.ts +++ b/src/scraper/youporn/youpornSearchController.ts @@ -29,7 +29,7 @@ export async function scrapeContent(url: string) { title: lust.removeHtmlTagWithoutSpace(title), image: image, duration: this.dur[i], - views: null, + views: "None", video: `https://www.youporn.com/embed/${id}`, }; }).get(); From 3fbce4e3cb8ca5a5935bb87c092c9648ffeb62b0 Mon Sep 17 00:00:00 2001 From: sinkaroid Date: Wed, 19 Apr 2023 21:31:54 +0700 Subject: [PATCH 6/6] pre release --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 1557ec4..20c91ba 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "lustpress", - "version": "1.6.1-alpha", + "version": "1.6.2-alpha", "description": "RESTful and experimental API for PornHub and other porn sites, which official is lack even isn't exist.", "main": "build/src/index.js", "scripts": {