Skip to content
This repository has been archived by the owner on Dec 2, 2022. It is now read-only.

Commit

Permalink
Implemented #17
Browse files Browse the repository at this point in the history
  • Loading branch information
@DazornSama
DazornSama committed Aug 13, 2021
1 parent 4f201ec commit d9b3b49
Show file tree
Hide file tree
Showing 5 changed files with 75 additions and 55 deletions.
2 changes: 2 additions & 0 deletions SyncService/Models/AnimeMatching.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,5 +13,7 @@ public class AnimeMatching
public int Score { get; set; }
public string Path { get; set; }
public List<EpisodeMatching> Episodes { get; set; } = new List<EpisodeMatching>();
public string SourceVariant { get; set; }
public AnimeMatching Linked { get; set; }
}
}
89 changes: 53 additions & 36 deletions SyncService/Models/IWebsiteScraper.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,8 @@ public abstract class IWebsiteScraper
protected Thread Thread { get; private set; }
public bool Working { get; private set; } = false;

protected List<EpisodeMatching> EpisodeMatchings = new List<EpisodeMatching>();

public IWebsiteScraper(WebsiteScraperService service)
{
this.Service = service;
Expand Down Expand Up @@ -174,43 +176,16 @@ private async Task run()

try
{
for (int i = 1; i <= _anime.EpisodesCount; i++)
AnimeMatching linked = matching.Linked;

await getEpisodes(browser, matching);

while (linked != null)
{
browser = await ProxyHelper.Instance.GetBrowser();
try
{
using (Page webPage = await ProxyHelper.Instance.GetBestProxy(browser, this.Website.CanBlockRequests))
{
EpisodeMatching episode = await this.GetEpisode(webPage, matching, i);

if (episode != null)
{
Episode ep = new Episode()
{
AnimeID = _anime.Id,
Source = this.Website.Name,
Number = episode.Number,
Title = episode.Title,
Video = episode.Source,
Locale = this.Website.Localization
};

if (this._episodeCollection.Exists(ref ep))
{
this._episodeCollection.Edit(ref ep);
}
else
{
this._episodeCollection.Add(ref ep);
}
}
}
}
catch { }
finally
{
await browser.CloseAsync();
}
EpisodeMatchings.Clear();

await getEpisodes(browser, linked);
linked = linked.Linked;
}
}
catch
Expand Down Expand Up @@ -264,6 +239,48 @@ private async Task run()
}
}

private async Task getEpisodes(Browser browser, AnimeMatching matching)
{
for (int i = 1; i <= _anime.EpisodesCount; i++)
{
browser = await ProxyHelper.Instance.GetBrowser();
try
{
using (Page webPage = await ProxyHelper.Instance.GetBestProxy(browser, this.Website.CanBlockRequests))
{
EpisodeMatching episode = await this.GetEpisode(webPage, matching, i);

if (episode != null)
{
Episode ep = new Episode()
{
AnimeID = _anime.Id,
Source = $"{this.Website.Name}{matching.SourceVariant}",
Number = episode.Number,
Title = episode.Title,
Video = episode.Source,
Locale = this.Website.Localization
};

if (this._episodeCollection.Exists(ref ep))
{
this._episodeCollection.Edit(ref ep);
}
else
{
this._episodeCollection.Add(ref ep);
}
}
}
}
catch { }
finally
{
await browser.CloseAsync();
}
}
}

private bool animeNeedWork(bool forceReload = false)
{
if (forceReload)
Expand Down
10 changes: 4 additions & 6 deletions SyncService/Models/WebsiteScrapers/AnimeworldScraper.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,16 +21,14 @@ public AnimeworldScraper(WebsiteScraperService service) : base(service)

protected override Type WebsiteType => typeof(AnimeworldScraper);

private List<EpisodeMatching> episodesMatchings = new List<EpisodeMatching>();

protected override async Task<AnimeMatching> GetMatching(Page webPage, string animeTitle)
{
AnimeMatching matching = null;

string url = $"{this.Website.SiteUrl}search?keyword={animeTitle}";
await ProxyHelper.NavigateAsync(webPage, url);

episodesMatchings.Clear();
EpisodeMatchings.Clear();

await webPage.WaitForSelectorAsync(".film-list", new WaitForSelectorOptions()
{
Expand Down Expand Up @@ -62,7 +60,7 @@ protected override async Task<EpisodeMatching> GetEpisode(Page webPage, AnimeMat
{
string url;

if (episodesMatchings.Count == 0)
if (EpisodeMatchings.Count == 0)
{
url = this.Website.SiteUrl.Substring(0, this.Website.SiteUrl.Length - 1);
url = $"{url}{matching.Path}";
Expand Down Expand Up @@ -94,7 +92,7 @@ protected override async Task<EpisodeMatching> GetEpisode(Page webPage, AnimeMat
string path = await info.EvaluateFunctionAsync<string>("e => e.getAttribute('href')");
string title = (await info.EvaluateFunctionAsync<string>("e => e.innerText")).Trim();

episodesMatchings.Add(new EpisodeMatching()
EpisodeMatchings.Add(new EpisodeMatching()
{
Path = path,
Title = title
Expand All @@ -105,7 +103,7 @@ protected override async Task<EpisodeMatching> GetEpisode(Page webPage, AnimeMat
}
}

EpisodeMatching episode = episodesMatchings[number - 1];
EpisodeMatching episode = EpisodeMatchings[number - 1];

if (episode != null)
{
Expand Down
10 changes: 4 additions & 6 deletions SyncService/Models/WebsiteScrapers/DreamsubScraper.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ protected override async Task<AnimeMatching> GetMatching(Page webPage, string an
{
AnimeMatching matching = null;

episodesMatchings.Clear();
EpisodeMatchings.Clear();

string url = $"{this.Website.SiteUrl}/search/?q={animeTitle}";
await ProxyHelper.NavigateAsync(webPage, url);
Expand Down Expand Up @@ -56,13 +56,11 @@ protected override async Task<AnimeMatching> GetMatching(Page webPage, string an
return null;
}

private List<EpisodeMatching> episodesMatchings = new List<EpisodeMatching>();

protected override async Task<EpisodeMatching> GetEpisode(Page webPage, AnimeMatching matching, int number)
{
string url;

if(episodesMatchings.Count == 0)
if(EpisodeMatchings.Count == 0)
{
url = $"{this.Website.SiteUrl}{matching.Path}";
await ProxyHelper.NavigateAsync(webPage, url);
Expand All @@ -83,15 +81,15 @@ protected override async Task<EpisodeMatching> GetEpisode(Page webPage, AnimeMat
string path = await info.EvaluateFunctionAsync<string>("e => e.getAttribute('href')");
string title = (await info.EvaluateFunctionAsync<string>("e => e.innerText")).Trim().Split(": ")[1];

episodesMatchings.Add(new EpisodeMatching()
EpisodeMatchings.Add(new EpisodeMatching()
{
Path = path,
Title = title
});
}
}

EpisodeMatching episode = episodesMatchings[number - 1];
EpisodeMatching episode = EpisodeMatchings[number - 1];

if(episode != null)
{
Expand Down
19 changes: 12 additions & 7 deletions SyncService/Models/WebsiteScrapers/GogoanimeScraper.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,6 @@ public GogoanimeScraper(WebsiteScraperService service) : base(service)

protected override Type WebsiteType => typeof(GogoanimeScraper);

private List<EpisodeMatching> episodesMatchings = new List<EpisodeMatching>();

protected override async Task<AnimeMatching> GetMatching(Page webPage, string animeTitle)
{
AnimeMatching matching = null;
Expand All @@ -44,7 +42,7 @@ protected override async Task<AnimeMatching> GetMatching(Page webPage, string an
new WebsiteCollection().Edit(ref website);
}

episodesMatchings.Clear();
EpisodeMatchings.Clear();

await webPage.WaitForSelectorAsync(".last_episodes", new WaitForSelectorOptions()
{
Expand All @@ -65,6 +63,13 @@ protected override async Task<AnimeMatching> GetMatching(Page webPage, string an
ElementHandle path = await title.QuerySelectorAsync("a");
matching.Path = (await path.EvaluateFunctionAsync<string>("e => e.getAttribute('href')")).Trim();

matching.Linked = new AnimeMatching()
{
Title = matching.Title,
Path = $"{matching.Path}-dub",
SourceVariant = "_dub"
};

return matching;
}
}
Expand All @@ -76,7 +81,7 @@ protected override async Task<EpisodeMatching> GetEpisode(Page webPage, AnimeMat
{
string url;

if (episodesMatchings.Count == 0)
if (EpisodeMatchings.Count == 0)
{
url = this.Website.SiteUrl.Substring(0, this.Website.SiteUrl.Length - 1);
url = $"{url}{matching.Path}";
Expand Down Expand Up @@ -114,16 +119,16 @@ protected override async Task<EpisodeMatching> GetEpisode(Page webPage, AnimeMat
string path = await info.EvaluateFunctionAsync<string>("e => e.getAttribute('href')");
string title = (await info.QuerySelectorAsync(".name").EvaluateFunctionAsync<string>("e => e.innerText")).Trim();

episodesMatchings.Add(new EpisodeMatching()
EpisodeMatchings.Add(new EpisodeMatching()
{
Path = path,
Title = title
});
}
episodesMatchings.Reverse();
EpisodeMatchings.Reverse();
}

EpisodeMatching episode = episodesMatchings[number - 1];
EpisodeMatching episode = EpisodeMatchings[number - 1];

if (episode != null)
{
Expand Down

0 comments on commit d9b3b49

Please sign in to comment.