Filter out invalid apps and playtests

FreePackages/Data/SharedExternalResource.cs

@@ -0,0 +1,37 @@
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+
+// This resource may be used zero or more times independently and, when used, needs to be fetched from an external source.
+// If it's used zero times we don't fetch it at all.
+// If it's used once or more then we only fetch it once.
+
+namespace FreePackages {
+	internal sealed class SharedExternalResource<T> {
+		private SemaphoreSlim FetchSemaphore = new SemaphoreSlim(1, 1);
+		private T? Resource;
+		private bool Fetched = false;
+
+		internal SharedExternalResource() {}
+
+		internal async Task<T?> Fetch(Func<Task<T?>> fetchResource) {
+			if (Fetched) {
+				return Resource;
+			}
+
+			await FetchSemaphore.WaitAsync().ConfigureAwait(false);
+			try {
+				if (Fetched) {
+					return Resource;
+				}
+
+				Resource = await fetchResource().ConfigureAwait(false);
+				Fetched = true;
+
+				return Resource;
+			} finally {
+				FetchSemaphore.Release();
+			}
+		}
+	}
+}

FreePackages/Handlers/PackageFilter.cs

@@ -1,7 +1,10 @@
 using System;
 using System.Collections.Generic;
 using System.Linq;
+using AngleSharp.Dom;
 using ArchiSteamFarm.Core;
+using ArchiSteamFarm.Steam.Integration;
+using ArchiSteamFarm.Web.Responses;
 using SteamKit2;
 
 namespace FreePackages {
@@ -188,6 +191,34 @@ internal bool IsAppIgnoredByFilter(FilterableApp app, FilterConfig filter) {
 			return false;
 		}
 
+		internal bool IsAppFreeAndValidOnStore(AppDetails? appDetails) {
+			if (appDetails == null) {
+				// Indeterminate, assume the app is free and valid
+				return true;
+			}
+
+			if (!appDetails.Success) {
+				// App doesn't have a store page
+				// Usually this is true, but not always.  Example: https://store.steampowered.com/api/appdetails/?appids=317780 (on May 13, 2024)
+				// App 317780, also passes all of the checks below, but cannot be activated and doesn't have a store page.  It's type is listed as "advertising".
+				return false;
+			}
+
+			bool isFree = appDetails?.Data?.IsFree ?? false;
+			if (!isFree) {
+				// App is not free
+				return false;
+			}
+
+			bool isComingSoon = appDetails?.Data?.ReleaseDate?.ComingSoon ?? true;
+			if (isComingSoon) {
+				// App is not released yet
+				return false;
+			}
+
+			return true;
+		}
+
 		internal bool IsRedeemablePackage(FilterablePackage package) {			
 			if (UserData == null) {
 				throw new InvalidOperationException(nameof(UserData));
@@ -269,7 +300,7 @@ internal bool IsPackageIgnoredByFilter(FilterablePackage package, FilterConfig f
 		internal bool IsRedeemablePlaytest(FilterableApp app) {
 			// More than half of playtests we try to join will be invalid.
 			// Some of these will be becase there's no free packages (which we can't determine here), Ex: playtest is activated by key: https://steamdb.info/sub/858277/
-			// For most, There seems to be no difference at all between invalid playtest and valid ones.  The only way to resolve these would be to scrape the parent's store page.
+			// For most, There seems to be no difference at all between invalid playtest and valid ones.  The only way to resolve these is to scrape the parent's store page.
 
 			if (app.Parent == null) {
 				return false;
@@ -323,6 +354,32 @@ internal bool IsPlaytestWantedByFilter(FilterableApp app, FilterConfig filter) {
 			return true;
 		}
 
+		internal bool IsPlaytestValidOnStore(HtmlDocumentResponse? storePage) {
+			if (storePage == null) {
+				// Indeterminate, assume the playtest is valid
+				return true;
+			}
+
+			bool hasStorePage = storePage.FinalUri != ArchiWebHandler.SteamStoreURL;
+			if (!hasStorePage) {
+				// App doesnt have a store page (redirects to homepage)
+				return false;
+			}
+
+			if (storePage.Content == null || !storePage.StatusCode.IsSuccessCode()) {
+				// Indeterminate (this will catch age gated store pages), assume the playtest is valid
+				return true;
+			}
+
+			bool hasPlaytestButton = storePage.Content.SelectNodes("//script").Any(static node => node.TextContent.Contains("RequestPlaytestAccess"));
+			if (!hasPlaytestButton) {
+				// Playtest is not active (doesn't have a "Request Access" button on store page)
+				return false;
+			}
+
+			return true;
+		}
+
 		internal bool FilterOnlyAllowsPackages(FilterConfig filter) {
 			if (filter.NoCostOnly) {
 				// NoCost is a property value that only applies to packages, so ignore all non-packages

FreePackages/Handlers/PackageHandler.cs

@@ -6,6 +6,7 @@
 using System.Threading.Tasks;
 using ArchiSteamFarm.Core;
 using ArchiSteamFarm.Steam;
+using ArchiSteamFarm.Web.Responses;
 using FreePackages.Localization;
 using SteamKit2;
 
@@ -195,9 +196,11 @@ private async static Task HandleProductInfo(List<SteamApps.PICSProductInfoCallba
 				// Add wanted apps to the queue
 				apps.ForEach(app => {
 					if (app.Type == EAppType.Beta) {
-						Handlers.Values.ToList().ForEach(x => x.HandlePlaytest(app));
+						SharedExternalResource<HtmlDocumentResponse> storePageResource = new();
+						Handlers.Values.ToList().ForEach(x => Utilities.InBackground(async() => await x.HandlePlaytest(app, storePageResource).ConfigureAwait(false)));
 					} else {
-						Handlers.Values.ToList().ForEach(x => x.HandleFreeApp(app));
+						SharedExternalResource<AppDetails> appDetailsResource = new();
+						Handlers.Values.ToList().ForEach(x => Utilities.InBackground(async() => await x.HandleFreeApp(app, appDetailsResource).ConfigureAwait(false)));
 					}
 				});
 			}
@@ -289,7 +292,7 @@ private async static Task HandleProductInfo(List<SteamApps.PICSProductInfoCallba
 			Handlers.Values.ToList().ForEach(x => x.BotCache.SaveChanges());
 		}
 
-		private void HandleFreeApp(FilterableApp app) {
+		private async Task HandleFreeApp(FilterableApp app, SharedExternalResource<AppDetails> appDetailsResource) {
 			if (!BotCache.ChangedApps.Contains(app.ID)) {
 				return;
 			}
@@ -307,6 +310,10 @@ private void HandleFreeApp(FilterableApp app) {
 					return;
 				}
 
+				if (!PackageFilter.IsAppFreeAndValidOnStore(await appDetailsResource.Fetch(async() => await WebRequest.GetAppDetails(Bot, app.ID).ConfigureAwait(false)).ConfigureAwait(false))) {
+					return;
+				}
+
 				PackageQueue.AddPackage(new Package(EPackageType.App, app.ID));				
 			} finally {
 				BotCache.RemoveChange(appID: app.ID);
@@ -337,7 +344,7 @@ private void HandleFreePackage(FilterablePackage package) {
 			}
 		}
 
-		private void HandlePlaytest(FilterableApp app) {
+		private async Task HandlePlaytest(FilterableApp app, SharedExternalResource<HtmlDocumentResponse> storePageResource) {
 			if (!BotCache.ChangedApps.Contains(app.ID)) {
 				return;
 			}
@@ -359,6 +366,10 @@ private void HandlePlaytest(FilterableApp app) {
 					return;
 				}
 
+				if (!PackageFilter.IsPlaytestValidOnStore(await storePageResource.Fetch(async() => await WebRequest.GetStorePage(Bot, app.Parent?.ID).ConfigureAwait(false)).ConfigureAwait(false))) {
+					return;
+				}
+
 				PackageQueue.AddPackage(new Package(EPackageType.Playtest, app.Parent.ID));
 			} finally {
 				BotCache.RemoveChange(appID: app.ID);

FreePackages/WebRequest.cs

@@ -6,13 +6,16 @@
 using ArchiSteamFarm.Core;
 using ArchiSteamFarm.Steam;
 using ArchiSteamFarm.Steam.Integration;
+using ArchiSteamFarm.Web;
 using ArchiSteamFarm.Web.Responses;
 using SteamKit2;
 
 namespace FreePackages {
 	internal static class WebRequest {
 		private static SemaphoreSlim AppDetailsSemaphore = new SemaphoreSlim(1, 1);
+		private static SemaphoreSlim StorePageSemaphore = new SemaphoreSlim(1, 1);
 		private const int AppDetailsDelaySeconds = 2;
+		private const int StorePageDelaySeconds = 2;
 
 		internal static async Task<UserData?> GetUserData(Bot bot) {
 			Uri request = new(ArchiWebHandler.SteamStoreURL, "/dynamicstore/userdata/");
@@ -26,7 +29,7 @@ internal static class WebRequest {
 			await AppDetailsSemaphore.WaitAsync().ConfigureAwait(false);
 			try {
 				Uri request = new(ArchiWebHandler.SteamStoreURL, String.Format("/api/appdetails/?appids={0}", appID));
-				ObjectResponse<Dictionary<uint, AppDetails>>? appDetailsResponse = await bot.ArchiWebHandler.UrlGetToJsonObjectWithSession<Dictionary<uint, AppDetails>>(request).ConfigureAwait(false);
+				ObjectResponse<Dictionary<uint, AppDetails>>? appDetailsResponse = await bot.ArchiWebHandler.UrlGetToJsonObjectWithSession<Dictionary<uint, AppDetails>>(request, maxTries: 1).ConfigureAwait(false);
 
 				return appDetailsResponse?.Content?[appID];
 			} finally {
@@ -62,5 +65,23 @@ internal static class WebRequest {
 
 			return playtestAccessResponse?.Content;
 		}
+
+		internal static async Task<HtmlDocumentResponse?> GetStorePage(Bot bot, uint? appID) {
+			ArgumentNullException.ThrowIfNull(appID);
+
+			await StorePageSemaphore.WaitAsync().ConfigureAwait(false);
+			try {
+				Uri request = new(ArchiWebHandler.SteamStoreURL, String.Format("/app/{0}", appID));
+
+				return await bot.ArchiWebHandler.UrlGetToHtmlDocumentWithSession(request, maxTries: 1, requestOptions: WebBrowser.ERequestOptions.ReturnRedirections);
+			} finally {
+				Utilities.InBackground(
+					async() => {
+						await Task.Delay(TimeSpan.FromSeconds(StorePageDelaySeconds)).ConfigureAwait(false);
+						StorePageSemaphore.Release();
+					}
+				);
+			}
+		}
 	}
 }