Implement helper method for asset reference validation

Refresh.Core/Helpers/ResourceValidationHelper.cs

@@ -0,0 +1,120 @@
+using System.Diagnostics;
+using Bunkum.Core;
+using NotEnoughLogs;
+using Refresh.Common.Verification;
+using Refresh.Core.Types.Assets.Validation;
+using Refresh.Database.Models.Assets;
+using Refresh.Database.Models.Authentication;
+
+namespace Refresh.Core.Helpers;
+
+public abstract class ResourceValidationHelper
+{
+    /// <summary>
+    /// Validates the given asset reference (hash/guid/blank) using the given parameters, if necessary also by reading the referenced asset
+    /// or getting data about it from database (see <see cref="GameAsset"/> and <see cref="DisallowedAsset"/>).
+    /// </summary>
+    public static ValidatedAssetResult ValidateReference(AssetValidationParameters parameters, Logger logger)
+    {
+        string assetTypeStr = parameters.AssetContextTypeStr ?? (parameters.MustBeTexture ? "image asset" : "asset");
+        GameAsset? asset = null;
+        bool existsInDataStore = false;
+        bool isPSP = parameters.GameToUseIn == TokenGame.LittleBigPlanetPSP;
+        Action<string>? onNewAssetRefCallback = parameters.OnNewAssetRefCallback;
+
+        if (parameters.AssetRef.IsBlankHash())
+        {
+            if (!parameters.MayBeBlank) return new(BadRequest, "0", $"The {assetTypeStr} must be set.", onNewAssetRefCallback);
+            else return new(OK, "0", null, onNewAssetRefCallback);
+        }
+
+        else if (parameters.AssetRef.StartsWith('g'))
+        {
+            if (!parameters.MayBeGuid) return new(BadRequest, null, $"The {assetTypeStr} may not be an in-game asset.", onNewAssetRefCallback);
+            if (parameters.AssetRef.Length < 2) return new(BadRequest, null, $"The used in-game {assetTypeStr} is invalid (empty GUID).", onNewAssetRefCallback);
+
+            // This should only happen if the user is messing with mods/the API/beta builds, so give them a more detailed response
+            bool canParseGuid = long.TryParse(parameters.AssetRef[1..], out long guid);
+            if (!canParseGuid)
+                return new(BadRequest, null, $"The used in-game {assetTypeStr} is invalid (badly formatted GUID).", onNewAssetRefCallback);
+
+            if (parameters.MustBeTexture && !parameters.GuidChecker.IsTextureGuid(parameters.GameToUseIn, guid))
+                return new(BadRequest, null, $"The used in-game {assetTypeStr} was not a valid image (unknown GUID).", onNewAssetRefCallback);
+        }
+
+        // At this point the reference is a hash
+        else if (!parameters.MayBeHash)
+        {
+            return new(BadRequest, null, $"The {assetTypeStr} may not be a custom asset.", onNewAssetRefCallback);
+        }
+
+        else if (!CommonPatterns.Sha1Regex().IsMatch(parameters.AssetRef))
+        {
+            // This should only happen if a player is messing with mods/the API, so give them a more detailed response
+            return new(BadRequest, null, $"The used {assetTypeStr} had an invalid hash.", onNewAssetRefCallback);
+        }
+
+        else
+        {
+            DisallowedAsset? disallowed = parameters.Database.GetDisallowedAssetInfo(parameters.AssetRef);
+            if (disallowed != null)
+            {
+                logger.LogWarning(BunkumCategory.UserContent, $"{parameters.User} tried to use a manually disallowed {assetTypeStr}.");
+                return new(Unauthorized, disallowanceInfo: disallowed, onNewAssetRefCallback: onNewAssetRefCallback);
+            }
+
+            string filename = isPSP ? $"psp/{parameters.AssetRef}" : parameters.AssetRef;
+            existsInDataStore = parameters.DataStore.ExistsInStore(filename);
+
+            if (!existsInDataStore)
+            {
+                logger.LogDebug(BunkumCategory.UserContent, $"Referenced asset '{filename}' could not be found in data store.");
+
+                if (parameters.MustBeInDataStoreIfHash)
+                    return new(NotFound, null, $"The used {assetTypeStr} did not exist on the server.", onNewAssetRefCallback);
+            }
+
+            asset = parameters.Cache.GetAssetInfo(parameters.AssetRef, parameters.Database);
+
+            // Only try to import if the asset exists in the data store
+            if (existsInDataStore && asset == null)
+            {
+                logger.LogInfo(BunkumCategory.UserContent, $"Referenced asset '{filename}' exists in data store but not in database, attempting to import automatically...");
+                Stopwatch sw = new();
+                sw.Start();
+
+                if (!parameters.DataStore.TryGetDataFromStore(filename, out byte[]? assetData) || assetData == null)
+                {
+                    sw.Stop();
+                    logger.LogError(BunkumCategory.UserContent, $"Failed to read '{filename}' from data store!");
+                    logger.LogDebug(BunkumCategory.UserContent, $"Failed to get '{filename}' after {sw.ElapsedMilliseconds}ms.");
+                    return new(InternalServerError, null, $"Failed to read {assetTypeStr} internally. Please report this to the server owner.", onNewAssetRefCallback, existsInDataStore: existsInDataStore);
+                }
+
+                asset = parameters.AssetImporter.ReadAndVerifyAsset(parameters.AssetRef, assetData, parameters.PlatformToUseIn, parameters.Database);
+                if (asset == null) 
+                {
+                    sw.Stop();
+                    logger.LogDebug(BunkumCategory.UserContent, $"Failed to get '{filename}' after {sw.ElapsedMilliseconds}ms.");
+                    return new(BadRequest, null, $"The used {assetTypeStr} was invalid or corrupt.", onNewAssetRefCallback, existsInDataStore: existsInDataStore);
+                }
+
+                sw.Stop();
+                logger.LogInfo(BunkumCategory.UserContent, $"Successfully imported '{filename}' in {sw.ElapsedMilliseconds}ms.");
+            }
+
+            // FIXME: for some reason, PSP texture detection/conversion broke so we can no longer tell if a PSP texture is actually a texture, so skip this for PSP
+            if (asset != null && !isPSP)
+            {
+                bool isHashedTexture = (asset.AssetFlags & AssetFlags.Imagery) != 0;
+
+                if (parameters.MustBeTexture && !isHashedTexture)
+                    return new(BadRequest, null, $"The used {assetTypeStr} was not a valid custom image.", onNewAssetRefCallback, assetInfo: asset, existsInDataStore: existsInDataStore);
+
+                // TODO: actually use AIPI to scan image if not null
+            }
+        }
+
+        return new(OK, parameters.AssetRef, null, onNewAssetRefCallback, assetInfo: asset, existsInDataStore: existsInDataStore);
+    }
+}

Refresh.Core/Importing/Importer.cs

@@ -107,32 +107,40 @@ private static bool MatchesMagic(ReadOnlySpan<byte> data, ulong magic)
     /// <returns>Whether the file is likely of TGA format</returns>
     private static bool IsPspTga(ReadOnlySpan<byte> data)
     {
-        byte imageIdLength = data[0];
-        byte colorMapType = data[1];
-        byte imageType = data[2];
-        ReadOnlySpan<byte> colorMapSpecification = data[3..8];
-        ReadOnlySpan<byte> imageSpecification = data[8..18];
-        short xOrigin = BinaryPrimitives.ReadInt16LittleEndian(imageSpecification[..2]);
-        short yOrigin = BinaryPrimitives.ReadInt16LittleEndian(imageSpecification[2..4]);
-        ushort width = BinaryPrimitives.ReadUInt16LittleEndian(imageSpecification[4..6]);
-        ushort height = BinaryPrimitives.ReadUInt16LittleEndian(imageSpecification[6..8]);
-        byte depth = imageSpecification[8];
-        byte descriptor = imageSpecification[9];
-
-        //PSP does not seem to fill out this information
-        if (imageIdLength != 0) return false;
-        if (xOrigin != 0) return false;
-        if (yOrigin != 0) return false;
-        //These are the fields set by PSP, that shouldn't change from image to image
-        if (colorMapType != 1) return false;
-        if (descriptor != 0) return false;
-        if (imageType != 1) return false;
-        if (depth != 8) return false;
-        //Reasonable validation checks (PSP seems to only send images of max size 480x272)
-        if (width > 500) return false;
-        if (height > 300) return false;
-
-        return true;
+        try
+        {
+            byte imageIdLength = data[0];
+            byte colorMapType = data[1];
+            byte imageType = data[2];
+            ReadOnlySpan<byte> colorMapSpecification = data[3..8];
+            ReadOnlySpan<byte> imageSpecification = data[8..18];
+            short xOrigin = BinaryPrimitives.ReadInt16LittleEndian(imageSpecification[..2]);
+            short yOrigin = BinaryPrimitives.ReadInt16LittleEndian(imageSpecification[2..4]);
+            ushort width = BinaryPrimitives.ReadUInt16LittleEndian(imageSpecification[4..6]);
+            ushort height = BinaryPrimitives.ReadUInt16LittleEndian(imageSpecification[6..8]);
+            byte depth = imageSpecification[8];
+            byte descriptor = imageSpecification[9];
+
+            //PSP does not seem to fill out this information
+            if (imageIdLength != 0) return false;
+            if (xOrigin != 0) return false;
+            if (yOrigin != 0) return false;
+            //These are the fields set by PSP, that shouldn't change from image to image
+            if (colorMapType != 1) return false;
+            if (descriptor != 0) return false;
+            if (imageType != 1) return false;
+            if (depth != 8) return false;
+            //Reasonable validation checks (PSP seems to only send images of max size 480x272)
+            if (width > 500) return false;
+            if (height > 300) return false;
+
+            return true;
+        }
+        catch
+        {
+            //If the data couldn't be read, it's not a TGA
+            return false;
+        }
     }
 
     private bool IsMip(Span<byte> rawData)

Refresh.Core/Types/Assets/Validation/AssetValidationParameters.cs

@@ -0,0 +1,65 @@
+using Bunkum.Core.Storage;
+using Refresh.Core.Importing;
+using Refresh.Core.Services;
+using Refresh.Core.Types.Data;
+using Refresh.Database;
+using Refresh.Database.Models.Authentication;
+using Refresh.Database.Models.Users;
+
+namespace Refresh.Core.Types.Assets.Validation;
+
+public struct AssetValidationParameters
+{
+    /// <summary>
+    /// The reference (hash/guid/blank) to validate
+    /// </summary>
+    public string AssetRef { get; set; } = "0";
+    public GameUser? User { get; set; }
+    public TokenGame GameToUseIn { get; set; }
+    public TokenPlatform PlatformToUseIn { get; set; }
+    public GameDatabaseContext Database { get; set; } = null!;
+    public IDataStore DataStore { get; set; } = null!;
+    public CacheService Cache { get; set; } = null!;
+    public GuidCheckerService GuidChecker { get; set; } = null!;
+    public AssetImporter AssetImporter { get; set; } = null!;
+    public AipiService? Aipi { get; set; }
+
+    public bool MayBeBlank { get; set; } = true;
+    public bool MayBeGuid { get; set; } = true;
+    public bool MayBeHash { get; set; } = true;
+    public bool MustBeInDataStoreIfHash { get; set; } = true;
+    public bool MustBeTexture { get; set; } = false;
+
+    /// <summary>
+    /// What the asset should be referred as in user-faced error messages and in logs, e.g. "planet asset" or "icon".
+    /// If null, we will default to calling it "asset" or "image" depending on MustBeTexture.
+    /// </summary>
+    public string? AssetContextTypeStr { get; set; }
+
+    /// <summary>
+    /// Callback which is called with the new asset reference as parameter when constructing <see cref="ValidatedAssetResult"/>;
+    /// useful to update asset references of entities during validation without requiring the caller to manually reassign them after validation;
+    /// this way similar attributes like photo images or face icons can simply be iterated.
+    /// If null, this will be skipped.
+    /// </summary>
+    public Action<string>? OnNewAssetRefCallback { get; set; }
+
+    public AssetValidationParameters(string assetKey, DataContext dataContext, AssetImporter assetImporter, AipiService? aipi = null)
+    {
+        this.AssetRef = assetKey;
+        this.User = dataContext.User;
+        this.GameToUseIn = dataContext.Game;
+        this.PlatformToUseIn = dataContext.Platform;
+        this.Database = dataContext.Database;
+        this.DataStore = dataContext.DataStore;
+        this.Cache = dataContext.Cache;
+        this.GuidChecker = dataContext.GuidChecker;
+        this.AssetImporter = assetImporter;
+        this.Aipi = aipi;
+    }
+
+    public AssetValidationParameters()
+    {
+
+    }
+}

Refresh.Core/Types/Assets/Validation/ValidatedAssetResult.cs

@@ -0,0 +1,41 @@
+using System.Net;
+using Refresh.Database.Models.Assets;
+
+namespace Refresh.Core.Types.Assets.Validation;
+
+public struct ValidatedAssetResult
+{
+    /// <summary>
+    /// HTTP code to return if validation failed.
+    /// OK: don't cancel request and proceed.
+    /// </summary>
+    public HttpStatusCode Status { get; set; }
+
+    /// <summary>
+    /// new reference (hash/guid/blank) to use
+    /// </summary>
+    public string NewAssetRef { get; set; }
+
+    /// <summary>
+    /// message to show to the user.
+    /// null: don't show anything.
+    /// </summary>
+    public string? ErrorMessage { get; set; }
+
+    public GameAsset? AssetInfo { get; set; }
+    public DisallowedAsset? DisallowanceInfo { get; set; }
+    public bool ExistsInDataStore { get; set; }
+
+    public ValidatedAssetResult(HttpStatusCode status, string? newAssetRef = null, string? errorMessage = null, Action<string>? onNewAssetRefCallback = null,
+        GameAsset? assetInfo = null, DisallowedAsset? disallowanceInfo = null, bool existsInDataStore = false)
+    {
+        this.Status = status;
+        this.NewAssetRef = newAssetRef ?? "0";
+        this.ErrorMessage = errorMessage;
+        this.AssetInfo = assetInfo;
+        this.DisallowanceInfo = disallowanceInfo;
+        this.ExistsInDataStore = existsInDataStore;
+
+        onNewAssetRefCallback?.Invoke(this.NewAssetRef);
+    }
+}

Refresh.Database/Models/Assets/AssetFlags.cs

@@ -20,6 +20,10 @@ public enum AssetFlags
     /// A planet is considered modded if it depends on this asset, or if the asset already has the Modded flag above.
     /// </summary>
     ModdedOnPlanets = 1 << 3,
+    /// <summary>
+    /// This asset is a texture or contains imagery, for now only given to asset types handled by image conversion.
+    /// </summary>
+    Imagery = 1 << 4,
 }
 
 public static class AssetSafetyLevelExtensions
@@ -30,33 +34,33 @@ public static AssetFlags FromAssetType(GameAssetType type, GameAssetFormat? meth
         {
             // Common asset types created by the game
             GameAssetType.Level => AssetFlags.None,
-            GameAssetType.StreamingLevelChunk => AssetFlags.None | AssetFlags.ModdedOnPlanets,
+            GameAssetType.StreamingLevelChunk => AssetFlags.ModdedOnPlanets,
             GameAssetType.Plan => AssetFlags.None,
-            GameAssetType.ThingRecording => AssetFlags.None | AssetFlags.ModdedOnPlanets,
-            GameAssetType.SyncedProfile => AssetFlags.None | AssetFlags.ModdedOnPlanets,
-            GameAssetType.GriefSongState => AssetFlags.None | AssetFlags.ModdedOnPlanets,
-            GameAssetType.Quest => AssetFlags.None | AssetFlags.ModdedOnPlanets,
-            GameAssetType.AdventureSharedData => AssetFlags.None | AssetFlags.ModdedOnPlanets,
-            GameAssetType.AdventureCreateProfile => AssetFlags.None | AssetFlags.ModdedOnPlanets,
-            GameAssetType.ChallengeGhost => AssetFlags.None | AssetFlags.ModdedOnPlanets,
+            GameAssetType.ThingRecording => AssetFlags.ModdedOnPlanets,
+            GameAssetType.SyncedProfile => AssetFlags.ModdedOnPlanets,
+            GameAssetType.GriefSongState => AssetFlags.ModdedOnPlanets,
+            GameAssetType.Quest => AssetFlags.ModdedOnPlanets,
+            GameAssetType.AdventureSharedData => AssetFlags.ModdedOnPlanets,
+            GameAssetType.AdventureCreateProfile => AssetFlags.ModdedOnPlanets,
+            GameAssetType.ChallengeGhost => AssetFlags.ModdedOnPlanets,
 
             // Common media types created by the game
             GameAssetType.VoiceRecording => AssetFlags.Media | AssetFlags.ModdedOnPlanets,
             GameAssetType.Painting => AssetFlags.Media,
-            GameAssetType.Texture => AssetFlags.Media,
-            GameAssetType.Jpeg => AssetFlags.Media,
-            GameAssetType.Png => AssetFlags.Media,
-            GameAssetType.Tga => AssetFlags.Media | AssetFlags.ModdedOnPlanets,
-            GameAssetType.Mip => AssetFlags.Media | AssetFlags.ModdedOnPlanets,
+            GameAssetType.Texture => AssetFlags.Media | AssetFlags.Imagery,
+            GameAssetType.Jpeg => AssetFlags.Media | AssetFlags.Imagery,
+            GameAssetType.Png => AssetFlags.Media | AssetFlags.Imagery,
+            GameAssetType.Tga => AssetFlags.Media | AssetFlags.Imagery | AssetFlags.ModdedOnPlanets,
+            GameAssetType.Mip => AssetFlags.Media | AssetFlags.Imagery | AssetFlags.ModdedOnPlanets,
 
             // Uncommon, but still vanilla assets created by the game in niche scenarios.
             // While not image/audio data like the other media types, GfxMaterial is marked as media because this file can contain full PS3 shaders.
             GameAssetType.GfxMaterial => AssetFlags.Media | AssetFlags.ModdedOnPlanets, 
-            GameAssetType.Material => AssetFlags.None | AssetFlags.ModdedOnPlanets,
-            GameAssetType.Bevel => AssetFlags.None | AssetFlags.ModdedOnPlanets,
+            GameAssetType.Material => AssetFlags.ModdedOnPlanets,
+            GameAssetType.Bevel => AssetFlags.ModdedOnPlanets,
 
             // Modded media types
-            GameAssetType.GameDataTexture => AssetFlags.Media | AssetFlags.Modded,
+            GameAssetType.GameDataTexture => AssetFlags.Media | AssetFlags.Modded | AssetFlags.Imagery,
             GameAssetType.AnimatedTexture => AssetFlags.Media | AssetFlags.Modded,
 
             // Normal modded assets