Asset Manager

- Class for threaded load and read assets from virtual file system

frontend/CMakeLists.txt

@@ -13,8 +13,8 @@ project (EEfrontend)
 
 # List sources
 
-set(HEADERS "Renderer.hpp")
-set(SOURCES "main.cpp" "Renderer.cpp")
+set(HEADERS "Renderer.hpp" "FileSystem.hpp" "compile_config.hpp" "AssetManager.hpp" "AssetData.hpp")
+set(SOURCES "main.cpp" "Renderer.cpp" "FileSystem.cpp" "AssetManager.cpp" "AssetData.cpp")
 
 list(TRANSFORM HEADERS PREPEND "include/")
 list(TRANSFORM SOURCES PREPEND "src/")

frontend/include/AssetData.hpp

@@ -0,0 +1,15 @@
+#pragma once
+#include <concepts>
+#include <cstdint>
+
+struct AssetData {};
+template<typename T> concept Asset = std::derived_from<T, AssetData>;
+
+struct AssetBinary : AssetData
+{
+    uint8_t* content = nullptr;
+    size_t size= 0;
+
+    AssetBinary(uint8_t* &source, size_t sourceSize);
+    ~AssetBinary();
+};

frontend/include/AssetManager.hpp

@@ -0,0 +1,122 @@
+#pragma once
+#include "FileSystem.hpp"
+#include "AssetData.hpp"
+
+#include <vector>
+#include <unordered_map>
+#include <filesystem>
+#include <thread>
+#include <atomic>
+#include <Boost/lockfree/spsc_queue.hpp>
+
+struct AssetAddress
+{
+    std::atomic<void*> resource;
+};
+
+struct AssetHandle
+{
+    size_t id = 0;
+};
+
+class AssetInfo
+{
+public:
+    enum Type
+    {
+        BINARY,
+
+        SHADER,
+        IMAGE
+    } type = BINARY; 
+
+    std::atomic<bool> isReady;
+    AssetAddress* resourceAddress;
+
+    AssetInfo(void *resource);
+
+    AssetInfo(const AssetInfo&) = delete;
+    AssetInfo& operator=(const AssetInfo&) = delete;
+
+    AssetInfo(AssetInfo&& other) noexcept;
+    AssetInfo& operator=(AssetInfo&& other) noexcept;
+};
+
+struct AssetLoadingEntry
+{
+    AssetHandle handle;
+    FileHandle* file;
+    IFileSystem* fileOwner;
+};
+
+struct AssetStreamCommand
+{
+    enum Type
+    {
+        NONE,
+
+        LOAD,
+        FIND,
+        GET,
+
+        COMPLETED
+    } type = NONE;
+
+    AssetHandle target;
+    std::filesystem::path targetPath;
+    AssetInfo::Type targetType = AssetInfo::BINARY;
+    std::atomic<void*>* out;
+};
+
+class AssetManager
+{
+public:
+    std::atomic<bool> isRunning;
+
+    AssetManager();
+    ~AssetManager();
+
+    void Start();
+    void Shutdown();
+
+    void Request(const AssetStreamCommand &command);
+    void Request(const AssetStreamCommand::Type &type);
+
+    void LoadAsync(AssetInfo::Type type, const std::filesystem::path &path);
+    AssetHandle& Find(const std::filesystem::path &path);
+    void Redirect(AssetHandle &handle, void *resource);
+    template <Asset T> T& Get(const AssetHandle &handle);
+    void Free(AssetHandle &handle);
+
+private:
+    std::stop_token *stoken;
+    std::jthread streamingThread;
+    boost::lockfree::spsc_queue<AssetStreamCommand, boost::lockfree::capacity<512>> commands;
+
+    IFileSystem *diskSystem;
+
+    std::vector<AssetInfo> activeAssets;
+    std::unordered_map<std::filesystem::path, AssetHandle> assetsMapping;
+    std::vector<size_t> freeAssetSlots;
+    std::vector<AssetLoadingEntry> loadingEntries;
+
+    std::vector<AssetBinary*> binaryStorage;
+
+    void Threading();
+    void Loop();
+    void ProcessCommand(AssetStreamCommand &command);
+    void Release();
+};
+
+template <Asset T> T& AssetManager::Get(const AssetHandle &handle)
+{
+    std::atomic<void*> result;
+    AssetStreamCommand command = {
+        .type = AssetStreamCommand::GET,
+        .target = handle,
+        .out = &result
+    };
+    Request(command);
+    while (command.out->load() == nullptr);
+    return *reinterpret_cast<T*>(command.out->load());
+}

frontend/include/FileSystem.hpp

@@ -0,0 +1,67 @@
+#pragma once
+#include <filesystem>
+#include <vector>
+#include <cstdint>
+
+#ifdef _WIN32
+    #include <windows.h>
+#endif
+
+#ifdef __linux__
+#endif
+
+#define FILE_OPEN_NONE 0
+#define FILE_OPEN_READ (1 << 0)
+#define FILE_OPEN_WRITE (1 << 1)
+#define FILE_OPEN_MMAP (1 << 2)
+
+#define FILE_READ_SEQUENTIAL 0
+#define FILE_READ_RANDOM_ACCESS (1 << 0)
+
+struct FileHandle
+{
+    size_t id = 0;
+
+    size_t size = 0;
+    size_t bufferSize = 0;
+    uint8_t *data = nullptr;
+    uint8_t *buffer = nullptr;
+
+    #ifndef NDEBUG
+    std::filesystem::path debugPath;
+    #endif
+
+    #ifdef _WIN32
+    HANDLE win32Handle = INVALID_HANDLE_VALUE;
+    OVERLAPPED overlapped;
+    #endif
+
+    #ifdef __linux__
+    #endif
+};
+
+class IFileSystem
+{
+public:
+    virtual FileHandle& Open(const std::filesystem::path &path, const uint32_t flags) = 0;
+    virtual void ReadAsync(FileHandle &file, const uint32_t bufferSize, const uint32_t flags) = 0;
+    virtual bool PollReady(FileHandle &file) = 0;
+    virtual void Close(const FileHandle &file) = 0;
+
+protected:
+    std::vector<FileHandle> activeFiles;
+    std::vector<FileHandle> failedFiles;
+    std::vector<size_t> freeFileSlots;
+};
+
+class NativeFileSystem : public IFileSystem
+{
+public:
+    FileHandle& Open(const std::filesystem::path &path, const uint32_t flags) override;
+    void ReadAsync(FileHandle &file, const uint32_t bufferSize, uint32_t flags) override;
+    bool PollReady(FileHandle &file) override;
+    void Close(const FileHandle &file) override;
+
+private:
+
+};

frontend/include/Renderer.hpp

@@ -59,9 +59,23 @@ struct SurfaceInfo
 
 class Renderer
 {
+public:
+    std::atomic<bool> isRunning;
+    std::atomic<int> screenWidth, screenHeight;
+
+    Renderer(SDL_Window *window);
+    ~Renderer();
+    
+    void Start();
+    void Shutdown();
+
+    void Request(const RenderCommand &command);
+    void Request(const RenderCommand::Type &type);
+
 private:
     std::stop_token *stoken;
     std::jthread renderThread;
+    boost::lockfree::spsc_queue<RenderCommand, boost::lockfree::capacity<512>> commands;
 
     SDL_Window *window;
 
@@ -79,8 +93,6 @@ private:
     const float defaultQueuePriority = 1.0f;
 
     Rml::Context *uiContext;
-
-    boost::lockfree::spsc_queue<RenderCommand, boost::lockfree::capacity<512>> commands;
     
     void InitVulkan();
     void SetupDevice();
@@ -91,16 +103,4 @@ private:
     void ProcessCommand(const RenderCommand &command);
     void Release();
 
-public:
-    std::atomic<bool> isRunning;
-    std::atomic<int> screenWidth, screenHeight;
-
-    Renderer(SDL_Window *window);
-    ~Renderer();
-    
-    void Start();
-    void Shutdown();
-
-    void Request(const RenderCommand &command);
-    void Request(RenderCommand::Type type);
 };

frontend/include/compile_config.hpp

@@ -1,12 +1,12 @@
 #pragma once
 
-#if !_DEBUG | NDEBUG
+#ifdef NDEBUG
 	#define P7_SINK L"/P7.Sink=FileTxt"
 #else
 	#define P7_SINK L"/P7.Sink=Console"
 #endif
 
-#if !APP_VERSION
+#ifndef APP_VERSION
 	#define MAJOR_VERSION 0
 	#define MINOR_VERSION 0
 	#define PATCH_VERSION 0

frontend/src/AssetData.cpp

@@ -0,0 +1,15 @@
+#include "AssetData.hpp"
+#include "allocator.hpp"
+
+AssetBinary::AssetBinary(uint8_t* &source, size_t sourceSize)
+{
+    content = source;
+    size = sourceSize;
+    source = nullptr;
+}
+
+AssetBinary::~AssetBinary()
+{
+    if (content != nullptr)
+        DeallocateArray(content);
+}

frontend/src/AssetManager.cpp

@@ -0,0 +1,285 @@
+#include "AssetManager.hpp"
+#include "allocator.hpp"
+#include "Logger.h"
+
+AssetInfo::AssetInfo(void *resource)
+{
+    this->resourceAddress = Allocate<AssetAddress>();
+    this->resourceAddress->resource.store(resource);
+}
+
+AssetInfo::AssetInfo(AssetInfo&& other) noexcept :
+    resourceAddress(other.resourceAddress)
+{
+    isReady.store(other.isReady.load(std::memory_order_relaxed), std::memory_order_relaxed);
+    
+    other.resourceAddress = nullptr;
+    other.isReady.store(false, std::memory_order_relaxed);
+}
+
+AssetInfo& AssetInfo::operator=(AssetInfo&& other) noexcept
+{
+    if (this != &other)
+    {
+        resourceAddress = other.resourceAddress;
+        isReady.store(other.isReady.load(std::memory_order_relaxed), std::memory_order_relaxed);
+
+        other.resourceAddress = nullptr;
+        other.isReady.store(false, std::memory_order_relaxed);
+    }
+    return *this;
+}
+
+void AssetManager::Redirect(AssetHandle &handle, void* resource)
+{
+    AssetInfo &asset = activeAssets[handle.id];
+    if (asset.resourceAddress == nullptr)
+    {
+        LOG(WARNING, "Attempt to redirect empty handle");
+        return;
+    }
+
+    if (asset.resourceAddress->resource.load() != nullptr)
+    {
+        void* oldResource = asset.resourceAddress->resource.exchange(nullptr, std::memory_order_acq_rel);
+        if (oldResource)
+            Deallocate(asset.resourceAddress->resource.load());
+    }
+
+    asset.resourceAddress->resource.store(resource);
+}
+
+void AssetManager::Free(AssetHandle &handle)
+{
+    AssetInfo &asset = activeAssets[handle.id];
+    Redirect(handle, nullptr);
+    Deallocate(asset.resourceAddress);
+    asset.resourceAddress = nullptr;
+
+    freeAssetSlots.push_back(handle.id);
+}
+
+AssetManager::AssetManager()
+{
+    diskSystem = Allocate<NativeFileSystem>();
+}
+
+AssetManager::~AssetManager()
+{
+    Deallocate(diskSystem);
+}
+
+void AssetManager::Request(const AssetStreamCommand &command)
+{
+	commands.push(command);
+}
+
+void AssetManager::Request(const AssetStreamCommand::Type &type)
+{
+	commands.push(AssetStreamCommand {.type = type});
+}
+
+void AssetManager::LoadAsync(AssetInfo::Type type, const std::filesystem::path &path)
+{
+    AssetStreamCommand command = {
+        .type = AssetStreamCommand::LOAD,
+        .targetPath = path,
+        .targetType = type
+    };
+    Request(command);
+}
+
+AssetHandle& AssetManager::Find(const std::filesystem::path &path)
+{
+    std::atomic<void*> result;
+    AssetStreamCommand command = {
+        .type = AssetStreamCommand::FIND,
+        .targetPath = path,
+        .out = &result
+    };
+    Request(command);
+    while (command.out->load() == nullptr);
+    return *reinterpret_cast<AssetHandle*>(command.out->load());
+}
+
+void AssetManager::Start()
+{
+    if (isRunning) throw std::exception("Asset Streaming can be started only once");
+
+    isRunning.store(true);
+    streamingThread = std::jthread([this](std::stop_token stop) {
+        this->stoken = &stop;
+        Threading();
+    });
+}
+
+void AssetManager::Shutdown()
+{
+    streamingThread.request_stop();
+}
+
+void AssetManager::Threading()
+{
+    Logger::trace->Register_Thread(L"Asset Stream", 0);
+    LOG(INFO, "Asset Streaming thread started");
+
+    Loop();
+
+    Release();
+}
+
+void AssetManager::Loop()
+{
+    AssetStreamCommand commandBuffer;
+    std::vector<AssetStreamCommand> incompleteCommands;
+    while (!stoken->stop_requested())
+    {
+		while (commands.pop(commandBuffer))
+		{
+			ProcessCommand(commandBuffer);
+            if (commandBuffer.type != AssetStreamCommand::COMPLETED)
+                incompleteCommands.push_back(commandBuffer);
+		}
+        for (AssetStreamCommand &command : incompleteCommands)
+        {
+            commands.push(command);
+        }
+        incompleteCommands.clear();
+
+		for (AssetLoadingEntry &entry : loadingEntries)
+        {
+            bool isReady = entry.fileOwner->PollReady(*entry.file);
+            if (isReady)
+            {
+                AssetInfo& asset = activeAssets[entry.handle.id];
+
+                //TODO: functions for each type
+                switch (asset.type)
+                {
+                    case AssetInfo::SHADER:
+                    {
+                        break;
+                    }
+
+                    case AssetInfo::IMAGE:
+                    {
+                        break;
+                    }
+
+                    case AssetInfo::BINARY:
+                    default:
+                    {
+                        AssetBinary* data = new AssetBinary(entry.file->data, entry.file->size);
+                        binaryStorage.push_back(data);
+                        Redirect(entry.handle, binaryStorage.back());
+                        break;
+                    }
+                }
+
+                asset.isReady.store(true);
+                entry.file = nullptr;
+            }
+        }
+
+        std::erase_if(loadingEntries, [](const AssetLoadingEntry &entry) {
+            return entry.file == nullptr;
+        });
+    }
+}
+void AssetManager::ProcessCommand(AssetStreamCommand &command)
+{
+    switch(command.type)
+	{
+		case AssetStreamCommand::LOAD:
+        {
+            LOG(INFO, "LOAD command");
+
+            /* TODO: process path to select FS */
+            /* TODO: select between async load and sync job task */
+
+            //Async load of full file without mmap in one thread
+            FileHandle &file = diskSystem->Open(command.targetPath, FILE_OPEN_READ);
+
+            diskSystem->ReadAsync(file, static_cast<uint32_t>(file.size), FILE_READ_SEQUENTIAL);
+            
+            AssetHandle handle;
+            AssetInfo asset(nullptr);
+            asset.type = command.targetType;
+            
+            size_t assetPos = 0;
+            if (freeAssetSlots.size() > 0)
+            {
+                assetPos = freeAssetSlots.back();
+                activeAssets[assetPos] = std::move(asset);
+                freeAssetSlots.pop_back();
+            }
+            else
+            {
+                activeAssets.push_back(std::move(asset));
+                assetPos = activeAssets.size() - 1;
+            }
+            handle.id = assetPos;
+            assetsMapping[command.targetPath] = handle;
+
+            AssetLoadingEntry loadingEntry = {
+                .handle = handle,
+                .file = &file,
+                .fileOwner = diskSystem
+            };
+            loadingEntries.push_back(loadingEntry);
+            
+            command.type = AssetStreamCommand::COMPLETED;
+            break;
+        }
+
+        case AssetStreamCommand::FIND:
+        {
+            std::unordered_map<std::filesystem::path, AssetHandle>::iterator mapping = assetsMapping.find(command.targetPath);
+            
+            if (mapping == assetsMapping.end())
+                break;
+
+            command.out->store(&mapping->second);
+
+            command.type = AssetStreamCommand::COMPLETED;
+            break;
+        }
+
+        case AssetStreamCommand::GET:
+        {
+            if (activeAssets.size() <= command.target.id)
+                break;
+            if (!activeAssets[command.target.id].isReady)
+                break;
+
+            command.out->store(activeAssets[command.target.id].resourceAddress->resource);
+
+            command.type = AssetStreamCommand::COMPLETED;
+            break;
+        }
+
+		default:
+		{
+			LOG(WARNING, "Unhandled command. Type id: %d", command.type);
+			break;
+		}
+	}
+}
+
+void AssetManager::Release()
+{
+    for (AssetInfo &asset : activeAssets)
+    {
+        if (asset.resourceAddress->resource.load() != nullptr)
+        {
+            void* oldResource = asset.resourceAddress->resource.exchange(nullptr, std::memory_order_acq_rel);
+            if (oldResource)
+                Deallocate(asset.resourceAddress->resource.load());
+        }
+        Deallocate(asset.resourceAddress);
+    }
+    activeAssets.clear();
+
+    LOG(INFO, "Asset Streaming is released");
+    isRunning.store(false);
+}

frontend/src/FileSystem.cpp

@@ -0,0 +1,117 @@
+#include "FileSystem.hpp"
+#include "Logger.h"
+#include "allocator.hpp"
+
+#ifdef _WIN32
+
+FileHandle& NativeFileSystem::Open(const std::filesystem::path &path, const uint32_t flags)
+{
+    FileHandle result;
+
+    #ifndef NDEBUG
+    result.debugPath = path;
+    #endif
+
+    DWORD createFileMode = 0;
+
+    if (flags & FILE_OPEN_READ)
+        createFileMode |= GENERIC_READ;
+    if (flags & FILE_OPEN_WRITE)
+        createFileMode |= GENERIC_WRITE;
+
+    HANDLE file = CreateFileW(
+        path.c_str(),
+        createFileMode,
+        NULL,
+        NULL,
+        OPEN_EXISTING,
+        FILE_FLAG_OVERLAPPED,
+        NULL
+    );
+
+    if (file == INVALID_HANDLE_VALUE)
+    {
+        LOG(ERROR, "Failed to open file %s. Error: %d", path.c_str(), GetLastError());
+        failedFiles.push_back(result);
+        return failedFiles[result.id];
+    }
+
+    result.win32Handle = file;
+    result.size = GetFileSize(file, NULL);
+    if (result.size == INVALID_FILE_SIZE)
+        LOG(ERROR, "Failed to get size of file %s. Error: %d", path.c_str(), GetLastError());
+
+    if (freeFileSlots.size() > 0)
+    {
+        activeFiles[freeFileSlots.back()] = result;
+        result.id = freeFileSlots.back();
+        freeFileSlots.pop_back();
+    }
+    else
+    {
+        activeFiles.push_back(result);
+        result.id = activeFiles.size() - 1;
+    }
+
+    return activeFiles[result.id];
+}
+
+void NativeFileSystem::ReadAsync(FileHandle &file, const uint32_t bufferSize, const uint32_t flags)
+{
+    file.overlapped = {};
+    file.buffer = AllocateArray<uint8_t>(bufferSize);
+
+    ReadFile(
+        file.win32Handle,
+        file.buffer,
+        bufferSize,
+        NULL,
+        &file.overlapped
+    );
+}
+
+bool NativeFileSystem::PollReady(FileHandle &file)
+{
+    DWORD transferredSize = 0L;
+    BOOL status = GetOverlappedResult(
+        file.win32Handle,
+        &file.overlapped,
+        &transferredSize,
+        FALSE
+    );
+    if (!status && GetLastError() != ERROR_IO_INCOMPLETE)
+    {
+        LOG(ERROR, "Error in file status. Code: %d", GetLastError());
+        return false;
+    }
+
+    if (file.size == transferredSize)
+    {
+        file.data = file.buffer;
+    }
+    else
+    {
+        /* TODO: reserve size for data, shift and move */
+    }
+    file.buffer = nullptr;
+
+    return status;
+}
+
+void NativeFileSystem::Close(const FileHandle &file)
+{
+    DeallocateArray(file.buffer);
+    DeallocateArray(file.data);
+
+    CloseHandle(file.win32Handle);
+    activeFiles[file.id].win32Handle = INVALID_HANDLE_VALUE;
+    freeFileSlots.push_back(file.id);
+}
+
+#endif
+
+#ifdef __linux__
+
+
+
+#endif

frontend/src/Renderer.cpp

@@ -1,4 +1,4 @@
-#if !APP_VERSION
+#ifndef APP_VERSION
 #include "compile_config.hpp"
 #endif
 
@@ -410,9 +410,13 @@ void Renderer::SetupSwapchain()
 
 void Renderer::Start()
 {
-	if (isRunning) throw std::exception("Renderer can be started only once");
+	if (isRunning)
+	{
+		LOG(CRITICAL, "Renderer can be started only once");
+		throw std::exception("Renderer can be started only once");
+	}
 
-    isRunning = true;
+    isRunning.store(true);
     renderThread = std::jthread([this](std::stop_token stop) {
         this->stoken = &stop;
         Threading();
@@ -482,7 +486,7 @@ void Renderer::Request(const RenderCommand &command)
 	commands.push(command);
 }
 
-void Renderer::Request(RenderCommand::Type type)
+void Renderer::Request(const RenderCommand::Type &type)
 {
 	commands.push(RenderCommand {.type = type});
 }

frontend/src/main.cpp

@@ -2,6 +2,7 @@
 #include <SDL2/SDL.h>
 
 #include "compile_config.hpp"
+#include "AssetManager.hpp"
 #include "Renderer.hpp"
 
 IP7_Trace* Logger::trace;
@@ -39,6 +40,10 @@ int SDL_main(int argc, char **argv)
 	}
 	else LOG(INFO, "SDL Window created");
 
+	// Setup assets
+	AssetManager assetManager;
+	assetManager.Start();
+
 	// Setup Renderer
 	Renderer renderer(window);
 	renderer.Start();
@@ -78,6 +83,7 @@ int SDL_main(int argc, char **argv)
 	// Free memory
 	LOG(INFO, "Exit...");
 	renderer.Shutdown(); while(renderer.isRunning.load());
+	assetManager.Shutdown(); while(assetManager.isRunning.load());
 	SDL_DestroyWindow(window);
 	SDL_Quit();
 	Logger::trace->Unregister_Thread(0);