remove d3dx dependency, severely improve memory usage if necessary (#19)

* basic tga/hdr image loading

* bgr tga tex

* convert to dds during loading
doesn't work yet, fails to create textures from dds memory

* remove ext member from TextureFileStruct

* do not convert images, leads to weird loading problems...

* use regex matching for tpf loading

* convert non-RGBA images to RGBA

* remove d3dx (mostly)

* fix moving wrong image

* move TextureFunction to module

* move FileLoader to ModfileLoader module

* move TextureClient to module

* spaces

* ifdef debug

* 1.6.0

* compress images to dxt5 (bc3_unorm) to use less memory

* remove unnecessary directxtex source code patch

* don't move unnecessarily

* only compress if mods in filesystem use up more than 400mb

* use std::future to prepare to multithread image loading
fails when using std::launch::async though, maybe something in the directxtex library is not thread safe

* remove d3dx sdk from workflows

* remove extern "C"

* catch exception when saving texture

* make loading async (call CoInitializeEx)

* support DXGI_FORMAT_BC1_UNORM (DXT1) compressed textures without warning

* don't warn for BC2 BC4 or BC5 either

* update README.md
This commit is contained in:
DubbleClick
2023-12-04 19:50:56 +01:00
committed by GitHub
parent b49428bafd
commit 9e593e83ee
24 changed files with 881 additions and 895 deletions
+26 -17
View File
@@ -3,27 +3,36 @@
//#define HashType DWORD64
using HashType = DWORD32;
#ifdef LOG_MESSAGE
#include <iostream>
struct TexEntry {
std::vector<BYTE> data{};
HashType crc_hash = 0; // hash value
std::string ext{};
};
struct TextureFileStruct {
std::vector<BYTE> data{};
HashType crc_hash = 0; // hash value
};
inline void Message(const char* format, ...)
{
#ifdef _DEBUG
#if 1
#define Message(...) { printf(__VA_ARGS__); }
#else
#define Message(...)
#if 0
va_list args;
va_start(args, format);
vprintf(format, args);
va_end(args);
#endif
#define Warning(...) { fprintf(stderr, __VA_ARGS__); }
#else
#define Message(...)
#define Warning(...)
#endif
}
inline void Warning(const char* format, ...)
{
#ifdef _DEBUG
va_list args;
va_start(args, format);
vfprintf(stderr, format, args);
va_end(args);
#endif
}
#ifndef _countof
#define _countof(arr) sizeof(arr) / sizeof(*arr)
#endif
-30
View File
@@ -1,30 +0,0 @@
#pragma once
#include <vector>
#include <string>
#include <libzippp.h>
class FileLoader {
std::string file_name;
const std::string TPF_PASSWORD{
0x73, 0x2A, 0x63, 0x7D, 0x5F, 0x0A, static_cast<char>(0xA6), static_cast<char>(0xBD),
0x7D, 0x65, 0x7E, 0x67, 0x61, 0x2A, 0x7F, 0x7F,
0x74, 0x61, 0x67, 0x5B, 0x60, 0x70, 0x45, 0x74,
0x5C, 0x22, 0x74, 0x5D, 0x6E, 0x6A, 0x73, 0x41,
0x77, 0x6E, 0x46, 0x47, 0x77, 0x49, 0x0C, 0x4B,
0x46, 0x6F
};
public:
FileLoader(const std::string& fileName);
std::vector<TextureFileStruct> GetContents();
private:
std::vector<TextureFileStruct> GetTpfContents();
std::vector<TextureFileStruct> GetFileContents();
void LoadEntries(libzippp::ZipArchive& archive, std::vector<TextureFileStruct>& entries);
};
+3 -4
View File
@@ -5,7 +5,10 @@
#include <cstdlib>
#include <cstdio>
#include <filesystem>
#include <fstream>
#include <future>
#include <map>
#include <ranges>
#include "Utils.h"
#include <d3d9.h>
@@ -13,8 +16,6 @@
#include "Defines.h"
#include "Error.h"
#include "Defines.h"
#include "dll_main.h"
#include "TextureFunction.h"
#include "uMod_IDirect3D9.h"
#include "uMod_IDirect3D9Ex.h"
@@ -26,8 +27,6 @@
#include "uMod_IDirect3DTexture9.h"
#include "uMod_IDirect3DVolumeTexture9.h"
#include "TextureClient.h"
#pragma warning(disable : 4477)
extern unsigned int gl_ErrorState;
-264
View File
@@ -1,264 +0,0 @@
#pragma once
#include "FileLoader.h"
#include "uMod_IDirect3DTexture9.h"
#include <DDSTextureLoader/DDSTextureLoader9.h>
#include <WICTextureLoader/WICTextureLoader9.h>
#include <d3dx9.h>
extern unsigned int gl_ErrorState;
struct TextureFileStruct {
std::vector<BYTE> data{};
HashType crc_hash = 0; // hash value
bool is_wic_texture = false;
};
template <typename T>
concept uModTexturePtr = requires(T a)
{
std::same_as<uMod_IDirect3DTexture9*, T> ||
std::same_as<uMod_IDirect3DVolumeTexture9*, T> ||
std::same_as<uMod_IDirect3DCubeTexture9*, T>;
};
template <typename T>
concept uModTexturePtrPtr = uModTexturePtr<std::remove_pointer_t<T>>;
/*
* An object of this class is owned by each d3d9 device.
* All other functions are called from the render thread instance of the game itself.
*/
class TextureClient {
public:
TextureClient(IDirect3DDevice9* device);
~TextureClient();
int AddTexture(uModTexturePtr auto pTexture);
int RemoveTexture(uModTexturePtr auto pTexture); //called from uMod_IDirect3DTexture9::Release()
int LookUpToMod(uModTexturePtr auto pTexture);
// called at the end AddTexture(...) and from Device->UpdateTexture(...)
int MergeUpdate(); //called from uMod_IDirect3DDevice9::BeginScene()
void Initialize();
// Add TextureFileStruct data, return size of data added. 0 on failure.
unsigned long AddFile(TextureFileStruct& entry);
std::vector<uMod_IDirect3DTexture9*> OriginalTextures;
// stores the pointer to the uMod_IDirect3DTexture9 objects created by the game
std::vector<uMod_IDirect3DVolumeTexture9*> OriginalVolumeTextures;
// stores the pointer to the uMod_IDirect3DVolumeTexture9 objects created by the game
std::vector<uMod_IDirect3DCubeTexture9*> OriginalCubeTextures;
// stores the pointer to the uMod_IDirect3DCubeTexture9 objects created by the game
private:
IDirect3DDevice9* D3D9Device;
// Cached info about whether this id a dx9ex device or not; used for proxy functions
bool isDirectXExDevice = false;
// DX9 proxy functions
void SetLastCreatedTexture(uMod_IDirect3DTexture9*);
void SetLastCreatedVolumeTexture(uMod_IDirect3DVolumeTexture9*);
void SetLastCreatedCubeTexture(uMod_IDirect3DCubeTexture9*);
bool should_update = false;
int LockMutex();
int UnlockMutex();
HANDLE mutex;
std::unordered_map<HashType, TextureFileStruct*> modded_textures;
// array which stores the file in memory and the hash of each texture to be modded
// called if a target texture is found
int LoadTexture(TextureFileStruct* file_in_memory, uModTexturePtrPtr auto ppTexture);
void LoadModsFromFile(const char* source);
};
int TextureClient::AddTexture(uModTexturePtr auto pTexture)
{
if constexpr (std::same_as<decltype(pTexture), uMod_IDirect3DTexture9*>) {
SetLastCreatedTexture(nullptr);
}
else if constexpr (std::same_as<decltype(pTexture), uMod_IDirect3DVolumeTexture9*>) {
SetLastCreatedVolumeTexture(nullptr);
}
else if constexpr (std::same_as<decltype(pTexture), uMod_IDirect3DCubeTexture9*>) {
SetLastCreatedCubeTexture(nullptr);
}
if (pTexture->FAKE) {
return RETURN_OK; // this is a fake texture
}
if (gl_ErrorState & uMod_ERROR_FATAL) {
return RETURN_FATAL_ERROR;
}
Message("TextureClient::AddTexture( Cube: %p): %p (thread: %u)\n", pTexture, this, GetCurrentThreadId());
pTexture->Hash = pTexture->GetHash();
if (!pTexture->Hash) {
return RETURN_FATAL_ERROR;
}
if constexpr (std::same_as<decltype(pTexture), uMod_IDirect3DTexture9*>) {
OriginalTextures.push_back(pTexture);
}
else if constexpr (std::same_as<decltype(pTexture), uMod_IDirect3DVolumeTexture9*>) {
OriginalVolumeTextures.push_back(pTexture);
}
else if constexpr (std::same_as<decltype(pTexture), uMod_IDirect3DCubeTexture9*>) {
OriginalCubeTextures.push_back(pTexture);
}
return LookUpToMod(pTexture); // check if this texture should be modded
}
int TextureClient::RemoveTexture(uModTexturePtr auto pTexture)
{
Message("TextureClient::RemoveTexture( %p, %#lX): %p\n", pTexture, pTexture->Hash, this);
if (gl_ErrorState & uMod_ERROR_FATAL) {
return RETURN_FATAL_ERROR;
}
if (!pTexture->FAKE) {
if constexpr (std::same_as<decltype(pTexture), uMod_IDirect3DTexture9*>) {
utils::erase_first(OriginalTextures, pTexture);
}
else if constexpr (std::same_as<decltype(pTexture), uMod_IDirect3DVolumeTexture9*>) {
utils::erase_first(OriginalVolumeTextures, pTexture);
}
else if constexpr (std::same_as<decltype(pTexture), uMod_IDirect3DCubeTexture9*>) {
utils::erase_first(OriginalCubeTextures, pTexture);
}
}
if (!pTexture->Reference)
return RETURN_OK; // Should this ever happen?
return RETURN_OK;
}
int TextureClient::LookUpToMod(uModTexturePtr auto pTexture)
{
Message("TextureClient::LookUpToMod( %p): hash: %#lX, %p\n", pTexture, pTexture->Hash, this);
int ret = RETURN_OK;
if (pTexture->CrossRef_D3Dtex != nullptr)
return ret; // bug, this texture is already switched
const auto found = modded_textures.find(pTexture->Hash);
if (found == modded_textures.end())
return ret;
const auto textureFileStruct = found->second;
decltype(pTexture) fake_Texture;
ret = LoadTexture(textureFileStruct, &fake_Texture);
if (ret != RETURN_OK)
return ret;
ret = SwitchTextures(fake_Texture, pTexture);
if (ret != RETURN_OK) {
Message("TextureClient::LookUpToMod(): textures not switched %#lX\n", textureFileStruct->crc_hash);
fake_Texture->Release();
return ret;
}
fake_Texture->Reference = textureFileStruct;
return ret;
}
extern HINSTANCE gl_hThisInstance;
int TextureClient::LoadTexture(TextureFileStruct* file_in_memory, uModTexturePtrPtr auto ppTexture)
{
Message("LoadTexture( %p, %p, %#lX): %p\n", file_in_memory, ppTexture, file_in_memory->crc_hash, this);
if (file_in_memory->is_wic_texture) {
Warning("(%#lX)\n", file_in_memory->crc_hash);
#if 0
if (D3D_OK != D3DXCreateTextureFromFileInMemoryEx(
D3D9Device, file_in_memory->data.data(),
file_in_memory->data.size(), D3DX_DEFAULT, D3DX_DEFAULT,
D3DX_DEFAULT, 0, D3DFMT_UNKNOWN, D3DPOOL_MANAGED,
D3DX_DEFAULT, D3DX_DEFAULT, 0, nullptr, nullptr,
reinterpret_cast<IDirect3DTexture9**>(ppTexture))) {
*ppTexture = nullptr;
Warning("LoadWICTexture(%p, %#lX): FAILED\n", *ppTexture, file_in_memory->crc_hash);
return RETURN_TEXTURE_NOT_LOADED;
}
D3DXIMAGE_INFO info;
D3DXGetImageInfoFromFileInMemory(file_in_memory->data.data(), file_in_memory->data.size(), &info);
char dllpath[MAX_PATH]{};
GetModuleFileName(gl_hThisInstance, dllpath, MAX_PATH); //ask for name and path of this dll
const auto dds_export_path = std::filesystem::path(dllpath).parent_path() / "d3dxout" / std::format("GW.EXE_{:X}.dds", file_in_memory->crc_hash);
if (!std::filesystem::exists(dds_export_path)) {
D3DXSaveTextureToFile(dds_export_path.string().c_str(), D3DXIFF_DDS, *ppTexture, nullptr);
}
#else
if (D3D_OK != DirectX::CreateWICTextureFromMemoryEx(
D3D9Device,
file_in_memory->data.data(),
file_in_memory->data.size(),
0, 0, D3DPOOL_MANAGED, DirectX::WIC_LOADER_MIP_AUTOGEN,
reinterpret_cast<IDirect3DTexture9**>(ppTexture))) {
*ppTexture = nullptr;
Warning("LoadWICTexture(%p, %#lX): FAILED\n", *ppTexture, file_in_memory->crc_hash);
return RETURN_TEXTURE_NOT_LOADED;
}
#endif
}
else if (D3D_OK != DirectX::CreateDDSTextureFromMemoryEx(
D3D9Device,
file_in_memory->data.data(),
file_in_memory->data.size(),
0, D3DPOOL_MANAGED, false,
reinterpret_cast<LPDIRECT3DTEXTURE9*>(ppTexture))) {
*ppTexture = nullptr;
Warning("LoadDDSTexture Normal(%p, %#lX): FAILED\n", *ppTexture, file_in_memory->crc_hash);
return RETURN_TEXTURE_NOT_LOADED;
}
if constexpr (std::same_as<decltype(ppTexture), uMod_IDirect3DTexture9**>) {
SetLastCreatedTexture(nullptr);
}
else if constexpr (std::same_as<decltype(ppTexture), uMod_IDirect3DVolumeTexture9**>) {
SetLastCreatedVolumeTexture(nullptr);
}
else if constexpr (std::same_as<decltype(ppTexture), uMod_IDirect3DCubeTexture9**>) {
SetLastCreatedCubeTexture(nullptr);
}
(*ppTexture)->FAKE = true;
Message("LoadTexture( %p, %#lX): DONE\n", *ppTexture, file_in_memory->crc_hash);
return RETURN_OK;
}
template <typename T> requires uModTexturePtr<T>
void UnswitchTextures(T pTexture)
{
decltype(pTexture) CrossRef = pTexture->CrossRef_D3Dtex;
if (CrossRef != nullptr) {
std::swap(pTexture->m_D3Dtex, CrossRef->m_D3Dtex);
// cancel the link
CrossRef->CrossRef_D3Dtex = nullptr;
pTexture->CrossRef_D3Dtex = nullptr;
}
}
template <typename T> requires uModTexturePtr<T>
int SwitchTextures(T pTexture1, T pTexture2)
{
if (pTexture1->m_D3Ddev == pTexture2->m_D3Ddev && pTexture1->CrossRef_D3Dtex == nullptr && pTexture2->CrossRef_D3Dtex == nullptr) {
// make cross reference
pTexture1->CrossRef_D3Dtex = pTexture2;
pTexture2->CrossRef_D3Dtex = pTexture1;
// switch textures
std::swap(pTexture1->m_D3Dtex, pTexture2->m_D3Dtex);
return RETURN_OK;
}
return RETURN_TEXTURE_NOT_SWITCHED;
}
-138
View File
@@ -1,138 +0,0 @@
#pragma once
constexpr auto crc32_poly = 0xEDB88320u;
constexpr auto ul_crc_in = 0xffffffff;
inline unsigned int GetCRC32(char* pcDatabuf, unsigned int ulDatalen)
{
unsigned int crc = ul_crc_in;
for (unsigned int idx = 0u; idx < ulDatalen; idx++) {
unsigned int data = *pcDatabuf++;
for (unsigned int bit = 0u; bit < 8u; bit++, data >>= 1) {
crc = crc >> 1 ^ ((crc ^ data) & 1 ? crc32_poly : 0);
}
}
return crc;
}
/*
case D3DFMT_MULTI2_ARGB8:
case D3DFMT_VERTEXDATA:
*/
inline int GetBitsFromFormat(D3DFORMAT format)
{
switch (format) //switch trough the formats to calculate the size of the raw data
{
case D3DFMT_A1: // 1-bit monochrome.
{
return 1;
break;
}
case D3DFMT_R3G3B2: // 8-bit RGB texture format using 3 bits for red, 3 bits for green, and 2 bits for blue.
case D3DFMT_A8: // 8-bit alpha only.
case D3DFMT_A8P8: // 8-bit color indexed with 8 bits of alpha.
case D3DFMT_P8: // 8-bit color indexed.
case D3DFMT_L8: // 8-bit luminance only.
case D3DFMT_A4L4: // 8-bit using 4 bits each for alpha and luminance.
case D3DFMT_FORCE_DWORD:
case D3DFMT_S8_LOCKABLE: // A lockable 8-bit stencil buffer.
{
return 8;
break;
}
case D3DFMT_D16_LOCKABLE: //16-bit z-buffer bit depth.
case D3DFMT_D15S1: // 16-bit z-buffer bit depth where 15 bits are reserved for the depth channel and 1 bit is reserved for the stencil channel.
case D3DFMT_L6V5U5: // 16-bit bump-map format with luminance using 6 bits for luminance, and 5 bits each for v and u.
case D3DFMT_V8U8: // 16-bit bump-map format using 8 bits each for u and v data.
case D3DFMT_CxV8U8: // 16-bit normal compression format. The texture sampler computes the C channel from: C = sqrt(1 - U2 - V2).
case D3DFMT_R5G6B5: // 16-bit RGB pixel format with 5 bits for red, 6 bits for green, and 5 bits for blue.
case D3DFMT_X1R5G5B5: // 16-bit pixel format where 5 bits are reserved for each color.
case D3DFMT_A1R5G5B5: // 16-bit pixel format where 5 bits are reserved for each color and 1 bit is reserved for alpha.
case D3DFMT_A4R4G4B4: // 16-bit ARGB pixel format with 4 bits for each channel.
case D3DFMT_A8R3G3B2: // 16-bit ARGB texture format using 8 bits for alpha, 3 bits each for red and green, and 2 bits for blue.
case D3DFMT_X4R4G4B4: // 16-bit RGB pixel format using 4 bits for each color.
case D3DFMT_L16: // 16-bit luminance only.
case D3DFMT_R16F: // 16-bit float format using 16 bits for the red channel.
case D3DFMT_A8L8: // 16-bit using 8 bits each for alpha and luminance.
case D3DFMT_D16: // 16-bit z-buffer bit depth.
case D3DFMT_INDEX16: // 16-bit index buffer bit depth.
case D3DFMT_G8R8_G8B8: // ??
case D3DFMT_R8G8_B8G8: // ??
case D3DFMT_UYVY: // ??
case D3DFMT_YUY2: // ??
{
return 16;
break;
}
case D3DFMT_R8G8B8: //24-bit RGB pixel format with 8 bits per channel.
{
return 24;
break;
}
case D3DFMT_R32F: // 32-bit float format using 32 bits for the red channel.
case D3DFMT_X8L8V8U8: // 32-bit bump-map format with luminance using 8 bits for each channel.
case D3DFMT_A2W10V10U10: // 32-bit bump-map format using 2 bits for alpha and 10 bits each for w, v, and u.
case D3DFMT_Q8W8V8U8: // 32-bit bump-map format using 8 bits for each channel.
case D3DFMT_V16U16: // 32-bit bump-map format using 16 bits for each channel.
case D3DFMT_A8R8G8B8: // 32-bit ARGB pixel format with alpha, using 8 bits per channel.
case D3DFMT_X8R8G8B8: // 32-bit RGB pixel format, where 8 bits are reserved for each color.
case D3DFMT_A2B10G10R10: // 32-bit pixel format using 10 bits for each color and 2 bits for alpha.
case D3DFMT_A8B8G8R8: // 32-bit ARGB pixel format with alpha, using 8 bits per channel.
case D3DFMT_X8B8G8R8: // 32-bit RGB pixel format, where 8 bits are reserved for each color.
case D3DFMT_G16R16: // 32-bit pixel format using 16 bits each for green and red.
case D3DFMT_G16R16F: // 32-bit float format using 16 bits for the red channel and 16 bits for the green channel.
case D3DFMT_A2R10G10B10: // 32-bit pixel format using 10 bits each for red, green, and blue, and 2 bits for alpha.
case D3DFMT_D32: // 32-bit z-buffer bit depth.
case D3DFMT_D24S8: // 32-bit z-buffer bit depth using 24 bits for the depth channel and 8 bits for the stencil channel.
case D3DFMT_D24X8: //32-bit z-buffer bit depth using 24 bits for the depth channel.
case D3DFMT_D24X4S4: // 32-bit z-buffer bit depth using 24 bits for the depth channel and 4 bits for the stencil channel.
case D3DFMT_D32F_LOCKABLE: // A lockable format where the depth value is represented as a standard IEEE floating-point number.
case D3DFMT_D24FS8: // A non-lockable format that contains 24 bits of depth (in a 24-bit floating point format - 20e4) and 8 bits of stencil.
case D3DFMT_D32_LOCKABLE: // A lockable 32-bit depth buffer.
case D3DFMT_INDEX32: // 32-bit index buffer bit depth.
//case : //
//case : //
//case : //
//case : //
{
return 32;
break;
}
case D3DFMT_G32R32F: // 64-bit float format using 32 bits for the red channel and 32 bits for the green channel.
case D3DFMT_Q16W16V16U16: // 64-bit bump-map format using 16 bits for each component.
case D3DFMT_A16B16G16R16: // 64-bit pixel format using 16 bits for each component.
case D3DFMT_A16B16G16R16F: // 64-bit float format using 16 bits for the each channel (alpha, blue, green, red).
{
return 64;
break;
}
case D3DFMT_A32B32G32R32F: // 128-bit float format using 32 bits for the each channel (alpha, blue, green, red).
{
return 128;
break;
}
case D3DFMT_DXT2:
case D3DFMT_DXT3:
case D3DFMT_DXT4:
case D3DFMT_DXT5: {
return 8;
break;
}
case D3DFMT_DXT1: {
return 4;
break;
}
default: //compressed formats
{
return 4;
break;
}
}
}
-59
View File
@@ -1,59 +0,0 @@
#pragma once
#include <fstream>
#include <vector>
class TpfReader {
public:
TpfReader(const std::string& path)
{
file_stream = std::ifstream(path, std::ios::binary);
if (!file_stream.seekg(0, std::ios::end).good() || !file_stream.seekg(0, std::ios::beg).good()) {
throw std::invalid_argument("Provided stream needs to have SEEK set to True");
}
}
~TpfReader()
{
file_stream.close();
}
std::vector<char> ReadToEnd()
{
file_stream.seekg(0, std::ios::end);
line_length = file_stream.tellg();
file_stream.seekg(0, std::ios::beg); // Go to the beginning of the stream
std::vector<char> data(line_length);
file_stream.read(data.data(), line_length);
for (auto i = 0; i < data.size(); i++) {
data[i] = XOR(data[i], i);
}
for (int i = data.size() - 1; i > 0 && data[i] != 0; i--) {
data[i] = 0;
}
// in the other zip libraries, these had to be cut off, with libzip we need to zero them out
// cutting them off makes the archive invalid
//data.resize(last_zero);
return data;
}
private:
std::ifstream file_stream;
long line_length;
[[nodiscard]] char XOR(const char b, const long position) const
{
if (position > line_length - 4) {
return b ^ TPF_XOREven;
}
return position % 2 == 0 ? b ^ TPF_XOREven : b ^ TPF_XOROdd;
}
static constexpr char TPF_XOROdd = 0x3F;
static constexpr char TPF_XOREven = static_cast<char>(0xA4);
};
-8
View File
@@ -1,8 +0,0 @@
#pragma once
#include <d3d9.h>
void InitInstance(HINSTANCE hModule);
void ExitInstance();
HMODULE LoadOriginalDll();
+1 -1
View File
@@ -4,8 +4,8 @@
#include "uMod_IDirect3DTexture9.h"
#include "uMod_IDirect3DVolumeTexture9.h"
#include "uMod_IDirect3DCubeTexture9.h"
#include "TextureClient.h"
class TextureClient;
class uMod_IDirect3DDevice9 : public IDirect3DDevice9 {
public:
+2 -2
View File
@@ -1,11 +1,11 @@
#pragma once
#include <d3d9.h>
#include "uMod_IDirect3DTexture9.h"
#include "uMod_IDirect3DVolumeTexture9.h"
#include "uMod_IDirect3DCubeTexture9.h"
#include "TextureClient.h"
class TextureClient;
class uMod_IDirect3DDevice9Ex : public IDirect3DDevice9Ex {
public: