implement UTF-8 for TodStringFile, ImageFont, DescParser

src/Sexy.TodLib/TodStringFile.cpp

@@ -1,8 +1,9 @@
 #include "TodDebug.h"
-#include "TodCommon.h"
-#include "TodStringFile.h"
 #include "../PakLib/PakInterface.h"
+#include "../SexyAppFramework/Encoding.h"
 #include "../SexyAppFramework/Font.h"
+#include "TodCommon.h"
+#include "TodStringFile.h"
 
 int gTodStringFormatCount;
 TodStringListFormat *gTodStringFormats;
@@ -125,8 +126,10 @@ bool TodStringListReadItems(const char *theFileText)
 	}
 }
 
+
 bool TodStringListReadFile(const char *theFileName)
 {
+    TodTrace("[TodLib] - Loading String File '%s'", theFileName);
 	PFILE *pFile = p_fopen(theFileName, "rb");
 	if (pFile == nullptr)
 	{
@@ -135,24 +138,34 @@ bool TodStringListReadFile(const char *theFileName)
 	}
 
 	p_fseek(pFile, 0, SEEK_END);
-	int aSize = p_ftell(pFile);	
+    auto aSize = p_ftell(pFile);
 	p_fseek(pFile, 0, SEEK_SET);
-	char *aFileText = new char[aSize + 1];
-	bool aSuccess = true;
-	if (p_fread(aFileText, sizeof(char), aSize, pFile) <= 0)
+
+    if (aSize <= 0)
+    {
+		TodTrace("[TodLib] - String file is empty: '%s'", theFileName);
+		p_fclose(pFile);
+		return false;
+    }
+
+	std::string aBytes;
+	aBytes.resize(aSize);
+	if (aSize > 0 && p_fread(&aBytes[0], sizeof(char), aSize, pFile) <= 0)
 	{
 		TodTrace("[TodLib] - Failed to read '%s'", theFileName);
-		aSuccess = false;
-	}
-	aFileText[aSize] = '\0';
-	std::string aFixedContent = ANSIToUTF8(aFileText);
-	if (aSuccess)
-	{
-		aSuccess = TodStringListReadItems(aFixedContent.c_str());
+		p_fclose(pFile);
+		return false;
 	}
+
 	p_fclose(pFile);
-	delete[] aFileText;
 
+    std::optional<std::string> aDecoded = Sexy::ConvertToUtf8IfNeeded(aBytes);
+    bool aSuccess = aDecoded
+        ? TodStringListReadItems(aDecoded->c_str())
+        : TodStringListReadItems(aBytes.c_str());
+    if (!aSuccess) {
+		TodTrace("[TodLib] - Failed to read list items from '%s'", theFileName);
+    }
 	return aSuccess;
 }
 
@@ -190,7 +203,7 @@ SexyString TodStringTranslate(const SexyChar *theString)
 {
 	if (theString != nullptr)
 	{
-		int aLen = sizeof(theString) / sizeof(theString[0]);
+		int aLen = StringLength(theString);
 		if (aLen >= 3 && theString[0] == '[')
 		{
 			SexyString aName = SexyCharToString(theString, 1, aLen - 2);

src/SexyAppFramework/Common.cpp

@@ -1,4 +1,4 @@
-#include "Common.h"
+#include "Common.h"
 #include "MTRand.h"
 #include <filesystem>
 #include <chrono>
@@ -17,7 +17,7 @@ std::string gAppDataFolder = std::filesystem::path(std::getenv("HOME")).string()
 #else
 std::string gAppDataFolder = std::filesystem::path(std::getenv("HOME")).string() + "/.config/";
 #endif
-}
+} // namespace Sexy
 
 int Sexy::Rand()
 {
@@ -51,7 +51,7 @@ void Sexy::SetAppDataFolder(const std::string &thePath)
 
 std::string Sexy::URLEncode(const std::string &theString)
 {
-	char *aHexChars = "0123456789ABCDEF";
+	const char *aHexChars = "0123456789ABCDEF";
 
 	std::string aString;
 
@@ -85,8 +85,16 @@ std::string Sexy::StringToUpper(const std::string &theString)
 {
 	std::string aString;
 
-	for (unsigned i = 0; i < theString.length(); i++)
-		aString += toupper(theString[i]);
+	auto it = theString.begin();
+	auto end = theString.end();
+	while (it != end)
+	{
+		uint32_t aCodepoint = utf8::next(it, end);
+		// only touch ascii
+		if (aCodepoint < 0x80)
+			aCodepoint = toupper(aCodepoint);
+		utf8::append(aCodepoint, aString);
+	}
 
 	return aString;
 }
@@ -95,8 +103,16 @@ std::string Sexy::StringToLower(const std::string &theString)
 {
 	std::string aString;
 
-	for (unsigned i = 0; i < theString.length(); i++)
-		aString += tolower(theString[i]);
+	auto it = theString.begin();
+	auto end = theString.end();
+	while (it != end)
+	{
+		uint32_t aCodepoint = utf8::next(it, end);
+		// only touch ascii
+		if (aCodepoint < 0x80)
+			aCodepoint = tolower(aCodepoint);
+		utf8::append(aCodepoint, aString);
+	}
 
 	return aString;
 }
@@ -113,15 +129,30 @@ std::string Sexy::SexyStringToString(const SexyString &theString)
 
 std::string Sexy::Trim(const std::string &theString)
 {
-	int aStartPos = 0;
-	while (aStartPos < (int)theString.length() && isspace(theString[aStartPos]))
-		aStartPos++;
+	auto aStart = theString.begin();
+	auto anEnd = theString.end();
 
-	int anEndPos = theString.length() - 1;
-	while (anEndPos >= 0 && isspace(theString[anEndPos]))
-		anEndPos--;
+	while (aStart != anEnd)
+	{
+		auto it = aStart;
+		uint32_t aCodepoint = utf8::next(it, anEnd);
+		// treat non-ascii as not whitespace
+		if (aCodepoint >= 0x80 || !isspace(aCodepoint))
+			break;
+		aStart = it;
+	}
+
+	while (anEnd != aStart)
+	{
+		auto it = anEnd;
+		uint32_t aCodepoint = utf8::prior(it, aStart);
+		// treat non-ascii as not whitespace
+		if (aCodepoint >= 0x80 || !isspace(aCodepoint))
+			break;
+		anEnd = it;
+	}
 
-	return theString.substr(aStartPos, anEndPos - aStartPos + 1);
+	return std::string(aStart, anEnd);
 }
 
 bool Sexy::StringToInt(const std::string theString, int *theIntVal)
@@ -378,24 +409,17 @@ bool Sexy::AllowAllAccess(const std::string &theFileName)
 	if (aLib == NULL)
 		return false;
 
-	BOOL(WINAPI * fnSetFileSecurity)(
-		LPCTSTR lpFileName, SECURITY_INFORMATION SecurityInformation, PSECURITY_DESCRIPTOR pSecurityDescriptor);
-	BOOL(WINAPI * fnSetSecurityDescriptorDacl)(
-		PSECURITY_DESCRIPTOR pSecurityDescriptor, BOOL bDaclPresent, PACL pDacl, BOOL bDaclDefaulted);
+	BOOL(WINAPI * fnSetFileSecurity)(LPCTSTR lpFileName, SECURITY_INFORMATION SecurityInformation,
+									 PSECURITY_DESCRIPTOR pSecurityDescriptor);
+	BOOL(WINAPI * fnSetSecurityDescriptorDacl)(PSECURITY_DESCRIPTOR pSecurityDescriptor, BOOL bDaclPresent, PACL pDacl,
+											   BOOL bDaclDefaulted);
 	BOOL(WINAPI * fnInitializeSecurityDescriptor)(PSECURITY_DESCRIPTOR pSecurityDescriptor, DWORD dwRevision);
-	BOOL(WINAPI * fnAllocateAndInitializeSid)(PSID_IDENTIFIER_AUTHORITY pIdentifierAuthority,
-											  BYTE nSubAuthorityCount,
-											  DWORD dwSubAuthority0,
-											  DWORD dwSubAuthority1,
-											  DWORD dwSubAuthority2,
-											  DWORD dwSubAuthority3,
-											  DWORD dwSubAuthority4,
-											  DWORD dwSubAuthority5,
-											  DWORD dwSubAuthority6,
-											  DWORD dwSubAuthority7,
-											  PSID * pSid);
-	DWORD(WINAPI * fnSetEntriesInAcl)(
-		ULONG cCountOfExplicitEntries, PEXPLICIT_ACCESS pListOfExplicitEntries, PACL OldAcl, PACL * NewAcl);
+	BOOL(WINAPI * fnAllocateAndInitializeSid)(PSID_IDENTIFIER_AUTHORITY pIdentifierAuthority, BYTE nSubAuthorityCount,
+											  DWORD dwSubAuthority0, DWORD dwSubAuthority1, DWORD dwSubAuthority2,
+											  DWORD dwSubAuthority3, DWORD dwSubAuthority4, DWORD dwSubAuthority5,
+											  DWORD dwSubAuthority6, DWORD dwSubAuthority7, PSID * pSid);
+	DWORD(WINAPI * fnSetEntriesInAcl)(ULONG cCountOfExplicitEntries, PEXPLICIT_ACCESS pListOfExplicitEntries,
+									  PACL OldAcl, PACL * NewAcl);
 	PVOID(WINAPI * fnFreeSid)(PSID pSid);
 
 	*(void **)&fnSetFileSecurity = (void *)GetProcAddress(aLib, "SetFileSecurityA");
@@ -458,7 +482,6 @@ bool Sexy::AllowAllAccess(const std::string &theFileName)
 #else
 	return false;
 #endif // WIN32
-
 }
 
 SexyString Sexy::SexyStringFromChar(SexyChar theChar)
@@ -668,7 +691,8 @@ std::string Sexy::AddTrailingSlash(const std::string &theDirectory, bool backSla
 uint64_t Sexy::GetLastWriteFileDate(const std::string &theFileName)
 {
 	auto ftime = std::filesystem::last_write_time(theFileName);
-	auto sctp = std::chrono::time_point_cast<std::chrono::system_clock::duration>(ftime - std::filesystem::file_time_type::clock::now() + std::chrono::system_clock::now());
+	auto sctp = std::chrono::time_point_cast<std::chrono::system_clock::duration>(
+		ftime - std::filesystem::file_time_type::clock::now() + std::chrono::system_clock::now());
 	return std::chrono::system_clock::to_time_t(sctp);
 }
 
@@ -763,8 +787,8 @@ std::string Sexy::Evaluate(const std::string &theString, const DefinesMap &theDe
 
 		anEvaluatedString.erase(anEvaluatedString.begin() + aPercentPos,
 								anEvaluatedString.begin() + aSecondPercentPos + 1);
-		anEvaluatedString.insert(
-			anEvaluatedString.begin() + aPercentPos, aValue.begin(), aValue.begin() + aValue.length());
+		anEvaluatedString.insert(anEvaluatedString.begin() + aPercentPos, aValue.begin(),
+								 aValue.begin() + aValue.length());
 	}
 
 	return anEvaluatedString;

src/SexyAppFramework/DescParser.cpp

@@ -1,6 +1,7 @@
 #include "DescParser.h"
 #include "../PakLib/PakInterface.h"
 #include "Common.h"
+#include "Encoding.h"
 
 using namespace Sexy;
 
@@ -288,26 +289,19 @@ bool DescParser::DataToDoubleVector(DataElement *theSource, DoubleVector *theDou
 	return true;
 }
 
-bool DescParser::ParseToList(const std::string &theString,
-							 ListDataElement *theList,
-							 bool expectListEnd,
-							 int *theStringPos)
+bool DescParser::ParseToList(std::string::const_iterator &it, const std::string::const_iterator &end,
+							 ListDataElement *theList, bool expectListEnd)
 {
 	bool inSingleQuotes = false;
 	bool inDoubleQuotes = false;
 	bool escaped = false;
 
 	SingleDataElement *aCurSingleDataElement = NULL;
 
-	int aStringPos = 0;
-
-	if (theStringPos == NULL)
-		theStringPos = &aStringPos;
-
-	while (*theStringPos < (int)theString.length())
+	while (it != end)
 	{
 		bool addSingleChar = false;
-		char aChar = theString[(*theStringPos)++];
+		uint32_t aChar = utf8::next(it, end); // really a Codepoint
 
 		bool isSeperator = (aChar == ' ') || (aChar == '\t') || (aChar == '\n') || (aChar == ',');
 
@@ -350,7 +344,7 @@ bool DescParser::ParseToList(const std::string &theString,
 					{
 						ListDataElement *aChildList = new ListDataElement();
 
-						if (!ParseToList(theString, aChildList, true, theStringPos))
+						if (!ParseToList(it, end, aChildList, true))
 							return false;
 
 						theList->mElementVector.push_back(aChildList);
@@ -376,7 +370,7 @@ bool DescParser::ParseToList(const std::string &theString,
 				theList->mElementVector.push_back(aCurSingleDataElement);
 			}
 
-			aCurSingleDataElement->mString += aChar;
+			utf8::append(aChar, aCurSingleDataElement->mString);
 		}
 	}
 
@@ -404,7 +398,8 @@ bool DescParser::ParseToList(const std::string &theString,
 bool DescParser::ParseDescriptorLine(const std::string &theDescriptorLine)
 {
 	ListDataElement aParams;
-	if (!ParseToList(theDescriptorLine, &aParams, false, NULL))
+	auto it = theDescriptorLine.begin();
+	if (!ParseToList(it, theDescriptorLine.end(), &aParams, false))
 		return false;
 
 	if (aParams.mElementVector.size() > 0)
@@ -433,15 +428,40 @@ bool DescParser::LoadDescriptor(const std::string &theFileName)
 	mError.erase();
 	mError.erase(mError.begin());
 
-	PFILE *aStream = p_fopen(theFileName.c_str(), "r");
-	if (aStream == NULL)
+	PFILE *pFile = p_fopen(theFileName.c_str(), "rb");
+	if (pFile == nullptr)
+	{
+		return false;
+	}
+	p_fseek(pFile, 0, SEEK_END);
+	auto aSize = p_ftell(pFile);
+	p_fseek(pFile, 0, SEEK_SET);
+
+	if (aSize <= 0)
+	{
+		p_fclose(pFile);
 		return false;
+	}
 
-	char aBuffChar = 0;
+	std::string aBytes;
+	aBytes.resize(aSize);
 
-	while (!p_feof(aStream))
+	std::size_t aReadSize = p_fread(&aBytes[0], sizeof(char), aSize, pFile);
+	p_fclose(pFile);
+
+	if (aReadSize == 0)
+		return false;
+
+	auto aDecoded = Sexy::ConvertToUtf8IfNeeded(aBytes);
+	const auto &aStream = aDecoded ? *aDecoded : aBytes;
+	auto it = aStream.begin();
+	auto end = aStream.end();
+
+	uint32_t aBuffChar = 0; // really a Codepoint
+
+	while (it != end)
 	{
-		int aChar;
+		uint32_t aChar; // really a Codepoint
 
 		bool skipLine = false;
 		bool atLineStart = true;
@@ -459,9 +479,11 @@ bool DescParser::LoadDescriptor(const std::string &theFileName)
 			}
 			else
 			{
-				aChar = p_fgetc(aStream);
-				if (aChar == EOF)
+				if (it == end)
+				{
 					break;
+				}
+				aChar = utf8::next(it, end);
 			}
 
 			if (aChar != '\r')
@@ -524,13 +546,12 @@ bool DescParser::LoadDescriptor(const std::string &theFileName)
 							if (mCurrentLine.size() == 0)
 								mCurrentLineNum = aLineCount + 1;
 
-							mCurrentLine += aChar;
+							utf8::append(aChar, mCurrentLine);
 						}
 					}
 				}
 			}
 		}
-		mCurrentLine = ANSIToUTF8(mCurrentLine); //fix encoding cause fuck windows!!!
 
 		if (mCurrentLine.length() > 0)
 		{
@@ -551,6 +572,5 @@ bool DescParser::LoadDescriptor(const std::string &theFileName)
 	mCurrentLine.erase();
 	mCurrentLineNum = 0;
 
-	p_fclose(aStream);
 	return !hasErrors;
 }