From 596c0535856827cd0cd80a2d0009cf64261d4fed Mon Sep 17 00:00:00 2001 From: textGamex Date: Thu, 10 Jul 2025 17:53:55 +0800 Subject: [PATCH 1/5] Add NUnit testing framework references to project --- CWToolsCSTests/CWToolsCSTests.csproj | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/CWToolsCSTests/CWToolsCSTests.csproj b/CWToolsCSTests/CWToolsCSTests.csproj index a233a263..04937442 100644 --- a/CWToolsCSTests/CWToolsCSTests.csproj +++ b/CWToolsCSTests/CWToolsCSTests.csproj @@ -24,5 +24,14 @@ PreserveNewest + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + \ No newline at end of file From dfe70df6a63fe02d9c4d4179e6d7cf9540ab022a Mon Sep 17 00:00:00 2001 From: textGamex Date: Thu, 10 Jul 2025 17:54:12 +0800 Subject: [PATCH 2/5] Remove commented-out code and clean up YAMLLocalisationParser.fs --- .../Localisation/YAMLLocalisationParser.fs | 55 +------------------ 1 file changed, 1 insertion(+), 54 deletions(-) diff --git a/CWTools/Localisation/YAMLLocalisationParser.fs b/CWTools/Localisation/YAMLLocalisationParser.fs index eab0a287..00e0b1ea 100644 --- a/CWTools/Localisation/YAMLLocalisationParser.fs +++ b/CWTools/Localisation/YAMLLocalisationParser.fs @@ -9,13 +9,6 @@ open CWTools.Utilities.Utils module YAMLLocalisationParser = open FParsec - // type Entry = { - // key : string - // value : char option - // desc : string - // position : Position - // } - type LocFile = { key: string; entries: Entry list } let inline isLocValueChar (c: char) = @@ -30,19 +23,13 @@ module YAMLLocalisationParser = || (c >= '\u3000' && c <= '\u30FF') || (c >= '\uFF00' && c <= '\uFFEF') - //let key = charsTillString ":" true 1000 .>> spaces "key" let key = many1Satisfy ((=) ':' >> not) .>> pchar ':' .>> spaces "key" - //let descInner = (charsTillString "§") - //let stringThenEscaped = pipe2 (manyCharsTill (noneOf ['"']) (pchar '§')) (manyCharsTill anyChar (pstring "§!")) (+) "escaped" - //let desc = pipe2 (pchar '"') (many ((attempt stringThenEscaped) <|> manyCharsTill anyChar (pchar '"')) |>> List.reduce (+)) (fun a b -> string a + b) - //let desc = between (pchar '"') (pchar '"') (charsTillString "\"" false 10000) .>> spaces "desc" - //let desc = pipe3 (pchar '"' |>> string) (many (attempt stringThenEscaped) |>> List.fold (+) "") (manyCharsTill (noneOf ['§']) (pchar '"')) (fun a b c -> string a + b + c) "string" let desc = many1Satisfy isLocValueChar .>>. getPosition .>>. restOfLine false "desc" let value = digit .>> spaces "version" - let getRange (start: FParsec.Position) (endp: FParsec.Position) = + let getRange (start: Position) (endp: Position) = mkRange start.StreamName (mkPos (int start.Line) (int start.Column)) (mkPos (int endp.Line) (int endp.Column)) let entry = @@ -81,27 +68,6 @@ module YAMLLocalisationParser = let parseLocText text name = runParserOnString file () name text type YAMLLocalisationService<'L>(files: (string * string) list, keyToLanguage, gameLang) = - // let localisationFolder : string = localisationSettings.folder - // let language : CK2Lang = - // match localisationSettings.language with - // | CK2 l -> l - // | _ -> failwith "Wrong language for localisation" - - // let languageKey = - // match language with - // |CK2Lang.English -> "l_english" - // |CK2Lang.French -> "l_french" - // |CK2Lang.Spanish -> "l_spanish" - // |CK2Lang.German -> "l_german" - // |_ -> failwith "Unknown language enum value" - - // let keyToLanguage = - // function - // |"l_english" -> Some EU4Lang.English - // |"l_french" -> Some EU4Lang.French - // |"l_spanish" -> Some EU4Lang.Spanish - // |"l_german" -> Some EU4Lang.German - // |_ -> None let mutable results: Results = upcast new Dictionary() @@ -120,16 +86,6 @@ module YAMLLocalisationParser = | None -> (true, entries.Length, "", None) | Failure(msg, p, _) -> (false, 0, msg, Some p.Position) - // let addFile f = - // match parseLocFile f with - // | Success({key = key; entries = entries}, _, _) -> - // match keyToLanguage key with - // |Some l -> - // let es = entries |> List.map (fun e -> e, CK2 l) - // records <- es@records; (true, es.Length, "") - // |None -> (true, entries.Length, "") - // | Failure(msg, _, _) -> (false, 0, msg) - // let addFiles (x : string list) = List.map (fun f -> (f, addFile f)) x let addFiles (x: (string * string) list) = List.map (fun (f, t) -> (f, addFile f t)) x @@ -159,13 +115,6 @@ module YAMLLocalisationParser = records <- recordsL |> Array.ofList recordsL <- [] - - // do - // match Directory.Exists(localisationFolder) with - // | true -> - // let files = Directory.EnumerateFiles localisationFolder |> List.ofSeq |> List.sort - // results <- addFiles files |> dict - // | false -> () new(localisationSettings: LocalisationSettings<'L>) = log (sprintf "Loading %s localisation in %s" localisationSettings.gameName localisationSettings.folder) @@ -190,9 +139,7 @@ module YAMLLocalisationParser = | false -> log (sprintf "%s not found" localisationSettings.folder) YAMLLocalisationService([], localisationSettings.keyToLanguage, localisationSettings.gameToLang) - //new (settings : CK2Settings) = HOI4LocalisationService(settings.HOI4Directory.localisationDirectory, settings.ck2Language) - //new (settings : CK2Settings) = EU4LocalisationService(settings.EU4Directory.localisationDirectory, settings.ck2Language) member __.Api lang = { new ILocalisationAPI with member __.Results = results From 126b89c987a1ddd4fc8e258a16ce4bd46656cf92 Mon Sep 17 00:00:00 2001 From: textGamex Date: Thu, 10 Jul 2025 18:33:04 +0800 Subject: [PATCH 3/5] Fix Localization Text Errors Including Comments & Optimize Parsing Performance --- .../Localisation/YAMLLocalisationParser.fs | 44 ++++++----- CWTools/Parser/SharedParsers.fs | 2 +- .../Parser/YAMLLocalisationParserTests.cs | 73 +++++++++++++++++++ CWToolsCSTests/Program.cs | 2 +- 4 files changed, 96 insertions(+), 25 deletions(-) create mode 100644 CWToolsCSTests/Parser/YAMLLocalisationParserTests.cs diff --git a/CWTools/Localisation/YAMLLocalisationParser.fs b/CWTools/Localisation/YAMLLocalisationParser.fs index 00e0b1ea..6f6c12a8 100644 --- a/CWTools/Localisation/YAMLLocalisationParser.fs +++ b/CWTools/Localisation/YAMLLocalisationParser.fs @@ -1,6 +1,7 @@ namespace CWTools.Localisation open CWTools.Common +open CWTools.Parser.SharedParsers open CWTools.Utilities.Position open System.Collections.Generic open System.IO @@ -23,9 +24,12 @@ module YAMLLocalisationParser = || (c >= '\u3000' && c <= '\u30FF') || (c >= '\uFF00' && c <= '\uFFEF') - let key = many1Satisfy ((=) ':' >> not) .>> pchar ':' .>> spaces "key" - let desc = - many1Satisfy isLocValueChar .>>. getPosition .>>. restOfLine false "desc" + let key = many1Satisfy ((=) ':' >> not) .>> skipChar ':' .>> spaces "key" + + let private desc = + between (skipChar '"') (skipChar '"') (manyStrings (quotedCharSnippet <|> escapedChar)) + .>>. getPosition + "desc" let value = digit .>> spaces "version" @@ -33,32 +37,26 @@ module YAMLLocalisationParser = mkRange start.StreamName (mkPos (int start.Line) (int start.Column)) (mkPos (int endp.Line) (int endp.Column)) let entry = - pipe5 - getPosition - key - (opt value) - desc - (getPosition .>> spaces) - (fun s k v ((validDesc, endofValid), invalidDesc) e -> - let errorRange = - if endofValid <> e then - Some(getRange endofValid e) - else - None - - { key = k - value = v - desc = validDesc + invalidDesc - position = getRange s e - errorRange = errorRange }) + pipe5 getPosition key (opt value) desc (getPosition .>> spaces) (fun s k v (validDesc, endofValid) e -> + let errorRange = + if endofValid <> e then + Some(getRange endofValid e) + else + None + + { key = k + value = v + desc = validDesc + position = getRange s e + errorRange = errorRange }) "entry" - let comment = pstring "#" >>. restOfLine true .>> spaces "comment" + let comment = skipChar '#' >>. skipRestOfLine true .>> spaces "comment" let file = spaces >>. many (attempt comment) - >>. pipe2 key (many ((attempt comment |>> (fun _ -> None)) <|> (entry |>> Some)) .>> eof) (fun k es -> + >>. pipe2 key (many ((attempt comment >>% None) <|> (entry |>> Some)) .>> eof) (fun k es -> { key = k; entries = List.choose id es }) "file" diff --git a/CWTools/Parser/SharedParsers.fs b/CWTools/Parser/SharedParsers.fs index 4d9488e0..79c012b8 100644 --- a/CWTools/Parser/SharedParsers.fs +++ b/CWTools/Parser/SharedParsers.fs @@ -144,7 +144,7 @@ module internal SharedParsers = betweenL (chSkip '{' "opening brace") (skipChar '}' "closing brace") inner "clause" let quotedCharSnippet = many1Satisfy (fun c -> c <> '\\' && c <> '"') - let escapedChar = (pstring "\\\"" <|> pstring "\\") |>> string + let escapedChar = (stringReturn "\\\"" "\"" <|> pstring "\\") let metaprogrammingCharSnippet = many1Satisfy (fun c -> c <> ']' && c <> '\\') let getRange (start: FParsec.Position) (endp: FParsec.Position) = diff --git a/CWToolsCSTests/Parser/YAMLLocalisationParserTests.cs b/CWToolsCSTests/Parser/YAMLLocalisationParserTests.cs new file mode 100644 index 00000000..338df3d1 --- /dev/null +++ b/CWToolsCSTests/Parser/YAMLLocalisationParserTests.cs @@ -0,0 +1,73 @@ +using System.Linq; +using CWTools.CSharp; +using CWTools.Localisation; +using NUnit.Framework; + +namespace CWToolsCSTests.Parser; + +[TestFixture(TestOf = typeof(YAMLLocalisationParser))] +public sealed class YAMLLocalisationParserTests +{ + private const string Text = """ + #comment + l_simp_chinese:#comment + #comment + key1: " value1" #comment1 + key2:2 "value2" + key3: "" #comment2 + key4: "va\"lue4" #comment3 + key5:5 "va\"lue5" #comment4 + key6:"value6"#comment4 + #comment5 + """; + + private const string Failure = """ + l_simp_chinese: + key1: + """; + + [Test] + public void ParseTest() + { + var result = YAMLLocalisationParser.parseLocText(Text, string.Empty); + var failure = YAMLLocalisationParser.parseLocText(Failure, string.Empty); + + Assert.That(result.IsSuccess, Is.True); + Assert.That(result.IsFailure, Is.False); + Assert.That(failure.IsSuccess, Is.False); + Assert.That(failure.IsFailure, Is.True); + } + + [Test] + public void ParseResultTest() + { + var result = YAMLLocalisationParser.parseLocText(Text, "test.txt").GetResult(); + + var key1 = result.entries.First(loc => loc.key == "key1"); + var key2 = result.entries.First(loc => loc.key == "key2"); + var key3 = result.entries.First(loc => loc.key == "key3"); + var key4 = result.entries.First(loc => loc.key == "key4"); + var key5 = result.entries.First(loc => loc.key == "key5"); + var key6 = result.entries.First(loc => loc.key == "key6"); + + Assert.That(key1.desc, Is.EqualTo(" value1")); + Assert.That(key1.value, Is.Null); + Assert.That(key2.desc, Is.EqualTo("value2")); + Assert.That(key2.value.Value, Is.EqualTo('2')); + Assert.That(key3.desc, Is.EqualTo("")); + Assert.That(key4.desc, Is.EqualTo("va\"lue4")); + Assert.That(key4.value, Is.Null); + Assert.That(key5.desc, Is.EqualTo("va\"lue5")); + Assert.That(key5.value.Value, Is.EqualTo('5')); + Assert.That(key6.desc, Is.EqualTo("value6")); + Assert.That(key1.position, Is.EqualTo(result.entries.First().position)); + } + + [Test] + public void ParseKeyTest() + { + var result = YAMLLocalisationParser.parseLocText(Text, "test.txt").GetResult(); + + Assert.That(result.key, Is.EqualTo("l_simp_chinese")); + } +} diff --git a/CWToolsCSTests/Program.cs b/CWToolsCSTests/Program.cs index 9caaa3bc..a7c141c2 100644 --- a/CWToolsCSTests/Program.cs +++ b/CWToolsCSTests/Program.cs @@ -56,7 +56,7 @@ public static MyNode MapToMyNode() { } class Program { - static void Main(string[] args) + static void main(string[] args) { //Parse event file // var parsed = CWTools.Parser.CKParser.parseEventFile("./testevent.txt"); From 05e1b7dc05e0d1ee7579759c6fdc6b8b76ea278c Mon Sep 17 00:00:00 2001 From: textGamex Date: Thu, 10 Jul 2025 19:41:15 +0800 Subject: [PATCH 4/5] replace many with skipMany in comment parser --- CWTools/Localisation/YAMLLocalisationParser.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CWTools/Localisation/YAMLLocalisationParser.fs b/CWTools/Localisation/YAMLLocalisationParser.fs index 6f6c12a8..a5ab237b 100644 --- a/CWTools/Localisation/YAMLLocalisationParser.fs +++ b/CWTools/Localisation/YAMLLocalisationParser.fs @@ -55,7 +55,7 @@ module YAMLLocalisationParser = let file = spaces - >>. many (attempt comment) + >>. skipMany (attempt comment) >>. pipe2 key (many ((attempt comment >>% None) <|> (entry |>> Some)) .>> eof) (fun k es -> { key = k; entries = List.choose id es }) "file" From a4b6381ce2d077b8f1055759158c0e8ba767ca8d Mon Sep 17 00:00:00 2001 From: Thomas Boby Date: Thu, 10 Jul 2025 19:00:17 +0100 Subject: [PATCH 5/5] Swap packaging to paket --- CWToolsCSTests/CWToolsCSTests.csproj | 14 -------------- CWToolsCSTests/paket.references | 5 ++++- paket.dependencies | 3 +++ paket.lock | 14 +++++++++++--- 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/CWToolsCSTests/CWToolsCSTests.csproj b/CWToolsCSTests/CWToolsCSTests.csproj index 04937442..1f2f4e1f 100644 --- a/CWToolsCSTests/CWToolsCSTests.csproj +++ b/CWToolsCSTests/CWToolsCSTests.csproj @@ -7,11 +7,6 @@ dotnet paket - - - - CWTools.fsproj @@ -24,14 +19,5 @@ PreserveNewest - - - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - - - \ No newline at end of file diff --git a/CWToolsCSTests/paket.references b/CWToolsCSTests/paket.references index eac14b87..7e126f91 100644 --- a/CWToolsCSTests/paket.references +++ b/CWToolsCSTests/paket.references @@ -1,5 +1,8 @@ -Expecto FParsec System.Text.Encoding.CodePages FsPickler FSharpx.Collections +NUnit +NUnit.Analyzers +NUnit3TestAdapter +Microsoft.NET.Test.Sdk diff --git a/paket.dependencies b/paket.dependencies index e1318180..708ab21b 100644 --- a/paket.dependencies +++ b/paket.dependencies @@ -22,6 +22,9 @@ nuget CWTools 0.4.0-alpha nuget VDS.Common github giraffe-fsharp/Giraffe:612aa186c68101f532196817f67ebaa083c16146 src/Giraffe/GiraffeViewEngine.fs nuget Microsoft.NET.Test.Sdk +nuget NUnit +nuget NUnit.Analyzers +nuget NUnit3TestAdapter nuget YoloDev.Expecto.TestSdk diff --git a/paket.lock b/paket.lock index e45373b9..37f6ad22 100644 --- a/paket.lock +++ b/paket.lock @@ -62,14 +62,14 @@ NUGET Microsoft.Testing.Platform (>= 1.7.3) - restriction: >= netstandard2.0 Microsoft.Testing.Extensions.TrxReport.Abstractions (1.7.3) - restriction: >= net8.0 Microsoft.Testing.Platform (>= 1.7.3) - restriction: >= netstandard2.0 - Microsoft.Testing.Extensions.VSTestBridge (1.7.3) - restriction: >= net8.0 + Microsoft.Testing.Extensions.VSTestBridge (1.7.3) - restriction: || (>= net462) (>= netcoreapp3.1) Microsoft.Testing.Extensions.Telemetry (>= 1.7.3) - restriction: >= netstandard2.0 Microsoft.Testing.Extensions.TrxReport.Abstractions (>= 1.7.3) - restriction: >= netstandard2.0 Microsoft.Testing.Platform (>= 1.7.3) - restriction: >= netstandard2.0 Microsoft.TestPlatform.AdapterUtilities (>= 17.13) - restriction: >= netstandard2.0 Microsoft.TestPlatform.ObjectModel (>= 17.13) - restriction: >= netstandard2.0 Microsoft.Testing.Platform (1.7.3) - restriction: >= net8.0 - Microsoft.Testing.Platform.MSBuild (1.7.3) - restriction: >= net8.0 + Microsoft.Testing.Platform.MSBuild (1.7.3) - restriction: || (>= net462) (>= netcoreapp3.1) Microsoft.Testing.Platform (>= 1.7.3) - restriction: >= netstandard2.0 Microsoft.TestPlatform.AdapterUtilities (17.14.1) - restriction: >= net8.0 Microsoft.TestPlatform.ObjectModel (17.14.1) - restriction: >= net8.0 @@ -129,6 +129,14 @@ NUGET System.Xml.XDocument (>= 4.3) - restriction: || (&& (< net45) (>= netstandard1.1) (< netstandard1.2) (< win8)) (&& (< net45) (>= netstandard1.2) (< netstandard1.3) (< win8) (< wpa81)) (&& (< net45) (>= netstandard1.3) (< netstandard1.4) (< win8) (< wpa81)) (&& (< net45) (>= netstandard1.4) (< netstandard1.5) (< win8) (< wpa81)) (&& (< net45) (>= netstandard1.5) (< netstandard1.6) (< win8) (< wpa81)) (&& (< net45) (>= netstandard1.6) (< netstandard2.0) (< win8) (< wpa81)) (&& (>= netstandard1.0) (< portable-net45+win8+wpa81) (< wp8)) (&& (< netstandard1.5) (>= uap10.0) (< uap10.1)) Newtonsoft.Json (13.0.3) - restriction: >= net8.0 NuGet.CommandLine (6.14) + NUnit (4.3.2) + System.Memory (>= 4.6) - restriction: >= net462 + System.Threading.Tasks.Extensions (>= 4.5.4) - restriction: >= net462 + System.ValueTuple (>= 4.5) - restriction: >= net462 + NUnit.Analyzers (4.9.2) + NUnit3TestAdapter (5.0) + Microsoft.Testing.Extensions.VSTestBridge (>= 1.5.3) - restriction: || (>= net462) (>= netcoreapp3.1) + Microsoft.Testing.Platform.MSBuild (>= 1.5.3) - restriction: || (>= net462) (>= netcoreapp3.1) Pastel (6.0.1) System.Memory (>= 4.6) - restriction: >= net462 System.Runtime.InteropServices.RuntimeInformation (>= 4.3) - restriction: >= net462 @@ -550,7 +558,7 @@ NUGET Microsoft.NETCore.Platforms (>= 1.1) - restriction: || (&& (< monoandroid) (< monotouch) (< net45) (>= netstandard1.3) (< win8) (< wpa81) (< xamarinios) (< xamarinmac) (< xamarintvos) (< xamarinwatchos)) (&& (< monoandroid) (< net45) (>= netstandard1.0) (< netstandard1.3) (< win8) (< wp8) (< wpa81)) Microsoft.NETCore.Targets (>= 1.1) - restriction: || (&& (< monoandroid) (< monotouch) (< net45) (>= netstandard1.3) (< win8) (< wpa81) (< xamarinios) (< xamarinmac) (< xamarintvos) (< xamarinwatchos)) (&& (< monoandroid) (< net45) (>= netstandard1.0) (< netstandard1.3) (< win8) (< wp8) (< wpa81)) System.Runtime (>= 4.3) - restriction: || (&& (< monoandroid) (< monotouch) (< net45) (>= netstandard1.3) (< win8) (< wpa81) (< xamarinios) (< xamarinmac) (< xamarintvos) (< xamarinwatchos)) (&& (< monoandroid) (< net45) (>= netstandard1.0) (< netstandard1.3) (< win8) (< wp8) (< wpa81)) - System.Threading.Tasks.Extensions (4.6.3) - restriction: || (&& (< monoandroid) (< monotouch) (< net40) (>= netstandard1.6) (< netstandard2.0) (< win8) (< wpa81) (< xamarintvos) (< xamarinwatchos)) (&& (< monoandroid) (< net40) (< netstandard1.2) (>= netstandard1.3) (< win8)) (&& (< monoandroid) (< net40) (>= netstandard1.3) (< netstandard1.4) (< win8) (< wpa81)) (&& (< monoandroid) (< net40) (>= netstandard1.4) (< netstandard1.5) (< win8) (< wpa81)) (&& (< monoandroid) (< net40) (>= netstandard1.5) (< netstandard1.6) (< win8) (< wpa81)) (&& (< net40) (>= net46) (< netstandard1.4)) (&& (< net40) (>= netstandard1.3) (< portable-net45+win8+wpa81)) (&& (< netstandard1.5) (>= uap10.0) (< win8) (< wpa81)) (&& (>= netstandard2.0) (< netstandard2.1)) + System.Threading.Tasks.Extensions (4.6.3) - restriction: || (&& (< monoandroid) (< monotouch) (< net40) (>= netstandard1.6) (< netstandard2.0) (< win8) (< wpa81) (< xamarintvos) (< xamarinwatchos)) (&& (< monoandroid) (< net40) (< netstandard1.2) (>= netstandard1.3) (< win8)) (&& (< monoandroid) (< net40) (>= netstandard1.3) (< netstandard1.4) (< win8) (< wpa81)) (&& (< monoandroid) (< net40) (>= netstandard1.4) (< netstandard1.5) (< win8) (< wpa81)) (&& (< monoandroid) (< net40) (>= netstandard1.5) (< netstandard1.6) (< win8) (< wpa81)) (&& (< net40) (>= net46) (< netstandard1.4)) (&& (< net40) (>= netstandard1.3) (< portable-net45+win8+wpa81)) (>= net462) (&& (< netstandard1.5) (>= uap10.0) (< win8) (< wpa81)) (&& (>= netstandard2.0) (< netstandard2.1)) System.Runtime.CompilerServices.Unsafe (>= 6.1.2) - restriction: || (>= net462) (&& (< netcoreapp2.1) (>= netstandard2.0) (< netstandard2.1)) System.Threading.Timer (4.3) - restriction: || (&& (< net40) (>= netstandard1.2) (< netstandard1.3) (< win8) (< wpa81)) (&& (< net40) (>= netstandard1.3) (< netstandard1.4) (< win8) (< wpa81)) (&& (< net40) (>= netstandard1.4) (< netstandard1.5) (< win8) (< wpa81)) (&& (< net40) (>= netstandard1.5) (< netstandard1.6) (< win8) (< wpa81)) (&& (< net40) (>= netstandard1.6) (< netstandard2.0) (< win8) (< wpa81)) (&& (< netstandard1.5) (>= uap10.0) (< uap10.1)) Microsoft.NETCore.Platforms (>= 1.1) - restriction: && (< monoandroid) (< monotouch) (< net451) (>= netstandard1.2) (< win81) (< wpa81) (< xamarinios) (< xamarinmac) (< xamarintvos) (< xamarinwatchos)