cwtools · tboby · Jul 11, 2025 · Jul 10, 2025 · Jul 10, 2025 · Jul 10, 2025
diff --git a/CWTools/Localisation/YAMLLocalisationParser.fs b/CWTools/Localisation/YAMLLocalisationParser.fs
@@ -1,6 +1,7 @@
 namespace CWTools.Localisation
 
 open CWTools.Common
+open CWTools.Parser.SharedParsers
 open CWTools.Utilities.Position
 open System.Collections.Generic
 open System.IO
@@ -9,13 +10,6 @@ open CWTools.Utilities.Utils
 module YAMLLocalisationParser =
     open FParsec
 
-    // type Entry = {
-    //     key : string
-    //     value : char option
-    //     desc : string
-    //     position : Position
-    // }
-
     type LocFile = { key: string; entries: Entry list }
 
     let inline isLocValueChar (c: char) =
@@ -30,48 +24,39 @@ module YAMLLocalisationParser =
         || (c >= '\u3000' && c <= '\u30FF')
         || (c >= '\uFF00' && c <= '\uFFEF')
 
-    //let key = charsTillString ":" true 1000 .>> spaces <?> "key"
-    let key = many1Satisfy ((=) ':' >> not) .>> pchar ':' .>> spaces <?> "key"
-    //let descInner = (charsTillString "§")
-    //let stringThenEscaped = pipe2 (manyCharsTill (noneOf ['"']) (pchar '§')) (manyCharsTill anyChar (pstring "§!")) (+) <?> "escaped"
-    //let desc = pipe2 (pchar '"') (many ((attempt stringThenEscaped) <|> manyCharsTill anyChar (pchar '"')) |>> List.reduce (+)) (fun a b -> string a + b)
-    //let desc = between (pchar '"') (pchar '"') (charsTillString "\"" false 10000) .>> spaces <?> "desc"
-    //let desc = pipe3 (pchar '"' |>> string) (many (attempt stringThenEscaped) |>> List.fold (+) "")  (manyCharsTill (noneOf ['§']) (pchar '"')) (fun a b c -> string a + b + c) <?> "string"
-    let desc =
-        many1Satisfy isLocValueChar .>>. getPosition .>>. restOfLine false <?> "desc"
+    let key = many1Satisfy ((=) ':' >> not) .>> skipChar ':' .>> spaces <?> "key"
+
+    let private desc =
+        between (skipChar '"') (skipChar '"') (manyStrings (quotedCharSnippet <|> escapedChar))
+        .>>. getPosition
+        <?> "desc"
 
     let value = digit .>> spaces <?> "version"
 
-    let getRange (start: FParsec.Position) (endp: FParsec.Position) =
+    let getRange (start: Position) (endp: Position) =
         mkRange start.StreamName (mkPos (int start.Line) (int start.Column)) (mkPos (int endp.Line) (int endp.Column))
 
     let entry =
-        pipe5
-            getPosition
-            key
-            (opt value)
-            desc
-            (getPosition .>> spaces)
-            (fun s k v ((validDesc, endofValid), invalidDesc) e ->
-                let errorRange =
-                    if endofValid <> e then
-                        Some(getRange endofValid e)
-                    else
-                        None
-
-                { key = k
-                  value = v
-                  desc = validDesc + invalidDesc
-                  position = getRange s e
-                  errorRange = errorRange })
+        pipe5 getPosition key (opt value) desc (getPosition .>> spaces) (fun s k v (validDesc, endofValid) e ->
+            let errorRange =
+                if endofValid <> e then
+                    Some(getRange endofValid e)
+                else
+                    None
+
+            { key = k
+              value = v
+              desc = validDesc
+              position = getRange s e
+              errorRange = errorRange })
         <?> "entry"
 
-    let comment = pstring "#" >>. restOfLine true .>> spaces <?> "comment"
+    let comment = skipChar '#' >>. skipRestOfLine true .>> spaces <?> "comment"
 
     let file =
         spaces
-        >>. many (attempt comment)
-        >>. pipe2 key (many ((attempt comment |>> (fun _ -> None)) <|> (entry |>> Some)) .>> eof) (fun k es ->
+        >>. skipMany (attempt comment)
+        >>. pipe2 key (many ((attempt comment >>% None) <|> (entry |>> Some)) .>> eof) (fun k es ->
             { key = k; entries = List.choose id es })
         <?> "file"
 
@@ -81,27 +66,6 @@ module YAMLLocalisationParser =
     let parseLocText text name = runParserOnString file () name text
 
     type YAMLLocalisationService<'L>(files: (string * string) list, keyToLanguage, gameLang) =
-        // let localisationFolder : string = localisationSettings.folder
-        // let language : CK2Lang =
-        //     match localisationSettings.language with
-        //         | CK2 l -> l
-        //         | _ -> failwith "Wrong language for localisation"
-
-        // let languageKey =
-        //     match language with
-        //     |CK2Lang.English -> "l_english"
-        //     |CK2Lang.French -> "l_french"
-        //     |CK2Lang.Spanish -> "l_spanish"
-        //     |CK2Lang.German -> "l_german"
-        //     |_ -> failwith "Unknown language enum value"
-
-        // let keyToLanguage =
-        //     function
-        //     |"l_english" -> Some EU4Lang.English
-        //     |"l_french" -> Some EU4Lang.French
-        //     |"l_spanish" -> Some EU4Lang.Spanish
-        //     |"l_german" -> Some EU4Lang.German
-        //     |_ -> None
         let mutable results: Results =
             upcast new Dictionary<string, bool * int * string * Position option>()
 
@@ -120,16 +84,6 @@ module YAMLLocalisationParser =
                 | None -> (true, entries.Length, "", None)
             | Failure(msg, p, _) -> (false, 0, msg, Some p.Position)
 
-        // let addFile f =
-        //     match parseLocFile f with
-        //     | Success({key = key; entries = entries}, _, _) ->
-        //         match keyToLanguage key with
-        //         |Some l ->
-        //             let es = entries |> List.map (fun e -> e, CK2 l)
-        //             records <- es@records; (true, es.Length, "")
-        //         |None -> (true, entries.Length, "")
-        //     | Failure(msg, _, _) -> (false, 0, msg)
-        // let addFiles (x : string list) = List.map (fun f -> (f, addFile f)) x
         let addFiles (x: (string * string) list) =
             List.map (fun (f, t) -> (f, addFile f t)) x
 
@@ -159,13 +113,6 @@ module YAMLLocalisationParser =
             records <- recordsL |> Array.ofList
             recordsL <- []
 
-
-        // do
-        //     match Directory.Exists(localisationFolder) with
-        //     | true ->
-        //                 let files = Directory.EnumerateFiles localisationFolder |> List.ofSeq |> List.sort
-        //                 results <- addFiles files |> dict
-        //     | false -> ()
         new(localisationSettings: LocalisationSettings<'L>) =
             log (sprintf "Loading %s localisation in %s" localisationSettings.gameName localisationSettings.folder)
 
@@ -190,9 +137,7 @@ module YAMLLocalisationParser =
             | false ->
                 log (sprintf "%s not found" localisationSettings.folder)
                 YAMLLocalisationService([], localisationSettings.keyToLanguage, localisationSettings.gameToLang)
-        //new (settings : CK2Settings) = HOI4LocalisationService(settings.HOI4Directory.localisationDirectory, settings.ck2Language)
 
-        //new (settings : CK2Settings) = EU4LocalisationService(settings.EU4Directory.localisationDirectory, settings.ck2Language)
         member __.Api lang =
             { new ILocalisationAPI with
                 member __.Results = results

diff --git a/CWTools/Parser/SharedParsers.fs b/CWTools/Parser/SharedParsers.fs
@@ -144,7 +144,7 @@ module internal SharedParsers =
         betweenL (chSkip '{' <?> "opening brace") (skipChar '}' <?> "closing brace") inner "clause"
 
     let quotedCharSnippet = many1Satisfy (fun c -> c <> '\\' && c <> '"')
-    let escapedChar = (pstring "\\\"" <|> pstring "\\") |>> string
+    let escapedChar = (stringReturn "\\\"" "\"" <|> pstring "\\")
     let metaprogrammingCharSnippet = many1Satisfy (fun c -> c <> ']' && c <> '\\')
 
     let getRange (start: FParsec.Position) (endp: FParsec.Position) =

diff --git a/CWToolsCSTests/CWToolsCSTests.csproj b/CWToolsCSTests/CWToolsCSTests.csproj
@@ -7,11 +7,6 @@
     <PaketCommand>dotnet paket</PaketCommand>
   </PropertyGroup>
   <ItemGroup>
-    <!-- <PackageReference Include="System.Text.Encoding.CodePages" Version="4.4.0" /> -->
-    <!-- <PackageReference Include="FParsec" Version="1.0.4-RC3" /> -->
-    <!-- <PackageReference Include="Microsoft.AspNetCore.All" Version="2.0.0" /> -->
-    <!-- <PackageReference Include="cwtools" Version="0.3.0"/>
-    <PackageReference Include="FSharp.Core" Version="4.5.0"/> -->
     <ProjectReference Include="..\CWTools\CWTools.fsproj">
     <Name>CWTools.fsproj</Name>
     </ProjectReference>

diff --git a/CWToolsCSTests/Parser/YAMLLocalisationParserTests.cs b/CWToolsCSTests/Parser/YAMLLocalisationParserTests.cs
@@ -0,0 +1,73 @@
+using System.Linq;
+using CWTools.CSharp;
+using CWTools.Localisation;
+using NUnit.Framework;
+
+namespace CWToolsCSTests.Parser;
+
+[TestFixture(TestOf = typeof(YAMLLocalisationParser))]
+public sealed class YAMLLocalisationParserTests
+{
+    private const string Text = """
+        #comment
+        l_simp_chinese:#comment
+        #comment
+        key1: " value1" #comment1
+        key2:2 "value2"
+        key3: "" #comment2
+        key4: "va\"lue4" #comment3
+        key5:5 "va\"lue5" #comment4
+        key6:"value6"#comment4
+        #comment5
+        """;
+
+    private const string Failure = """
+        l_simp_chinese:
+         key1:
+        """;
+
+    [Test]
+    public void ParseTest()
+    {
+        var result = YAMLLocalisationParser.parseLocText(Text, string.Empty);
+        var failure = YAMLLocalisationParser.parseLocText(Failure, string.Empty);
+
+        Assert.That(result.IsSuccess, Is.True);
+        Assert.That(result.IsFailure, Is.False);
+        Assert.That(failure.IsSuccess, Is.False);
+        Assert.That(failure.IsFailure, Is.True);
+    }
+
+    [Test]
+    public void ParseResultTest()
+    {
+        var result = YAMLLocalisationParser.parseLocText(Text, "test.txt").GetResult();
+
+        var key1 = result.entries.First(loc => loc.key == "key1");
+        var key2 = result.entries.First(loc => loc.key == "key2");
+        var key3 = result.entries.First(loc => loc.key == "key3");
+        var key4 = result.entries.First(loc => loc.key == "key4");
+        var key5 = result.entries.First(loc => loc.key == "key5");
+        var key6 = result.entries.First(loc => loc.key == "key6");
+
+        Assert.That(key1.desc, Is.EqualTo(" value1"));
+        Assert.That(key1.value, Is.Null);
+        Assert.That(key2.desc, Is.EqualTo("value2"));
+        Assert.That(key2.value.Value, Is.EqualTo('2'));
+        Assert.That(key3.desc, Is.EqualTo(""));
+        Assert.That(key4.desc, Is.EqualTo("va\"lue4"));
+        Assert.That(key4.value, Is.Null);
+        Assert.That(key5.desc, Is.EqualTo("va\"lue5"));
+        Assert.That(key5.value.Value, Is.EqualTo('5'));
+        Assert.That(key6.desc, Is.EqualTo("value6"));
+        Assert.That(key1.position, Is.EqualTo(result.entries.First().position));
+    }
+
+    [Test]
+    public void ParseKeyTest()
+    {
+        var result = YAMLLocalisationParser.parseLocText(Text, "test.txt").GetResult();
+
+        Assert.That(result.key, Is.EqualTo("l_simp_chinese"));
+    }
+}
diff --git a/CWToolsCSTests/Program.cs b/CWToolsCSTests/Program.cs
@@ -56,7 +56,7 @@ public static MyNode MapToMyNode() {
     }
     class Program
     {
-        static void Main(string[] args)
+        static void main(string[] args)
         {
             //Parse event file
             // var parsed = CWTools.Parser.CKParser.parseEventFile("./testevent.txt");

diff --git a/CWToolsCSTests/paket.references b/CWToolsCSTests/paket.references
@@ -1,5 +1,8 @@
-Expecto
 FParsec
 System.Text.Encoding.CodePages
 FsPickler
 FSharpx.Collections
+NUnit
+NUnit.Analyzers
+NUnit3TestAdapter
+Microsoft.NET.Test.Sdk
diff --git a/paket.dependencies b/paket.dependencies
@@ -22,6 +22,9 @@ nuget CWTools 0.4.0-alpha
 nuget VDS.Common
 github giraffe-fsharp/Giraffe:612aa186c68101f532196817f67ebaa083c16146 src/Giraffe/GiraffeViewEngine.fs
 nuget Microsoft.NET.Test.Sdk
+nuget NUnit
+nuget NUnit.Analyzers
+nuget NUnit3TestAdapter
 nuget YoloDev.Expecto.TestSdk
 
 

diff --git a/paket.lock b/paket.lock
@@ -62,14 +62,14 @@ NUGET
       Microsoft.Testing.Platform (>= 1.7.3) - restriction: >= netstandard2.0
     Microsoft.Testing.Extensions.TrxReport.Abstractions (1.7.3) - restriction: >= net8.0
       Microsoft.Testing.Platform (>= 1.7.3) - restriction: >= netstandard2.0
-    Microsoft.Testing.Extensions.VSTestBridge (1.7.3) - restriction: >= net8.0
+    Microsoft.Testing.Extensions.VSTestBridge (1.7.3) - restriction: || (>= net462) (>= netcoreapp3.1)
       Microsoft.Testing.Extensions.Telemetry (>= 1.7.3) - restriction: >= netstandard2.0
       Microsoft.Testing.Extensions.TrxReport.Abstractions (>= 1.7.3) - restriction: >= netstandard2.0
       Microsoft.Testing.Platform (>= 1.7.3) - restriction: >= netstandard2.0
       Microsoft.TestPlatform.AdapterUtilities (>= 17.13) - restriction: >= netstandard2.0
       Microsoft.TestPlatform.ObjectModel (>= 17.13) - restriction: >= netstandard2.0
     Microsoft.Testing.Platform (1.7.3) - restriction: >= net8.0
-    Microsoft.Testing.Platform.MSBuild (1.7.3) - restriction: >= net8.0
+    Microsoft.Testing.Platform.MSBuild (1.7.3) - restriction: || (>= net462) (>= netcoreapp3.1)
       Microsoft.Testing.Platform (>= 1.7.3) - restriction: >= netstandard2.0
     Microsoft.TestPlatform.AdapterUtilities (17.14.1) - restriction: >= net8.0
     Microsoft.TestPlatform.ObjectModel (17.14.1) - restriction: >= net8.0
@@ -129,6 +129,14 @@ NUGET
       System.Xml.XDocument (>= 4.3) - restriction: || (&& (< net45) (>= netstandard1.1) (< netstandard1.2) (< win8)) (&& (< net45) (>= netstandard1.2) (< netstandard1.3) (< win8) (< wpa81)) (&& (< net45) (>= netstandard1.3) (< netstandard1.4) (< win8) (< wpa81)) (&& (< net45) (>= netstandard1.4) (< netstandard1.5) (< win8) (< wpa81)) (&& (< net45) (>= netstandard1.5) (< netstandard1.6) (< win8) (< wpa81)) (&& (< net45) (>= netstandard1.6) (< netstandard2.0) (< win8) (< wpa81)) (&& (>= netstandard1.0) (< portable-net45+win8+wpa81) (< wp8)) (&& (< netstandard1.5) (>= uap10.0) (< uap10.1))
     Newtonsoft.Json (13.0.3) - restriction: >= net8.0
     NuGet.CommandLine (6.14)
+    NUnit (4.3.2)
+      System.Memory (>= 4.6) - restriction: >= net462
+      System.Threading.Tasks.Extensions (>= 4.5.4) - restriction: >= net462
+      System.ValueTuple (>= 4.5) - restriction: >= net462
+    NUnit.Analyzers (4.9.2)
+    NUnit3TestAdapter (5.0)
+      Microsoft.Testing.Extensions.VSTestBridge (>= 1.5.3) - restriction: || (>= net462) (>= netcoreapp3.1)
+      Microsoft.Testing.Platform.MSBuild (>= 1.5.3) - restriction: || (>= net462) (>= netcoreapp3.1)
     Pastel (6.0.1)
       System.Memory (>= 4.6) - restriction: >= net462
       System.Runtime.InteropServices.RuntimeInformation (>= 4.3) - restriction: >= net462
@@ -550,7 +558,7 @@ NUGET
       Microsoft.NETCore.Platforms (>= 1.1) - restriction: || (&& (< monoandroid) (< monotouch) (< net45) (>= netstandard1.3) (< win8) (< wpa81) (< xamarinios) (< xamarinmac) (< xamarintvos) (< xamarinwatchos)) (&& (< monoandroid) (< net45) (>= netstandard1.0) (< netstandard1.3) (< win8) (< wp8) (< wpa81))
       Microsoft.NETCore.Targets (>= 1.1) - restriction: || (&& (< monoandroid) (< monotouch) (< net45) (>= netstandard1.3) (< win8) (< wpa81) (< xamarinios) (< xamarinmac) (< xamarintvos) (< xamarinwatchos)) (&& (< monoandroid) (< net45) (>= netstandard1.0) (< netstandard1.3) (< win8) (< wp8) (< wpa81))
       System.Runtime (>= 4.3) - restriction: || (&& (< monoandroid) (< monotouch) (< net45) (>= netstandard1.3) (< win8) (< wpa81) (< xamarinios) (< xamarinmac) (< xamarintvos) (< xamarinwatchos)) (&& (< monoandroid) (< net45) (>= netstandard1.0) (< netstandard1.3) (< win8) (< wp8) (< wpa81))
-    System.Threading.Tasks.Extensions (4.6.3) - restriction: || (&& (< monoandroid) (< monotouch) (< net40) (>= netstandard1.6) (< netstandard2.0) (< win8) (< wpa81) (< xamarintvos) (< xamarinwatchos)) (&& (< monoandroid) (< net40) (< netstandard1.2) (>= netstandard1.3) (< win8)) (&& (< monoandroid) (< net40) (>= netstandard1.3) (< netstandard1.4) (< win8) (< wpa81)) (&& (< monoandroid) (< net40) (>= netstandard1.4) (< netstandard1.5) (< win8) (< wpa81)) (&& (< monoandroid) (< net40) (>= netstandard1.5) (< netstandard1.6) (< win8) (< wpa81)) (&& (< net40) (>= net46) (< netstandard1.4)) (&& (< net40) (>= netstandard1.3) (< portable-net45+win8+wpa81)) (&& (< netstandard1.5) (>= uap10.0) (< win8) (< wpa81)) (&& (>= netstandard2.0) (< netstandard2.1))
+    System.Threading.Tasks.Extensions (4.6.3) - restriction: || (&& (< monoandroid) (< monotouch) (< net40) (>= netstandard1.6) (< netstandard2.0) (< win8) (< wpa81) (< xamarintvos) (< xamarinwatchos)) (&& (< monoandroid) (< net40) (< netstandard1.2) (>= netstandard1.3) (< win8)) (&& (< monoandroid) (< net40) (>= netstandard1.3) (< netstandard1.4) (< win8) (< wpa81)) (&& (< monoandroid) (< net40) (>= netstandard1.4) (< netstandard1.5) (< win8) (< wpa81)) (&& (< monoandroid) (< net40) (>= netstandard1.5) (< netstandard1.6) (< win8) (< wpa81)) (&& (< net40) (>= net46) (< netstandard1.4)) (&& (< net40) (>= netstandard1.3) (< portable-net45+win8+wpa81)) (>= net462) (&& (< netstandard1.5) (>= uap10.0) (< win8) (< wpa81)) (&& (>= netstandard2.0) (< netstandard2.1))
       System.Runtime.CompilerServices.Unsafe (>= 6.1.2) - restriction: || (>= net462) (&& (< netcoreapp2.1) (>= netstandard2.0) (< netstandard2.1))
     System.Threading.Timer (4.3) - restriction: || (&& (< net40) (>= netstandard1.2) (< netstandard1.3) (< win8) (< wpa81)) (&& (< net40) (>= netstandard1.3) (< netstandard1.4) (< win8) (< wpa81)) (&& (< net40) (>= netstandard1.4) (< netstandard1.5) (< win8) (< wpa81)) (&& (< net40) (>= netstandard1.5) (< netstandard1.6) (< win8) (< wpa81)) (&& (< net40) (>= netstandard1.6) (< netstandard2.0) (< win8) (< wpa81)) (&& (< netstandard1.5) (>= uap10.0) (< uap10.1))
       Microsoft.NETCore.Platforms (>= 1.1) - restriction: && (< monoandroid) (< monotouch) (< net451) (>= netstandard1.2) (< win81) (< wpa81) (< xamarinios) (< xamarinmac) (< xamarintvos) (< xamarinwatchos)