Base64 performance improvement; almost ready to release version 4.2; …

…etc.

CBOR/CBOR.csproj

@@ -3,7 +3,7 @@
   <PropertyGroup>
     <TargetFrameworks>netstandard1.0</TargetFrameworks>
     <GeneratePackageOnBuild>True</GeneratePackageOnBuild>
-    <Version>4.2.0-a0</Version>
+    <Version>4.2.0</Version>
     <Owners>Peter Occil</Owners>
     <Description>A C# implementation of Concise Binary Object Representation (CBOR), a general-purpose binary data format defined in RFC 7049.</Description>
     <Summary>A C# implementation of Concise Binary Object Representation (CBOR), a general-purpose binary data format defined in RFC 7049. </Summary>
@@ -13,9 +13,14 @@
     <PackageLicenseExpression>CC0-1.0</PackageLicenseExpression>
     <PackageProjectUrl>https://github.com/peteroupc/CBOR</PackageProjectUrl>
     <PackageReleaseNotes>
-Version 4.1:
+Version 4.2:
 
-- JSONOptions string constructor now sets ReplaceSurrogates to false by default (previously, it was inadvertently true).
+- Some arithmetic methods in CBORNumber do basic overflow checks.
+- Add char array and byte array overloads to ParseJSONNumber
+- Support implementations of IList in CBORObject deserialization
+- Internally, the code avoids storing doubles (64-bit floating-point numbers) directly in CBORNumbers, uses sorted maps rather than hash tables in some CBOR objects, and can now store text strings as UTF-8 byte arrays.  This could help avoid unnecessary string conversions in many case.
+- Bug fixes and performance improvements
+- Now uses Numbers library version 1.7.3
 
 </PackageReleaseNotes>
     <PackageTags>cbor data serialization binary json</PackageTags>
@@ -42,4 +47,4 @@ Version 4.1:
 
 <PackageReference Include='PeterO.Numbers' Version='1.7.3'/>
 <PackageReference Include='Microsoft.CodeAnalysis.FxCopAnalyzers' PrivateAssets='All' Version='3.0.0'/></ItemGroup>
-               </Project>
+                  </Project>

CBOR/PeterO/Cbor/Base64.cs

@@ -67,43 +67,48 @@ private static void WriteBase64(
       string alphabet = classic ? Base64Classic : Base64URL;
       int length = offset + count;
       int i = offset;
-      var buffer = new char[4];
+      var buffer = new byte[32];
+      var bufferOffset = 0;
       for (i = offset; i < (length - 2); i += 3) {
-        buffer[0] = (char)alphabet[(data[i] >> 2) & 63];
-        buffer[1] = (char)alphabet[((data[i] & 3) << 4) +
+        if (bufferOffset >= buffer.Length) {
+           writer.WriteAscii(buffer, 0, bufferOffset);
+           bufferOffset = 0;
+        }
+        buffer[bufferOffset++] = (byte)alphabet[(data[i] >> 2) & 63];
+        buffer[bufferOffset++] = (byte)alphabet[((data[i] & 3) << 4) +
             ((data[i + 1] >> 4) & 15)];
-        buffer[2] = (char)alphabet[((data[i + 1] & 15) << 2) + ((data[i +
+        buffer[bufferOffset++] = (byte)alphabet[((data[i + 1] & 15) << 2) +
+((data[i +
                   2] >> 6) & 3)];
-        buffer[3] = (char)alphabet[data[i + 2] & 63];
-        writer.WriteCodePoint((int)buffer[0]);
-        writer.WriteCodePoint((int)buffer[1]);
-        writer.WriteCodePoint((int)buffer[2]);
-        writer.WriteCodePoint((int)buffer[3]);
+        buffer[bufferOffset++] = (byte)alphabet[data[i + 2] & 63];
       }
       int lenmod3 = count % 3;
       if (lenmod3 != 0) {
+        if (bufferOffset >= buffer.Length) {
+           writer.WriteAscii(buffer, 0, bufferOffset);
+           bufferOffset = 0;
+        }
         i = length - lenmod3;
-        buffer[0] = (char)alphabet[(data[i] >> 2) & 63];
+        buffer[bufferOffset++] = (byte)alphabet[(data[i] >> 2) & 63];
         if (lenmod3 == 2) {
-          buffer[1] = (char)alphabet[((data[i] & 3) << 4) + ((data[i + 1] >>
+          buffer[bufferOffset++] = (byte)alphabet[((data[i] & 3) << 4) +
+((data[i + 1] >>
                   4) & 15)];
-          buffer[2] = (char)alphabet[(data[i + 1] & 15) << 2];
-          writer.WriteCodePoint((int)buffer[0]);
-          writer.WriteCodePoint((int)buffer[1]);
-          writer.WriteCodePoint((int)buffer[2]);
+          buffer[bufferOffset++] = (byte)alphabet[(data[i + 1] & 15) << 2];
           if (padding) {
-            writer.WriteCodePoint((int)'=');
+            buffer[bufferOffset++] = (byte)'=';
           }
         } else {
-          buffer[1] = (char)alphabet[(data[i] & 3) << 4];
-          writer.WriteCodePoint((int)buffer[0]);
-          writer.WriteCodePoint((int)buffer[1]);
+          buffer[bufferOffset++] = (byte)alphabet[(data[i] & 3) << 4];
           if (padding) {
-            writer.WriteCodePoint((int)'=');
-            writer.WriteCodePoint((int)'=');
+            buffer[bufferOffset++] = (byte)'=';
+            buffer[bufferOffset++] = (byte)'=';
           }
         }
       }
+      if (bufferOffset >= 0) {
+        writer.WriteAscii(buffer, 0, bufferOffset);
+      }
     }
   }
 }

CBOR/PeterO/Cbor/CBORJsonWriter.cs

@@ -23,6 +23,9 @@ internal static void WriteJSONStringUnquoted(
         sb.WriteString(str, 0, i);
         return;
       }
+      // int bufferlen = Math.Min(Math.Max(4, str.Length), 64);
+      // byte[] buffer = new byte[bufferlen];
+      // int bufferpos = 0;
       for (; i < str.Length; ++i) {
         char c = str[i];
         if (c == '\\' || c == '"') {

CBOR/PeterO/Cbor/StringOutput.cs

@@ -65,6 +65,49 @@ public void WriteString(string str, int index, int length) {
       }
     }
 
+    public void WriteAscii(byte[] bytes, int index, int length) {
+      if (bytes == null) {
+        throw new ArgumentNullException(nameof(bytes));
+      }
+      if (index < 0) {
+        throw new ArgumentException("\"index\" (" + index + ") is not" +
+"\u0020greater or equal to 0");
+      }
+      if (index > bytes.Length) {
+        throw new ArgumentException("\"index\" (" + index + ") is not less" +
+"\u0020or equal to " + bytes.Length);
+      }
+      if (length < 0) {
+        throw new ArgumentException(" (" + length + ") is not greater or" +
+"\u0020equal to 0");
+      }
+      if (length > bytes.Length) {
+        throw new ArgumentException(" (" + length + ") is not less or equal" +
+"\u0020to " + bytes.Length);
+      }
+      if (bytes.Length - index < length) {
+        throw new ArgumentException("\"bytes\" + \"'s length minus \" +" +
+"\u0020index (" + (bytes.Length - index) + ") is not greater or equal to " +
+length);
+      }
+      if (this.outputStream == null) {
+          DataUtilities.ReadUtf8FromBytes(
+            bytes,
+            index,
+            length,
+            this.builder,
+            false);
+      } else {
+          for (var i = 0; i < length; ++i) {
+             byte b = bytes[i + index];
+             if ((((int)b) & 0x7f) != b) {
+                throw new ArgumentException("str is non-ASCII");
+             }
+          }
+          this.outputStream.Write(bytes, index, length);
+      }
+    }
+
     public void WriteCodePoint(int codePoint) {
       if ((codePoint >> 7) == 0) {
         // Code point is in the Basic Latin range (U+0000 to U+007F)

CBOR20/CBOR20.csproj

@@ -213,4 +213,6 @@
     <Error Condition="!Exists('../packages/Microsoft.NetFramework.Analyzers.3.0.0/build/Microsoft.NetFramework.Analyzers.props')" Text="$([System.String]::Format('$(ErrorText)', '../packages/Microsoft.NetFramework.Analyzers.3.0.0/build/Microsoft.NetFramework.Analyzers.props'))" />
     <Error Condition="!Exists('../packages/Microsoft.CodeAnalysis.FxCopAnalyzers.3.0.0/build/Microsoft.CodeAnalysis.FxCopAnalyzers.props')" Text="$([System.String]::Format('$(ErrorText)', '../packages/Microsoft.CodeAnalysis.FxCopAnalyzers.3.0.0/build/Microsoft.CodeAnalysis.FxCopAnalyzers.props'))" />
   </Target>
-</Project>
+   <Target Name="DisableAnalyzers" BeforeTargets="CoreCompile" Condition="'$(UseRoslynAnalyzers)' == 'false'">
+   <ItemGroup><Analyzer Remove="@(Analyzer)" /></ItemGroup>
+   </Target></Project>

CBOR40/CBOR40.csproj

@@ -213,4 +213,6 @@
     <Error Condition="!Exists('../packages/Microsoft.NetFramework.Analyzers.3.0.0/build/Microsoft.NetFramework.Analyzers.props')" Text="$([System.String]::Format('$(ErrorText)', '../packages/Microsoft.NetFramework.Analyzers.3.0.0/build/Microsoft.NetFramework.Analyzers.props'))" />
     <Error Condition="!Exists('../packages/Microsoft.CodeAnalysis.FxCopAnalyzers.3.0.0/build/Microsoft.CodeAnalysis.FxCopAnalyzers.props')" Text="$([System.String]::Format('$(ErrorText)', '../packages/Microsoft.CodeAnalysis.FxCopAnalyzers.3.0.0/build/Microsoft.CodeAnalysis.FxCopAnalyzers.props'))" />
   </Target>
-</Project>
+   <Target Name="DisableAnalyzers" BeforeTargets="CoreCompile" Condition="'$(UseRoslynAnalyzers)' == 'false'">
+   <ItemGroup><Analyzer Remove="@(Analyzer)" /></ItemGroup>
+   </Target></Project>

CBORTest/CBORTest.cs

@@ -1970,7 +1970,7 @@ public void TestParseDecimalStrings() {
     }
 
     [Test]
-    [Timeout(50000)]
+    [Timeout(200000)]
     public void TestRandomData() {
       var rand = new RandomGenerator();
       CBORObject obj;
@@ -2240,18 +2240,9 @@ private static string ToByteArrayStringFrom(byte[] array, int pos) {
     public void TestRandomNonsense() {
       var rand = new RandomGenerator();
       for (var i = 0; i < 1000; ++i) {
-        var array = new byte[rand.UniformInt(1000000) + 1];
-        for (int j = 0; j < array.Length; ++j) {
-          if (j + 3 <= array.Length) {
-            int r = rand.UniformInt(0x1000000);
-            array[j] = (byte)(r & (byte)0xff);
-            array[j + 1] = (byte)((r >> 8) & (byte)0xff);
-            array[j + 2] = (byte)((r >> 16) & (byte)0xff);
-            j += 2;
-          } else {
-            array[j] = (byte)rand.UniformInt(256);
-          }
-        }
+        Console.WriteLine(i + ": " + DateTime.UtcNow);
+        var array = new byte[rand.UniformInt(100000) + 1];
+        rand.GetBytes(array, 0, array.Length);
         TestRandomOne(array);
       }
     }
@@ -2280,10 +2271,19 @@ public static void TestRandomOne(byte[] array) {
         while (inputStream.Position != inputStream.Length) {
           try {
             CBORObject o;
+            long oldPos = inputStream.Position;
             o = CBORObject.Read(inputStream);
+            long cborlen = inputStream.Position - oldPos;
+            if (cborlen > 3000) {
+               Console.WriteLine("pos=" + inputStream.Position + " of " +
+inputStream.Length + ", cborlen=" + cborlen);
+            }
             byte[] encodedBytes = (o == null) ? null : o.EncodeToBytes();
             try {
               CBORObject.DecodeFromBytes(encodedBytes);
+            if (cborlen > 3000) {
+                Console.WriteLine("end DecodeFromBytes");
+              }
             } catch (Exception ex) {
               throw new InvalidOperationException(ex.Message, ex);
             }
@@ -2294,14 +2294,29 @@ public static void TestRandomOne(byte[] array) {
               }
               if (o != null) {
                 try {
+if (cborlen > 3000) {
+                    Console.WriteLine("toJSONString " + DateTime.UtcNow);
+                  }
                   jsonString = o.ToJSONString();
+if (cborlen > 3000) {
+                    Console.WriteLine("jsonStringLen = " + jsonString.Length);
+                  }
                 } catch (CBORException ex) {
                   Console.WriteLine(ex.Message);
                   jsonString = String.Empty;
                 }
                 if (jsonString.Length > 0) {
+                  if (cborlen > 3000) {
+                    Console.WriteLine("fromJSONString " + DateTime.UtcNow);
+                  }
                   CBORObject.FromJSONString(jsonString);
+                if (cborlen > 3000) {
+                    Console.WriteLine("writeToJSON " + DateTime.UtcNow);
+                  }
                   TestWriteToJSON(o);
+                if (cborlen > 3000) {
+                    Console.WriteLine("endJSON " + DateTime.UtcNow);
+                  }
                 }
               }
             } catch (Exception ex) {

CBORTest/CBORTest.csproj

@@ -27,4 +27,6 @@
         <PackageReference Include='StyleCop.Analyzers' PrivateAssets='All' Version='1.2.0-beta.164'/>
         <PackageReference Include='PeterO.Numbers' Version='1.7.3'/>
       </ItemGroup>
-</Project>
+   <Target Name="DisableAnalyzers" BeforeTargets="CoreCompile" Condition="'$(UseRoslynAnalyzers)' == 'false'">
+   <ItemGroup><Analyzer Remove="@(Analyzer)" /></ItemGroup>
+   </Target></Project>

CBORTest/Runner.cs

@@ -34,14 +34,19 @@ private static bool HasAttribute(MethodInfo mi, Type t) {
     }
 
     public static bool Extra() {
-  new CBORDataUtilitiesTest().TestParseJSONNumber();
-  new CBORDataUtilitiesTest().TestParseJSONNumberObsolete();
-  new CBORObjectTest().TestFromJSONString();
-  new CBORObjectTest().TestFromJsonStringLongKindIntOrFloat();
-  new CBORObjectTest().TestFromJsonStringLongKindIntOrFloatFromDouble();
-  new CBORObjectTest().TestFromJsonStringLongKindFullBad();
-  new CBORObjectTest().TestFromJsonStringFastCases();
-  return false;
+      var rand = new RandomGenerator();
+      for (var i = 0; i < 20; ++i) {
+        var array = new byte[rand.UniformInt(100000) + 1];
+        rand.GetBytes(array, 0, array.Length);
+        DateTime utcn = DateTime.UtcNow;
+        CBORTest.TestRandomOne(array);
+        var span = DateTime.UtcNow - utcn;
+        if (span.Seconds > 3) {
+           Console.WriteLine("----" + i + ": " + span.Seconds + " " +
+array.Length);
+        }
+      }
+      return false;
     }
 
     public static void Main() {

CBORTest20/CBORTest20.csproj

@@ -183,4 +183,6 @@
     <Error Condition="!Exists('../packages/Microsoft.NetFramework.Analyzers.3.0.0/build/Microsoft.NetFramework.Analyzers.props')" Text="$([System.String]::Format('$(ErrorText)', '../packages/Microsoft.NetFramework.Analyzers.3.0.0/build/Microsoft.NetFramework.Analyzers.props'))" />
     <Error Condition="!Exists('../packages/Microsoft.CodeAnalysis.FxCopAnalyzers.3.0.0/build/Microsoft.CodeAnalysis.FxCopAnalyzers.props')" Text="$([System.String]::Format('$(ErrorText)', '../packages/Microsoft.CodeAnalysis.FxCopAnalyzers.3.0.0/build/Microsoft.CodeAnalysis.FxCopAnalyzers.props'))" />
   </Target>
-</Project>
+   <Target Name="DisableAnalyzers" BeforeTargets="CoreCompile" Condition="'$(UseRoslynAnalyzers)' == 'false'">
+   <ItemGroup><Analyzer Remove="@(Analyzer)" /></ItemGroup>
+   </Target></Project>

CBORTest40/CBORTest40.csproj

@@ -186,4 +186,6 @@
     <Error Condition="!Exists('../packages/Microsoft.NetFramework.Analyzers.3.0.0/build/Microsoft.NetFramework.Analyzers.props')" Text="$([System.String]::Format('$(ErrorText)', '../packages/Microsoft.NetFramework.Analyzers.3.0.0/build/Microsoft.NetFramework.Analyzers.props'))" />
     <Error Condition="!Exists('../packages/Microsoft.CodeAnalysis.FxCopAnalyzers.3.0.0/build/Microsoft.CodeAnalysis.FxCopAnalyzers.props')" Text="$([System.String]::Format('$(ErrorText)', '../packages/Microsoft.CodeAnalysis.FxCopAnalyzers.3.0.0/build/Microsoft.CodeAnalysis.FxCopAnalyzers.props'))" />
   </Target>
-</Project>
+   <Target Name="DisableAnalyzers" BeforeTargets="CoreCompile" Condition="'$(UseRoslynAnalyzers)' == 'false'">
+   <ItemGroup><Analyzer Remove="@(Analyzer)" /></ItemGroup>
+   </Target></Project>

History.md

@@ -1,6 +1,15 @@
 Release notes
 ---------------------
 
+### Version 4.2
+
+- Some arithmetic methods in CBORNumber do basic overflow checks.
+- Add char array and byte array overloads to ParseJSONNumber
+- Support implementations of IList in CBORObject deserialization
+- Internally, the code avoids storing doubles (64-bit floating-point numbers) directly in CBORNumbers, uses sorted maps rather than hash tables in some CBOR objects, and can now store text strings as UTF-8 byte arrays.  This could help avoid unnecessary string conversions in many case.
+- Bug fixes and performance improvements
+- Now uses Numbers library version 1.7.3
+
 ### Version 4.1.1
 
 - Fix issue where some non-basic characters in JSON strings encoded in UTF-8 were read incorrectly by the CBORObject.FromJSONBytes method