From e8f7a928413d2bb997ecc088ce4bf9fca6cb1bb8 Mon Sep 17 00:00:00 2001 From: Jan Pluskal Date: Thu, 21 Nov 2019 11:50:17 +0100 Subject: [PATCH 01/34] Add EditorConfig with default MS configuration. --- .editorconfig | 153 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 153 insertions(+) create mode 100644 .editorconfig diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..595e8b2 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,153 @@ +############################### +# Core EditorConfig Options # +############################### + +root = true + +# All files +[*] +indent_style = space + +# Code files +[*.{cs,csx,vb,vbx}] +indent_size = 4 +insert_final_newline = true +charset = utf-8-bom + +############################### +# .NET Coding Conventions # +############################### + +[*.{cs,vb}] +# Organize usings +dotnet_sort_system_directives_first = true +dotnet_separate_import_directive_groups = false + +# this. preferences +dotnet_style_qualification_for_field = false:silent +dotnet_style_qualification_for_property = false:silent +dotnet_style_qualification_for_method = false:silent +dotnet_style_qualification_for_event = false:silent + +# Language keywords vs BCL types preferences +dotnet_style_predefined_type_for_locals_parameters_members = true:silent +dotnet_style_predefined_type_for_member_access = true:silent + +# Parentheses preferences +dotnet_style_parentheses_in_arithmetic_binary_operators = always_for_clarity:silent +dotnet_style_parentheses_in_relational_binary_operators = always_for_clarity:silent +dotnet_style_parentheses_in_other_binary_operators = always_for_clarity:silent +dotnet_style_parentheses_in_other_operators = never_if_unnecessary:silent + +# Modifier preferences +dotnet_style_require_accessibility_modifiers = for_non_interface_members:silent +dotnet_style_readonly_field = true:suggestion + +# Expression-level preferences +dotnet_style_object_initializer = true:suggestion +dotnet_style_collection_initializer = true:suggestion +dotnet_style_explicit_tuple_names = true:suggestion +dotnet_style_null_propagation = true:suggestion +dotnet_style_coalesce_expression = true:suggestion +dotnet_style_prefer_is_null_check_over_reference_equality_method = true:silent +dotnet_style_prefer_inferred_tuple_names = true:suggestion +dotnet_style_prefer_inferred_anonymous_type_member_names = true:suggestion +dotnet_style_prefer_auto_properties = true:silent +dotnet_style_prefer_conditional_expression_over_assignment = true:silent +dotnet_style_prefer_conditional_expression_over_return = true:silent + +############################### +# Naming Conventions # +############################### + +# Style Definitions +dotnet_naming_style.pascal_case_style.capitalization = pascal_case + +# Use PascalCase for constant fields +dotnet_naming_rule.constant_fields_should_be_pascal_case.severity = suggestion +dotnet_naming_rule.constant_fields_should_be_pascal_case.symbols = constant_fields +dotnet_naming_rule.constant_fields_should_be_pascal_case.style = pascal_case_style +dotnet_naming_symbols.constant_fields.applicable_kinds = field +dotnet_naming_symbols.constant_fields.applicable_accessibilities = * +dotnet_naming_symbols.constant_fields.required_modifiers = const + +############################### +# C# Code Style Rules # +############################### + +[*.cs] +# var preferences +csharp_style_var_for_built_in_types = true:silent +csharp_style_var_when_type_is_apparent = true:silent +csharp_style_var_elsewhere = true:silent + +# Expression-bodied members +csharp_style_expression_bodied_methods = false:silent +csharp_style_expression_bodied_constructors = false:silent +csharp_style_expression_bodied_operators = false:silent +csharp_style_expression_bodied_properties = true:silent +csharp_style_expression_bodied_indexers = true:silent +csharp_style_expression_bodied_accessors = true:silent + +# Pattern-matching preferences +csharp_style_pattern_matching_over_is_with_cast_check = true:suggestion +csharp_style_pattern_matching_over_as_with_null_check = true:suggestion + +# Null-checking preferences +csharp_style_throw_expression = true:suggestion +csharp_style_conditional_delegate_call = true:suggestion + +# Modifier preferences +csharp_preferred_modifier_order = public,private,protected,internal,static,extern,new,virtual,abstract,sealed,override,readonly,unsafe,volatile,async:suggestion + +# Expression-level preferences +csharp_prefer_braces = true:silent +csharp_style_deconstructed_variable_declaration = true:suggestion +csharp_prefer_simple_default_expression = true:suggestion +csharp_style_pattern_local_over_anonymous_function = true:suggestion +csharp_style_inlined_variable_declaration = true:suggestion + +############################### +# C# Formatting Rules # +############################### + +# New line preferences +csharp_new_line_before_open_brace = all +csharp_new_line_before_else = true +csharp_new_line_before_catch = true +csharp_new_line_before_finally = true +csharp_new_line_before_members_in_object_initializers = true +csharp_new_line_before_members_in_anonymous_types = true +csharp_new_line_between_query_expression_clauses = true + +# Indentation preferences +csharp_indent_case_contents = true +csharp_indent_switch_labels = true +csharp_indent_labels = flush_left + +# Space preferences +csharp_space_after_cast = false +csharp_space_after_keywords_in_control_flow_statements = true +csharp_space_between_method_call_parameter_list_parentheses = false +csharp_space_between_method_declaration_parameter_list_parentheses = false +csharp_space_between_parentheses = false +csharp_space_before_colon_in_inheritance_clause = true +csharp_space_after_colon_in_inheritance_clause = true +csharp_space_around_binary_operators = before_and_after +csharp_space_between_method_declaration_empty_parameter_list_parentheses = false +csharp_space_between_method_call_name_and_opening_parenthesis = false +csharp_space_between_method_call_empty_parameter_list_parentheses = false +csharp_space_after_comma = true +csharp_space_after_dot = false + +# Wrapping preferences +csharp_preserve_single_line_statements = true +csharp_preserve_single_line_blocks = true + +################################## +# Visual Basic Code Style Rules # +################################## + +[*.vb] +# Modifier preferences +visual_basic_preferred_modifier_order = Partial,Default,Private,Protected,Public,Friend,NotOverridable,Overridable,MustOverride,Overloads,Overrides,MustInherit,NotInheritable,Static,Shared,Shadows,ReadOnly,WriteOnly,Dim,Const,WithEvents,Widening,Narrowing,Custom,Async:suggestion \ No newline at end of file From be2c108eaaae2553ab85b607d8fcbc02ec587725 Mon Sep 17 00:00:00 2001 From: Jan Pluskal Date: Thu, 21 Nov 2019 11:51:26 +0100 Subject: [PATCH 02/34] Split types to separate files --- Exception/KaitaiStructError.cs | 20 +++++++ Exception/ValidationFailedError.cs | 36 ++++++++++++ Exception/ValidationNotEqualError.cs | 30 ++++++++++ Interface/CustomDecoder.cs | 17 ++++++ KaitaiStruct.cs | 86 +--------------------------- 5 files changed, 104 insertions(+), 85 deletions(-) create mode 100644 Exception/KaitaiStructError.cs create mode 100644 Exception/ValidationFailedError.cs create mode 100644 Exception/ValidationNotEqualError.cs create mode 100644 Interface/CustomDecoder.cs diff --git a/Exception/KaitaiStructError.cs b/Exception/KaitaiStructError.cs new file mode 100644 index 0000000..61b6349 --- /dev/null +++ b/Exception/KaitaiStructError.cs @@ -0,0 +1,20 @@ +using System; + +namespace Kaitai +{ + /// + /// Common ancestor for all error originating from Kaitai Struct usage. + /// Stores KSY source path, pointing to an element supposedly guilty of + /// an error. + /// + public class KaitaiStructError : Exception + { + protected string srcPath; + + public KaitaiStructError(string msg, string srcPath) + : base(srcPath + ": " + msg) + { + this.srcPath = srcPath; + } + } +} diff --git a/Exception/ValidationFailedError.cs b/Exception/ValidationFailedError.cs new file mode 100644 index 0000000..a75c8b6 --- /dev/null +++ b/Exception/ValidationFailedError.cs @@ -0,0 +1,36 @@ +using System.Text; + +namespace Kaitai +{ + /// + /// Common ancestor for all validation failures. Stores pointer to + /// KaitaiStream IO object which was involved in an error. + /// + public class ValidationFailedError : KaitaiStructError + { + protected KaitaiStream io; + + public ValidationFailedError(string msg, KaitaiStream io, string srcPath) + : base("at pos " + io.Pos + ": validation failed: " + msg, srcPath) + { + this.io = io; + } + + protected static string ByteArrayToHex(byte[] arr) + { + StringBuilder sb = new StringBuilder("["); + for (int i = 0; i < arr.Length; i++) + { + if (i > 0) + { + sb.Append(' '); + } + + sb.Append(string.Format("{0:X2}", arr[i])); + } + + sb.Append(']'); + return sb.ToString(); + } + } +} diff --git a/Exception/ValidationNotEqualError.cs b/Exception/ValidationNotEqualError.cs new file mode 100644 index 0000000..2137122 --- /dev/null +++ b/Exception/ValidationNotEqualError.cs @@ -0,0 +1,30 @@ +using System; + +namespace Kaitai +{ + /// + /// Signals validation failure: we required "actual" value to be equal to + /// "expected", but it turned out that it's not. + /// + public class ValidationNotEqualError : ValidationFailedError + { + protected Object actual; + + protected Object expected; + + public ValidationNotEqualError(byte[] expected, byte[] actual, KaitaiStream io, string srcPath) + : base("not equal, expected " + ByteArrayToHex(expected) + ", but got " + ByteArrayToHex(actual), io, + srcPath) + { + this.expected = expected; + this.actual = actual; + } + + public ValidationNotEqualError(Object expected, Object actual, KaitaiStream io, string srcPath) + : base("not equal, expected " + expected + ", but got " + actual, io, srcPath) + { + this.expected = expected; + this.actual = actual; + } + } +} diff --git a/Interface/CustomDecoder.cs b/Interface/CustomDecoder.cs new file mode 100644 index 0000000..33a6cae --- /dev/null +++ b/Interface/CustomDecoder.cs @@ -0,0 +1,17 @@ +namespace Kaitai +{ + /// + /// A custom decoder interface. Implementing classes can be called from + /// inside a .ksy file using `process: XXX` syntax. + /// + public interface CustomDecoder + { + /// + /// Decodes a given byte array, according to some custom algorithm + /// (specific to implementing class) and parameters given in the + /// constructor, returning another byte array. + /// + /// Source byte array. + byte[] Decode(byte[] src); + } +} diff --git a/KaitaiStruct.cs b/KaitaiStruct.cs index 0fe152f..5051480 100644 --- a/KaitaiStruct.cs +++ b/KaitaiStruct.cs @@ -1,7 +1,4 @@ -using System; -using System.Text; - -namespace Kaitai +namespace Kaitai { public abstract class KaitaiStruct { @@ -20,85 +17,4 @@ public KaitaiStruct(KaitaiStream io) m_io = io; } } - - /// - /// A custom decoder interface. Implementing classes can be called from - /// inside a .ksy file using `process: XXX` syntax. - /// - public interface CustomDecoder - { - /// - /// Decodes a given byte array, according to some custom algorithm - /// (specific to implementing class) and parameters given in the - /// constructor, returning another byte array. - /// - /// Source byte array. - byte[] Decode(byte[] src); - } - - /// - /// Common ancestor for all error originating from Kaitai Struct usage. - /// Stores KSY source path, pointing to an element supposedly guilty of - /// an error. - /// - public class KaitaiStructError : Exception { - public KaitaiStructError(string msg, string srcPath) - : base(srcPath + ": " + msg) - { - this.srcPath = srcPath; - } - - protected string srcPath; - } - - /// - /// Common ancestor for all validation failures. Stores pointer to - /// KaitaiStream IO object which was involved in an error. - /// - public class ValidationFailedError : KaitaiStructError { - public ValidationFailedError(string msg, KaitaiStream io, string srcPath) - : base("at pos " + io.Pos + ": validation failed: " + msg, srcPath) - { - this.io = io; - } - - protected KaitaiStream io; - - protected static string ByteArrayToHex(byte[] arr) { - StringBuilder sb = new StringBuilder("["); - for (int i = 0; i < arr.Length; i++) - { - if (i > 0) - { - sb.Append(' '); - } - sb.Append(string.Format("{0:X2}", arr[i])); - } - sb.Append(']'); - return sb.ToString(); - } - } - - /// - /// Signals validation failure: we required "actual" value to be equal to - /// "expected", but it turned out that it's not. - /// - public class ValidationNotEqualError : ValidationFailedError { - public ValidationNotEqualError(byte[] expected, byte[] actual, KaitaiStream io, string srcPath) - : base("not equal, expected " + ByteArrayToHex(expected) + ", but got " + ByteArrayToHex(actual), io, srcPath) - { - this.expected = expected; - this.actual = actual; - } - - public ValidationNotEqualError(Object expected, Object actual, KaitaiStream io, string srcPath) - : base("not equal, expected " + expected + ", but got " + actual, io, srcPath) - { - this.expected = expected; - this.actual = actual; - } - - protected Object expected; - protected Object actual; - } } From bf275d0d22a970c151a28e8b00431df29a3d8258 Mon Sep 17 00:00:00 2001 From: Jan Pluskal Date: Thu, 21 Nov 2019 11:52:40 +0100 Subject: [PATCH 03/34] Extract IKaitaiStream interface. --- Interface/IKaitaiStream.cs | 209 +++++++++++++++++++++++++++++++++++++ KaitaiStream.cs | 4 +- 2 files changed, 211 insertions(+), 2 deletions(-) create mode 100644 Interface/IKaitaiStream.cs diff --git a/Interface/IKaitaiStream.cs b/Interface/IKaitaiStream.cs new file mode 100644 index 0000000..2c44284 --- /dev/null +++ b/Interface/IKaitaiStream.cs @@ -0,0 +1,209 @@ +namespace Kaitai +{ + public interface IKaitaiStream + { + /// + /// Check if the stream position is at the end of the stream + /// + bool IsEof { get; } + + /// + /// Get the current position in the stream + /// + long Pos { get; } + + /// + /// Get the total length of the stream (ie. file size) + /// + long Size { get; } + + /// + /// Seek to a specific position from the beginning of the stream + /// + /// The position to seek to + void Seek(long position); + + /// + /// Read a signed byte from the stream + /// + /// + sbyte ReadS1(); + + /// + /// Read a signed short from the stream (big endian) + /// + /// + short ReadS2be(); + + /// + /// Read a signed int from the stream (big endian) + /// + /// + int ReadS4be(); + + /// + /// Read a signed long from the stream (big endian) + /// + /// + long ReadS8be(); + + /// + /// Read a signed short from the stream (little endian) + /// + /// + short ReadS2le(); + + /// + /// Read a signed int from the stream (little endian) + /// + /// + int ReadS4le(); + + /// + /// Read a signed long from the stream (little endian) + /// + /// + long ReadS8le(); + + /// + /// Read an unsigned byte from the stream + /// + /// + byte ReadU1(); + + /// + /// Read an unsigned short from the stream (big endian) + /// + /// + ushort ReadU2be(); + + /// + /// Read an unsigned int from the stream (big endian) + /// + /// + uint ReadU4be(); + + /// + /// Read an unsigned long from the stream (big endian) + /// + /// + ulong ReadU8be(); + + /// + /// Read an unsigned short from the stream (little endian) + /// + /// + ushort ReadU2le(); + + /// + /// Read an unsigned int from the stream (little endian) + /// + /// + uint ReadU4le(); + + /// + /// Read an unsigned long from the stream (little endian) + /// + /// + ulong ReadU8le(); + + /// + /// Read a single-precision floating point value from the stream (big endian) + /// + /// + float ReadF4be(); + + /// + /// Read a double-precision floating point value from the stream (big endian) + /// + /// + double ReadF8be(); + + /// + /// Read a single-precision floating point value from the stream (little endian) + /// + /// + float ReadF4le(); + + /// + /// Read a double-precision floating point value from the stream (little endian) + /// + /// + double ReadF8le(); + + void AlignToByte(); + ulong ReadBitsInt(int n); + ulong ReadBitsIntLe(int n); + + /// + /// Read a fixed number of bytes from the stream + /// + /// The number of bytes to read + /// + byte[] ReadBytes(long count); + + /// + /// Read a fixed number of bytes from the stream + /// + /// The number of bytes to read + /// + byte[] ReadBytes(ulong count); + + /// + /// Read all the remaining bytes from the stream until the end is reached + /// + /// + byte[] ReadBytesFull(); + + /// + /// Read a terminated string from the stream + /// + /// The string terminator value + /// True to include the terminator in the returned string + /// True to consume the terminator byte before returning + /// True to throw an error when the EOS was reached before the terminator + /// + byte[] ReadBytesTerm(byte terminator, bool includeTerminator, bool consumeTerminator, bool eosError); + + /// + /// Read a specific set of bytes and assert that they are the same as an expected result + /// + /// The expected result + /// + byte[] EnsureFixedContents(byte[] expected); + + /// + /// Performs XOR processing with given data, XORing every byte of the input with a single value. + /// + /// The data toe process + /// The key value to XOR with + /// Processed data + byte[] ProcessXor(byte[] value, int key); + + /// + /// Performs XOR processing with given data, XORing every byte of the input with a key + /// array, repeating from the beginning of the key array if necessary + /// + /// The data toe process + /// The key array to XOR with + /// Processed data + byte[] ProcessXor(byte[] value, byte[] key); + + /// + /// Performs a circular left rotation shift for a given buffer by a given amount of bits. + /// Pass a negative amount to rotate right. + /// + /// The data to rotate + /// The number of bytes to rotate by + /// + /// + byte[] ProcessRotateLeft(byte[] data, int amount, int groupSize); + + /// + /// Inflates a deflated zlib byte stream + /// + /// The data to deflate + /// The deflated result + byte[] ProcessZlib(byte[] data); + } +} diff --git a/KaitaiStream.cs b/KaitaiStream.cs index a3a80d1..5e2207a 100644 --- a/KaitaiStream.cs +++ b/KaitaiStream.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.IO; using System.IO.Compression; @@ -10,7 +10,7 @@ namespace Kaitai /// The base Kaitai stream which exposes an API for the Kaitai Struct framework. /// It's based off a BinaryReader, which is a little-endian reader. /// - public partial class KaitaiStream : BinaryReader + public partial class KaitaiStream : BinaryReader, IKaitaiStream { #region Constructors From c61561a6e6507528535f2241899cf843d7d75d5f Mon Sep 17 00:00:00 2001 From: Jan Pluskal Date: Thu, 21 Nov 2019 11:59:43 +0100 Subject: [PATCH 04/34] Use composition instead of inheritance with BinaryReader --- KaitaiStream.cs | 38 ++++++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/KaitaiStream.cs b/KaitaiStream.cs index 5e2207a..8b6ec92 100644 --- a/KaitaiStream.cs +++ b/KaitaiStream.cs @@ -10,32 +10,42 @@ namespace Kaitai /// The base Kaitai stream which exposes an API for the Kaitai Struct framework. /// It's based off a BinaryReader, which is a little-endian reader. /// - public partial class KaitaiStream : BinaryReader, IKaitaiStream + public partial class KaitaiStream : IKaitaiStream { #region Constructors - public KaitaiStream(Stream stream) : base(stream) + static readonly bool IsLittleEndian = BitConverter.IsLittleEndian; + + private ulong Bits = 0; + private int BitsLeft = 0; + private BinaryReader m_binaryReader; + + protected Stream BaseStream; + + public KaitaiStream(Stream stream) { + BaseStream = stream; } /// /// Creates a KaitaiStream backed by a file (RO) /// - public KaitaiStream(string file) : base(File.Open(file, FileMode.Open, FileAccess.Read, FileShare.Read)) + public KaitaiStream(string file) : this(File.Open(file, FileMode.Open, FileAccess.Read, FileShare.Read)) { } /// ///Creates a KaitaiStream backed by a byte buffer /// - public KaitaiStream(byte[] bytes) : base(new MemoryStream(bytes)) + public KaitaiStream(byte[] bytes) : this(new MemoryStream(bytes)) { } - - private ulong Bits = 0; - private int BitsLeft = 0; - - static readonly bool IsLittleEndian = BitConverter.IsLittleEndian; + + protected BinaryReader BinaryReader + { + get => m_binaryReader ?? (BinaryReader = new BinaryReader(BaseStream)); + set => m_binaryReader = value; + } #endregion @@ -86,7 +96,7 @@ public long Size /// public sbyte ReadS1() { - return ReadSByte(); + return BinaryReader.ReadSByte(); } #region Big-endian @@ -161,7 +171,7 @@ public long ReadS8le() /// public byte ReadU1() { - return ReadByte(); + return BinaryReader.ReadByte(); } #region Big-endian @@ -370,7 +380,7 @@ public byte[] ReadBytes(long count) { if (count < 0 || count > Int32.MaxValue) throw new ArgumentOutOfRangeException("requested " + count + " bytes, while only non-negative int32 amount of bytes possible"); - byte[] bytes = base.ReadBytes((int) count); + byte[] bytes = BinaryReader.ReadBytes((int) count); if (bytes.Length < count) throw new EndOfStreamException("requested " + count + " bytes, but got only " + bytes.Length + " bytes"); return bytes; @@ -385,7 +395,7 @@ public byte[] ReadBytes(ulong count) { if (count > Int32.MaxValue) throw new ArgumentOutOfRangeException("requested " + count + " bytes, while only non-negative int32 amount of bytes possible"); - byte[] bytes = base.ReadBytes((int)count); + byte[] bytes = BinaryReader.ReadBytes((int)count); if (bytes.Length < (int)count) throw new EndOfStreamException("requested " + count + " bytes, but got only " + bytes.Length + " bytes"); return bytes; @@ -443,7 +453,7 @@ public byte[] ReadBytesTerm(byte terminator, bool includeTerminator, bool consum break; } - byte b = ReadByte(); + byte b = BinaryReader.ReadByte(); if (b == terminator) { if (includeTerminator) bytes.Add(b); From 6e597f8794b10f8b413e589f556ff6658b052db2 Mon Sep 17 00:00:00 2001 From: Jan Pluskal Date: Thu, 12 Dec 2019 13:14:06 +0100 Subject: [PATCH 05/34] Add source and EOL in .editorconfig --- .editorconfig | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.editorconfig b/.editorconfig index 595e8b2..43cdaa5 100644 --- a/.editorconfig +++ b/.editorconfig @@ -1,4 +1,6 @@ -############################### +# Source: https://docs.microsoft.com/en-us/visualstudio/ide/editorconfig-code-style-settings-reference?view=vs-2019 + +############################### # Core EditorConfig Options # ############################### @@ -150,4 +152,4 @@ csharp_preserve_single_line_blocks = true [*.vb] # Modifier preferences -visual_basic_preferred_modifier_order = Partial,Default,Private,Protected,Public,Friend,NotOverridable,Overridable,MustOverride,Overloads,Overrides,MustInherit,NotInheritable,Static,Shared,Shadows,ReadOnly,WriteOnly,Dim,Const,WithEvents,Widening,Narrowing,Custom,Async:suggestion \ No newline at end of file +visual_basic_preferred_modifier_order = Partial,Default,Private,Protected,Public,Friend,NotOverridable,Overridable,MustOverride,Overloads,Overrides,MustInherit,NotInheritable,Static,Shared,Shadows,ReadOnly,WriteOnly,Dim,Const,WithEvents,Widening,Narrowing,Custom,Async:suggestion From a748b58c094d70d700a417c412778f51b16d2b82 Mon Sep 17 00:00:00 2001 From: Jan Pluskal Date: Thu, 12 Dec 2019 14:17:01 +0100 Subject: [PATCH 06/34] Fix XML comment indentation. --- Exception/KaitaiStructError.cs | 6 +-- Exception/ValidationFailedError.cs | 4 +- Exception/ValidationNotEqualError.cs | 4 +- Interface/CustomDecoder.cs | 10 ++--- Interface/IKaitaiStream.cs | 66 ++++++++++++++-------------- KaitaiStream.cs | 10 ++--- 6 files changed, 50 insertions(+), 50 deletions(-) diff --git a/Exception/KaitaiStructError.cs b/Exception/KaitaiStructError.cs index 61b6349..225f6fc 100644 --- a/Exception/KaitaiStructError.cs +++ b/Exception/KaitaiStructError.cs @@ -3,9 +3,9 @@ namespace Kaitai { /// - /// Common ancestor for all error originating from Kaitai Struct usage. - /// Stores KSY source path, pointing to an element supposedly guilty of - /// an error. + /// Common ancestor for all error originating from Kaitai Struct usage. + /// Stores KSY source path, pointing to an element supposedly guilty of + /// an error. /// public class KaitaiStructError : Exception { diff --git a/Exception/ValidationFailedError.cs b/Exception/ValidationFailedError.cs index a75c8b6..f29cbe9 100644 --- a/Exception/ValidationFailedError.cs +++ b/Exception/ValidationFailedError.cs @@ -3,8 +3,8 @@ namespace Kaitai { /// - /// Common ancestor for all validation failures. Stores pointer to - /// KaitaiStream IO object which was involved in an error. + /// Common ancestor for all validation failures. Stores pointer to + /// KaitaiStream IO object which was involved in an error. /// public class ValidationFailedError : KaitaiStructError { diff --git a/Exception/ValidationNotEqualError.cs b/Exception/ValidationNotEqualError.cs index 2137122..8d73eca 100644 --- a/Exception/ValidationNotEqualError.cs +++ b/Exception/ValidationNotEqualError.cs @@ -3,8 +3,8 @@ namespace Kaitai { /// - /// Signals validation failure: we required "actual" value to be equal to - /// "expected", but it turned out that it's not. + /// Signals validation failure: we required "actual" value to be equal to + /// "expected", but it turned out that it's not. /// public class ValidationNotEqualError : ValidationFailedError { diff --git a/Interface/CustomDecoder.cs b/Interface/CustomDecoder.cs index 33a6cae..6afb3c9 100644 --- a/Interface/CustomDecoder.cs +++ b/Interface/CustomDecoder.cs @@ -1,15 +1,15 @@ namespace Kaitai { /// - /// A custom decoder interface. Implementing classes can be called from - /// inside a .ksy file using `process: XXX` syntax. + /// A custom decoder interface. Implementing classes can be called from + /// inside a .ksy file using `process: XXX` syntax. /// public interface CustomDecoder { /// - /// Decodes a given byte array, according to some custom algorithm - /// (specific to implementing class) and parameters given in the - /// constructor, returning another byte array. + /// Decodes a given byte array, according to some custom algorithm + /// (specific to implementing class) and parameters given in the + /// constructor, returning another byte array. /// /// Source byte array. byte[] Decode(byte[] src); diff --git a/Interface/IKaitaiStream.cs b/Interface/IKaitaiStream.cs index 2c44284..1d1b363 100644 --- a/Interface/IKaitaiStream.cs +++ b/Interface/IKaitaiStream.cs @@ -3,130 +3,130 @@ public interface IKaitaiStream { /// - /// Check if the stream position is at the end of the stream + /// Check if the stream position is at the end of the stream /// bool IsEof { get; } /// - /// Get the current position in the stream + /// Get the current position in the stream /// long Pos { get; } /// - /// Get the total length of the stream (ie. file size) + /// Get the total length of the stream (ie. file size) /// long Size { get; } /// - /// Seek to a specific position from the beginning of the stream + /// Seek to a specific position from the beginning of the stream /// /// The position to seek to void Seek(long position); /// - /// Read a signed byte from the stream + /// Read a signed byte from the stream /// /// sbyte ReadS1(); /// - /// Read a signed short from the stream (big endian) + /// Read a signed short from the stream (big endian) /// /// short ReadS2be(); /// - /// Read a signed int from the stream (big endian) + /// Read a signed int from the stream (big endian) /// /// int ReadS4be(); /// - /// Read a signed long from the stream (big endian) + /// Read a signed long from the stream (big endian) /// /// long ReadS8be(); /// - /// Read a signed short from the stream (little endian) + /// Read a signed short from the stream (little endian) /// /// short ReadS2le(); /// - /// Read a signed int from the stream (little endian) + /// Read a signed int from the stream (little endian) /// /// int ReadS4le(); /// - /// Read a signed long from the stream (little endian) + /// Read a signed long from the stream (little endian) /// /// long ReadS8le(); /// - /// Read an unsigned byte from the stream + /// Read an unsigned byte from the stream /// /// byte ReadU1(); /// - /// Read an unsigned short from the stream (big endian) + /// Read an unsigned short from the stream (big endian) /// /// ushort ReadU2be(); /// - /// Read an unsigned int from the stream (big endian) + /// Read an unsigned int from the stream (big endian) /// /// uint ReadU4be(); /// - /// Read an unsigned long from the stream (big endian) + /// Read an unsigned long from the stream (big endian) /// /// ulong ReadU8be(); /// - /// Read an unsigned short from the stream (little endian) + /// Read an unsigned short from the stream (little endian) /// /// ushort ReadU2le(); /// - /// Read an unsigned int from the stream (little endian) + /// Read an unsigned int from the stream (little endian) /// /// uint ReadU4le(); /// - /// Read an unsigned long from the stream (little endian) + /// Read an unsigned long from the stream (little endian) /// /// ulong ReadU8le(); /// - /// Read a single-precision floating point value from the stream (big endian) + /// Read a single-precision floating point value from the stream (big endian) /// /// float ReadF4be(); /// - /// Read a double-precision floating point value from the stream (big endian) + /// Read a double-precision floating point value from the stream (big endian) /// /// double ReadF8be(); /// - /// Read a single-precision floating point value from the stream (little endian) + /// Read a single-precision floating point value from the stream (little endian) /// /// float ReadF4le(); /// - /// Read a double-precision floating point value from the stream (little endian) + /// Read a double-precision floating point value from the stream (little endian) /// /// double ReadF8le(); @@ -136,27 +136,27 @@ public interface IKaitaiStream ulong ReadBitsIntLe(int n); /// - /// Read a fixed number of bytes from the stream + /// Read a fixed number of bytes from the stream /// /// The number of bytes to read /// byte[] ReadBytes(long count); /// - /// Read a fixed number of bytes from the stream + /// Read a fixed number of bytes from the stream /// /// The number of bytes to read /// byte[] ReadBytes(ulong count); /// - /// Read all the remaining bytes from the stream until the end is reached + /// Read all the remaining bytes from the stream until the end is reached /// /// byte[] ReadBytesFull(); /// - /// Read a terminated string from the stream + /// Read a terminated string from the stream /// /// The string terminator value /// True to include the terminator in the returned string @@ -166,14 +166,14 @@ public interface IKaitaiStream byte[] ReadBytesTerm(byte terminator, bool includeTerminator, bool consumeTerminator, bool eosError); /// - /// Read a specific set of bytes and assert that they are the same as an expected result + /// Read a specific set of bytes and assert that they are the same as an expected result /// /// The expected result /// byte[] EnsureFixedContents(byte[] expected); /// - /// Performs XOR processing with given data, XORing every byte of the input with a single value. + /// Performs XOR processing with given data, XORing every byte of the input with a single value. /// /// The data toe process /// The key value to XOR with @@ -181,8 +181,8 @@ public interface IKaitaiStream byte[] ProcessXor(byte[] value, int key); /// - /// Performs XOR processing with given data, XORing every byte of the input with a key - /// array, repeating from the beginning of the key array if necessary + /// Performs XOR processing with given data, XORing every byte of the input with a key + /// array, repeating from the beginning of the key array if necessary /// /// The data toe process /// The key array to XOR with @@ -190,8 +190,8 @@ public interface IKaitaiStream byte[] ProcessXor(byte[] value, byte[] key); /// - /// Performs a circular left rotation shift for a given buffer by a given amount of bits. - /// Pass a negative amount to rotate right. + /// Performs a circular left rotation shift for a given buffer by a given amount of bits. + /// Pass a negative amount to rotate right. /// /// The data to rotate /// The number of bytes to rotate by @@ -200,7 +200,7 @@ public interface IKaitaiStream byte[] ProcessRotateLeft(byte[] data, int amount, int groupSize); /// - /// Inflates a deflated zlib byte stream + /// Inflates a deflated zlib byte stream /// /// The data to deflate /// The deflated result diff --git a/KaitaiStream.cs b/KaitaiStream.cs index 8b6ec92..f4d96de 100644 --- a/KaitaiStream.cs +++ b/KaitaiStream.cs @@ -27,16 +27,16 @@ public KaitaiStream(Stream stream) BaseStream = stream; } - /// + /// /// Creates a KaitaiStream backed by a file (RO) - /// + /// public KaitaiStream(string file) : this(File.Open(file, FileMode.Open, FileAccess.Read, FileShare.Read)) { } - /// - ///Creates a KaitaiStream backed by a byte buffer - /// + /// + /// Creates a KaitaiStream backed by a byte buffer + /// public KaitaiStream(byte[] bytes) : this(new MemoryStream(bytes)) { } From d634aabb2965d71b2e3872fdfa3a948aeba738df Mon Sep 17 00:00:00 2001 From: Jan Pluskal Date: Thu, 12 Dec 2019 14:23:34 +0100 Subject: [PATCH 07/34] Change file layout -> fileds, constructors, properties. Chage abstract class constructor visibility to protected. --- KaitaiStruct.cs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/KaitaiStruct.cs b/KaitaiStruct.cs index 5051480..4895751 100644 --- a/KaitaiStruct.cs +++ b/KaitaiStruct.cs @@ -4,6 +4,11 @@ public abstract class KaitaiStruct { protected KaitaiStream m_io; + protected KaitaiStruct(KaitaiStream io) + { + m_io = io; + } + public KaitaiStream M_Io { get @@ -11,10 +16,5 @@ public KaitaiStream M_Io return m_io; } } - - public KaitaiStruct(KaitaiStream io) - { - m_io = io; - } } } From aa93ab735b3c4888b9973b8b349c75435d062edb Mon Sep 17 00:00:00 2001 From: Jan Pluskal Date: Thu, 12 Dec 2019 14:36:29 +0100 Subject: [PATCH 08/34] Use string interpolation and clean up redundant code. --- Exception/KaitaiStructError.cs | 2 +- Exception/ValidationFailedError.cs | 4 +- Exception/ValidationNotEqualError.cs | 4 +- KaitaiStream.cs | 59 +++++++++++++++++----------- 4 files changed, 40 insertions(+), 29 deletions(-) diff --git a/Exception/KaitaiStructError.cs b/Exception/KaitaiStructError.cs index 225f6fc..4a28cb5 100644 --- a/Exception/KaitaiStructError.cs +++ b/Exception/KaitaiStructError.cs @@ -12,7 +12,7 @@ public class KaitaiStructError : Exception protected string srcPath; public KaitaiStructError(string msg, string srcPath) - : base(srcPath + ": " + msg) + : base($"srcPath: {msg}") { this.srcPath = srcPath; } diff --git a/Exception/ValidationFailedError.cs b/Exception/ValidationFailedError.cs index f29cbe9..4ac5992 100644 --- a/Exception/ValidationFailedError.cs +++ b/Exception/ValidationFailedError.cs @@ -11,7 +11,7 @@ public class ValidationFailedError : KaitaiStructError protected KaitaiStream io; public ValidationFailedError(string msg, KaitaiStream io, string srcPath) - : base("at pos " + io.Pos + ": validation failed: " + msg, srcPath) + : base($"at pos {io.Pos}: validation failed: {msg}", srcPath) { this.io = io; } @@ -26,7 +26,7 @@ protected static string ByteArrayToHex(byte[] arr) sb.Append(' '); } - sb.Append(string.Format("{0:X2}", arr[i])); + sb.Append($"{arr[i]:X2}"); } sb.Append(']'); diff --git a/Exception/ValidationNotEqualError.cs b/Exception/ValidationNotEqualError.cs index 8d73eca..c7eb300 100644 --- a/Exception/ValidationNotEqualError.cs +++ b/Exception/ValidationNotEqualError.cs @@ -13,7 +13,7 @@ public class ValidationNotEqualError : ValidationFailedError protected Object expected; public ValidationNotEqualError(byte[] expected, byte[] actual, KaitaiStream io, string srcPath) - : base("not equal, expected " + ByteArrayToHex(expected) + ", but got " + ByteArrayToHex(actual), io, + : base($"not equal, expected {ByteArrayToHex(expected)}, but got {ByteArrayToHex(actual)}", io, srcPath) { this.expected = expected; @@ -21,7 +21,7 @@ public ValidationNotEqualError(byte[] expected, byte[] actual, KaitaiStream io, } public ValidationNotEqualError(Object expected, Object actual, KaitaiStream io, string srcPath) - : base("not equal, expected " + expected + ", but got " + actual, io, srcPath) + : base($"not equal, expected {expected}, but got {actual}", io, srcPath) { this.expected = expected; this.actual = actual; diff --git a/KaitaiStream.cs b/KaitaiStream.cs index f4d96de..6d1b58c 100644 --- a/KaitaiStream.cs +++ b/KaitaiStream.cs @@ -2,7 +2,6 @@ using System.Collections.Generic; using System.IO; using System.IO.Compression; -using System.Linq; namespace Kaitai { @@ -16,8 +15,8 @@ public partial class KaitaiStream : IKaitaiStream static readonly bool IsLittleEndian = BitConverter.IsLittleEndian; - private ulong Bits = 0; - private int BitsLeft = 0; + private ulong Bits; + private int BitsLeft; private BinaryReader m_binaryReader; protected Stream BaseStream; @@ -40,7 +39,7 @@ public KaitaiStream(string file) : this(File.Open(file, FileMode.Open, FileAcces public KaitaiStream(byte[] bytes) : this(new MemoryStream(bytes)) { } - + protected BinaryReader BinaryReader { get => m_binaryReader ?? (BinaryReader = new BinaryReader(BaseStream)); @@ -304,7 +303,7 @@ public ulong ReadBitsInt(int n) // 1 bit => 1 byte // 8 bits => 1 byte // 9 bits => 2 bytes - int bytesNeeded = ((bitsNeeded - 1) / 8) + 1; + int bytesNeeded = (bitsNeeded - 1) / 8 + 1; byte[] buf = ReadBytes(bytesNeeded); for (int i = 0; i < buf.Length; i++) { @@ -339,11 +338,11 @@ public ulong ReadBitsIntLe(int n) // 1 bit => 1 byte // 8 bits => 1 byte // 9 bits => 2 bytes - int bytesNeeded = ((bitsNeeded - 1) / 8) + 1; + int bytesNeeded = (bitsNeeded - 1) / 8 + 1; byte[] buf = ReadBytes(bytesNeeded); for (int i = 0; i < buf.Length; i++) { - ulong v = (ulong)((ulong)buf[i] << BitsLeft); + ulong v = (ulong)buf[i] << BitsLeft; Bits |= v; BitsLeft += 8; } @@ -353,7 +352,7 @@ public ulong ReadBitsIntLe(int n) ulong mask = GetMaskOnes(n); // derive reading result - ulong res = (Bits & mask); + ulong res = Bits & mask; // remove bottom bits that we've just read by shifting Bits >>= n; @@ -379,10 +378,10 @@ private static ulong GetMaskOnes(int n) public byte[] ReadBytes(long count) { if (count < 0 || count > Int32.MaxValue) - throw new ArgumentOutOfRangeException("requested " + count + " bytes, while only non-negative int32 amount of bytes possible"); - byte[] bytes = BinaryReader.ReadBytes((int) count); + throw new ArgumentOutOfRangeException($"requested {count} bytes, while only non-negative int32 amount of bytes possible"); + byte[] bytes = BinaryReader.ReadBytes((int)count); if (bytes.Length < count) - throw new EndOfStreamException("requested " + count + " bytes, but got only " + bytes.Length + " bytes"); + throw new EndOfStreamException($"requested {count} bytes, but got only {bytes.Length} bytes"); return bytes; } @@ -394,10 +393,10 @@ public byte[] ReadBytes(long count) public byte[] ReadBytes(ulong count) { if (count > Int32.MaxValue) - throw new ArgumentOutOfRangeException("requested " + count + " bytes, while only non-negative int32 amount of bytes possible"); + throw new ArgumentOutOfRangeException($"requested {count} bytes, while only non-negative int32 amount of bytes possible"); byte[] bytes = BinaryReader.ReadBytes((int)count); if (bytes.Length < (int)count) - throw new EndOfStreamException("requested " + count + " bytes, but got only " + bytes.Length + " bytes"); + throw new EndOfStreamException($"requested {count} bytes, but got only {bytes.Length} bytes"); return bytes; } @@ -449,7 +448,8 @@ public byte[] ReadBytesTerm(byte terminator, bool includeTerminator, bool consum { if (IsEof) { - if (eosError) throw new EndOfStreamException(string.Format("End of stream reached, but no terminator `{0}` found", terminator)); + if (eosError) + throw new EndOfStreamException($"End of stream reached, but no terminator `{terminator}` found"); break; } @@ -460,8 +460,10 @@ public byte[] ReadBytesTerm(byte terminator, bool includeTerminator, bool consum if (!consumeTerminator) Seek(Pos - 1); break; } + bytes.Add(b); } + return bytes.ToArray(); } @@ -476,13 +478,14 @@ public byte[] EnsureFixedContents(byte[] expected) if (bytes.Length != expected.Length) { - throw new Exception(string.Format("Expected bytes: {0} ({1} bytes), Instead got: {2} ({3} bytes)", Convert.ToBase64String(expected), expected.Length, Convert.ToBase64String(bytes), bytes.Length)); + throw new Exception($"Expected bytes: {Convert.ToBase64String(expected)} ({expected.Length} bytes), Instead got: {Convert.ToBase64String(bytes)} ({bytes.Length} bytes)"); } + for (int i = 0; i < bytes.Length; i++) { if (bytes[i] != expected[i]) { - throw new Exception(string.Format("Expected bytes: {0} ({1} bytes), Instead got: {2} ({3} bytes)", Convert.ToBase64String(expected), expected.Length, Convert.ToBase64String(bytes), bytes.Length)); + throw new Exception($"Expected bytes: {Convert.ToBase64String(expected)} ({expected.Length} bytes), Instead got: {Convert.ToBase64String(bytes)} ({bytes.Length} bytes)"); } } @@ -533,6 +536,7 @@ public byte[] ProcessXor(byte[] value, int key) { result[i] = (byte)(value[i] ^ key); } + return result; } @@ -551,6 +555,7 @@ public byte[] ProcessXor(byte[] value, byte[] key) { result[i] = (byte)(value[i] ^ key[j]); } + return result; } @@ -564,7 +569,8 @@ public byte[] ProcessXor(byte[] value, byte[] key) /// public byte[] ProcessRotateLeft(byte[] data, int amount, int groupSize) { - if (amount > 7 || amount < -7) throw new ArgumentException("Rotation of more than 7 cannot be performed.", "amount"); + if (amount > 7 || amount < -7) + throw new ArgumentException("Rotation of more than 7 cannot be performed.", "amount"); if (amount < 0) amount += 8; // Rotation of -2 is the same as rotation of +6 byte[] r = new byte[data.Length]; @@ -575,12 +581,14 @@ public byte[] ProcessRotateLeft(byte[] data, int amount, int groupSize) { byte bits = data[i]; // http://stackoverflow.com/a/812039 - r[i] = (byte) ((bits << amount) | (bits >> (8 - amount))); + r[i] = (byte)((bits << amount) | (bits >> (8 - amount))); } + break; default: - throw new NotImplementedException(string.Format("Unable to rotate a group of {0} bytes yet", groupSize)); + throw new NotImplementedException($"Unable to rotate a group of {groupSize} bytes yet"); } + return r; } @@ -597,7 +605,8 @@ public byte[] ProcessZlib(byte[] data) // There's also 4 checksum bytes at the end of the stream. byte zlibCmf = data[0]; - if ((zlibCmf & 0x0F) != 0x08) throw new NotSupportedException("Only the DEFLATE algorithm is supported for zlib data."); + if ((zlibCmf & 0x0F) != 0x08) + throw new NotSupportedException("Only the DEFLATE algorithm is supported for zlib data."); const int zlibFooter = 4; int zlibHeader = 2; @@ -674,18 +683,20 @@ public static int ByteArrayCompare(byte[] a, byte[] b) int al = a.Length; int bl = b.Length; int minLen = al < bl ? al : bl; - for (int i = 0; i < minLen; i++) { + for (int i = 0; i < minLen; i++) + { int cmp = a[i] - b[i]; if (cmp != 0) return cmp; } // Reached the end of at least one of the arrays - if (al == bl) { + if (al == bl) + { return 0; - } else { - return al - bl; } + + return al - bl; } #endregion From 638cc2dd767a1781c821e230af1b11d3e21ca643 Mon Sep 17 00:00:00 2001 From: Jan Pluskal Date: Fri, 3 Jan 2020 16:08:52 +0100 Subject: [PATCH 09/34] Add initial configuration for C# Async. Use C# naming conventions. --- .../Kaitai.Struct.Runtime.Async.Tests.csproj | 16 ++++++++ .../Kaitai.Struct.Runtime.Async.csproj | 7 ++++ .../.editorconfig | 0 .../Exception}/KaitaiStructError.cs | 0 .../Exception}/ValidationFailedError.cs | 0 .../Exception}/ValidationNotEqualError.cs | 0 .../Interface}/CustomDecoder.cs | 0 .../Interface}/IKaitaiStream.cs | 0 .../Kaitai.Struct.Runtime.csproj | 37 ++++++++++++++++++ .../Kaitai.Struct.sln.DotSettings | 3 ++ .../KaitaiStream.cs | 0 .../KaitaiStruct.cs | 0 .../LICENSE.txt | 0 README.md => Kaitai.Struct.Runtime/README.md | 0 icon.png => Kaitai.Struct.Runtime/icon.png | Bin Kaitai.Struct.sln | 37 ++++++++++++++++++ kaitai_struct_runtime_csharp.csproj | 36 ----------------- 17 files changed, 100 insertions(+), 36 deletions(-) create mode 100644 Kaitai.Struct.Runtime.Async.Tests/Kaitai.Struct.Runtime.Async.Tests.csproj create mode 100644 Kaitai.Struct.Runtime.Async/Kaitai.Struct.Runtime.Async.csproj rename .editorconfig => Kaitai.Struct.Runtime/.editorconfig (100%) rename {Exception => Kaitai.Struct.Runtime/Exception}/KaitaiStructError.cs (100%) rename {Exception => Kaitai.Struct.Runtime/Exception}/ValidationFailedError.cs (100%) rename {Exception => Kaitai.Struct.Runtime/Exception}/ValidationNotEqualError.cs (100%) rename {Interface => Kaitai.Struct.Runtime/Interface}/CustomDecoder.cs (100%) rename {Interface => Kaitai.Struct.Runtime/Interface}/IKaitaiStream.cs (100%) create mode 100644 Kaitai.Struct.Runtime/Kaitai.Struct.Runtime.csproj create mode 100644 Kaitai.Struct.Runtime/Kaitai.Struct.sln.DotSettings rename KaitaiStream.cs => Kaitai.Struct.Runtime/KaitaiStream.cs (100%) rename KaitaiStruct.cs => Kaitai.Struct.Runtime/KaitaiStruct.cs (100%) rename LICENSE.txt => Kaitai.Struct.Runtime/LICENSE.txt (100%) rename README.md => Kaitai.Struct.Runtime/README.md (100%) rename icon.png => Kaitai.Struct.Runtime/icon.png (100%) create mode 100644 Kaitai.Struct.sln delete mode 100644 kaitai_struct_runtime_csharp.csproj diff --git a/Kaitai.Struct.Runtime.Async.Tests/Kaitai.Struct.Runtime.Async.Tests.csproj b/Kaitai.Struct.Runtime.Async.Tests/Kaitai.Struct.Runtime.Async.Tests.csproj new file mode 100644 index 0000000..a2352e4 --- /dev/null +++ b/Kaitai.Struct.Runtime.Async.Tests/Kaitai.Struct.Runtime.Async.Tests.csproj @@ -0,0 +1,16 @@ + + + + netcoreapp3.1 + + false + + + + + + + + + + diff --git a/Kaitai.Struct.Runtime.Async/Kaitai.Struct.Runtime.Async.csproj b/Kaitai.Struct.Runtime.Async/Kaitai.Struct.Runtime.Async.csproj new file mode 100644 index 0000000..cb63190 --- /dev/null +++ b/Kaitai.Struct.Runtime.Async/Kaitai.Struct.Runtime.Async.csproj @@ -0,0 +1,7 @@ + + + + netcoreapp3.1 + + + diff --git a/.editorconfig b/Kaitai.Struct.Runtime/.editorconfig similarity index 100% rename from .editorconfig rename to Kaitai.Struct.Runtime/.editorconfig diff --git a/Exception/KaitaiStructError.cs b/Kaitai.Struct.Runtime/Exception/KaitaiStructError.cs similarity index 100% rename from Exception/KaitaiStructError.cs rename to Kaitai.Struct.Runtime/Exception/KaitaiStructError.cs diff --git a/Exception/ValidationFailedError.cs b/Kaitai.Struct.Runtime/Exception/ValidationFailedError.cs similarity index 100% rename from Exception/ValidationFailedError.cs rename to Kaitai.Struct.Runtime/Exception/ValidationFailedError.cs diff --git a/Exception/ValidationNotEqualError.cs b/Kaitai.Struct.Runtime/Exception/ValidationNotEqualError.cs similarity index 100% rename from Exception/ValidationNotEqualError.cs rename to Kaitai.Struct.Runtime/Exception/ValidationNotEqualError.cs diff --git a/Interface/CustomDecoder.cs b/Kaitai.Struct.Runtime/Interface/CustomDecoder.cs similarity index 100% rename from Interface/CustomDecoder.cs rename to Kaitai.Struct.Runtime/Interface/CustomDecoder.cs diff --git a/Interface/IKaitaiStream.cs b/Kaitai.Struct.Runtime/Interface/IKaitaiStream.cs similarity index 100% rename from Interface/IKaitaiStream.cs rename to Kaitai.Struct.Runtime/Interface/IKaitaiStream.cs diff --git a/Kaitai.Struct.Runtime/Kaitai.Struct.Runtime.csproj b/Kaitai.Struct.Runtime/Kaitai.Struct.Runtime.csproj new file mode 100644 index 0000000..61fd5c9 --- /dev/null +++ b/Kaitai.Struct.Runtime/Kaitai.Struct.Runtime.csproj @@ -0,0 +1,37 @@ + + + + netstandard1.3;net45 + true + 1701;1702;CS1591 + {8339A750-C407-4CE8-8E31-51CB2EFD3A4B} + + + + KaitaiStruct.Runtime.CSharp + + Kaitai Project + This library implements Kaitai Struct API for C#. + Kaitai Struct Runtime + Copyright © Kaitai Project 2016-2019 + Kaitai.Struct.Runtime + Kaitai + http://kaitai.io/ + https://github.com/kaitai-io/kaitai_struct_csharp_runtime + Kaitai Struct File-Format Binary Protocols + LICENSE.txt + icon.png + + 0.8.0.0 + 0.8.0.0 + 0.8.0.0 + + + + + + + + + + diff --git a/Kaitai.Struct.Runtime/Kaitai.Struct.sln.DotSettings b/Kaitai.Struct.Runtime/Kaitai.Struct.sln.DotSettings new file mode 100644 index 0000000..98f62ff --- /dev/null +++ b/Kaitai.Struct.Runtime/Kaitai.Struct.sln.DotSettings @@ -0,0 +1,3 @@ + + True + True \ No newline at end of file diff --git a/KaitaiStream.cs b/Kaitai.Struct.Runtime/KaitaiStream.cs similarity index 100% rename from KaitaiStream.cs rename to Kaitai.Struct.Runtime/KaitaiStream.cs diff --git a/KaitaiStruct.cs b/Kaitai.Struct.Runtime/KaitaiStruct.cs similarity index 100% rename from KaitaiStruct.cs rename to Kaitai.Struct.Runtime/KaitaiStruct.cs diff --git a/LICENSE.txt b/Kaitai.Struct.Runtime/LICENSE.txt similarity index 100% rename from LICENSE.txt rename to Kaitai.Struct.Runtime/LICENSE.txt diff --git a/README.md b/Kaitai.Struct.Runtime/README.md similarity index 100% rename from README.md rename to Kaitai.Struct.Runtime/README.md diff --git a/icon.png b/Kaitai.Struct.Runtime/icon.png similarity index 100% rename from icon.png rename to Kaitai.Struct.Runtime/icon.png diff --git a/Kaitai.Struct.sln b/Kaitai.Struct.sln new file mode 100644 index 0000000..e9c9610 --- /dev/null +++ b/Kaitai.Struct.sln @@ -0,0 +1,37 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 16 +VisualStudioVersion = 16.0.29418.71 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{8339A750-C407-4CE8-8E31-51CB2EFD3A4B}") = "Kaitai.Struct.Runtime", "Kaitai.Struct.Runtime\Kaitai.Struct.Runtime.csproj", "{5E981335-06A9-46DE-9BD2-312B2CD329A4}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Kaitai.Struct.Runtime.Async", "Kaitai.Struct.Runtime.Async\Kaitai.Struct.Runtime.Async.csproj", "{33BAA91F-0777-466A-B379-65B85F8E7793}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Kaitai.Struct.Runtime.Async.Tests", "Kaitai.Struct.Runtime.Async.Tests\Kaitai.Struct.Runtime.Async.Tests.csproj", "{551D67ED-04AA-49CB-8430-33B6097854A6}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {5E981335-06A9-46DE-9BD2-312B2CD329A4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {5E981335-06A9-46DE-9BD2-312B2CD329A4}.Debug|Any CPU.Build.0 = Debug|Any CPU + {5E981335-06A9-46DE-9BD2-312B2CD329A4}.Release|Any CPU.ActiveCfg = Release|Any CPU + {5E981335-06A9-46DE-9BD2-312B2CD329A4}.Release|Any CPU.Build.0 = Release|Any CPU + {33BAA91F-0777-466A-B379-65B85F8E7793}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {33BAA91F-0777-466A-B379-65B85F8E7793}.Debug|Any CPU.Build.0 = Debug|Any CPU + {33BAA91F-0777-466A-B379-65B85F8E7793}.Release|Any CPU.ActiveCfg = Release|Any CPU + {33BAA91F-0777-466A-B379-65B85F8E7793}.Release|Any CPU.Build.0 = Release|Any CPU + {551D67ED-04AA-49CB-8430-33B6097854A6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {551D67ED-04AA-49CB-8430-33B6097854A6}.Debug|Any CPU.Build.0 = Debug|Any CPU + {551D67ED-04AA-49CB-8430-33B6097854A6}.Release|Any CPU.ActiveCfg = Release|Any CPU + {551D67ED-04AA-49CB-8430-33B6097854A6}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {22936F10-7B00-4CED-ADCF-2F9B4DA7F101} + EndGlobalSection +EndGlobal diff --git a/kaitai_struct_runtime_csharp.csproj b/kaitai_struct_runtime_csharp.csproj deleted file mode 100644 index 5670c7b..0000000 --- a/kaitai_struct_runtime_csharp.csproj +++ /dev/null @@ -1,36 +0,0 @@ - - - - netstandard1.3;net4.5 - true - 1701;1702;CS1591 - - - - KaitaiStruct.Runtime.CSharp - - Kaitai Project - This library implements Kaitai Struct API for C#. - Kaitai Struct Runtime - Copyright © Kaitai Project 2016-2019 - Kaitai.Struct.Runtime - Kaitai - http://kaitai.io/ - https://github.com/kaitai-io/kaitai_struct_csharp_runtime - Kaitai Struct File-Format Binary Protocols - LICENSE.txt - icon.png - - 0.8.0.0 - 0.8.0.0 - 0.8.0.0 - - - - - - - - - - From 9c85cf049de7b63ba4c38a583cf86be3236b629d Mon Sep 17 00:00:00 2001 From: Jan Pluskal Date: Fri, 3 Jan 2020 17:23:15 +0100 Subject: [PATCH 10/34] Extract non KaitaiStream related methods into static Utilities. --- Kaitai.Struct.Runtime/KaitaiStream.cs | 119 ++------------- Kaitai.Struct.Runtime/Utilities.cs | 208 ++++++++++++++++++++++++++ 2 files changed, 217 insertions(+), 110 deletions(-) create mode 100644 Kaitai.Struct.Runtime/Utilities.cs diff --git a/Kaitai.Struct.Runtime/KaitaiStream.cs b/Kaitai.Struct.Runtime/KaitaiStream.cs index 6d1b58c..a3e85b3 100644 --- a/Kaitai.Struct.Runtime/KaitaiStream.cs +++ b/Kaitai.Struct.Runtime/KaitaiStream.cs @@ -494,31 +494,13 @@ public byte[] EnsureFixedContents(byte[] expected) public static byte[] BytesStripRight(byte[] src, byte padByte) { - int newLen = src.Length; - while (newLen > 0 && src[newLen - 1] == padByte) - newLen--; - - byte[] dst = new byte[newLen]; - Array.Copy(src, dst, newLen); - return dst; + return Utilities.BytesStripRight(src, padByte); } public static byte[] BytesTerminate(byte[] src, byte terminator, bool includeTerminator) { - int newLen = 0; - int maxLen = src.Length; - - while (newLen < maxLen && src[newLen] != terminator) - newLen++; - - if (includeTerminator && newLen < maxLen) - newLen++; - - byte[] dst = new byte[newLen]; - Array.Copy(src, dst, newLen); - return dst; + return Utilities.BytesTerminate(src, terminator, includeTerminator); } - #endregion #region Byte array processing @@ -531,13 +513,7 @@ public static byte[] BytesTerminate(byte[] src, byte terminator, bool includeTer /// Processed data public byte[] ProcessXor(byte[] value, int key) { - byte[] result = new byte[value.Length]; - for (int i = 0; i < value.Length; i++) - { - result[i] = (byte)(value[i] ^ key); - } - - return result; + return Utilities.ProcessXor(value, key); } /// @@ -549,14 +525,7 @@ public byte[] ProcessXor(byte[] value, int key) /// Processed data public byte[] ProcessXor(byte[] value, byte[] key) { - int keyLen = key.Length; - byte[] result = new byte[value.Length]; - for (int i = 0, j = 0; i < value.Length; i++, j = (j + 1) % keyLen) - { - result[i] = (byte)(value[i] ^ key[j]); - } - - return result; + return Utilities.ProcessXor(value, key); } /// @@ -569,27 +538,7 @@ public byte[] ProcessXor(byte[] value, byte[] key) /// public byte[] ProcessRotateLeft(byte[] data, int amount, int groupSize) { - if (amount > 7 || amount < -7) - throw new ArgumentException("Rotation of more than 7 cannot be performed.", "amount"); - if (amount < 0) amount += 8; // Rotation of -2 is the same as rotation of +6 - - byte[] r = new byte[data.Length]; - switch (groupSize) - { - case 1: - for (int i = 0; i < data.Length; i++) - { - byte bits = data[i]; - // http://stackoverflow.com/a/812039 - r[i] = (byte)((bits << amount) | (bits >> (8 - amount))); - } - - break; - default: - throw new NotImplementedException($"Unable to rotate a group of {groupSize} bytes yet"); - } - - return r; + return Utilities.ProcessRotateLeft(data, amount, groupSize); } /// @@ -599,33 +548,7 @@ public byte[] ProcessRotateLeft(byte[] data, int amount, int groupSize) /// The deflated result public byte[] ProcessZlib(byte[] data) { - // See RFC 1950 (https://tools.ietf.org/html/rfc1950) - // zlib adds a header to DEFLATE streams - usually 2 bytes, - // but can be 6 bytes if FDICT is set. - // There's also 4 checksum bytes at the end of the stream. - - byte zlibCmf = data[0]; - if ((zlibCmf & 0x0F) != 0x08) - throw new NotSupportedException("Only the DEFLATE algorithm is supported for zlib data."); - - const int zlibFooter = 4; - int zlibHeader = 2; - - // If the FDICT bit (0x20) is 1, then the 4-byte dictionary is included in the header, we need to skip it - byte zlibFlg = data[1]; - if ((zlibFlg & 0x20) == 0x20) zlibHeader += 4; - - using (MemoryStream ms = new MemoryStream(data, zlibHeader, data.Length - (zlibHeader + zlibFooter))) - { - using (DeflateStream ds = new DeflateStream(ms, CompressionMode.Decompress)) - { - using (MemoryStream target = new MemoryStream()) - { - ds.CopyTo(target); - return target.ToArray(); - } - } - } + return Utilities.ProcessZlib(data); } #endregion @@ -645,10 +568,7 @@ public byte[] ProcessZlib(byte[] data) /// The result of the modulo opertion. Will always be positive. public static int Mod(int a, int b) { - if (b <= 0) throw new ArgumentException("Divisor of mod operation must be greater than zero.", "b"); - int r = a % b; - if (r < 0) r += b; - return r; + return Utilities.Mod(a, b); } /// @@ -664,10 +584,7 @@ public static int Mod(int a, int b) /// The result of the modulo opertion. Will always be positive. public static long Mod(long a, long b) { - if (b <= 0) throw new ArgumentException("Divisor of mod operation must be greater than zero.", "b"); - long r = a % b; - if (r < 0) r += b; - return r; + return Utilities.Mod(a, b); } /// @@ -678,25 +595,7 @@ public static long Mod(long a, long b) /// Second byte array to compare. public static int ByteArrayCompare(byte[] a, byte[] b) { - if (a == b) - return 0; - int al = a.Length; - int bl = b.Length; - int minLen = al < bl ? al : bl; - for (int i = 0; i < minLen; i++) - { - int cmp = a[i] - b[i]; - if (cmp != 0) - return cmp; - } - - // Reached the end of at least one of the arrays - if (al == bl) - { - return 0; - } - - return al - bl; + return Utilities.ByteArrayCompare(a, b); } #endregion diff --git a/Kaitai.Struct.Runtime/Utilities.cs b/Kaitai.Struct.Runtime/Utilities.cs new file mode 100644 index 0000000..19ff5d7 --- /dev/null +++ b/Kaitai.Struct.Runtime/Utilities.cs @@ -0,0 +1,208 @@ +using System; +using System.IO; +using System.IO.Compression; + +namespace Kaitai +{ + public class Utilities + { + /// + /// Performs XOR processing with given data, XORing every byte of the input with a single value. + /// + /// The data toe process + /// The key value to XOR with + /// Processed data + public static byte[] ProcessXor(byte[] value, int key) + { + byte[] result = new byte[value.Length]; + for (int i = 0; i < value.Length; i++) + { + result[i] = (byte)(value[i] ^ key); + } + + return result; + } + + /// + /// Performs XOR processing with given data, XORing every byte of the input with a key + /// array, repeating from the beginning of the key array if necessary + /// + /// The data toe process + /// The key array to XOR with + /// Processed data + public static byte[] ProcessXor(byte[] value, byte[] key) + { + int keyLen = key.Length; + byte[] result = new byte[value.Length]; + for (int i = 0, j = 0; i < value.Length; i++, j = (j + 1) % keyLen) + { + result[i] = (byte)(value[i] ^ key[j]); + } + + return result; + } + + /// + /// Performs a circular left rotation shift for a given buffer by a given amount of bits. + /// Pass a negative amount to rotate right. + /// + /// The data to rotate + /// The number of bytes to rotate by + /// + /// + public static byte[] ProcessRotateLeft(byte[] data, int amount, int groupSize) + { + if (amount > 7 || amount < -7) + throw new ArgumentException("Rotation of more than 7 cannot be performed.", "amount"); + if (amount < 0) amount += 8; // Rotation of -2 is the same as rotation of +6 + + byte[] r = new byte[data.Length]; + switch (groupSize) + { + case 1: + for (int i = 0; i < data.Length; i++) + { + byte bits = data[i]; + // http://stackoverflow.com/a/812039 + r[i] = (byte)((bits << amount) | (bits >> (8 - amount))); + } + + break; + default: + throw new NotImplementedException($"Unable to rotate a group of {groupSize} bytes yet"); + } + + return r; + } + + /// + /// Inflates a deflated zlib byte stream + /// + /// The data to deflate + /// The deflated result + public static byte[] ProcessZlib(byte[] data) + { + // See RFC 1950 (https://tools.ietf.org/html/rfc1950) + // zlib adds a header to DEFLATE streams - usually 2 bytes, + // but can be 6 bytes if FDICT is set. + // There's also 4 checksum bytes at the end of the stream. + + byte zlibCmf = data[0]; + if ((zlibCmf & 0x0F) != 0x08) + throw new NotSupportedException("Only the DEFLATE algorithm is supported for zlib data."); + + const int zlibFooter = 4; + int zlibHeader = 2; + + // If the FDICT bit (0x20) is 1, then the 4-byte dictionary is included in the header, we need to skip it + byte zlibFlg = data[1]; + if ((zlibFlg & 0x20) == 0x20) zlibHeader += 4; + + using (MemoryStream ms = new MemoryStream(data, zlibHeader, data.Length - (zlibHeader + zlibFooter))) + { + using (DeflateStream ds = new DeflateStream(ms, CompressionMode.Decompress)) + { + using (MemoryStream target = new MemoryStream()) + { + ds.CopyTo(target); + return target.ToArray(); + } + } + } + } + + public static byte[] BytesStripRight(byte[] src, byte padByte) + { + int newLen = src.Length; + while (newLen > 0 && src[newLen - 1] == padByte) + newLen--; + + byte[] dst = new byte[newLen]; + Array.Copy(src, dst, newLen); + return dst; + } + + public static byte[] BytesTerminate(byte[] src, byte terminator, bool includeTerminator) + { + int newLen = 0; + int maxLen = src.Length; + + while (newLen < maxLen && src[newLen] != terminator) + newLen++; + + if (includeTerminator && newLen < maxLen) + newLen++; + + byte[] dst = new byte[newLen]; + Array.Copy(src, dst, newLen); + return dst; + } + + /// + /// Performs modulo operation between two integers. + /// + /// + /// This method is required because C# lacks a "true" modulo + /// operator, the % operator rather being the "remainder" + /// operator. We want mod operations to always be positive. + /// + /// The value to be divided + /// The value to divide by. Must be greater than zero. + /// The result of the modulo opertion. Will always be positive. + public static int Mod(int a, int b) + { + if (b <= 0) throw new ArgumentException("Divisor of mod operation must be greater than zero.", "b"); + int r = a % b; + if (r < 0) r += b; + return r; + } + + /// + /// Performs modulo operation between two integers. + /// + /// + /// This method is required because C# lacks a "true" modulo + /// operator, the % operator rather being the "remainder" + /// operator. We want mod operations to always be positive. + /// + /// The value to be divided + /// The value to divide by. Must be greater than zero. + /// The result of the modulo opertion. Will always be positive. + public static long Mod(long a, long b) + { + if (b <= 0) throw new ArgumentException("Divisor of mod operation must be greater than zero.", "b"); + long r = a % b; + if (r < 0) r += b; + return r; + } + + /// + /// Compares two byte arrays in lexicographical order. + /// + /// negative number if a is less than b, 0 if a is equal to b, positive number if a is greater than b. + /// First byte array to compare + /// Second byte array to compare. + public static int ByteArrayCompare(byte[] a, byte[] b) + { + if (a == b) + return 0; + int al = a.Length; + int bl = b.Length; + int minLen = al < bl ? al : bl; + for (int i = 0; i < minLen; i++) + { + int cmp = a[i] - b[i]; + if (cmp != 0) + return cmp; + } + + // Reached the end of at least one of the arrays + if (al == bl) + { + return 0; + } + + return al - bl; + } + } +} From 34366235e0c418f9b6bee1edeb682b57d061ceaa Mon Sep 17 00:00:00 2001 From: Jan Pluskal Date: Fri, 3 Jan 2020 17:33:17 +0100 Subject: [PATCH 11/34] Extract KaitaiStreamBase containing utility methods. --- .../Interface/IKaitaiStream.cs | 52 +------ .../Interface/IKaitaiStreamBase.cs | 56 ++++++++ .../Kaitai.Struct.Runtime.csproj.DotSettings | 2 + Kaitai.Struct.Runtime/KaitaiStream.cs | 130 ++---------------- .../{Utilities.cs => KaitaiStreamBase.cs} | 85 +++++++----- 5 files changed, 121 insertions(+), 204 deletions(-) create mode 100644 Kaitai.Struct.Runtime/Interface/IKaitaiStreamBase.cs create mode 100644 Kaitai.Struct.Runtime/Kaitai.Struct.Runtime.csproj.DotSettings rename Kaitai.Struct.Runtime/{Utilities.cs => KaitaiStreamBase.cs} (88%) diff --git a/Kaitai.Struct.Runtime/Interface/IKaitaiStream.cs b/Kaitai.Struct.Runtime/Interface/IKaitaiStream.cs index 1d1b363..d66a014 100644 --- a/Kaitai.Struct.Runtime/Interface/IKaitaiStream.cs +++ b/Kaitai.Struct.Runtime/Interface/IKaitaiStream.cs @@ -1,22 +1,7 @@ namespace Kaitai { - public interface IKaitaiStream + public interface IKaitaiStream : IKaitaiStreamBase { - /// - /// Check if the stream position is at the end of the stream - /// - bool IsEof { get; } - - /// - /// Get the current position in the stream - /// - long Pos { get; } - - /// - /// Get the total length of the stream (ie. file size) - /// - long Size { get; } - /// /// Seek to a specific position from the beginning of the stream /// @@ -131,7 +116,6 @@ public interface IKaitaiStream /// double ReadF8le(); - void AlignToByte(); ulong ReadBitsInt(int n); ulong ReadBitsIntLe(int n); @@ -171,39 +155,5 @@ public interface IKaitaiStream /// The expected result /// byte[] EnsureFixedContents(byte[] expected); - - /// - /// Performs XOR processing with given data, XORing every byte of the input with a single value. - /// - /// The data toe process - /// The key value to XOR with - /// Processed data - byte[] ProcessXor(byte[] value, int key); - - /// - /// Performs XOR processing with given data, XORing every byte of the input with a key - /// array, repeating from the beginning of the key array if necessary - /// - /// The data toe process - /// The key array to XOR with - /// Processed data - byte[] ProcessXor(byte[] value, byte[] key); - - /// - /// Performs a circular left rotation shift for a given buffer by a given amount of bits. - /// Pass a negative amount to rotate right. - /// - /// The data to rotate - /// The number of bytes to rotate by - /// - /// - byte[] ProcessRotateLeft(byte[] data, int amount, int groupSize); - - /// - /// Inflates a deflated zlib byte stream - /// - /// The data to deflate - /// The deflated result - byte[] ProcessZlib(byte[] data); } } diff --git a/Kaitai.Struct.Runtime/Interface/IKaitaiStreamBase.cs b/Kaitai.Struct.Runtime/Interface/IKaitaiStreamBase.cs new file mode 100644 index 0000000..7693495 --- /dev/null +++ b/Kaitai.Struct.Runtime/Interface/IKaitaiStreamBase.cs @@ -0,0 +1,56 @@ +namespace Kaitai +{ + public interface IKaitaiStreamBase + { + /// + /// Check if the stream position is at the end of the stream + /// + bool IsEof { get; } + + /// + /// Get the current position in the stream + /// + long Pos { get; } + + /// + /// Get the total length of the stream (ie. file size) + /// + long Size { get; } + + void AlignToByte(); + + /// + /// Performs XOR processing with given data, XORing every byte of the input with a single value. + /// + /// The data toe process + /// The key value to XOR with + /// Processed data + byte[] ProcessXor(byte[] value, int key); + + /// + /// Performs XOR processing with given data, XORing every byte of the input with a key + /// array, repeating from the beginning of the key array if necessary + /// + /// The data toe process + /// The key array to XOR with + /// Processed data + byte[] ProcessXor(byte[] value, byte[] key); + + /// + /// Performs a circular left rotation shift for a given buffer by a given amount of bits. + /// Pass a negative amount to rotate right. + /// + /// The data to rotate + /// The number of bytes to rotate by + /// + /// + byte[] ProcessRotateLeft(byte[] data, int amount, int groupSize); + + /// + /// Inflates a deflated zlib byte stream + /// + /// The data to deflate + /// The deflated result + byte[] ProcessZlib(byte[] data); + } +} diff --git a/Kaitai.Struct.Runtime/Kaitai.Struct.Runtime.csproj.DotSettings b/Kaitai.Struct.Runtime/Kaitai.Struct.Runtime.csproj.DotSettings new file mode 100644 index 0000000..d19dc3e --- /dev/null +++ b/Kaitai.Struct.Runtime/Kaitai.Struct.Runtime.csproj.DotSettings @@ -0,0 +1,2 @@ + + True \ No newline at end of file diff --git a/Kaitai.Struct.Runtime/KaitaiStream.cs b/Kaitai.Struct.Runtime/KaitaiStream.cs index a3e85b3..3bd7964 100644 --- a/Kaitai.Struct.Runtime/KaitaiStream.cs +++ b/Kaitai.Struct.Runtime/KaitaiStream.cs @@ -1,7 +1,6 @@ using System; using System.Collections.Generic; using System.IO; -using System.IO.Compression; namespace Kaitai { @@ -9,12 +8,9 @@ namespace Kaitai /// The base Kaitai stream which exposes an API for the Kaitai Struct framework. /// It's based off a BinaryReader, which is a little-endian reader. /// - public partial class KaitaiStream : IKaitaiStream + public partial class KaitaiStream : KaitaiStreamBase, IKaitaiStream { #region Constructors - - static readonly bool IsLittleEndian = BitConverter.IsLittleEndian; - private ulong Bits; private int BitsLeft; private BinaryReader m_binaryReader; @@ -53,7 +49,7 @@ protected BinaryReader BinaryReader /// /// Check if the stream position is at the end of the stream /// - public bool IsEof + public override bool IsEof { get { return BaseStream.Position >= BaseStream.Length && BitsLeft == 0; } } @@ -70,7 +66,7 @@ public void Seek(long position) /// /// Get the current position in the stream /// - public long Pos + public override long Pos { get { return BaseStream.Position; } } @@ -78,7 +74,7 @@ public long Pos /// /// Get the total length of the stream (ie. file size) /// - public long Size + public override long Size { get { return BaseStream.Length; } } @@ -289,7 +285,7 @@ public double ReadF8le() #region Unaligned bit values - public void AlignToByte() + public override void AlignToByte() { Bits = 0; BitsLeft = 0; @@ -361,11 +357,6 @@ public ulong ReadBitsIntLe(int n) return res; } - private static ulong GetMaskOnes(int n) - { - return n == 64 ? 0xffffffffffffffffUL : (1UL << n) - 1; - } - #endregion #region Byte arrays @@ -441,7 +432,10 @@ public byte[] ReadBytesFull() /// True to consume the terminator byte before returning /// True to throw an error when the EOS was reached before the terminator /// - public byte[] ReadBytesTerm(byte terminator, bool includeTerminator, bool consumeTerminator, bool eosError) + public byte[] ReadBytesTerm(byte terminator, + bool includeTerminator, + bool consumeTerminator, + bool eosError) { List bytes = new List(); while (true) @@ -492,112 +486,6 @@ public byte[] EnsureFixedContents(byte[] expected) return bytes; } - public static byte[] BytesStripRight(byte[] src, byte padByte) - { - return Utilities.BytesStripRight(src, padByte); - } - - public static byte[] BytesTerminate(byte[] src, byte terminator, bool includeTerminator) - { - return Utilities.BytesTerminate(src, terminator, includeTerminator); - } - #endregion - - #region Byte array processing - - /// - /// Performs XOR processing with given data, XORing every byte of the input with a single value. - /// - /// The data toe process - /// The key value to XOR with - /// Processed data - public byte[] ProcessXor(byte[] value, int key) - { - return Utilities.ProcessXor(value, key); - } - - /// - /// Performs XOR processing with given data, XORing every byte of the input with a key - /// array, repeating from the beginning of the key array if necessary - /// - /// The data toe process - /// The key array to XOR with - /// Processed data - public byte[] ProcessXor(byte[] value, byte[] key) - { - return Utilities.ProcessXor(value, key); - } - - /// - /// Performs a circular left rotation shift for a given buffer by a given amount of bits. - /// Pass a negative amount to rotate right. - /// - /// The data to rotate - /// The number of bytes to rotate by - /// - /// - public byte[] ProcessRotateLeft(byte[] data, int amount, int groupSize) - { - return Utilities.ProcessRotateLeft(data, amount, groupSize); - } - - /// - /// Inflates a deflated zlib byte stream - /// - /// The data to deflate - /// The deflated result - public byte[] ProcessZlib(byte[] data) - { - return Utilities.ProcessZlib(data); - } - - #endregion - - #region Misc utility methods - - /// - /// Performs modulo operation between two integers. - /// - /// - /// This method is required because C# lacks a "true" modulo - /// operator, the % operator rather being the "remainder" - /// operator. We want mod operations to always be positive. - /// - /// The value to be divided - /// The value to divide by. Must be greater than zero. - /// The result of the modulo opertion. Will always be positive. - public static int Mod(int a, int b) - { - return Utilities.Mod(a, b); - } - - /// - /// Performs modulo operation between two integers. - /// - /// - /// This method is required because C# lacks a "true" modulo - /// operator, the % operator rather being the "remainder" - /// operator. We want mod operations to always be positive. - /// - /// The value to be divided - /// The value to divide by. Must be greater than zero. - /// The result of the modulo opertion. Will always be positive. - public static long Mod(long a, long b) - { - return Utilities.Mod(a, b); - } - - /// - /// Compares two byte arrays in lexicographical order. - /// - /// negative number if a is less than b, 0 if a is equal to b, positive number if a is greater than b. - /// First byte array to compare - /// Second byte array to compare. - public static int ByteArrayCompare(byte[] a, byte[] b) - { - return Utilities.ByteArrayCompare(a, b); - } - #endregion } } diff --git a/Kaitai.Struct.Runtime/Utilities.cs b/Kaitai.Struct.Runtime/KaitaiStreamBase.cs similarity index 88% rename from Kaitai.Struct.Runtime/Utilities.cs rename to Kaitai.Struct.Runtime/KaitaiStreamBase.cs index 19ff5d7..df3748b 100644 --- a/Kaitai.Struct.Runtime/Utilities.cs +++ b/Kaitai.Struct.Runtime/KaitaiStreamBase.cs @@ -4,15 +4,61 @@ namespace Kaitai { - public class Utilities + public abstract class KaitaiStreamBase : IKaitaiStreamBase { + protected static readonly bool IsLittleEndian = BitConverter.IsLittleEndian; + + /// + /// Check if the stream position is at the end of the stream + /// + public abstract bool IsEof { get; } + + /// + /// Get the current position in the stream + /// + public abstract long Pos { get; } + + /// + /// Get the total length of the stream (ie. file size) + /// + public abstract long Size { get; } + + public abstract void AlignToByte(); + + public static byte[] BytesStripRight(byte[] src, byte padByte) + { + int newLen = src.Length; + while (newLen > 0 && src[newLen - 1] == padByte) + newLen--; + + byte[] dst = new byte[newLen]; + Array.Copy(src, dst, newLen); + return dst; + } + + public static byte[] BytesTerminate(byte[] src, byte terminator, bool includeTerminator) + { + int newLen = 0; + int maxLen = src.Length; + + while (newLen < maxLen && src[newLen] != terminator) + newLen++; + + if (includeTerminator && newLen < maxLen) + newLen++; + + byte[] dst = new byte[newLen]; + Array.Copy(src, dst, newLen); + return dst; + } + /// /// Performs XOR processing with given data, XORing every byte of the input with a single value. /// /// The data toe process /// The key value to XOR with /// Processed data - public static byte[] ProcessXor(byte[] value, int key) + public byte[] ProcessXor(byte[] value, int key) { byte[] result = new byte[value.Length]; for (int i = 0; i < value.Length; i++) @@ -30,7 +76,7 @@ public static byte[] ProcessXor(byte[] value, int key) /// The data toe process /// The key array to XOR with /// Processed data - public static byte[] ProcessXor(byte[] value, byte[] key) + public byte[] ProcessXor(byte[] value, byte[] key) { int keyLen = key.Length; byte[] result = new byte[value.Length]; @@ -50,7 +96,7 @@ public static byte[] ProcessXor(byte[] value, byte[] key) /// The number of bytes to rotate by /// /// - public static byte[] ProcessRotateLeft(byte[] data, int amount, int groupSize) + public byte[] ProcessRotateLeft(byte[] data, int amount, int groupSize) { if (amount > 7 || amount < -7) throw new ArgumentException("Rotation of more than 7 cannot be performed.", "amount"); @@ -80,7 +126,7 @@ public static byte[] ProcessRotateLeft(byte[] data, int amount, int groupSize) /// /// The data to deflate /// The deflated result - public static byte[] ProcessZlib(byte[] data) + public byte[] ProcessZlib(byte[] data) { // See RFC 1950 (https://tools.ietf.org/html/rfc1950) // zlib adds a header to DEFLATE streams - usually 2 bytes, @@ -111,33 +157,6 @@ public static byte[] ProcessZlib(byte[] data) } } - public static byte[] BytesStripRight(byte[] src, byte padByte) - { - int newLen = src.Length; - while (newLen > 0 && src[newLen - 1] == padByte) - newLen--; - - byte[] dst = new byte[newLen]; - Array.Copy(src, dst, newLen); - return dst; - } - - public static byte[] BytesTerminate(byte[] src, byte terminator, bool includeTerminator) - { - int newLen = 0; - int maxLen = src.Length; - - while (newLen < maxLen && src[newLen] != terminator) - newLen++; - - if (includeTerminator && newLen < maxLen) - newLen++; - - byte[] dst = new byte[newLen]; - Array.Copy(src, dst, newLen); - return dst; - } - /// /// Performs modulo operation between two integers. /// @@ -204,5 +223,7 @@ public static int ByteArrayCompare(byte[] a, byte[] b) return al - bl; } + + protected static ulong GetMaskOnes(int n) => n == 64 ? 0xffffffffffffffffUL : (1UL << n) - 1; } } From f534e90dc515629ece79c3735abcd6bce4967531 Mon Sep 17 00:00:00 2001 From: Jan Pluskal Date: Fri, 3 Jan 2020 18:15:11 +0100 Subject: [PATCH 12/34] Add simple Async implementation. --- .../Interface/IKaitaiAsyncStream.cs | 161 +++++++ .../Kaitai.Struct.Runtime.Async.csproj | 11 +- .../KaitaiAsyncStream.cs | 424 ++++++++++++++++++ .../KaitaiAsyncStruct.cs | 15 + .../Exception/ValidationFailedError.cs | 4 +- .../Exception/ValidationNotEqualError.cs | 4 +- Kaitai.Struct.sln.DotSettings | 3 + 7 files changed, 617 insertions(+), 5 deletions(-) create mode 100644 Kaitai.Struct.Runtime.Async/Interface/IKaitaiAsyncStream.cs create mode 100644 Kaitai.Struct.Runtime.Async/KaitaiAsyncStream.cs create mode 100644 Kaitai.Struct.Runtime.Async/KaitaiAsyncStruct.cs create mode 100644 Kaitai.Struct.sln.DotSettings diff --git a/Kaitai.Struct.Runtime.Async/Interface/IKaitaiAsyncStream.cs b/Kaitai.Struct.Runtime.Async/Interface/IKaitaiAsyncStream.cs new file mode 100644 index 0000000..8000ad5 --- /dev/null +++ b/Kaitai.Struct.Runtime.Async/Interface/IKaitaiAsyncStream.cs @@ -0,0 +1,161 @@ +using System.Threading.Tasks; + +namespace Kaitai.Async +{ + public interface IKaitaiAsyncStream : IKaitaiStreamBase + { + /// + /// Seek to a specific position from the beginning of the stream + /// + /// The position to seek to + Task SeekAsync(long position); + + /// + /// Read a signed byte from the stream + /// + /// + Task ReadS1Async(); + + /// + /// Read a signed short from the stream (big endian) + /// + /// + Task ReadS2beAsync(); + + /// + /// Read a signed int from the stream (big endian) + /// + /// + Task ReadS4beAsync(); + + /// + /// Read a signed long from the stream (big endian) + /// + /// + Task ReadS8beAsync(); + + /// + /// Read a signed short from the stream (little endian) + /// + /// + Task ReadS2leAsync(); + + /// + /// Read a signed int from the stream (little endian) + /// + /// + Task ReadS4leAsync(); + + /// + /// Read a signed long from the stream (little endian) + /// + /// + Task ReadS8leAsync(); + + /// + /// Read an unsigned byte from the stream + /// + /// + Task ReadU1Async(); + + /// + /// Read an unsigned short from the stream (big endian) + /// + /// + Task ReadU2beAsync(); + + /// + /// Read an unsigned int from the stream (big endian) + /// + /// + Task ReadU4beAsync(); + + /// + /// Read an unsigned long from the stream (big endian) + /// + /// + Task ReadU8beAsync(); + + /// + /// Read an unsigned short from the stream (little endian) + /// + /// + Task ReadU2leAsync(); + + /// + /// Read an unsigned int from the stream (little endian) + /// + /// + Task ReadU4leAsync(); + + /// + /// Read an unsigned long from the stream (little endian) + /// + /// + Task ReadU8leAsync(); + + /// + /// Read a single-precision floating point value from the stream (big endian) + /// + /// + Task ReadF4beAsync(); + + /// + /// Read a double-precision floating point value from the stream (big endian) + /// + /// + Task ReadF8beAsync(); + + /// + /// Read a single-precision floating point value from the stream (little endian) + /// + /// + Task ReadF4leAsync(); + + /// + /// Read a double-precision floating point value from the stream (little endian) + /// + /// + Task ReadF8leAsync(); + + Task ReadBitsIntAsync(int n); + Task ReadBitsIntLeAsync(int n); + + /// + /// Read a fixed number of bytes from the stream + /// + /// The number of bytes to read + /// + Task ReadBytesAsync(long count); + + /// + /// Read a fixed number of bytes from the stream + /// + /// The number of bytes to read + /// + Task ReadBytesAsync(ulong count); + + /// + /// Read all the remaining bytes from the stream until the end is reached + /// + /// + Task ReadBytesFullAsync(); + + /// + /// Read a terminated string from the stream + /// + /// The string terminator value + /// True to include the terminator in the returned string + /// True to consume the terminator byte before returning + /// True to throw an error when the EOS was reached before the terminator + /// + Task ReadBytesTermAsync(byte terminator, bool includeTerminator, bool consumeTerminator, bool eosError); + + /// + /// Read a specific set of bytes and assert that they are the same as an expected result + /// + /// The expected result + /// + Task EnsureFixedContentsAsync(byte[] expected); + } +} diff --git a/Kaitai.Struct.Runtime.Async/Kaitai.Struct.Runtime.Async.csproj b/Kaitai.Struct.Runtime.Async/Kaitai.Struct.Runtime.Async.csproj index cb63190..848a9b2 100644 --- a/Kaitai.Struct.Runtime.Async/Kaitai.Struct.Runtime.Async.csproj +++ b/Kaitai.Struct.Runtime.Async/Kaitai.Struct.Runtime.Async.csproj @@ -1,7 +1,16 @@ - + netcoreapp3.1 + Kaitai.Async + + + + + + + + diff --git a/Kaitai.Struct.Runtime.Async/KaitaiAsyncStream.cs b/Kaitai.Struct.Runtime.Async/KaitaiAsyncStream.cs new file mode 100644 index 0000000..dc4e37d --- /dev/null +++ b/Kaitai.Struct.Runtime.Async/KaitaiAsyncStream.cs @@ -0,0 +1,424 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.IO.Compression; +using System.Threading.Tasks; +using Overby.Extensions.AsyncBinaryReaderWriter; + +namespace Kaitai.Async +{ + public partial class KaitaiAsyncStream : KaitaiStreamBase, IKaitaiAsyncStream + { + protected Stream BaseStream; + + private ulong _bits = 0; + private int _bitsLeft = 0; + private AsyncBinaryReader _asyncBinaryReader; + + protected AsyncBinaryReader AsyncBinaryReader + { + get => _asyncBinaryReader ?? (_asyncBinaryReader = new AsyncBinaryReader(BaseStream)); + set => _asyncBinaryReader = value; + } + + #region Constructors + + public KaitaiAsyncStream(Stream stream) + { + BaseStream = stream; + } + + /// + /// Creates a IKaitaiAsyncStream backed by a file (RO) + /// + public KaitaiAsyncStream(string file) : this(File.Open(file, FileMode.Open, FileAccess.Read, FileShare.Read)) + { + } + + /// + ///Creates a IKaitaiAsyncStream backed by a byte buffer + /// + public KaitaiAsyncStream(byte[] bytes) : this(new MemoryStream(bytes)) + { + } + + #endregion + + #region Stream positioning + + /// + /// Check if the stream position is at the end of the stream + /// + public override bool IsEof => BaseStream.Position >= BaseStream.Length && _bitsLeft == 0; + + /// + /// Seek to a specific position from the beginning of the stream + /// + /// The position to seek to + /// .NET Stream API does not support SeekAsync, therefore, consumer needs to define his own implementation. + public virtual Task SeekAsync(long position) + { + BaseStream.Seek(position, SeekOrigin.Begin); + return Task.CompletedTask; + } + + /// + /// Get the current position in the stream + /// + public override long Pos => BaseStream.Position; + + /// + /// Get the total length of the stream (ie. file size) + /// + public override long Size => BaseStream.Length; + + #endregion + + #region Integer types + + #region Signed + + /// + /// Read a signed byte from the stream + /// + /// + public async Task ReadS1Async() => await AsyncBinaryReader.ReadSByteAsync(); + + #region Big-endian + + /// + /// Read a signed short from the stream (big endian) + /// + /// + public async Task ReadS2beAsync() => BitConverter.ToInt16(await ReadBytesNormalisedBigEndianAsync(2), 0); + + /// + /// Read a signed int from the stream (big endian) + /// + /// + public async Task ReadS4beAsync() => BitConverter.ToInt32(await ReadBytesNormalisedBigEndianAsync(4), 0); + + /// + /// Read a signed long from the stream (big endian) + /// + /// + public async Task ReadS8beAsync() => BitConverter.ToInt64(await ReadBytesNormalisedBigEndianAsync(8), 0); + + #endregion + + #region Little-endian + + /// + /// Read a signed short from the stream (little endian) + /// + /// + public async Task ReadS2leAsync() => BitConverter.ToInt16(await ReadBytesNormalisedLittleEndianAsync(2), 0); + + /// + /// Read a signed int from the stream (little endian) + /// + /// + public async Task ReadS4leAsync() => BitConverter.ToInt32(await ReadBytesNormalisedLittleEndianAsync(4), 0); + + /// + /// Read a signed long from the stream (little endian) + /// + /// + public async Task ReadS8leAsync() => BitConverter.ToInt64(await ReadBytesNormalisedLittleEndianAsync(8), 0); + + #endregion + + #endregion + + #region Unsigned + + /// + /// Read an unsigned byte from the stream + /// + /// + public async Task ReadU1Async() => await AsyncBinaryReader.ReadByteAsync(); + + #region Big-endian + + /// + /// Read an unsigned short from the stream (big endian) + /// + /// + public async Task ReadU2beAsync() => BitConverter.ToUInt16(await ReadBytesNormalisedBigEndianAsync(2), 0); + + /// + /// Read an unsigned int from the stream (big endian) + /// + /// + public async Task ReadU4beAsync() => BitConverter.ToUInt32(await ReadBytesNormalisedBigEndianAsync(4), 0); + + /// + /// Read an unsigned long from the stream (big endian) + /// + /// + public async Task ReadU8beAsync() => BitConverter.ToUInt64(await ReadBytesNormalisedBigEndianAsync(8), 0); + + #endregion + + #region Little-endian + + /// + /// Read an unsigned short from the stream (little endian) + /// + /// + public async Task ReadU2leAsync() => BitConverter.ToUInt16(await ReadBytesNormalisedLittleEndianAsync(2), 0); + + /// + /// Read an unsigned int from the stream (little endian) + /// + /// + public async Task ReadU4leAsync() => BitConverter.ToUInt32(await ReadBytesNormalisedLittleEndianAsync(4), 0); + + /// + /// Read an unsigned long from the stream (little endian) + /// + /// + public async Task ReadU8leAsync() => BitConverter.ToUInt64(await ReadBytesNormalisedLittleEndianAsync(8), 0); + + #endregion + + #endregion + + #endregion + + #region Floating point types + + #region Big-endian + + /// + /// Read a single-precision floating point value from the stream (big endian) + /// + /// + public async Task ReadF4beAsync() => BitConverter.ToSingle(await ReadBytesNormalisedBigEndianAsync(4), 0); + + /// + /// Read a double-precision floating point value from the stream (big endian) + /// + /// + public async Task ReadF8beAsync() => BitConverter.ToDouble(await ReadBytesNormalisedBigEndianAsync(8), 0); + + #endregion + + #region Little-endian + + /// + /// Read a single-precision floating point value from the stream (little endian) + /// + /// + public async Task ReadF4leAsync() => BitConverter.ToSingle(await ReadBytesNormalisedLittleEndianAsync(4), 0); + + /// + /// Read a double-precision floating point value from the stream (little endian) + /// + /// + public async Task ReadF8leAsync() => BitConverter.ToDouble(await ReadBytesNormalisedLittleEndianAsync(8), 0); + + #endregion + + #endregion + + #region Unaligned bit values + + public override void AlignToByte() + { + _bits = 0; + _bitsLeft = 0; + } + + public async Task ReadBitsIntAsync(int n) + { + int bitsNeeded = n - _bitsLeft; + if (bitsNeeded > 0) + { + // 1 bit => 1 byte + // 8 bits => 1 byte + // 9 bits => 2 bytes + int bytesNeeded = ((bitsNeeded - 1) / 8) + 1; + byte[] buf = await ReadBytesAsync(bytesNeeded); + for (int i = 0; i < buf.Length; i++) + { + _bits <<= 8; + _bits |= buf[i]; + _bitsLeft += 8; + } + } + + // raw mask with required number of 1s, starting from lowest bit + ulong mask = GetMaskOnes(n); + // shift mask to align with highest bits available in "bits" + int shiftBits = _bitsLeft - n; + mask = mask << shiftBits; + // derive reading result + ulong res = (_bits & mask) >> shiftBits; + // clear top bits that we've just read => AND with 1s + _bitsLeft -= n; + mask = GetMaskOnes(_bitsLeft); + _bits &= mask; + + return res; + } + + //Method ported from algorithm specified @ issue#155 + public async Task ReadBitsIntLeAsync(int n) + { + int bitsNeeded = n - _bitsLeft; + + if (bitsNeeded > 0) + { + // 1 bit => 1 byte + // 8 bits => 1 byte + // 9 bits => 2 bytes + int bytesNeeded = ((bitsNeeded - 1) / 8) + 1; + byte[] buf = await ReadBytesAsync(bytesNeeded); + for (int i = 0; i < buf.Length; i++) + { + ulong v = (ulong)((ulong)buf[i] << _bitsLeft); + _bits |= v; + _bitsLeft += 8; + } + } + + // raw mask with required number of 1s, starting from lowest bit + ulong mask = GetMaskOnes(n); + + // derive reading result + ulong res = (_bits & mask); + + // remove bottom bits that we've just read by shifting + _bits >>= n; + _bitsLeft -= n; + + return res; + } + + #endregion + + #region Byte arrays + + /// + /// Read a fixed number of bytes from the stream + /// + /// The number of bytes to read + /// + public async Task ReadBytesAsync(long count) + { + if (count < 0 || count > Int32.MaxValue) + throw new ArgumentOutOfRangeException("requested " + count + " bytes, while only non-negative int32 amount of bytes possible"); + byte[] bytes = await AsyncBinaryReader.ReadBytesAsync((int)count); + if (bytes.Length < count) + throw new EndOfStreamException("requested " + count + " bytes, but got only " + bytes.Length + " bytes"); + return bytes; + } + + /// + /// Read a fixed number of bytes from the stream + /// + /// The number of bytes to read + /// + public async Task ReadBytesAsync(ulong count) + { + if (count > Int32.MaxValue) + throw new ArgumentOutOfRangeException("requested " + count + " bytes, while only non-negative int32 amount of bytes possible"); + byte[] bytes = await AsyncBinaryReader.ReadBytesAsync((int)count); + if (bytes.Length < (int)count) + throw new EndOfStreamException("requested " + count + " bytes, but got only " + bytes.Length + " bytes"); + return bytes; + } + + /// + /// Read bytes from the stream in little endian format and convert them to the endianness of the current platform + /// + /// The number of bytes to read + /// An array of bytes that matches the endianness of the current platform + protected async Task ReadBytesNormalisedLittleEndianAsync(int count) + { + byte[] bytes = await ReadBytesAsync(count); + if (!IsLittleEndian) Array.Reverse(bytes); + return bytes; + } + + /// + /// Read bytes from the stream in big endian format and convert them to the endianness of the current platform + /// + /// The number of bytes to read + /// An array of bytes that matches the endianness of the current platform + protected async Task ReadBytesNormalisedBigEndianAsync(int count) + { + byte[] bytes = await ReadBytesAsync(count); + if (IsLittleEndian) Array.Reverse(bytes); + return bytes; + } + + /// + /// Read all the remaining bytes from the stream until the end is reached + /// + /// + /// //TODO Handle asynchronously, BaseStream.Length is troublesome + public virtual async Task ReadBytesFullAsync() => await ReadBytesAsync(BaseStream.Length - BaseStream.Position); + + /// + /// Read a terminated string from the stream + /// + /// The string terminator value + /// True to include the terminator in the returned string + /// True to consume the terminator byte before returning + /// True to throw an error when the EOS was reached before the terminator + /// + public async Task ReadBytesTermAsync(byte terminator, bool includeTerminator, bool consumeTerminator, bool eosError) + { + List bytes = new List(); + while (true) + { + if (IsEof) + { + if (eosError) throw new EndOfStreamException( + $"End of stream reached, but no terminator `{terminator}` found"); + break; + } + + byte b = await AsyncBinaryReader.ReadByteAsync(); + if (b == terminator) + { + if (includeTerminator) bytes.Add(b); + if (!consumeTerminator) await SeekAsync(Pos - 1); + break; + } + bytes.Add(b); + } + return bytes.ToArray(); + } + + /// + /// Read a specific set of bytes and assert that they are the same as an expected result + /// + /// The expected result + /// + public async Task EnsureFixedContentsAsync(byte[] expected) + { + byte[] bytes = await ReadBytesAsync(expected.Length); + + if (bytes.Length != expected.Length) + { + throw new Exception( + $"Expected bytes: {Convert.ToBase64String(expected)} ({expected.Length} bytes), Instead got: {Convert.ToBase64String(bytes)} ({bytes.Length} bytes)"); + } + for (int i = 0; i < bytes.Length; i++) + { + if (bytes[i] != expected[i]) + { + throw new Exception( + $"Expected bytes: {Convert.ToBase64String(expected)} ({expected.Length} bytes), Instead got: {Convert.ToBase64String(bytes)} ({bytes.Length} bytes)"); + } + } + + return bytes; + } + + #endregion + } +} diff --git a/Kaitai.Struct.Runtime.Async/KaitaiAsyncStruct.cs b/Kaitai.Struct.Runtime.Async/KaitaiAsyncStruct.cs new file mode 100644 index 0000000..7fb9253 --- /dev/null +++ b/Kaitai.Struct.Runtime.Async/KaitaiAsyncStruct.cs @@ -0,0 +1,15 @@ +namespace Kaitai.Async +{ + public abstract class KaitaiAsyncStruct + { + protected readonly KaitaiAsyncStream m_io; + + protected KaitaiAsyncStruct(KaitaiAsyncStream kaitaiStream) + { + m_io = kaitaiStream; + } + + // ReSharper disable once InconsistentNaming + public KaitaiAsyncStream M_Io => m_io; + } +} \ No newline at end of file diff --git a/Kaitai.Struct.Runtime/Exception/ValidationFailedError.cs b/Kaitai.Struct.Runtime/Exception/ValidationFailedError.cs index 4ac5992..373af24 100644 --- a/Kaitai.Struct.Runtime/Exception/ValidationFailedError.cs +++ b/Kaitai.Struct.Runtime/Exception/ValidationFailedError.cs @@ -8,9 +8,9 @@ namespace Kaitai /// public class ValidationFailedError : KaitaiStructError { - protected KaitaiStream io; + protected IKaitaiStreamBase io; - public ValidationFailedError(string msg, KaitaiStream io, string srcPath) + public ValidationFailedError(string msg, IKaitaiStreamBase io, string srcPath) : base($"at pos {io.Pos}: validation failed: {msg}", srcPath) { this.io = io; diff --git a/Kaitai.Struct.Runtime/Exception/ValidationNotEqualError.cs b/Kaitai.Struct.Runtime/Exception/ValidationNotEqualError.cs index c7eb300..e97520e 100644 --- a/Kaitai.Struct.Runtime/Exception/ValidationNotEqualError.cs +++ b/Kaitai.Struct.Runtime/Exception/ValidationNotEqualError.cs @@ -12,7 +12,7 @@ public class ValidationNotEqualError : ValidationFailedError protected Object expected; - public ValidationNotEqualError(byte[] expected, byte[] actual, KaitaiStream io, string srcPath) + public ValidationNotEqualError(byte[] expected, byte[] actual, IKaitaiStreamBase io, string srcPath) : base($"not equal, expected {ByteArrayToHex(expected)}, but got {ByteArrayToHex(actual)}", io, srcPath) { @@ -20,7 +20,7 @@ public ValidationNotEqualError(byte[] expected, byte[] actual, KaitaiStream io, this.actual = actual; } - public ValidationNotEqualError(Object expected, Object actual, KaitaiStream io, string srcPath) + public ValidationNotEqualError(Object expected, Object actual, IKaitaiStreamBase io, string srcPath) : base($"not equal, expected {expected}, but got {actual}", io, srcPath) { this.expected = expected; diff --git a/Kaitai.Struct.sln.DotSettings b/Kaitai.Struct.sln.DotSettings new file mode 100644 index 0000000..72e13e3 --- /dev/null +++ b/Kaitai.Struct.sln.DotSettings @@ -0,0 +1,3 @@ + + True + True \ No newline at end of file From a91e516b96928c4ad9a504fcd659e26543f41db0 Mon Sep 17 00:00:00 2001 From: Jan Pluskal Date: Wed, 8 Jan 2020 09:07:25 +0100 Subject: [PATCH 13/34] Remove ambigous comments. --- .../Interface/IKaitaiAsyncStream.cs | 1 + .../KaitaiAsyncStream.cs | 96 -------- .../KaitaiAsyncStruct.cs | 1 - .../Interface/IKaitaiStream.cs | 1 + Kaitai.Struct.Runtime/KaitaiStream.cs | 211 ++---------------- Kaitai.Struct.Runtime/KaitaiStreamBase.cs | 37 +-- Kaitai.Struct.Runtime/KaitaiStruct.cs | 5 +- 7 files changed, 27 insertions(+), 325 deletions(-) diff --git a/Kaitai.Struct.Runtime.Async/Interface/IKaitaiAsyncStream.cs b/Kaitai.Struct.Runtime.Async/Interface/IKaitaiAsyncStream.cs index 8000ad5..9898416 100644 --- a/Kaitai.Struct.Runtime.Async/Interface/IKaitaiAsyncStream.cs +++ b/Kaitai.Struct.Runtime.Async/Interface/IKaitaiAsyncStream.cs @@ -119,6 +119,7 @@ public interface IKaitaiAsyncStream : IKaitaiStreamBase Task ReadF8leAsync(); Task ReadBitsIntAsync(int n); + Task ReadBitsIntLeAsync(int n); /// diff --git a/Kaitai.Struct.Runtime.Async/KaitaiAsyncStream.cs b/Kaitai.Struct.Runtime.Async/KaitaiAsyncStream.cs index dc4e37d..e2500d8 100644 --- a/Kaitai.Struct.Runtime.Async/KaitaiAsyncStream.cs +++ b/Kaitai.Struct.Runtime.Async/KaitaiAsyncStream.cs @@ -46,30 +46,16 @@ public KaitaiAsyncStream(byte[] bytes) : this(new MemoryStream(bytes)) #region Stream positioning - /// - /// Check if the stream position is at the end of the stream - /// public override bool IsEof => BaseStream.Position >= BaseStream.Length && _bitsLeft == 0; - /// - /// Seek to a specific position from the beginning of the stream - /// - /// The position to seek to - /// .NET Stream API does not support SeekAsync, therefore, consumer needs to define his own implementation. public virtual Task SeekAsync(long position) { BaseStream.Seek(position, SeekOrigin.Begin); return Task.CompletedTask; } - /// - /// Get the current position in the stream - /// public override long Pos => BaseStream.Position; - /// - /// Get the total length of the stream (ie. file size) - /// public override long Size => BaseStream.Length; #endregion @@ -78,52 +64,24 @@ public virtual Task SeekAsync(long position) #region Signed - /// - /// Read a signed byte from the stream - /// - /// public async Task ReadS1Async() => await AsyncBinaryReader.ReadSByteAsync(); #region Big-endian - /// - /// Read a signed short from the stream (big endian) - /// - /// public async Task ReadS2beAsync() => BitConverter.ToInt16(await ReadBytesNormalisedBigEndianAsync(2), 0); - /// - /// Read a signed int from the stream (big endian) - /// - /// public async Task ReadS4beAsync() => BitConverter.ToInt32(await ReadBytesNormalisedBigEndianAsync(4), 0); - /// - /// Read a signed long from the stream (big endian) - /// - /// public async Task ReadS8beAsync() => BitConverter.ToInt64(await ReadBytesNormalisedBigEndianAsync(8), 0); #endregion #region Little-endian - /// - /// Read a signed short from the stream (little endian) - /// - /// public async Task ReadS2leAsync() => BitConverter.ToInt16(await ReadBytesNormalisedLittleEndianAsync(2), 0); - /// - /// Read a signed int from the stream (little endian) - /// - /// public async Task ReadS4leAsync() => BitConverter.ToInt32(await ReadBytesNormalisedLittleEndianAsync(4), 0); - /// - /// Read a signed long from the stream (little endian) - /// - /// public async Task ReadS8leAsync() => BitConverter.ToInt64(await ReadBytesNormalisedLittleEndianAsync(8), 0); #endregion @@ -132,52 +90,24 @@ public virtual Task SeekAsync(long position) #region Unsigned - /// - /// Read an unsigned byte from the stream - /// - /// public async Task ReadU1Async() => await AsyncBinaryReader.ReadByteAsync(); #region Big-endian - /// - /// Read an unsigned short from the stream (big endian) - /// - /// public async Task ReadU2beAsync() => BitConverter.ToUInt16(await ReadBytesNormalisedBigEndianAsync(2), 0); - /// - /// Read an unsigned int from the stream (big endian) - /// - /// public async Task ReadU4beAsync() => BitConverter.ToUInt32(await ReadBytesNormalisedBigEndianAsync(4), 0); - /// - /// Read an unsigned long from the stream (big endian) - /// - /// public async Task ReadU8beAsync() => BitConverter.ToUInt64(await ReadBytesNormalisedBigEndianAsync(8), 0); #endregion #region Little-endian - /// - /// Read an unsigned short from the stream (little endian) - /// - /// public async Task ReadU2leAsync() => BitConverter.ToUInt16(await ReadBytesNormalisedLittleEndianAsync(2), 0); - /// - /// Read an unsigned int from the stream (little endian) - /// - /// public async Task ReadU4leAsync() => BitConverter.ToUInt32(await ReadBytesNormalisedLittleEndianAsync(4), 0); - /// - /// Read an unsigned long from the stream (little endian) - /// - /// public async Task ReadU8leAsync() => BitConverter.ToUInt64(await ReadBytesNormalisedLittleEndianAsync(8), 0); #endregion @@ -190,32 +120,16 @@ public virtual Task SeekAsync(long position) #region Big-endian - /// - /// Read a single-precision floating point value from the stream (big endian) - /// - /// public async Task ReadF4beAsync() => BitConverter.ToSingle(await ReadBytesNormalisedBigEndianAsync(4), 0); - /// - /// Read a double-precision floating point value from the stream (big endian) - /// - /// public async Task ReadF8beAsync() => BitConverter.ToDouble(await ReadBytesNormalisedBigEndianAsync(8), 0); #endregion #region Little-endian - /// - /// Read a single-precision floating point value from the stream (little endian) - /// - /// public async Task ReadF4leAsync() => BitConverter.ToSingle(await ReadBytesNormalisedLittleEndianAsync(4), 0); - /// - /// Read a double-precision floating point value from the stream (little endian) - /// - /// public async Task ReadF8leAsync() => BitConverter.ToDouble(await ReadBytesNormalisedLittleEndianAsync(8), 0); #endregion @@ -300,11 +214,6 @@ public async Task ReadBitsIntLeAsync(int n) #region Byte arrays - /// - /// Read a fixed number of bytes from the stream - /// - /// The number of bytes to read - /// public async Task ReadBytesAsync(long count) { if (count < 0 || count > Int32.MaxValue) @@ -315,11 +224,6 @@ public async Task ReadBytesAsync(long count) return bytes; } - /// - /// Read a fixed number of bytes from the stream - /// - /// The number of bytes to read - /// public async Task ReadBytesAsync(ulong count) { if (count > Int32.MaxValue) diff --git a/Kaitai.Struct.Runtime.Async/KaitaiAsyncStruct.cs b/Kaitai.Struct.Runtime.Async/KaitaiAsyncStruct.cs index 7fb9253..faf626d 100644 --- a/Kaitai.Struct.Runtime.Async/KaitaiAsyncStruct.cs +++ b/Kaitai.Struct.Runtime.Async/KaitaiAsyncStruct.cs @@ -9,7 +9,6 @@ protected KaitaiAsyncStruct(KaitaiAsyncStream kaitaiStream) m_io = kaitaiStream; } - // ReSharper disable once InconsistentNaming public KaitaiAsyncStream M_Io => m_io; } } \ No newline at end of file diff --git a/Kaitai.Struct.Runtime/Interface/IKaitaiStream.cs b/Kaitai.Struct.Runtime/Interface/IKaitaiStream.cs index d66a014..1f5df27 100644 --- a/Kaitai.Struct.Runtime/Interface/IKaitaiStream.cs +++ b/Kaitai.Struct.Runtime/Interface/IKaitaiStream.cs @@ -117,6 +117,7 @@ public interface IKaitaiStream : IKaitaiStreamBase double ReadF8le(); ulong ReadBitsInt(int n); + ulong ReadBitsIntLe(int n); /// diff --git a/Kaitai.Struct.Runtime/KaitaiStream.cs b/Kaitai.Struct.Runtime/KaitaiStream.cs index 3bd7964..dbb5263 100644 --- a/Kaitai.Struct.Runtime/KaitaiStream.cs +++ b/Kaitai.Struct.Runtime/KaitaiStream.cs @@ -46,38 +46,13 @@ protected BinaryReader BinaryReader #region Stream positioning - /// - /// Check if the stream position is at the end of the stream - /// - public override bool IsEof - { - get { return BaseStream.Position >= BaseStream.Length && BitsLeft == 0; } - } + public override bool IsEof => BaseStream.Position >= BaseStream.Length && BitsLeft == 0; - /// - /// Seek to a specific position from the beginning of the stream - /// - /// The position to seek to - public void Seek(long position) - { - BaseStream.Seek(position, SeekOrigin.Begin); - } + public void Seek(long position) => BaseStream.Seek(position, SeekOrigin.Begin); - /// - /// Get the current position in the stream - /// - public override long Pos - { - get { return BaseStream.Position; } - } + public override long Pos => BaseStream.Position; - /// - /// Get the total length of the stream (ie. file size) - /// - public override long Size - { - get { return BaseStream.Length; } - } + public override long Size => BaseStream.Length; #endregion @@ -85,74 +60,25 @@ public override long Size #region Signed - /// - /// Read a signed byte from the stream - /// - /// - public sbyte ReadS1() - { - return BinaryReader.ReadSByte(); - } + public sbyte ReadS1() => BinaryReader.ReadSByte(); #region Big-endian - /// - /// Read a signed short from the stream (big endian) - /// - /// - public short ReadS2be() - { - return BitConverter.ToInt16(ReadBytesNormalisedBigEndian(2), 0); - } + public short ReadS2be() => BitConverter.ToInt16(ReadBytesNormalisedBigEndian(2), 0); - /// - /// Read a signed int from the stream (big endian) - /// - /// - public int ReadS4be() - { - return BitConverter.ToInt32(ReadBytesNormalisedBigEndian(4), 0); - } + public int ReadS4be() => BitConverter.ToInt32(ReadBytesNormalisedBigEndian(4), 0); - /// - /// Read a signed long from the stream (big endian) - /// - /// - public long ReadS8be() - { - return BitConverter.ToInt64(ReadBytesNormalisedBigEndian(8), 0); - } + public long ReadS8be() => BitConverter.ToInt64(ReadBytesNormalisedBigEndian(8), 0); #endregion #region Little-endian - /// - /// Read a signed short from the stream (little endian) - /// - /// - public short ReadS2le() - { - return BitConverter.ToInt16(ReadBytesNormalisedLittleEndian(2), 0); - } + public short ReadS2le() => BitConverter.ToInt16(ReadBytesNormalisedLittleEndian(2), 0); - /// - /// Read a signed int from the stream (little endian) - /// - /// - public int ReadS4le() - { - return BitConverter.ToInt32(ReadBytesNormalisedLittleEndian(4), 0); - } + public int ReadS4le() => BitConverter.ToInt32(ReadBytesNormalisedLittleEndian(4), 0); - /// - /// Read a signed long from the stream (little endian) - /// - /// - public long ReadS8le() - { - return BitConverter.ToInt64(ReadBytesNormalisedLittleEndian(8), 0); - } + public long ReadS8le() => BitConverter.ToInt64(ReadBytesNormalisedLittleEndian(8), 0); #endregion @@ -160,74 +86,25 @@ public long ReadS8le() #region Unsigned - /// - /// Read an unsigned byte from the stream - /// - /// - public byte ReadU1() - { - return BinaryReader.ReadByte(); - } + public byte ReadU1() => BinaryReader.ReadByte(); #region Big-endian - /// - /// Read an unsigned short from the stream (big endian) - /// - /// - public ushort ReadU2be() - { - return BitConverter.ToUInt16(ReadBytesNormalisedBigEndian(2), 0); - } + public ushort ReadU2be() => BitConverter.ToUInt16(ReadBytesNormalisedBigEndian(2), 0); - /// - /// Read an unsigned int from the stream (big endian) - /// - /// - public uint ReadU4be() - { - return BitConverter.ToUInt32(ReadBytesNormalisedBigEndian(4), 0); - } + public uint ReadU4be() => BitConverter.ToUInt32(ReadBytesNormalisedBigEndian(4), 0); - /// - /// Read an unsigned long from the stream (big endian) - /// - /// - public ulong ReadU8be() - { - return BitConverter.ToUInt64(ReadBytesNormalisedBigEndian(8), 0); - } + public ulong ReadU8be() => BitConverter.ToUInt64(ReadBytesNormalisedBigEndian(8), 0); #endregion #region Little-endian - /// - /// Read an unsigned short from the stream (little endian) - /// - /// - public ushort ReadU2le() - { - return BitConverter.ToUInt16(ReadBytesNormalisedLittleEndian(2), 0); - } + public ushort ReadU2le() => BitConverter.ToUInt16(ReadBytesNormalisedLittleEndian(2), 0); - /// - /// Read an unsigned int from the stream (little endian) - /// - /// - public uint ReadU4le() - { - return BitConverter.ToUInt32(ReadBytesNormalisedLittleEndian(4), 0); - } + public uint ReadU4le() => BitConverter.ToUInt32(ReadBytesNormalisedLittleEndian(4), 0); - /// - /// Read an unsigned long from the stream (little endian) - /// - /// - public ulong ReadU8le() - { - return BitConverter.ToUInt64(ReadBytesNormalisedLittleEndian(8), 0); - } + public ulong ReadU8le() => BitConverter.ToUInt64(ReadBytesNormalisedLittleEndian(8), 0); #endregion @@ -239,45 +116,17 @@ public ulong ReadU8le() #region Big-endian - /// - /// Read a single-precision floating point value from the stream (big endian) - /// - /// - public float ReadF4be() - { - return BitConverter.ToSingle(ReadBytesNormalisedBigEndian(4), 0); - } + public float ReadF4be() => BitConverter.ToSingle(ReadBytesNormalisedBigEndian(4), 0); - /// - /// Read a double-precision floating point value from the stream (big endian) - /// - /// - public double ReadF8be() - { - return BitConverter.ToDouble(ReadBytesNormalisedBigEndian(8), 0); - } + public double ReadF8be() => BitConverter.ToDouble(ReadBytesNormalisedBigEndian(8), 0); #endregion #region Little-endian - /// - /// Read a single-precision floating point value from the stream (little endian) - /// - /// - public float ReadF4le() - { - return BitConverter.ToSingle(ReadBytesNormalisedLittleEndian(4), 0); - } + public float ReadF4le() => BitConverter.ToSingle(ReadBytesNormalisedLittleEndian(4), 0); - /// - /// Read a double-precision floating point value from the stream (little endian) - /// - /// - public double ReadF8le() - { - return BitConverter.ToDouble(ReadBytesNormalisedLittleEndian(8), 0); - } + public double ReadF8le() => BitConverter.ToDouble(ReadBytesNormalisedLittleEndian(8), 0); #endregion @@ -313,7 +162,7 @@ public ulong ReadBitsInt(int n) ulong mask = GetMaskOnes(n); // shift mask to align with highest bits available in "bits" int shiftBits = BitsLeft - n; - mask = mask << shiftBits; + mask <<= shiftBits; // derive reading result ulong res = (Bits & mask) >> shiftBits; // clear top bits that we've just read => AND with 1s @@ -361,11 +210,6 @@ public ulong ReadBitsIntLe(int n) #region Byte arrays - /// - /// Read a fixed number of bytes from the stream - /// - /// The number of bytes to read - /// public byte[] ReadBytes(long count) { if (count < 0 || count > Int32.MaxValue) @@ -376,11 +220,6 @@ public byte[] ReadBytes(long count) return bytes; } - /// - /// Read a fixed number of bytes from the stream - /// - /// The number of bytes to read - /// public byte[] ReadBytes(ulong count) { if (count > Int32.MaxValue) @@ -415,10 +254,6 @@ protected byte[] ReadBytesNormalisedBigEndian(int count) return bytes; } - /// - /// Read all the remaining bytes from the stream until the end is reached - /// - /// public byte[] ReadBytesFull() { return ReadBytes(BaseStream.Length - BaseStream.Position); diff --git a/Kaitai.Struct.Runtime/KaitaiStreamBase.cs b/Kaitai.Struct.Runtime/KaitaiStreamBase.cs index df3748b..c6487c3 100644 --- a/Kaitai.Struct.Runtime/KaitaiStreamBase.cs +++ b/Kaitai.Struct.Runtime/KaitaiStreamBase.cs @@ -8,19 +8,10 @@ public abstract class KaitaiStreamBase : IKaitaiStreamBase { protected static readonly bool IsLittleEndian = BitConverter.IsLittleEndian; - /// - /// Check if the stream position is at the end of the stream - /// public abstract bool IsEof { get; } - /// - /// Get the current position in the stream - /// public abstract long Pos { get; } - - /// - /// Get the total length of the stream (ie. file size) - /// + public abstract long Size { get; } public abstract void AlignToByte(); @@ -52,12 +43,6 @@ public static byte[] BytesTerminate(byte[] src, byte terminator, bool includeTer return dst; } - /// - /// Performs XOR processing with given data, XORing every byte of the input with a single value. - /// - /// The data toe process - /// The key value to XOR with - /// Processed data public byte[] ProcessXor(byte[] value, int key) { byte[] result = new byte[value.Length]; @@ -69,13 +54,6 @@ public byte[] ProcessXor(byte[] value, int key) return result; } - /// - /// Performs XOR processing with given data, XORing every byte of the input with a key - /// array, repeating from the beginning of the key array if necessary - /// - /// The data toe process - /// The key array to XOR with - /// Processed data public byte[] ProcessXor(byte[] value, byte[] key) { int keyLen = key.Length; @@ -88,14 +66,6 @@ public byte[] ProcessXor(byte[] value, byte[] key) return result; } - /// - /// Performs a circular left rotation shift for a given buffer by a given amount of bits. - /// Pass a negative amount to rotate right. - /// - /// The data to rotate - /// The number of bytes to rotate by - /// - /// public byte[] ProcessRotateLeft(byte[] data, int amount, int groupSize) { if (amount > 7 || amount < -7) @@ -121,11 +91,6 @@ public byte[] ProcessRotateLeft(byte[] data, int amount, int groupSize) return r; } - /// - /// Inflates a deflated zlib byte stream - /// - /// The data to deflate - /// The deflated result public byte[] ProcessZlib(byte[] data) { // See RFC 1950 (https://tools.ietf.org/html/rfc1950) diff --git a/Kaitai.Struct.Runtime/KaitaiStruct.cs b/Kaitai.Struct.Runtime/KaitaiStruct.cs index 4895751..9cdff29 100644 --- a/Kaitai.Struct.Runtime/KaitaiStruct.cs +++ b/Kaitai.Struct.Runtime/KaitaiStruct.cs @@ -11,10 +11,7 @@ protected KaitaiStruct(KaitaiStream io) public KaitaiStream M_Io { - get - { - return m_io; - } + get => m_io; } } } From 9812fe778ea420f87baf2e6eb792ca451a5ab535 Mon Sep 17 00:00:00 2001 From: Jan Pluskal Date: Wed, 8 Jan 2020 18:24:43 +0100 Subject: [PATCH 14/34] Add test for Kaitai.Struct.Async runtime. --- .../Data/BitsData.cs | 166 ++++++++++++++ .../Data/DecimalData.cs | 31 +++ .../Data/IntegralData.cs | 82 +++++++ .../Kaitai.Struct.Runtime.Async.Tests.csproj | 4 + .../KaitaiAsyncStreamBaseTests.cs | 97 ++++++++ .../KaitaiAsyncStructTests.cs | 28 +++ Kaitai.Struct.Runtime.Async.Tests/ReadBits.cs | 30 +++ .../ReadBytesAsync.cs | 208 ++++++++++++++++++ .../ReadDecimal.cs | 46 ++++ .../ReadSigned.cs | 73 ++++++ .../ReadUnSigned.cs | 73 ++++++ .../KaitaiAsyncStream.cs | 11 +- Kaitai.Struct.sln.DotSettings | 1 + 13 files changed, 843 insertions(+), 7 deletions(-) create mode 100644 Kaitai.Struct.Runtime.Async.Tests/Data/BitsData.cs create mode 100644 Kaitai.Struct.Runtime.Async.Tests/Data/DecimalData.cs create mode 100644 Kaitai.Struct.Runtime.Async.Tests/Data/IntegralData.cs create mode 100644 Kaitai.Struct.Runtime.Async.Tests/KaitaiAsyncStreamBaseTests.cs create mode 100644 Kaitai.Struct.Runtime.Async.Tests/KaitaiAsyncStructTests.cs create mode 100644 Kaitai.Struct.Runtime.Async.Tests/ReadBits.cs create mode 100644 Kaitai.Struct.Runtime.Async.Tests/ReadBytesAsync.cs create mode 100644 Kaitai.Struct.Runtime.Async.Tests/ReadDecimal.cs create mode 100644 Kaitai.Struct.Runtime.Async.Tests/ReadSigned.cs create mode 100644 Kaitai.Struct.Runtime.Async.Tests/ReadUnSigned.cs diff --git a/Kaitai.Struct.Runtime.Async.Tests/Data/BitsData.cs b/Kaitai.Struct.Runtime.Async.Tests/Data/BitsData.cs new file mode 100644 index 0000000..2f26fde --- /dev/null +++ b/Kaitai.Struct.Runtime.Async.Tests/Data/BitsData.cs @@ -0,0 +1,166 @@ +using System.Collections.Generic; +using System.Linq; + +namespace Kaitai.Struct.Runtime.Async.Tests +{ + public class BitsData + { + public static IEnumerable BitsBeData => + new List<(ulong expected, byte[] streamContent, int bitsCount)> + { + (0b_0000_0000, new byte[] {0x00}, 0), + (0b_0000_0000, new byte[] {0x00}, 1), + (0b_0000_0001, new byte[] {0b_1000_0000}, 1), + (0b_0000_0001, new byte[] {0b_1100_0000}, 1), + (0b_0000_0011, new byte[] {0b_1100_0000}, 2), + (0b_0000_0111, new byte[] {0b_1110_0000}, 3), + (0b_0000_1111, new byte[] {0b_1111_0000}, 4), + (0b_0001_1111, new byte[] {0b_1111_1000}, 5), + (0b_0011_1111, new byte[] {0b_1111_1100}, 6), + (0b_0111_1111, new byte[] {0b_1111_1110}, 7), + (0b_1111_1111, new byte[] {0b_1111_1111}, 8), + + (0b_0000_0001_1111_1111, new byte[] {0b_1111_1111, 0b_1000_0000}, 9), + (0b_0000_0011_1111_1111, new byte[] {0b_1111_1111, 0b_1100_0000}, 10), + (0b_0000_0111_1111_1111, new byte[] {0b_1111_1111, 0b_1110_0000}, 11), + (0b_0000_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_0000}, 12), + (0b_0001_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1000}, 13), + (0b_0011_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1100}, 14), + (0b_0111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1110}, 15), + (0b_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111}, 16), + + (0b_0000_0001_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111, 0b_1000_0000}, 17), + (0b_0000_0011_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111, 0b_1100_0000}, 18), + (0b_0000_0111_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111, 0b_1110_0000}, 19), + (0b_0000_1111_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111, 0b_1111_0000}, 20), + (0b_0001_1111_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111, 0b_1111_1000}, 21), + (0b_0011_1111_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111, 0b_1111_1100}, 22), + (0b_0111_1111_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111, 0b_1111_1110}, 23), + (0b_1111_1111_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111, 0b_1111_1111}, 24), + + (0b_0000_0001_1111_1111_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1000_0000}, 25), + (0b_0000_0011_1111_1111_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1100_0000}, 26), + (0b_0000_0111_1111_1111_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1110_0000}, 27), + (0b_0000_1111_1111_1111_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_0000}, 28), + (0b_0001_1111_1111_1111_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1000}, 29), + (0b_0011_1111_1111_1111_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1100}, 30), + (0b_0111_1111_1111_1111_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1110}, 31), + (0b_1111_1111_1111_1111_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111}, 32), + + (0b_0000_0001_1111_1111_1111_1111_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1000_0000}, 33), + (0b_0000_0011_1111_1111_1111_1111_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1100_0000}, 34), + (0b_0000_0111_1111_1111_1111_1111_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1110_0000}, 35), + (0b_0000_1111_1111_1111_1111_1111_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_0000}, 36), + (0b_0001_1111_1111_1111_1111_1111_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1000}, 37), + (0b_0011_1111_1111_1111_1111_1111_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1100}, 38), + (0b_0111_1111_1111_1111_1111_1111_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1110}, 39), + (0b_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111}, 40), + + (0b_0000_0001_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1000_0000}, 41), + (0b_0000_0011_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1100_0000}, 42), + (0b_0000_0111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1110_0000}, 43), + (0b_0000_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_0000}, 44), + (0b_0001_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1000}, 45), + (0b_0011_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1100}, 46), + (0b_0111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1110}, 47), + (0b_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111}, 48), + + (0b_0000_0001_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1000_0000}, 49), + (0b_0000_0011_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1100_0000}, 50), + (0b_0000_0111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1110_0000}, 51), + (0b_0000_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_0000}, 52), + (0b_0001_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1000}, 53), + (0b_0011_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1100}, 54), + (0b_0111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1110}, 55), + (0b_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111}, 56), + + (0b_0000_0001_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1000_0000}, 57), + (0b_0000_0011_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1100_0000}, 58), + (0b_0000_0111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1110_0000}, 59), + (0b_0000_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_0000}, 60), + (0b_0001_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1000}, 61), + (0b_0011_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1100}, 62), + (0b_0111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1110}, 63), + (0b_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111}, 64), + }.Select(t => new object[] { t.expected, t.streamContent, t.bitsCount }); + + public static IEnumerable BitsLeData => + new List<(ulong expected, byte[] streamContent, int bitsCount)> + { + (0b_0000_0000, new byte[] {0x00}, 0), + (0b_0000_0000, new byte[] {0x00}, 1), + (0b_0000_0001, new byte[] {0b_0000_0001}, 1), + (0b_0000_0001, new byte[] {0b_0000_0011}, 1), + (0b_0000_0011, new byte[] {0b_0000_0011}, 2), + (0b_0000_0111, new byte[] {0b_0000_0111}, 3), + (0b_0000_1111, new byte[] {0b_0000_1111}, 4), + (0b_0001_1111, new byte[] {0b_0001_1111}, 5), + (0b_0011_1111, new byte[] {0b_0011_1111}, 6), + (0b_0111_1111, new byte[] {0b_0111_1111}, 7), + (0b_1111_1111, new byte[] {0b_1111_1111}, 8), + + (0b_0000_0001_1111_1111, new byte[] {0b_1111_1111, 0b_0000_0001}, 9), + (0b_0000_0011_1111_1111, new byte[] {0b_1111_1111, 0b_0000_0011}, 10), + (0b_0000_0111_1111_1111, new byte[] {0b_1111_1111, 0b_0000_0111}, 11), + (0b_0000_1111_1111_1111, new byte[] {0b_1111_1111, 0b_0000_1111}, 12), + (0b_0001_1111_1111_1111, new byte[] {0b_1111_1111, 0b_0001_1111}, 13), + (0b_0011_1111_1111_1111, new byte[] {0b_1111_1111, 0b_0011_1111}, 14), + (0b_0111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_0111_1111}, 15), + (0b_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111}, 16), + + (0b_0000_0001_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111, 0b_0000_0001}, 17), + (0b_0000_0011_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111, 0b_0000_0011}, 18), + (0b_0000_0111_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111, 0b_0000_0111}, 19), + (0b_0000_1111_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111, 0b_0000_1111}, 20), + (0b_0001_1111_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111, 0b_0001_1111}, 21), + (0b_0011_1111_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111, 0b_0011_1111}, 22), + (0b_0111_1111_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111, 0b_0111_1111}, 23), + (0b_1111_1111_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111, 0b_1111_1111}, 24), + + (0b_0000_0001_1111_1111_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_0000_0001}, 25), + (0b_0000_0011_1111_1111_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_0000_0011}, 26), + (0b_0000_0111_1111_1111_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_0000_0111}, 27), + (0b_0000_1111_1111_1111_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_0000_1111}, 28), + (0b_0001_1111_1111_1111_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_0001_1111}, 29), + (0b_0011_1111_1111_1111_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_0011_1111}, 30), + (0b_0111_1111_1111_1111_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_0111_1111}, 31), + (0b_1111_1111_1111_1111_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111}, 32), + + (0b_0000_0001_1111_1111_1111_1111_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_0000_0001}, 33), + (0b_0000_0011_1111_1111_1111_1111_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_0000_0011}, 34), + (0b_0000_0111_1111_1111_1111_1111_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_0000_0111}, 35), + (0b_0000_1111_1111_1111_1111_1111_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_0000_1111}, 36), + (0b_0001_1111_1111_1111_1111_1111_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_0001_1111}, 37), + (0b_0011_1111_1111_1111_1111_1111_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_0011_1111}, 38), + (0b_0111_1111_1111_1111_1111_1111_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_0111_1111}, 39), + (0b_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111}, 40), + + (0b_0000_0001_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_0000_0001}, 41), + (0b_0000_0011_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_0000_0011}, 42), + (0b_0000_0111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_0000_0111}, 43), + (0b_0000_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_0000_1111}, 44), + (0b_0001_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_0001_1111}, 45), + (0b_0011_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_0011_1111}, 46), + (0b_0111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_0111_1111}, 47), + (0b_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111}, 48), + + (0b_0000_0001_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_0000_0001}, 49), + (0b_0000_0011_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_0000_0011}, 50), + (0b_0000_0111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_0000_0111}, 51), + (0b_0000_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_0000_1111}, 52), + (0b_0001_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_0001_1111}, 53), + (0b_0011_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_0011_1111}, 54), + (0b_0111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_0111_1111}, 55), + (0b_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111}, 56), + + (0b_0000_0001_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_0000_0001}, 57), + (0b_0000_0011_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_0000_0011}, 58), + (0b_0000_0111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_0000_0111}, 59), + (0b_0000_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_0000_1111}, 60), + (0b_0001_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_0001_1111}, 61), + (0b_0011_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_0011_1111}, 62), + (0b_0111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_0111_1111}, 63), + (0b_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111, new byte[] {0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111, 0b_1111_1111}, 64), + }.Select(t => new object[] { t.expected, t.streamContent, t.bitsCount }); + } +} \ No newline at end of file diff --git a/Kaitai.Struct.Runtime.Async.Tests/Data/DecimalData.cs b/Kaitai.Struct.Runtime.Async.Tests/Data/DecimalData.cs new file mode 100644 index 0000000..4cec9a0 --- /dev/null +++ b/Kaitai.Struct.Runtime.Async.Tests/Data/DecimalData.cs @@ -0,0 +1,31 @@ +using System; +using System.Collections.Generic; +using System.Linq; + +namespace Kaitai.Struct.Runtime.Async.Tests +{ + public class DecimalData + { + public static IEnumerable Decimal4Data => + new List<(float expected, byte[] streamContent)> + { + (0f, BitConverter.GetBytes(0f)), + (1f, BitConverter.GetBytes(1f)), + (0.1f, BitConverter.GetBytes(0.1f)), + (1.1f, BitConverter.GetBytes(1.1f)), + (float.MinValue, BitConverter.GetBytes(float.MinValue)), + (float.MaxValue, BitConverter.GetBytes(float.MaxValue)) + }.Select(t => new object[] {t.expected, t.streamContent}); + + public static IEnumerable Decimal8Data => + new List<(double expected, byte[] streamContent)> + { + (0d, BitConverter.GetBytes(0d)), + (1d, BitConverter.GetBytes(1d)), + (0.1d, BitConverter.GetBytes(0.1d)), + (1.1d, BitConverter.GetBytes(1.1d)), + (double.MinValue, BitConverter.GetBytes(double.MinValue)), + (double.MaxValue, BitConverter.GetBytes(double.MaxValue)) + }.Select(t => new object[] {t.expected, t.streamContent}); + } +} \ No newline at end of file diff --git a/Kaitai.Struct.Runtime.Async.Tests/Data/IntegralData.cs b/Kaitai.Struct.Runtime.Async.Tests/Data/IntegralData.cs new file mode 100644 index 0000000..ff5c52f --- /dev/null +++ b/Kaitai.Struct.Runtime.Async.Tests/Data/IntegralData.cs @@ -0,0 +1,82 @@ +using System.Collections.Generic; +using System.Linq; + +namespace Kaitai.Struct.Runtime.Async.Tests +{ + public class IntegralData + { + public static IEnumerable Integral1Data => + new List<(sbyte expected, byte[] streamContent)> + { + (0x00, new byte[] {0x00}), + (0x01, new byte[] {0x01}), + (0x7F, new byte[] {0x7F}), + (unchecked((sbyte) 0xFF), new byte[] {0xFF}) + }.Select(t => new object[] {t.expected, t.streamContent}); + + public static IEnumerable Integral2Data => + new List<(short expected, byte[] streamContent)> + { + (0x00, new byte[] {0x00, 0x00}), + (0x01, new byte[] {0x00, 0x01}), + (0xFF, new byte[] {0x00, 0xFF}), + (0x01_FF, new byte[] {0x01, 0xFF}), + (0x7F_FF, new byte[] {0x7F, 0xFF}), + (unchecked((short) 0xFF_FF), new byte[] {0xFF, 0xFF}) + }.Select(t => new object[] {t.expected, t.streamContent}); + + public static IEnumerable Integral4Data => + new List<(int expected, byte[] streamContent)> + { + (0x00, new byte[] {0x00, 0x00, 0x00, 0x00}), + (0x01, new byte[] {0x00, 0x00, 0x00, 0x01}), + (0xFF, new byte[] {0x00, 0x00, 0x00, 0xFF}), + (0x01_FF, new byte[] {0x00, 0x00, 0x01, 0xFF}), + (0x7F_FF, new byte[] {0x00, 0x00, 0x7F, 0xFF}), + (0xFF_FF, new byte[] {0x00, 0x00, 0xFF, 0xFF}), + (0x01_FF_FF, new byte[] {0x00, 0x01, 0xFF, 0xFF}), + (0x7F_FF_FF, new byte[] {0x00, 0x7F, 0xFF, 0xFF}), + (0xFF_FF_FF, new byte[] {0x00, 0xFF, 0xFF, 0xFF}), + (0x01_FF_FF_FF, new byte[] {0x01, 0xFF, 0xFF, 0xFF}), + (0x7F_FF_FF_FF, new byte[] {0x7F, 0xFF, 0xFF, 0xFF}), + (unchecked((int) 0xFF_FF_FF_FF), new byte[] {0xFF, 0xFF, 0xFF, 0xFF}) + }.Select(t => new object[] {t.expected, t.streamContent}); + + public static IEnumerable Integral8Data => + new List<(long expected, byte[] streamContent)> + { + (0, new byte[] {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}), + (1, new byte[] {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01}), + (0xFF, new byte[] {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xFF}), + + (0x01_FF, new byte[] {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0xFF}), + (0x7F_FF, new byte[] {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7F, 0xFF}), + (0xFF_FF, new byte[] {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xFF, 0xFF}), + + (0x01_FF_FF, new byte[] {0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0xFF, 0xFF}), + (0x7F_FF_FF, new byte[] {0x00, 0x00, 0x00, 0x00, 0x00, 0x7F, 0xFF, 0xFF}), + (0xFF_FF_FF, new byte[] {0x00, 0x00, 0x00, 0x00, 0x00, 0xFF, 0xFF, 0xFF}), + + (0x01_FF_FF_FF, new byte[] {0x00, 0x00, 0x00, 0x00, 0x01, 0xFF, 0xFF, 0xFF}), + (0x7F_FF_FF_FF, new byte[] {0x00, 0x00, 0x00, 0x00, 0x7F, 0xFF, 0xFF, 0xFF}), + (0xFF_FF_FF_FF, new byte[] {0x00, 0x00, 0x00, 0x00, 0xFF, 0xFF, 0xFF, 0xFF}), + + (0x01_FF_FF_FF_FF, new byte[] {0x00, 0x00, 0x00, 0x01, 0xFF, 0xFF, 0xFF, 0xFF}), + (0x7F_FF_FF_FF_FF, new byte[] {0x00, 0x00, 0x00, 0x7F, 0xFF, 0xFF, 0xFF, 0xFF}), + (0xFF_FF_FF_FF_FF, new byte[] {0x00, 0x00, 0x00, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}), + + (0x01_FF_FF_FF_FF_FF, new byte[] {0x00, 0x00, 0x01, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}), + (0x7F_FF_FF_FF_FF_FF, new byte[] {0x00, 0x00, 0x7F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}), + (0xFF_FF_FF_FF_FF_FF, new byte[] {0x00, 0x00, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}), + + (0x01_FF_FF_FF_FF_FF_FF, new byte[] {0x00, 0x01, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}), + (0x7F_FF_FF_FF_FF_FF_FF, new byte[] {0x00, 0x7F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}), + (0xFF_FF_FF_FF_FF_FF_FF, new byte[] {0x00, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}), + + (0x01_FF_FF_FF_FF_FF_FF_FF, new byte[] {0x01, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}), + (0x7F_FF_FF_FF_FF_FF_FF_FF, new byte[] {0x7F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}), + (unchecked((long) 0xFF_FF_FF_FF_FF_FF_FF_FF), + new byte[] {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}) + }.Select(t => new object[] {t.expected, t.streamContent}); + } +} \ No newline at end of file diff --git a/Kaitai.Struct.Runtime.Async.Tests/Kaitai.Struct.Runtime.Async.Tests.csproj b/Kaitai.Struct.Runtime.Async.Tests/Kaitai.Struct.Runtime.Async.Tests.csproj index a2352e4..847abad 100644 --- a/Kaitai.Struct.Runtime.Async.Tests/Kaitai.Struct.Runtime.Async.Tests.csproj +++ b/Kaitai.Struct.Runtime.Async.Tests/Kaitai.Struct.Runtime.Async.Tests.csproj @@ -13,4 +13,8 @@ + + + + diff --git a/Kaitai.Struct.Runtime.Async.Tests/KaitaiAsyncStreamBaseTests.cs b/Kaitai.Struct.Runtime.Async.Tests/KaitaiAsyncStreamBaseTests.cs new file mode 100644 index 0000000..3241584 --- /dev/null +++ b/Kaitai.Struct.Runtime.Async.Tests/KaitaiAsyncStreamBaseTests.cs @@ -0,0 +1,97 @@ +using System.Threading.Tasks; +using Kaitai.Async; +using Xunit; + +namespace Kaitai.Struct.Runtime.Async.Tests +{ + public class KaitaiAsyncStreamBaseTests + { + [Fact] + public async Task AlignToByte_Test() + { + //Arrange + var kaitaiStreamSUT = new KaitaiAsyncStream(new byte[]{0b_1000_0000}); + + var read = await kaitaiStreamSUT.ReadBitsIntAsync(1); + Assert.Equal(1u, read); + + //Act + kaitaiStreamSUT.AlignToByte(); + //Assert + Assert.Equal(1, kaitaiStreamSUT.Pos); + } + + [Theory] + [InlineData(true, 0, 0)] + [InlineData(false, 1, 0)] + [InlineData(false, 1, 1)] + [InlineData(false, 1, 2)] + [InlineData(false, 1, 3)] + [InlineData(false, 1, 4)] + [InlineData(false, 1, 5)] + [InlineData(false, 1, 6)] + [InlineData(false, 1, 7)] + [InlineData(true, 1, 8)] + public async Task Eof_Test(bool shouldBeEof, int streamSize, int readBitsAmount) + { + var kaitaiStreamSUT = new KaitaiAsyncStream(new byte[streamSize]); + + await kaitaiStreamSUT.ReadBitsIntAsync(readBitsAmount); + + if (shouldBeEof) + Assert.True(kaitaiStreamSUT.IsEof); + else + Assert.False(kaitaiStreamSUT.IsEof); + } + + [Theory] + [InlineData(0, 0)] + [InlineData(1, 1)] + public async Task Pos_ByRead_Test(int expectedPos, int readBitsAmount) + { + var kaitaiStreamSUT = new KaitaiAsyncStream(new byte[1]); + + await kaitaiStreamSUT.ReadBytesAsync(readBitsAmount); + + Assert.Equal(expectedPos, kaitaiStreamSUT.Pos); + } + + [Theory] + [InlineData(0, 0)] + [InlineData(1, 1)] + public async Task Pos_BySeek_Test(int expectedPos, int position) + { + var kaitaiStreamSUT = new KaitaiAsyncStream(new byte[1]); + + await kaitaiStreamSUT.SeekAsync(position); + + Assert.Equal(expectedPos, kaitaiStreamSUT.Pos); + } + + [Theory] + [InlineData(0)] + [InlineData(1)] + public void Size_Test(int streamSize) + { + var kaitaiStreamSUT = new KaitaiAsyncStream(new byte[streamSize]); + + Assert.Equal(streamSize, kaitaiStreamSUT.Size); + } + + [Fact] + public void EmptyStream_NoRead_NoSeek_IsEof_ShouldBe_True() + { + var kaitaiStreamSUT = new KaitaiAsyncStream(new byte[0]); + + Assert.True(kaitaiStreamSUT.IsEof); + } + + [Fact] + public void EmptyStream_NoRead_NoSeek_Pos_ShouldBe_0() + { + var kaitaiStreamSUT = new KaitaiAsyncStream(new byte[0]); + + Assert.Equal(0, kaitaiStreamSUT.Pos); + } + } +} \ No newline at end of file diff --git a/Kaitai.Struct.Runtime.Async.Tests/KaitaiAsyncStructTests.cs b/Kaitai.Struct.Runtime.Async.Tests/KaitaiAsyncStructTests.cs new file mode 100644 index 0000000..9803a62 --- /dev/null +++ b/Kaitai.Struct.Runtime.Async.Tests/KaitaiAsyncStructTests.cs @@ -0,0 +1,28 @@ +using System; +using System.Collections.Generic; +using System.Text; +using Kaitai.Async; +using Xunit; + +namespace Kaitai.Struct.Runtime.Async.Tests +{ + + public class KaitaiAsyncStructTests + { + [Fact] + public void M_Io_IsSet() + { + var kaitaiAsyncStream = new KaitaiAsyncStream(new byte[0]); + var kaitaiAsyncStructSUT = new FooKaitaiAsyncStruct(kaitaiAsyncStream); + + Assert.Equal(kaitaiAsyncStream, kaitaiAsyncStructSUT.M_Io); + } + + private class FooKaitaiAsyncStruct : KaitaiAsyncStruct + { + public FooKaitaiAsyncStruct(KaitaiAsyncStream kaitaiStream) : base(kaitaiStream) + { + } + } + } +} diff --git a/Kaitai.Struct.Runtime.Async.Tests/ReadBits.cs b/Kaitai.Struct.Runtime.Async.Tests/ReadBits.cs new file mode 100644 index 0000000..bb89c64 --- /dev/null +++ b/Kaitai.Struct.Runtime.Async.Tests/ReadBits.cs @@ -0,0 +1,30 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Kaitai.Async; +using Xunit; + +namespace Kaitai.Struct.Runtime.Async.Tests +{ + public class ReadBits + { + [Theory] + [MemberData(nameof(BitsData.BitsBeData), MemberType = typeof(BitsData))] + public async Task ReadBitsIntAsync_Test(ulong expected, byte[] streamContent, int bitsCount) + { + var kaitaiStreamSUT = new KaitaiAsyncStream(streamContent); + + Assert.Equal(expected, await kaitaiStreamSUT.ReadBitsIntAsync(bitsCount)); + } + + [Theory] + [MemberData(nameof(BitsData.BitsLeData), MemberType = typeof(BitsData))] + public async Task ReadBitsIntLeAsync_Test(ulong expected, byte[] streamContent, int bitsCount) + { + var kaitaiStreamSUT = new KaitaiAsyncStream(streamContent); + + Assert.Equal(expected, await kaitaiStreamSUT.ReadBitsIntLeAsync(bitsCount)); + } + } +} \ No newline at end of file diff --git a/Kaitai.Struct.Runtime.Async.Tests/ReadBytesAsync.cs b/Kaitai.Struct.Runtime.Async.Tests/ReadBytesAsync.cs new file mode 100644 index 0000000..f1fad8f --- /dev/null +++ b/Kaitai.Struct.Runtime.Async.Tests/ReadBytesAsync.cs @@ -0,0 +1,208 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Kaitai.Async; +using Xunit; + +namespace Kaitai.Struct.Runtime.Async.Tests +{ + public class ReadBytesAsync + { + public static IEnumerable BytesData => + new List<(byte[] streamContent, int bytesCount)> + { + (new byte[] {0b_1101_0101}, 0), + (new byte[] {0b_1101_0101}, 1), + (new byte[] {0b_1101_0101, 0b_1101_0101}, 1), + (new byte[] {0b_1101_0101, 0b_1101_0101}, 2), + }.Select(t => new object[] { t.streamContent, t.bytesCount }); + + + [Theory] + [MemberData(nameof(BytesData))] + public async Task ReadBytesAsync_long_Test(byte[] streamContent, long bytesCount) + { + var kaitaiStreamSUT = new KaitaiAsyncStream(streamContent); + + Assert.Equal(streamContent.Take((int)bytesCount), await kaitaiStreamSUT.ReadBytesAsync(bytesCount)); + } + + [Theory] + [MemberData(nameof(BytesData))] + public async Task ReadBytesAsync_ulong_Test(byte[] streamContent, ulong bytesCount) + { + var kaitaiStreamSUT = new KaitaiAsyncStream(streamContent); + + Assert.Equal(streamContent.Take((int)bytesCount), await kaitaiStreamSUT.ReadBytesAsync(bytesCount)); + } + + [Fact] + public async Task ReadBytesAsyncLong_NegativeInvoke_ThrowsArgumentOutOfRangeException() + { + var kaitaiStreamSUT = new KaitaiAsyncStream(new byte[0]); + + await Assert.ThrowsAsync(async () => + await kaitaiStreamSUT.ReadBytesAsync((long) -1)); + } + + [Fact] + public async Task ReadBytesAsyncLong_LargerThanInt32Invoke_ThrowsArgumentOutOfRangeException() + { + var kaitaiStreamSUT = new KaitaiAsyncStream(new byte[0]); + + await Assert.ThrowsAsync(async () => + await kaitaiStreamSUT.ReadBytesAsync((long)Int32.MaxValue+1)); + } + + [Fact] + public async Task ReadBytesAsyncLong_LargerThanBufferInvoke_ThrowsArgumentOutOfRangeException() + { + var kaitaiStreamSUT = new KaitaiAsyncStream(new byte[0]); + + await Assert.ThrowsAsync(async () => + await kaitaiStreamSUT.ReadBytesAsync((long)1)); + } + + [Fact] + public async Task ReadBytesAsyncULong_LargerThanInt32Invoke_ThrowsArgumentOutOfRangeException() + { + var kaitaiStreamSUT = new KaitaiAsyncStream(new byte[0]); + + await Assert.ThrowsAsync(async () => + await kaitaiStreamSUT.ReadBytesAsync((ulong)Int32.MaxValue + 1)); + } + + [Fact] + public async Task ReadBytesAsyncULong_LargerThanBufferInvoke_ThrowsArgumentOutOfRangeException() + { + var kaitaiStreamSUT = new KaitaiAsyncStream(new byte[0]); + + await Assert.ThrowsAsync(async () => + await kaitaiStreamSUT.ReadBytesAsync((ulong)1)); + } + + public static IEnumerable StringData => + new List + { + "", + "ABC", + "ABCDEFGHIJKLMNOPQRSTUVWXYZ", + }.Select(t => new []{ Encoding.ASCII.GetBytes(t)}); + + + [Theory] + [MemberData(nameof(StringData))] + public async Task ReadBytesFullAsync_Test(byte[] streamContent) + { + var kaitaiStreamSUT = new KaitaiAsyncStream(streamContent); + + Assert.Equal(streamContent, await kaitaiStreamSUT.ReadBytesFullAsync()); + } + + [Theory] + [MemberData(nameof(StringData))] + public async Task EnsureFixedContentsAsync_Test(byte[] streamContent) + { + var kaitaiStreamSUT = new KaitaiAsyncStream(streamContent); + + Assert.Equal(streamContent, await kaitaiStreamSUT.EnsureFixedContentsAsync(streamContent)); + } + + [Theory] + [MemberData(nameof(StringData))] + public async Task EnsureFixedContentsAsync_ThrowsIfByteIsChanged(byte[] streamContent) + { + if(streamContent.Length == 0) return; + + var kaitaiStreamSUT = new KaitaiAsyncStream(streamContent); + + var expected = streamContent.ToArray(); + expected[0] = (byte)~expected[0]; + + await Assert.ThrowsAsync(async () => await kaitaiStreamSUT.EnsureFixedContentsAsync(expected)); + } + + public static IEnumerable StringWithTerminatorsData => + new List<(string streamContent, string expected, char terminator, bool isPresent, bool shouldInclude)> + { + ("", "", '\0', false, false), + ("", "", '\0', false, true), + + ("ABC", "ABC", '\0', false, false), + ("ABC", "ABC", '\0', false, true), + + ("ABC", "", 'A', true, false), + ("ABC", "A", 'A', true, true), + + ("ABC", "A", 'B', true, false), + ("ABC", "AB", 'B', true, true), + + ("ABC", "AB", 'C', true, false), + ("ABC", "ABC", 'C', true, true), + }.Select(t => new[] { Encoding.ASCII.GetBytes(t.streamContent), Encoding.ASCII.GetBytes(t.expected), (object)(byte)t.terminator, t.isPresent, t.shouldInclude}); + + [Theory] + [MemberData(nameof(StringWithTerminatorsData))] + public async Task ReadBytesTermAsync(byte[] streamContent, byte[] expected, byte terminator, bool _, bool shouldInclude) + { + var kaitaiStreamSUT = new KaitaiAsyncStream(streamContent); + + Assert.Equal(expected, await kaitaiStreamSUT.ReadBytesTermAsync(terminator, shouldInclude, false, false)); + } + + [Theory] + [MemberData(nameof(StringWithTerminatorsData))] + public async Task ReadBytesTermAsync_ThrowsIsTerminatorNotPresent(byte[] streamContent, byte[] expected, byte terminator, bool terminatorIsPresent, bool shouldInclude) + { + var kaitaiStreamSUT = new KaitaiAsyncStream(streamContent); + + if(terminatorIsPresent) return; + + await Assert.ThrowsAsync(async()=> await kaitaiStreamSUT.ReadBytesTermAsync(terminator, shouldInclude, false, true)); + } + + [Theory] + [MemberData(nameof(StringWithTerminatorsData))] + public async Task ReadBytesTermAsync_ShouldNotConsumeTerminator(byte[] streamContent, byte[] expected, byte terminator, bool terminatorIsPresent, bool shouldInclude) + { + //Arrange + var kaitaiStreamSUT = new KaitaiAsyncStream(streamContent); + + //Act + await kaitaiStreamSUT.ReadBytesTermAsync(terminator, shouldInclude, false, false); + + //Assert + var amountToConsume = expected.Length; + if (expected.Length > 0 && shouldInclude && terminatorIsPresent) + { + amountToConsume--; + } + + Assert.Equal(amountToConsume, kaitaiStreamSUT.Pos); + } + + [Theory] + [MemberData(nameof(StringWithTerminatorsData))] + public async Task ReadBytesTermAsync_ShouldConsumeTerminator(byte[] streamContent, byte[] expected, byte terminator, bool terminatorIsPresent, bool shouldInclude) + { + //Arrange + var kaitaiStreamSUT = new KaitaiAsyncStream(streamContent); + + //Act + await kaitaiStreamSUT.ReadBytesTermAsync(terminator, shouldInclude, true, false); + + //Assert + var amountToConsume = expected.Length; + if (!shouldInclude && terminatorIsPresent) + { + amountToConsume++; + } + + Assert.Equal(amountToConsume, kaitaiStreamSUT.Pos); + } + + } +} \ No newline at end of file diff --git a/Kaitai.Struct.Runtime.Async.Tests/ReadDecimal.cs b/Kaitai.Struct.Runtime.Async.Tests/ReadDecimal.cs new file mode 100644 index 0000000..9a9487c --- /dev/null +++ b/Kaitai.Struct.Runtime.Async.Tests/ReadDecimal.cs @@ -0,0 +1,46 @@ +using System.Linq; +using System.Threading.Tasks; +using Kaitai.Async; +using Xunit; + +namespace Kaitai.Struct.Runtime.Async.Tests +{ + public class ReadDecimal + { + [Theory] + [MemberData(nameof(DecimalData.Decimal4Data), MemberType = typeof(DecimalData))] + public async Task ReadF4beAsync_Test(float expected, byte[] streamContent) + { + var kaitaiStreamSUT = new KaitaiAsyncStream(streamContent.Reverse().ToArray()); + + Assert.Equal(expected, await kaitaiStreamSUT.ReadF4beAsync()); + } + + [Theory] + [MemberData(nameof(DecimalData.Decimal4Data), MemberType = typeof(DecimalData))] + public async Task ReadF4leAsync_Test(float expected, byte[] streamContent) + { + var kaitaiStreamSUT = new KaitaiAsyncStream(streamContent); + + Assert.Equal(expected, await kaitaiStreamSUT.ReadF4leAsync()); + } + + [Theory] + [MemberData(nameof(DecimalData.Decimal8Data), MemberType = typeof(DecimalData))] + public async Task ReadF8beAsync_Test(double expected, byte[] streamContent) + { + var kaitaiStreamSUT = new KaitaiAsyncStream(streamContent.Reverse().ToArray()); + + Assert.Equal(expected, await kaitaiStreamSUT.ReadF8beAsync()); + } + + [Theory] + [MemberData(nameof(DecimalData.Decimal8Data), MemberType = typeof(DecimalData))] + public async Task ReadF8leAsync_Test(double expected, byte[] streamContent) + { + var kaitaiStreamSUT = new KaitaiAsyncStream(streamContent); + + Assert.Equal(expected, await kaitaiStreamSUT.ReadF8leAsync()); + } + } +} \ No newline at end of file diff --git a/Kaitai.Struct.Runtime.Async.Tests/ReadSigned.cs b/Kaitai.Struct.Runtime.Async.Tests/ReadSigned.cs new file mode 100644 index 0000000..0207c42 --- /dev/null +++ b/Kaitai.Struct.Runtime.Async.Tests/ReadSigned.cs @@ -0,0 +1,73 @@ +using System.Linq; +using System.Threading.Tasks; +using Kaitai.Async; +using Xunit; + +namespace Kaitai.Struct.Runtime.Async.Tests +{ + public class ReadSigned + { + [Theory] + [MemberData(nameof(IntegralData.Integral1Data), MemberType = typeof(IntegralData))] + public async Task ReadS1Async_Test(sbyte expected, byte[] streamContent) + { + var kaitaiStreamSUT = new KaitaiAsyncStream(streamContent); + + Assert.Equal(expected, await kaitaiStreamSUT.ReadS1Async()); + } + + [Theory] + [MemberData(nameof(IntegralData.Integral2Data), MemberType = typeof(IntegralData))] + public async Task ReadS2beAsync_Test(short expected, byte[] streamContent) + { + var kaitaiStreamSUT = new KaitaiAsyncStream(streamContent); + + Assert.Equal(expected, await kaitaiStreamSUT.ReadS2beAsync()); + } + + [Theory] + [MemberData(nameof(IntegralData.Integral4Data), MemberType = typeof(IntegralData))] + public async Task ReadS4beAsync_Test(int expected, byte[] streamContent) + { + var kaitaiStreamSUT = new KaitaiAsyncStream(streamContent); + + Assert.Equal(expected, await kaitaiStreamSUT.ReadS4beAsync()); + } + + [Theory] + [MemberData(nameof(IntegralData.Integral8Data), MemberType = typeof(IntegralData))] + public async Task ReadS8beAsync_Test(long expected, byte[] streamContent) + { + var kaitaiStreamSUT = new KaitaiAsyncStream(streamContent); + + Assert.Equal(expected, await kaitaiStreamSUT.ReadS8beAsync()); + } + + [Theory] + [MemberData(nameof(IntegralData.Integral2Data), MemberType = typeof(IntegralData))] + public async Task ReadS2leAsync_Test(short expected, byte[] streamContent) + { + var kaitaiStreamSUT = new KaitaiAsyncStream(streamContent.Reverse().ToArray()); + + Assert.Equal(expected, await kaitaiStreamSUT.ReadS2leAsync()); + } + + [Theory] + [MemberData(nameof(IntegralData.Integral4Data), MemberType = typeof(IntegralData))] + public async Task ReadS4leAsync_Test(int expected, byte[] streamContent) + { + var kaitaiStreamSUT = new KaitaiAsyncStream(streamContent.Reverse().ToArray()); + + Assert.Equal(expected, await kaitaiStreamSUT.ReadS4leAsync()); + } + + [Theory] + [MemberData(nameof(IntegralData.Integral8Data), MemberType = typeof(IntegralData))] + public async Task ReadS8leAsync_Test(long expected, byte[] streamContent) + { + var kaitaiStreamSUT = new KaitaiAsyncStream(streamContent.Reverse().ToArray()); + + Assert.Equal(expected, await kaitaiStreamSUT.ReadS8leAsync()); + } + } +} \ No newline at end of file diff --git a/Kaitai.Struct.Runtime.Async.Tests/ReadUnSigned.cs b/Kaitai.Struct.Runtime.Async.Tests/ReadUnSigned.cs new file mode 100644 index 0000000..9428724 --- /dev/null +++ b/Kaitai.Struct.Runtime.Async.Tests/ReadUnSigned.cs @@ -0,0 +1,73 @@ +using System.Linq; +using System.Threading.Tasks; +using Kaitai.Async; +using Xunit; + +namespace Kaitai.Struct.Runtime.Async.Tests +{ + public class ReadUnSigned + { + [Theory] + [MemberData(nameof(IntegralData.Integral1Data), MemberType = typeof(IntegralData))] + public async Task ReadU1Async_Test( /*u*/ sbyte expected, byte[] streamContent) + { + var kaitaiStreamSUT = new KaitaiAsyncStream(streamContent); + + Assert.Equal((byte) expected, await kaitaiStreamSUT.ReadU1Async()); + } + + [Theory] + [MemberData(nameof(IntegralData.Integral2Data), MemberType = typeof(IntegralData))] + public async Task ReadU2beAsync_Test( /*u*/ short expected, byte[] streamContent) + { + var kaitaiStreamSUT = new KaitaiAsyncStream(streamContent); + + Assert.Equal((ushort) expected, await kaitaiStreamSUT.ReadU2beAsync()); + } + + [Theory] + [MemberData(nameof(IntegralData.Integral4Data), MemberType = typeof(IntegralData))] + public async Task ReadU4beAsync_Test( /*u*/ int expected, byte[] streamContent) + { + var kaitaiStreamSUT = new KaitaiAsyncStream(streamContent); + + Assert.Equal((uint) expected, await kaitaiStreamSUT.ReadU4beAsync()); + } + + [Theory] + [MemberData(nameof(IntegralData.Integral8Data), MemberType = typeof(IntegralData))] + public async Task ReadU8beAsync_Test( /*u*/ long expected, byte[] streamContent) + { + var kaitaiStreamSUT = new KaitaiAsyncStream(streamContent); + + Assert.Equal((ulong) expected, await kaitaiStreamSUT.ReadU8beAsync()); + } + + [Theory] + [MemberData(nameof(IntegralData.Integral2Data), MemberType = typeof(IntegralData))] + public async Task ReadU2leAsync_Test( /*u*/ short expected, byte[] streamContent) + { + var kaitaiStreamSUT = new KaitaiAsyncStream(streamContent.Reverse().ToArray()); + + Assert.Equal((ushort) expected, await kaitaiStreamSUT.ReadU2leAsync()); + } + + [Theory] + [MemberData(nameof(IntegralData.Integral4Data), MemberType = typeof(IntegralData))] + public async Task ReadU4leAsync_Test( /*u*/ int expected, byte[] streamContent) + { + var kaitaiStreamSUT = new KaitaiAsyncStream(streamContent.Reverse().ToArray()); + + Assert.Equal((uint) expected, await kaitaiStreamSUT.ReadU4leAsync()); + } + + [Theory] + [MemberData(nameof(IntegralData.Integral8Data), MemberType = typeof(IntegralData))] + public async Task ReadU8leAsync_Test( /*u*/ long expected, byte[] streamContent) + { + var kaitaiStreamSUT = new KaitaiAsyncStream(streamContent.Reverse().ToArray()); + + Assert.Equal((ulong) expected, await kaitaiStreamSUT.ReadU8leAsync()); + } + } +} \ No newline at end of file diff --git a/Kaitai.Struct.Runtime.Async/KaitaiAsyncStream.cs b/Kaitai.Struct.Runtime.Async/KaitaiAsyncStream.cs index e2500d8..9227593 100644 --- a/Kaitai.Struct.Runtime.Async/KaitaiAsyncStream.cs +++ b/Kaitai.Struct.Runtime.Async/KaitaiAsyncStream.cs @@ -13,19 +13,16 @@ public partial class KaitaiAsyncStream : KaitaiStreamBase, IKaitaiAsyncStream private ulong _bits = 0; private int _bitsLeft = 0; - private AsyncBinaryReader _asyncBinaryReader; - protected AsyncBinaryReader AsyncBinaryReader - { - get => _asyncBinaryReader ?? (_asyncBinaryReader = new AsyncBinaryReader(BaseStream)); - set => _asyncBinaryReader = value; - } + protected AsyncBinaryReader AsyncBinaryReader { get; } #region Constructors public KaitaiAsyncStream(Stream stream) { BaseStream = stream; + AsyncBinaryReader = new AsyncBinaryReader(BaseStream); + } /// @@ -306,7 +303,7 @@ public async Task EnsureFixedContentsAsync(byte[] expected) { byte[] bytes = await ReadBytesAsync(expected.Length); - if (bytes.Length != expected.Length) + if (bytes.Length != expected.Length) //TODO Is this necessary? { throw new Exception( $"Expected bytes: {Convert.ToBase64String(expected)} ({expected.Length} bytes), Instead got: {Convert.ToBase64String(bytes)} ({bytes.Length} bytes)"); diff --git a/Kaitai.Struct.sln.DotSettings b/Kaitai.Struct.sln.DotSettings index 72e13e3..2c5120e 100644 --- a/Kaitai.Struct.sln.DotSettings +++ b/Kaitai.Struct.sln.DotSettings @@ -1,3 +1,4 @@  + SUT True True \ No newline at end of file From c161b7fc4eba64bc388ad203f06809af34878f0a Mon Sep 17 00:00:00 2001 From: Jan Pluskal Date: Wed, 22 Jan 2020 18:25:06 +0000 Subject: [PATCH 15/34] Pipe reader --- .../KaitaiAsyncStreamBaseTests.cs | 196 ++++--- .../KaitaiAsyncStructTests.cs | 48 +- Kaitai.Struct.Runtime.Async.Tests/ReadBits.cs | 30 -- .../ReadBitsAsyncTests.cs | 41 ++ .../ReadBytesAsync.cs | 208 ------- .../ReadBytesAsyncTests.cs | 247 +++++++++ .../ReadDecimal.cs | 46 -- .../ReadDecimalAsyncTests.cs | 60 +++ .../ReadSigned.cs | 73 --- .../ReadSignedAsyncTests.cs | 87 +++ .../ReadUnSigned.cs | 73 --- .../ReadUnSignedAsyncTests.cs | 87 +++ .../Interface/IReaderContext.cs | 15 + .../Kaitai.Struct.Runtime.Async.csproj | 1 + ...ai.Struct.Runtime.Async.csproj.DotSettings | 2 + .../KaitaiAsyncStream.cs | 508 +++++++++--------- .../ReaderContext/PipeReaderContext.cs | 148 +++++ .../ReaderContext/StreamReaderContext.cs | 53 ++ 18 files changed, 1143 insertions(+), 780 deletions(-) delete mode 100644 Kaitai.Struct.Runtime.Async.Tests/ReadBits.cs create mode 100644 Kaitai.Struct.Runtime.Async.Tests/ReadBitsAsyncTests.cs delete mode 100644 Kaitai.Struct.Runtime.Async.Tests/ReadBytesAsync.cs create mode 100644 Kaitai.Struct.Runtime.Async.Tests/ReadBytesAsyncTests.cs delete mode 100644 Kaitai.Struct.Runtime.Async.Tests/ReadDecimal.cs create mode 100644 Kaitai.Struct.Runtime.Async.Tests/ReadDecimalAsyncTests.cs delete mode 100644 Kaitai.Struct.Runtime.Async.Tests/ReadSigned.cs create mode 100644 Kaitai.Struct.Runtime.Async.Tests/ReadSignedAsyncTests.cs delete mode 100644 Kaitai.Struct.Runtime.Async.Tests/ReadUnSigned.cs create mode 100644 Kaitai.Struct.Runtime.Async.Tests/ReadUnSignedAsyncTests.cs create mode 100644 Kaitai.Struct.Runtime.Async/Interface/IReaderContext.cs create mode 100644 Kaitai.Struct.Runtime.Async/Kaitai.Struct.Runtime.Async.csproj.DotSettings create mode 100644 Kaitai.Struct.Runtime.Async/ReaderContext/PipeReaderContext.cs create mode 100644 Kaitai.Struct.Runtime.Async/ReaderContext/StreamReaderContext.cs diff --git a/Kaitai.Struct.Runtime.Async.Tests/KaitaiAsyncStreamBaseTests.cs b/Kaitai.Struct.Runtime.Async.Tests/KaitaiAsyncStreamBaseTests.cs index 3241584..1d61599 100644 --- a/Kaitai.Struct.Runtime.Async.Tests/KaitaiAsyncStreamBaseTests.cs +++ b/Kaitai.Struct.Runtime.Async.Tests/KaitaiAsyncStreamBaseTests.cs @@ -1,97 +1,115 @@ -using System.Threading.Tasks; +using System.IO; +using System.Threading.Tasks; using Kaitai.Async; using Xunit; namespace Kaitai.Struct.Runtime.Async.Tests { - public class KaitaiAsyncStreamBaseTests + public class StreamKaitaiAsyncStreamBaseTests : KaitaiAsyncStreamBaseTests { - [Fact] - public async Task AlignToByte_Test() - { - //Arrange - var kaitaiStreamSUT = new KaitaiAsyncStream(new byte[]{0b_1000_0000}); - - var read = await kaitaiStreamSUT.ReadBitsIntAsync(1); - Assert.Equal(1u, read); - - //Act - kaitaiStreamSUT.AlignToByte(); - //Assert - Assert.Equal(1, kaitaiStreamSUT.Pos); - } - - [Theory] - [InlineData(true, 0, 0)] - [InlineData(false, 1, 0)] - [InlineData(false, 1, 1)] - [InlineData(false, 1, 2)] - [InlineData(false, 1, 3)] - [InlineData(false, 1, 4)] - [InlineData(false, 1, 5)] - [InlineData(false, 1, 6)] - [InlineData(false, 1, 7)] - [InlineData(true, 1, 8)] - public async Task Eof_Test(bool shouldBeEof, int streamSize, int readBitsAmount) - { - var kaitaiStreamSUT = new KaitaiAsyncStream(new byte[streamSize]); - - await kaitaiStreamSUT.ReadBitsIntAsync(readBitsAmount); - - if (shouldBeEof) - Assert.True(kaitaiStreamSUT.IsEof); - else - Assert.False(kaitaiStreamSUT.IsEof); - } - - [Theory] - [InlineData(0, 0)] - [InlineData(1, 1)] - public async Task Pos_ByRead_Test(int expectedPos, int readBitsAmount) - { - var kaitaiStreamSUT = new KaitaiAsyncStream(new byte[1]); - - await kaitaiStreamSUT.ReadBytesAsync(readBitsAmount); - - Assert.Equal(expectedPos, kaitaiStreamSUT.Pos); - } - - [Theory] - [InlineData(0, 0)] - [InlineData(1, 1)] - public async Task Pos_BySeek_Test(int expectedPos, int position) - { - var kaitaiStreamSUT = new KaitaiAsyncStream(new byte[1]); - - await kaitaiStreamSUT.SeekAsync(position); - - Assert.Equal(expectedPos, kaitaiStreamSUT.Pos); - } - - [Theory] - [InlineData(0)] - [InlineData(1)] - public void Size_Test(int streamSize) - { - var kaitaiStreamSUT = new KaitaiAsyncStream(new byte[streamSize]); - - Assert.Equal(streamSize, kaitaiStreamSUT.Size); - } - - [Fact] - public void EmptyStream_NoRead_NoSeek_IsEof_ShouldBe_True() - { - var kaitaiStreamSUT = new KaitaiAsyncStream(new byte[0]); - - Assert.True(kaitaiStreamSUT.IsEof); - } - - [Fact] - public void EmptyStream_NoRead_NoSeek_Pos_ShouldBe_0() - { - var kaitaiStreamSUT = new KaitaiAsyncStream(new byte[0]); - - Assert.Equal(0, kaitaiStreamSUT.Pos); - } + protected override KaitaiAsyncStream Create(byte[] data) => new KaitaiAsyncStream(data); + } + + public class PipeReaderKaitaiAsyncStreamBaseTests : KaitaiAsyncStreamBaseTests + { + protected override KaitaiAsyncStream Create(byte[] data) => + new KaitaiAsyncStream(System.IO.Pipelines.PipeReader.Create(new MemoryStream(data))); + } + + public abstract class KaitaiAsyncStreamBaseTests + { + protected abstract KaitaiAsyncStream Create(byte[] data); + + [Theory] + [InlineData(true, 0, 0)] + [InlineData(false, 1, 0)] + [InlineData(false, 1, 1)] + [InlineData(false, 1, 2)] + [InlineData(false, 1, 3)] + [InlineData(false, 1, 4)] + [InlineData(false, 1, 5)] + [InlineData(false, 1, 6)] + [InlineData(false, 1, 7)] + [InlineData(true, 1, 8)] + public async Task Eof_Test(bool shouldBeEof, int streamSize, int readBitsAmount) + { + var kaitaiStreamSUT = Create(new byte[streamSize]); + + await kaitaiStreamSUT.ReadBitsIntAsync(readBitsAmount); + + if (shouldBeEof) + { + Assert.True(kaitaiStreamSUT.IsEof); + } + else + { + Assert.False(kaitaiStreamSUT.IsEof); + } + } + + [Theory] + [InlineData(0, 0)] + [InlineData(1, 1)] + public async Task Pos_ByRead_Test(int expectedPos, int readBitsAmount) + { + var kaitaiStreamSUT = Create(new byte[1]); + + await kaitaiStreamSUT.ReadBytesAsync(readBitsAmount); + + Assert.Equal(expectedPos, kaitaiStreamSUT.Pos); + } + + [Theory] + [InlineData(0, 0)] + [InlineData(1, 1)] + public async Task Pos_BySeek_Test(int expectedPos, int position) + { + var kaitaiStreamSUT = Create(new byte[1]); + + await kaitaiStreamSUT.SeekAsync(position); + + Assert.Equal(expectedPos, kaitaiStreamSUT.Pos); + } + + [Theory] + [InlineData(0)] + [InlineData(1)] + public void Size_Test(int streamSize) + { + var kaitaiStreamSUT = Create(new byte[streamSize]); + + Assert.Equal(streamSize, kaitaiStreamSUT.Size); + } + + [Fact] + public async Task AlignToByte_Test() + { + //Arrange + var kaitaiStreamSUT = Create(new byte[] {0b_1000_0000}); + + ulong read = await kaitaiStreamSUT.ReadBitsIntAsync(1); + Assert.Equal(1u, read); + + //Act + kaitaiStreamSUT.AlignToByte(); + //Assert + Assert.Equal(1, kaitaiStreamSUT.Pos); + } + + [Fact] + public void EmptyStream_NoRead_NoSeek_IsEof_ShouldBe_True() + { + var kaitaiStreamSUT = Create(new byte[0]); + + Assert.True(kaitaiStreamSUT.IsEof); + } + + [Fact] + public void EmptyStream_NoRead_NoSeek_Pos_ShouldBe_0() + { + var kaitaiStreamSUT = Create(new byte[0]); + + Assert.Equal(0, kaitaiStreamSUT.Pos); } + } } \ No newline at end of file diff --git a/Kaitai.Struct.Runtime.Async.Tests/KaitaiAsyncStructTests.cs b/Kaitai.Struct.Runtime.Async.Tests/KaitaiAsyncStructTests.cs index 9803a62..f64dc20 100644 --- a/Kaitai.Struct.Runtime.Async.Tests/KaitaiAsyncStructTests.cs +++ b/Kaitai.Struct.Runtime.Async.Tests/KaitaiAsyncStructTests.cs @@ -1,28 +1,38 @@ -using System; -using System.Collections.Generic; -using System.Text; +using System.IO; using Kaitai.Async; using Xunit; namespace Kaitai.Struct.Runtime.Async.Tests { - - public class KaitaiAsyncStructTests + public class StreamKaitaiAsyncStructTests : KaitaiAsyncStructTests + { + protected override KaitaiAsyncStream Create(byte[] data) => new KaitaiAsyncStream(data); + } + + public class PipeReaderKaitaiAsyncStructTests : KaitaiAsyncStructTests + { + protected override KaitaiAsyncStream Create(byte[] data) => + new KaitaiAsyncStream(System.IO.Pipelines.PipeReader.Create(new MemoryStream(data))); + } + + public abstract class KaitaiAsyncStructTests + { + protected abstract KaitaiAsyncStream Create(byte[] data); + + private class FooKaitaiAsyncStruct : KaitaiAsyncStruct { - [Fact] - public void M_Io_IsSet() - { - var kaitaiAsyncStream = new KaitaiAsyncStream(new byte[0]); - var kaitaiAsyncStructSUT = new FooKaitaiAsyncStruct(kaitaiAsyncStream); + public FooKaitaiAsyncStruct(KaitaiAsyncStream kaitaiStream) : base(kaitaiStream) + { + } + } - Assert.Equal(kaitaiAsyncStream, kaitaiAsyncStructSUT.M_Io); - } + [Fact] + public void M_Io_IsSet() + { + var kaitaiAsyncStream = Create(new byte[0]); + var kaitaiAsyncStructSUT = new FooKaitaiAsyncStruct(kaitaiAsyncStream); - private class FooKaitaiAsyncStruct : KaitaiAsyncStruct - { - public FooKaitaiAsyncStruct(KaitaiAsyncStream kaitaiStream) : base(kaitaiStream) - { - } - } + Assert.Equal(kaitaiAsyncStream, kaitaiAsyncStructSUT.M_Io); } -} + } +} \ No newline at end of file diff --git a/Kaitai.Struct.Runtime.Async.Tests/ReadBits.cs b/Kaitai.Struct.Runtime.Async.Tests/ReadBits.cs deleted file mode 100644 index bb89c64..0000000 --- a/Kaitai.Struct.Runtime.Async.Tests/ReadBits.cs +++ /dev/null @@ -1,30 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading.Tasks; -using Kaitai.Async; -using Xunit; - -namespace Kaitai.Struct.Runtime.Async.Tests -{ - public class ReadBits - { - [Theory] - [MemberData(nameof(BitsData.BitsBeData), MemberType = typeof(BitsData))] - public async Task ReadBitsIntAsync_Test(ulong expected, byte[] streamContent, int bitsCount) - { - var kaitaiStreamSUT = new KaitaiAsyncStream(streamContent); - - Assert.Equal(expected, await kaitaiStreamSUT.ReadBitsIntAsync(bitsCount)); - } - - [Theory] - [MemberData(nameof(BitsData.BitsLeData), MemberType = typeof(BitsData))] - public async Task ReadBitsIntLeAsync_Test(ulong expected, byte[] streamContent, int bitsCount) - { - var kaitaiStreamSUT = new KaitaiAsyncStream(streamContent); - - Assert.Equal(expected, await kaitaiStreamSUT.ReadBitsIntLeAsync(bitsCount)); - } - } -} \ No newline at end of file diff --git a/Kaitai.Struct.Runtime.Async.Tests/ReadBitsAsyncTests.cs b/Kaitai.Struct.Runtime.Async.Tests/ReadBitsAsyncTests.cs new file mode 100644 index 0000000..baa4b56 --- /dev/null +++ b/Kaitai.Struct.Runtime.Async.Tests/ReadBitsAsyncTests.cs @@ -0,0 +1,41 @@ +using System.IO; +using System.Threading.Tasks; +using Kaitai.Async; +using Xunit; + +namespace Kaitai.Struct.Runtime.Async.Tests +{ + public class StreamKaitaiReadBitsAsyncTests : ReadBitsAsyncTests + { + protected override KaitaiAsyncStream Create(byte[] data) => new KaitaiAsyncStream(data); + } + + public class PipeReaderReadBitsAsyncTests : ReadBitsAsyncTests + { + protected override KaitaiAsyncStream Create(byte[] data) => + new KaitaiAsyncStream(System.IO.Pipelines.PipeReader.Create(new MemoryStream(data))); + } + + public abstract class ReadBitsAsyncTests + { + protected abstract KaitaiAsyncStream Create(byte[] data); + + [Theory] + [MemberData(nameof(BitsData.BitsBeData), MemberType = typeof(BitsData))] + public async Task ReadBitsIntAsync_Test(ulong expected, byte[] streamContent, int bitsCount) + { + var kaitaiStreamSUT = Create(streamContent); + + Assert.Equal(expected, await kaitaiStreamSUT.ReadBitsIntAsync(bitsCount)); + } + + [Theory] + [MemberData(nameof(BitsData.BitsLeData), MemberType = typeof(BitsData))] + public async Task ReadBitsIntLeAsync_Test(ulong expected, byte[] streamContent, int bitsCount) + { + var kaitaiStreamSUT = Create(streamContent); + + Assert.Equal(expected, await kaitaiStreamSUT.ReadBitsIntLeAsync(bitsCount)); + } + } +} \ No newline at end of file diff --git a/Kaitai.Struct.Runtime.Async.Tests/ReadBytesAsync.cs b/Kaitai.Struct.Runtime.Async.Tests/ReadBytesAsync.cs deleted file mode 100644 index f1fad8f..0000000 --- a/Kaitai.Struct.Runtime.Async.Tests/ReadBytesAsync.cs +++ /dev/null @@ -1,208 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using Kaitai.Async; -using Xunit; - -namespace Kaitai.Struct.Runtime.Async.Tests -{ - public class ReadBytesAsync - { - public static IEnumerable BytesData => - new List<(byte[] streamContent, int bytesCount)> - { - (new byte[] {0b_1101_0101}, 0), - (new byte[] {0b_1101_0101}, 1), - (new byte[] {0b_1101_0101, 0b_1101_0101}, 1), - (new byte[] {0b_1101_0101, 0b_1101_0101}, 2), - }.Select(t => new object[] { t.streamContent, t.bytesCount }); - - - [Theory] - [MemberData(nameof(BytesData))] - public async Task ReadBytesAsync_long_Test(byte[] streamContent, long bytesCount) - { - var kaitaiStreamSUT = new KaitaiAsyncStream(streamContent); - - Assert.Equal(streamContent.Take((int)bytesCount), await kaitaiStreamSUT.ReadBytesAsync(bytesCount)); - } - - [Theory] - [MemberData(nameof(BytesData))] - public async Task ReadBytesAsync_ulong_Test(byte[] streamContent, ulong bytesCount) - { - var kaitaiStreamSUT = new KaitaiAsyncStream(streamContent); - - Assert.Equal(streamContent.Take((int)bytesCount), await kaitaiStreamSUT.ReadBytesAsync(bytesCount)); - } - - [Fact] - public async Task ReadBytesAsyncLong_NegativeInvoke_ThrowsArgumentOutOfRangeException() - { - var kaitaiStreamSUT = new KaitaiAsyncStream(new byte[0]); - - await Assert.ThrowsAsync(async () => - await kaitaiStreamSUT.ReadBytesAsync((long) -1)); - } - - [Fact] - public async Task ReadBytesAsyncLong_LargerThanInt32Invoke_ThrowsArgumentOutOfRangeException() - { - var kaitaiStreamSUT = new KaitaiAsyncStream(new byte[0]); - - await Assert.ThrowsAsync(async () => - await kaitaiStreamSUT.ReadBytesAsync((long)Int32.MaxValue+1)); - } - - [Fact] - public async Task ReadBytesAsyncLong_LargerThanBufferInvoke_ThrowsArgumentOutOfRangeException() - { - var kaitaiStreamSUT = new KaitaiAsyncStream(new byte[0]); - - await Assert.ThrowsAsync(async () => - await kaitaiStreamSUT.ReadBytesAsync((long)1)); - } - - [Fact] - public async Task ReadBytesAsyncULong_LargerThanInt32Invoke_ThrowsArgumentOutOfRangeException() - { - var kaitaiStreamSUT = new KaitaiAsyncStream(new byte[0]); - - await Assert.ThrowsAsync(async () => - await kaitaiStreamSUT.ReadBytesAsync((ulong)Int32.MaxValue + 1)); - } - - [Fact] - public async Task ReadBytesAsyncULong_LargerThanBufferInvoke_ThrowsArgumentOutOfRangeException() - { - var kaitaiStreamSUT = new KaitaiAsyncStream(new byte[0]); - - await Assert.ThrowsAsync(async () => - await kaitaiStreamSUT.ReadBytesAsync((ulong)1)); - } - - public static IEnumerable StringData => - new List - { - "", - "ABC", - "ABCDEFGHIJKLMNOPQRSTUVWXYZ", - }.Select(t => new []{ Encoding.ASCII.GetBytes(t)}); - - - [Theory] - [MemberData(nameof(StringData))] - public async Task ReadBytesFullAsync_Test(byte[] streamContent) - { - var kaitaiStreamSUT = new KaitaiAsyncStream(streamContent); - - Assert.Equal(streamContent, await kaitaiStreamSUT.ReadBytesFullAsync()); - } - - [Theory] - [MemberData(nameof(StringData))] - public async Task EnsureFixedContentsAsync_Test(byte[] streamContent) - { - var kaitaiStreamSUT = new KaitaiAsyncStream(streamContent); - - Assert.Equal(streamContent, await kaitaiStreamSUT.EnsureFixedContentsAsync(streamContent)); - } - - [Theory] - [MemberData(nameof(StringData))] - public async Task EnsureFixedContentsAsync_ThrowsIfByteIsChanged(byte[] streamContent) - { - if(streamContent.Length == 0) return; - - var kaitaiStreamSUT = new KaitaiAsyncStream(streamContent); - - var expected = streamContent.ToArray(); - expected[0] = (byte)~expected[0]; - - await Assert.ThrowsAsync(async () => await kaitaiStreamSUT.EnsureFixedContentsAsync(expected)); - } - - public static IEnumerable StringWithTerminatorsData => - new List<(string streamContent, string expected, char terminator, bool isPresent, bool shouldInclude)> - { - ("", "", '\0', false, false), - ("", "", '\0', false, true), - - ("ABC", "ABC", '\0', false, false), - ("ABC", "ABC", '\0', false, true), - - ("ABC", "", 'A', true, false), - ("ABC", "A", 'A', true, true), - - ("ABC", "A", 'B', true, false), - ("ABC", "AB", 'B', true, true), - - ("ABC", "AB", 'C', true, false), - ("ABC", "ABC", 'C', true, true), - }.Select(t => new[] { Encoding.ASCII.GetBytes(t.streamContent), Encoding.ASCII.GetBytes(t.expected), (object)(byte)t.terminator, t.isPresent, t.shouldInclude}); - - [Theory] - [MemberData(nameof(StringWithTerminatorsData))] - public async Task ReadBytesTermAsync(byte[] streamContent, byte[] expected, byte terminator, bool _, bool shouldInclude) - { - var kaitaiStreamSUT = new KaitaiAsyncStream(streamContent); - - Assert.Equal(expected, await kaitaiStreamSUT.ReadBytesTermAsync(terminator, shouldInclude, false, false)); - } - - [Theory] - [MemberData(nameof(StringWithTerminatorsData))] - public async Task ReadBytesTermAsync_ThrowsIsTerminatorNotPresent(byte[] streamContent, byte[] expected, byte terminator, bool terminatorIsPresent, bool shouldInclude) - { - var kaitaiStreamSUT = new KaitaiAsyncStream(streamContent); - - if(terminatorIsPresent) return; - - await Assert.ThrowsAsync(async()=> await kaitaiStreamSUT.ReadBytesTermAsync(terminator, shouldInclude, false, true)); - } - - [Theory] - [MemberData(nameof(StringWithTerminatorsData))] - public async Task ReadBytesTermAsync_ShouldNotConsumeTerminator(byte[] streamContent, byte[] expected, byte terminator, bool terminatorIsPresent, bool shouldInclude) - { - //Arrange - var kaitaiStreamSUT = new KaitaiAsyncStream(streamContent); - - //Act - await kaitaiStreamSUT.ReadBytesTermAsync(terminator, shouldInclude, false, false); - - //Assert - var amountToConsume = expected.Length; - if (expected.Length > 0 && shouldInclude && terminatorIsPresent) - { - amountToConsume--; - } - - Assert.Equal(amountToConsume, kaitaiStreamSUT.Pos); - } - - [Theory] - [MemberData(nameof(StringWithTerminatorsData))] - public async Task ReadBytesTermAsync_ShouldConsumeTerminator(byte[] streamContent, byte[] expected, byte terminator, bool terminatorIsPresent, bool shouldInclude) - { - //Arrange - var kaitaiStreamSUT = new KaitaiAsyncStream(streamContent); - - //Act - await kaitaiStreamSUT.ReadBytesTermAsync(terminator, shouldInclude, true, false); - - //Assert - var amountToConsume = expected.Length; - if (!shouldInclude && terminatorIsPresent) - { - amountToConsume++; - } - - Assert.Equal(amountToConsume, kaitaiStreamSUT.Pos); - } - - } -} \ No newline at end of file diff --git a/Kaitai.Struct.Runtime.Async.Tests/ReadBytesAsyncTests.cs b/Kaitai.Struct.Runtime.Async.Tests/ReadBytesAsyncTests.cs new file mode 100644 index 0000000..1de942b --- /dev/null +++ b/Kaitai.Struct.Runtime.Async.Tests/ReadBytesAsyncTests.cs @@ -0,0 +1,247 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Kaitai.Async; +using Xunit; + +namespace Kaitai.Struct.Runtime.Async.Tests +{ + public class StreamReadBytesAsyncTests : ReadBytesAsyncTests + { + protected override KaitaiAsyncStream Create(byte[] data) => new KaitaiAsyncStream(data); + } + + public class PipeReaderReadBytesAsyncTests : ReadBytesAsyncTests + { + protected override KaitaiAsyncStream Create(byte[] data) => + new KaitaiAsyncStream(System.IO.Pipelines.PipeReader.Create(new MemoryStream(data))); + } + + public abstract class ReadBytesAsyncTests + { + protected abstract KaitaiAsyncStream Create(byte[] data); + + public static IEnumerable BytesData => + new List<(byte[] streamContent, int bytesCount)> + { + (new byte[] {0b_1101_0101}, 0), + (new byte[] {0b_1101_0101}, 1), + (new byte[] {0b_1101_0101, 0b_1101_0101}, 1), + (new byte[] {0b_1101_0101, 0b_1101_0101}, 2) + }.Select(t => new object[] {t.streamContent, t.bytesCount}); + + + [Theory] + [MemberData(nameof(BytesData))] + public async Task ReadBytesAsync_long_Test(byte[] streamContent, long bytesCount) + { + var kaitaiStreamSUT = Create(streamContent); + + Assert.Equal(streamContent.Take((int) bytesCount), await kaitaiStreamSUT.ReadBytesAsync(bytesCount)); + } + + [Theory] + [MemberData(nameof(BytesData))] + public async Task ReadBytesAsync_ulong_Test(byte[] streamContent, ulong bytesCount) + { + var kaitaiStreamSUT = Create(streamContent); + + Assert.Equal(streamContent.Take((int) bytesCount), await kaitaiStreamSUT.ReadBytesAsync(bytesCount)); + } + + public static IEnumerable StringData => + new List + { + "", + "ABC", + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + }.Select(t => new[] {Encoding.ASCII.GetBytes(t)}); + + + [Theory] + [MemberData(nameof(StringData))] + public async Task ReadBytesFullAsync_Test(byte[] streamContent) + { + var kaitaiStreamSUT = Create(streamContent); + + Assert.Equal(streamContent, await kaitaiStreamSUT.ReadBytesFullAsync()); + } + + [Theory] + [MemberData(nameof(StringData))] + public async Task EnsureFixedContentsAsync_Test(byte[] streamContent) + { + var kaitaiStreamSUT = Create(streamContent); + + Assert.Equal(streamContent, await kaitaiStreamSUT.EnsureFixedContentsAsync(streamContent)); + } + + [Theory] + [MemberData(nameof(StringData))] + public async Task EnsureFixedContentsAsync_ThrowsIfByteIsChanged(byte[] streamContent) + { + if (streamContent.Length == 0) + { + return; + } + + var kaitaiStreamSUT = Create(streamContent); + + var expected = streamContent.ToArray(); + expected[0] = (byte) ~expected[0]; + + await Assert.ThrowsAsync(async () => await kaitaiStreamSUT.EnsureFixedContentsAsync(expected)); + } + + public static IEnumerable StringWithTerminatorsData => + new List<(string streamContent, string expected, char terminator, bool isPresent, bool shouldInclude)> + { + ("", "", '\0', false, false), + ("", "", '\0', false, true), + + ("ABC", "ABC", '\0', false, false), + ("ABC", "ABC", '\0', false, true), + + ("ABC", "", 'A', true, false), + ("ABC", "A", 'A', true, true), + + ("ABC", "A", 'B', true, false), + ("ABC", "AB", 'B', true, true), + + ("ABC", "AB", 'C', true, false), + ("ABC", "ABC", 'C', true, true) + }.Select(t => new[] + { + Encoding.ASCII.GetBytes(t.streamContent), Encoding.ASCII.GetBytes(t.expected), (object) (byte) t.terminator, + t.isPresent, t.shouldInclude + }); + + [Theory] + [MemberData(nameof(StringWithTerminatorsData))] + public async Task ReadBytesTermAsync(byte[] streamContent, + byte[] expected, + byte terminator, + bool _, + bool shouldInclude) + { + var kaitaiStreamSUT = Create(streamContent); + + Assert.Equal(expected, await kaitaiStreamSUT.ReadBytesTermAsync(terminator, shouldInclude, false, false)); + } + + [Theory] + [MemberData(nameof(StringWithTerminatorsData))] + public async Task ReadBytesTermAsync_ThrowsIsTerminatorNotPresent(byte[] streamContent, + byte[] expected, + byte terminator, + bool terminatorIsPresent, + bool shouldInclude) + { + var kaitaiStreamSUT = Create(streamContent); + + if (terminatorIsPresent) + { + return; + } + + await Assert.ThrowsAsync(async () => + await kaitaiStreamSUT.ReadBytesTermAsync(terminator, shouldInclude, false, true)); + } + + [Theory] + [MemberData(nameof(StringWithTerminatorsData))] + public async Task ReadBytesTermAsync_ShouldNotConsumeTerminator(byte[] streamContent, + byte[] expected, + byte terminator, + bool terminatorIsPresent, + bool shouldInclude) + { + //Arrange + var kaitaiStreamSUT = Create(streamContent); + + //Act + await kaitaiStreamSUT.ReadBytesTermAsync(terminator, shouldInclude, false, false); + + //Assert + int amountToConsume = expected.Length; + if (expected.Length > 0 && shouldInclude && terminatorIsPresent) + { + amountToConsume--; + } + + Assert.Equal(amountToConsume, kaitaiStreamSUT.Pos); + } + + [Theory] + [MemberData(nameof(StringWithTerminatorsData))] + public async Task ReadBytesTermAsync_ShouldConsumeTerminator(byte[] streamContent, + byte[] expected, + byte terminator, + bool terminatorIsPresent, + bool shouldInclude) + { + //Arrange + var kaitaiStreamSUT = Create(streamContent); + + //Act + await kaitaiStreamSUT.ReadBytesTermAsync(terminator, shouldInclude, true, false); + + //Assert + int amountToConsume = expected.Length; + if (!shouldInclude && terminatorIsPresent) + { + amountToConsume++; + } + + Assert.Equal(amountToConsume, kaitaiStreamSUT.Pos); + } + + [Fact] + public async Task ReadBytesAsyncLong_LargerThanBufferInvoke_ThrowsArgumentOutOfRangeException() + { + var kaitaiStreamSUT = Create(new byte[0]); + + await Assert.ThrowsAsync(async () => + await kaitaiStreamSUT.ReadBytesAsync(1)); + } + + [Fact] + public async Task ReadBytesAsyncLong_LargerThanInt32Invoke_ThrowsArgumentOutOfRangeException() + { + var kaitaiStreamSUT = Create(new byte[0]); + + await Assert.ThrowsAsync(async () => + await kaitaiStreamSUT.ReadBytesAsync((long) int.MaxValue + 1)); + } + + [Fact] + public async Task ReadBytesAsyncLong_NegativeInvoke_ThrowsArgumentOutOfRangeException() + { + var kaitaiStreamSUT = Create(new byte[0]); + + await Assert.ThrowsAsync(async () => + await kaitaiStreamSUT.ReadBytesAsync(-1)); + } + + [Fact] + public async Task ReadBytesAsyncULong_LargerThanBufferInvoke_ThrowsArgumentOutOfRangeException() + { + var kaitaiStreamSUT = Create(new byte[0]); + + await Assert.ThrowsAsync(async () => + await kaitaiStreamSUT.ReadBytesAsync((ulong) 1)); + } + + [Fact] + public async Task ReadBytesAsyncULong_LargerThanInt32Invoke_ThrowsArgumentOutOfRangeException() + { + var kaitaiStreamSUT = Create(new byte[0]); + + await Assert.ThrowsAsync(async () => + await kaitaiStreamSUT.ReadBytesAsync((ulong) int.MaxValue + 1)); + } + } +} \ No newline at end of file diff --git a/Kaitai.Struct.Runtime.Async.Tests/ReadDecimal.cs b/Kaitai.Struct.Runtime.Async.Tests/ReadDecimal.cs deleted file mode 100644 index 9a9487c..0000000 --- a/Kaitai.Struct.Runtime.Async.Tests/ReadDecimal.cs +++ /dev/null @@ -1,46 +0,0 @@ -using System.Linq; -using System.Threading.Tasks; -using Kaitai.Async; -using Xunit; - -namespace Kaitai.Struct.Runtime.Async.Tests -{ - public class ReadDecimal - { - [Theory] - [MemberData(nameof(DecimalData.Decimal4Data), MemberType = typeof(DecimalData))] - public async Task ReadF4beAsync_Test(float expected, byte[] streamContent) - { - var kaitaiStreamSUT = new KaitaiAsyncStream(streamContent.Reverse().ToArray()); - - Assert.Equal(expected, await kaitaiStreamSUT.ReadF4beAsync()); - } - - [Theory] - [MemberData(nameof(DecimalData.Decimal4Data), MemberType = typeof(DecimalData))] - public async Task ReadF4leAsync_Test(float expected, byte[] streamContent) - { - var kaitaiStreamSUT = new KaitaiAsyncStream(streamContent); - - Assert.Equal(expected, await kaitaiStreamSUT.ReadF4leAsync()); - } - - [Theory] - [MemberData(nameof(DecimalData.Decimal8Data), MemberType = typeof(DecimalData))] - public async Task ReadF8beAsync_Test(double expected, byte[] streamContent) - { - var kaitaiStreamSUT = new KaitaiAsyncStream(streamContent.Reverse().ToArray()); - - Assert.Equal(expected, await kaitaiStreamSUT.ReadF8beAsync()); - } - - [Theory] - [MemberData(nameof(DecimalData.Decimal8Data), MemberType = typeof(DecimalData))] - public async Task ReadF8leAsync_Test(double expected, byte[] streamContent) - { - var kaitaiStreamSUT = new KaitaiAsyncStream(streamContent); - - Assert.Equal(expected, await kaitaiStreamSUT.ReadF8leAsync()); - } - } -} \ No newline at end of file diff --git a/Kaitai.Struct.Runtime.Async.Tests/ReadDecimalAsyncTests.cs b/Kaitai.Struct.Runtime.Async.Tests/ReadDecimalAsyncTests.cs new file mode 100644 index 0000000..dc4f652 --- /dev/null +++ b/Kaitai.Struct.Runtime.Async.Tests/ReadDecimalAsyncTests.cs @@ -0,0 +1,60 @@ +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using Kaitai.Async; +using Xunit; + +namespace Kaitai.Struct.Runtime.Async.Tests +{ + public class StreamReadDecimalAsyncTests : ReadDecimalAsyncTests + { + protected override KaitaiAsyncStream Create(byte[] data) => new KaitaiAsyncStream(data); + } + + public class PipeReaderReadDecimalAsyncTests : ReadDecimalAsyncTests + { + protected override KaitaiAsyncStream Create(byte[] data) => + new KaitaiAsyncStream(System.IO.Pipelines.PipeReader.Create(new MemoryStream(data))); + } + + public abstract class ReadDecimalAsyncTests + { + protected abstract KaitaiAsyncStream Create(byte[] data); + + [Theory] + [MemberData(nameof(DecimalData.Decimal4Data), MemberType = typeof(DecimalData))] + public async Task ReadF4beAsync_Test(float expected, byte[] streamContent) + { + var kaitaiStreamSUT = Create(streamContent.Reverse().ToArray()); + + Assert.Equal(expected, await kaitaiStreamSUT.ReadF4beAsync()); + } + + [Theory] + [MemberData(nameof(DecimalData.Decimal4Data), MemberType = typeof(DecimalData))] + public async Task ReadF4leAsync_Test(float expected, byte[] streamContent) + { + var kaitaiStreamSUT = Create(streamContent); + + Assert.Equal(expected, await kaitaiStreamSUT.ReadF4leAsync()); + } + + [Theory] + [MemberData(nameof(DecimalData.Decimal8Data), MemberType = typeof(DecimalData))] + public async Task ReadF8beAsync_Test(double expected, byte[] streamContent) + { + var kaitaiStreamSUT = Create(streamContent.Reverse().ToArray()); + + Assert.Equal(expected, await kaitaiStreamSUT.ReadF8beAsync()); + } + + [Theory] + [MemberData(nameof(DecimalData.Decimal8Data), MemberType = typeof(DecimalData))] + public async Task ReadF8leAsync_Test(double expected, byte[] streamContent) + { + var kaitaiStreamSUT = Create(streamContent); + + Assert.Equal(expected, await kaitaiStreamSUT.ReadF8leAsync()); + } + } +} \ No newline at end of file diff --git a/Kaitai.Struct.Runtime.Async.Tests/ReadSigned.cs b/Kaitai.Struct.Runtime.Async.Tests/ReadSigned.cs deleted file mode 100644 index 0207c42..0000000 --- a/Kaitai.Struct.Runtime.Async.Tests/ReadSigned.cs +++ /dev/null @@ -1,73 +0,0 @@ -using System.Linq; -using System.Threading.Tasks; -using Kaitai.Async; -using Xunit; - -namespace Kaitai.Struct.Runtime.Async.Tests -{ - public class ReadSigned - { - [Theory] - [MemberData(nameof(IntegralData.Integral1Data), MemberType = typeof(IntegralData))] - public async Task ReadS1Async_Test(sbyte expected, byte[] streamContent) - { - var kaitaiStreamSUT = new KaitaiAsyncStream(streamContent); - - Assert.Equal(expected, await kaitaiStreamSUT.ReadS1Async()); - } - - [Theory] - [MemberData(nameof(IntegralData.Integral2Data), MemberType = typeof(IntegralData))] - public async Task ReadS2beAsync_Test(short expected, byte[] streamContent) - { - var kaitaiStreamSUT = new KaitaiAsyncStream(streamContent); - - Assert.Equal(expected, await kaitaiStreamSUT.ReadS2beAsync()); - } - - [Theory] - [MemberData(nameof(IntegralData.Integral4Data), MemberType = typeof(IntegralData))] - public async Task ReadS4beAsync_Test(int expected, byte[] streamContent) - { - var kaitaiStreamSUT = new KaitaiAsyncStream(streamContent); - - Assert.Equal(expected, await kaitaiStreamSUT.ReadS4beAsync()); - } - - [Theory] - [MemberData(nameof(IntegralData.Integral8Data), MemberType = typeof(IntegralData))] - public async Task ReadS8beAsync_Test(long expected, byte[] streamContent) - { - var kaitaiStreamSUT = new KaitaiAsyncStream(streamContent); - - Assert.Equal(expected, await kaitaiStreamSUT.ReadS8beAsync()); - } - - [Theory] - [MemberData(nameof(IntegralData.Integral2Data), MemberType = typeof(IntegralData))] - public async Task ReadS2leAsync_Test(short expected, byte[] streamContent) - { - var kaitaiStreamSUT = new KaitaiAsyncStream(streamContent.Reverse().ToArray()); - - Assert.Equal(expected, await kaitaiStreamSUT.ReadS2leAsync()); - } - - [Theory] - [MemberData(nameof(IntegralData.Integral4Data), MemberType = typeof(IntegralData))] - public async Task ReadS4leAsync_Test(int expected, byte[] streamContent) - { - var kaitaiStreamSUT = new KaitaiAsyncStream(streamContent.Reverse().ToArray()); - - Assert.Equal(expected, await kaitaiStreamSUT.ReadS4leAsync()); - } - - [Theory] - [MemberData(nameof(IntegralData.Integral8Data), MemberType = typeof(IntegralData))] - public async Task ReadS8leAsync_Test(long expected, byte[] streamContent) - { - var kaitaiStreamSUT = new KaitaiAsyncStream(streamContent.Reverse().ToArray()); - - Assert.Equal(expected, await kaitaiStreamSUT.ReadS8leAsync()); - } - } -} \ No newline at end of file diff --git a/Kaitai.Struct.Runtime.Async.Tests/ReadSignedAsyncTests.cs b/Kaitai.Struct.Runtime.Async.Tests/ReadSignedAsyncTests.cs new file mode 100644 index 0000000..64ba08b --- /dev/null +++ b/Kaitai.Struct.Runtime.Async.Tests/ReadSignedAsyncTests.cs @@ -0,0 +1,87 @@ +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using Kaitai.Async; +using Xunit; + +namespace Kaitai.Struct.Runtime.Async.Tests +{ + public class StreamReadSignedAsyncTests : ReadSignedAsyncTests + { + protected override KaitaiAsyncStream Create(byte[] data) => new KaitaiAsyncStream(data); + } + + public class PipeReaderReadSignedAsyncTests : ReadSignedAsyncTests + { + protected override KaitaiAsyncStream Create(byte[] data) => + new KaitaiAsyncStream(System.IO.Pipelines.PipeReader.Create(new MemoryStream(data))); + } + + public abstract class ReadSignedAsyncTests + { + protected abstract KaitaiAsyncStream Create(byte[] data); + + [Theory] + [MemberData(nameof(IntegralData.Integral1Data), MemberType = typeof(IntegralData))] + public async Task ReadS1Async_Test(sbyte expected, byte[] streamContent) + { + var kaitaiStreamSUT = Create(streamContent); + + Assert.Equal(expected, await kaitaiStreamSUT.ReadS1Async()); + } + + [Theory] + [MemberData(nameof(IntegralData.Integral2Data), MemberType = typeof(IntegralData))] + public async Task ReadS2beAsync_Test(short expected, byte[] streamContent) + { + var kaitaiStreamSUT = Create(streamContent); + + Assert.Equal(expected, await kaitaiStreamSUT.ReadS2beAsync()); + } + + [Theory] + [MemberData(nameof(IntegralData.Integral4Data), MemberType = typeof(IntegralData))] + public async Task ReadS4beAsync_Test(int expected, byte[] streamContent) + { + var kaitaiStreamSUT = Create(streamContent); + + Assert.Equal(expected, await kaitaiStreamSUT.ReadS4beAsync()); + } + + [Theory] + [MemberData(nameof(IntegralData.Integral8Data), MemberType = typeof(IntegralData))] + public async Task ReadS8beAsync_Test(long expected, byte[] streamContent) + { + var kaitaiStreamSUT = Create(streamContent); + + Assert.Equal(expected, await kaitaiStreamSUT.ReadS8beAsync()); + } + + [Theory] + [MemberData(nameof(IntegralData.Integral2Data), MemberType = typeof(IntegralData))] + public async Task ReadS2leAsync_Test(short expected, byte[] streamContent) + { + var kaitaiStreamSUT = Create(streamContent.Reverse().ToArray()); + + Assert.Equal(expected, await kaitaiStreamSUT.ReadS2leAsync()); + } + + [Theory] + [MemberData(nameof(IntegralData.Integral4Data), MemberType = typeof(IntegralData))] + public async Task ReadS4leAsync_Test(int expected, byte[] streamContent) + { + var kaitaiStreamSUT = Create(streamContent.Reverse().ToArray()); + + Assert.Equal(expected, await kaitaiStreamSUT.ReadS4leAsync()); + } + + [Theory] + [MemberData(nameof(IntegralData.Integral8Data), MemberType = typeof(IntegralData))] + public async Task ReadS8leAsync_Test(long expected, byte[] streamContent) + { + var kaitaiStreamSUT = Create(streamContent.Reverse().ToArray()); + + Assert.Equal(expected, await kaitaiStreamSUT.ReadS8leAsync()); + } + } +} \ No newline at end of file diff --git a/Kaitai.Struct.Runtime.Async.Tests/ReadUnSigned.cs b/Kaitai.Struct.Runtime.Async.Tests/ReadUnSigned.cs deleted file mode 100644 index 9428724..0000000 --- a/Kaitai.Struct.Runtime.Async.Tests/ReadUnSigned.cs +++ /dev/null @@ -1,73 +0,0 @@ -using System.Linq; -using System.Threading.Tasks; -using Kaitai.Async; -using Xunit; - -namespace Kaitai.Struct.Runtime.Async.Tests -{ - public class ReadUnSigned - { - [Theory] - [MemberData(nameof(IntegralData.Integral1Data), MemberType = typeof(IntegralData))] - public async Task ReadU1Async_Test( /*u*/ sbyte expected, byte[] streamContent) - { - var kaitaiStreamSUT = new KaitaiAsyncStream(streamContent); - - Assert.Equal((byte) expected, await kaitaiStreamSUT.ReadU1Async()); - } - - [Theory] - [MemberData(nameof(IntegralData.Integral2Data), MemberType = typeof(IntegralData))] - public async Task ReadU2beAsync_Test( /*u*/ short expected, byte[] streamContent) - { - var kaitaiStreamSUT = new KaitaiAsyncStream(streamContent); - - Assert.Equal((ushort) expected, await kaitaiStreamSUT.ReadU2beAsync()); - } - - [Theory] - [MemberData(nameof(IntegralData.Integral4Data), MemberType = typeof(IntegralData))] - public async Task ReadU4beAsync_Test( /*u*/ int expected, byte[] streamContent) - { - var kaitaiStreamSUT = new KaitaiAsyncStream(streamContent); - - Assert.Equal((uint) expected, await kaitaiStreamSUT.ReadU4beAsync()); - } - - [Theory] - [MemberData(nameof(IntegralData.Integral8Data), MemberType = typeof(IntegralData))] - public async Task ReadU8beAsync_Test( /*u*/ long expected, byte[] streamContent) - { - var kaitaiStreamSUT = new KaitaiAsyncStream(streamContent); - - Assert.Equal((ulong) expected, await kaitaiStreamSUT.ReadU8beAsync()); - } - - [Theory] - [MemberData(nameof(IntegralData.Integral2Data), MemberType = typeof(IntegralData))] - public async Task ReadU2leAsync_Test( /*u*/ short expected, byte[] streamContent) - { - var kaitaiStreamSUT = new KaitaiAsyncStream(streamContent.Reverse().ToArray()); - - Assert.Equal((ushort) expected, await kaitaiStreamSUT.ReadU2leAsync()); - } - - [Theory] - [MemberData(nameof(IntegralData.Integral4Data), MemberType = typeof(IntegralData))] - public async Task ReadU4leAsync_Test( /*u*/ int expected, byte[] streamContent) - { - var kaitaiStreamSUT = new KaitaiAsyncStream(streamContent.Reverse().ToArray()); - - Assert.Equal((uint) expected, await kaitaiStreamSUT.ReadU4leAsync()); - } - - [Theory] - [MemberData(nameof(IntegralData.Integral8Data), MemberType = typeof(IntegralData))] - public async Task ReadU8leAsync_Test( /*u*/ long expected, byte[] streamContent) - { - var kaitaiStreamSUT = new KaitaiAsyncStream(streamContent.Reverse().ToArray()); - - Assert.Equal((ulong) expected, await kaitaiStreamSUT.ReadU8leAsync()); - } - } -} \ No newline at end of file diff --git a/Kaitai.Struct.Runtime.Async.Tests/ReadUnSignedAsyncTests.cs b/Kaitai.Struct.Runtime.Async.Tests/ReadUnSignedAsyncTests.cs new file mode 100644 index 0000000..e593961 --- /dev/null +++ b/Kaitai.Struct.Runtime.Async.Tests/ReadUnSignedAsyncTests.cs @@ -0,0 +1,87 @@ +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using Kaitai.Async; +using Xunit; + +namespace Kaitai.Struct.Runtime.Async.Tests +{ + public class StreamReadUnSignedAsyncTests : ReadUnSignedAsyncTests + { + protected override KaitaiAsyncStream Create(byte[] data) => new KaitaiAsyncStream(data); + } + + public class PipeReaderReadUnSignedAsyncTests : ReadUnSignedAsyncTests + { + protected override KaitaiAsyncStream Create(byte[] data) => + new KaitaiAsyncStream(System.IO.Pipelines.PipeReader.Create(new MemoryStream(data))); + } + + public abstract class ReadUnSignedAsyncTests + { + protected abstract KaitaiAsyncStream Create(byte[] data); + + [Theory] + [MemberData(nameof(IntegralData.Integral1Data), MemberType = typeof(IntegralData))] + public async Task ReadU1Async_Test( /*u*/ sbyte expected, byte[] streamContent) + { + var kaitaiStreamSUT = Create(streamContent); + + Assert.Equal((byte) expected, await kaitaiStreamSUT.ReadU1Async()); + } + + [Theory] + [MemberData(nameof(IntegralData.Integral2Data), MemberType = typeof(IntegralData))] + public async Task ReadU2beAsync_Test( /*u*/ short expected, byte[] streamContent) + { + var kaitaiStreamSUT = Create(streamContent); + + Assert.Equal((ushort) expected, await kaitaiStreamSUT.ReadU2beAsync()); + } + + [Theory] + [MemberData(nameof(IntegralData.Integral4Data), MemberType = typeof(IntegralData))] + public async Task ReadU4beAsync_Test( /*u*/ int expected, byte[] streamContent) + { + var kaitaiStreamSUT = Create(streamContent); + + Assert.Equal((uint) expected, await kaitaiStreamSUT.ReadU4beAsync()); + } + + [Theory] + [MemberData(nameof(IntegralData.Integral8Data), MemberType = typeof(IntegralData))] + public async Task ReadU8beAsync_Test( /*u*/ long expected, byte[] streamContent) + { + var kaitaiStreamSUT = Create(streamContent); + + Assert.Equal((ulong) expected, await kaitaiStreamSUT.ReadU8beAsync()); + } + + [Theory] + [MemberData(nameof(IntegralData.Integral2Data), MemberType = typeof(IntegralData))] + public async Task ReadU2leAsync_Test( /*u*/ short expected, byte[] streamContent) + { + var kaitaiStreamSUT = Create(streamContent.Reverse().ToArray()); + + Assert.Equal((ushort) expected, await kaitaiStreamSUT.ReadU2leAsync()); + } + + [Theory] + [MemberData(nameof(IntegralData.Integral4Data), MemberType = typeof(IntegralData))] + public async Task ReadU4leAsync_Test( /*u*/ int expected, byte[] streamContent) + { + var kaitaiStreamSUT = Create(streamContent.Reverse().ToArray()); + + Assert.Equal((uint) expected, await kaitaiStreamSUT.ReadU4leAsync()); + } + + [Theory] + [MemberData(nameof(IntegralData.Integral8Data), MemberType = typeof(IntegralData))] + public async Task ReadU8leAsync_Test( /*u*/ long expected, byte[] streamContent) + { + var kaitaiStreamSUT = Create(streamContent.Reverse().ToArray()); + + Assert.Equal((ulong) expected, await kaitaiStreamSUT.ReadU8leAsync()); + } + } +} \ No newline at end of file diff --git a/Kaitai.Struct.Runtime.Async/Interface/IReaderContext.cs b/Kaitai.Struct.Runtime.Async/Interface/IReaderContext.cs new file mode 100644 index 0000000..1f954d6 --- /dev/null +++ b/Kaitai.Struct.Runtime.Async/Interface/IReaderContext.cs @@ -0,0 +1,15 @@ +using System.Threading.Tasks; + +namespace Kaitai.Async +{ + public interface IReaderContext + { + long Position { get; } + ValueTask GetSize(); + ValueTask IsEof(); + ValueTask SeekAsync(long position); + ValueTask ReadByteAsync(); + ValueTask ReadBytesAsync(long count); + ValueTask ReadBytesFullAsync(); + } +} \ No newline at end of file diff --git a/Kaitai.Struct.Runtime.Async/Kaitai.Struct.Runtime.Async.csproj b/Kaitai.Struct.Runtime.Async/Kaitai.Struct.Runtime.Async.csproj index 848a9b2..a842e82 100644 --- a/Kaitai.Struct.Runtime.Async/Kaitai.Struct.Runtime.Async.csproj +++ b/Kaitai.Struct.Runtime.Async/Kaitai.Struct.Runtime.Async.csproj @@ -11,6 +11,7 @@ + diff --git a/Kaitai.Struct.Runtime.Async/Kaitai.Struct.Runtime.Async.csproj.DotSettings b/Kaitai.Struct.Runtime.Async/Kaitai.Struct.Runtime.Async.csproj.DotSettings new file mode 100644 index 0000000..ad21594 --- /dev/null +++ b/Kaitai.Struct.Runtime.Async/Kaitai.Struct.Runtime.Async.csproj.DotSettings @@ -0,0 +1,2 @@ + + True \ No newline at end of file diff --git a/Kaitai.Struct.Runtime.Async/KaitaiAsyncStream.cs b/Kaitai.Struct.Runtime.Async/KaitaiAsyncStream.cs index 9227593..804a11d 100644 --- a/Kaitai.Struct.Runtime.Async/KaitaiAsyncStream.cs +++ b/Kaitai.Struct.Runtime.Async/KaitaiAsyncStream.cs @@ -1,325 +1,349 @@ using System; using System.Collections.Generic; using System.IO; -using System.IO.Compression; +using System.IO.Pipelines; using System.Threading.Tasks; -using Overby.Extensions.AsyncBinaryReaderWriter; namespace Kaitai.Async { - public partial class KaitaiAsyncStream : KaitaiStreamBase, IKaitaiAsyncStream - { - protected Stream BaseStream; + public class KaitaiAsyncStream : KaitaiStreamBase, IKaitaiAsyncStream + { + private readonly IReaderContext _readerContext; + private ulong _bits; + private int _bitsLeft; - private ulong _bits = 0; - private int _bitsLeft = 0; + #region Constructors - protected AsyncBinaryReader AsyncBinaryReader { get; } + public KaitaiAsyncStream(IReaderContext readerContext) + { + _readerContext = readerContext; + } - #region Constructors + public KaitaiAsyncStream(PipeReader pipeReader) + { + _readerContext = new PipeReaderContext(pipeReader); + } - public KaitaiAsyncStream(Stream stream) - { - BaseStream = stream; - AsyncBinaryReader = new AsyncBinaryReader(BaseStream); + public KaitaiAsyncStream(Stream stream) + { + _readerContext = new StreamReaderContext(stream); + } - } + /// + /// Creates a IKaitaiAsyncStream backed by a file (RO) + /// + public KaitaiAsyncStream(string file) : this(File.Open(file, + FileMode.Open, + FileAccess.Read, + FileShare.Read)) + { + } - /// - /// Creates a IKaitaiAsyncStream backed by a file (RO) - /// - public KaitaiAsyncStream(string file) : this(File.Open(file, FileMode.Open, FileAccess.Read, FileShare.Read)) - { - } + /// + ///Creates a IKaitaiAsyncStream backed by a byte buffer + /// + public KaitaiAsyncStream(byte[] bytes) : this(new MemoryStream(bytes)) + { + } - /// - ///Creates a IKaitaiAsyncStream backed by a byte buffer - /// - public KaitaiAsyncStream(byte[] bytes) : this(new MemoryStream(bytes)) - { - } + #endregion - #endregion + #region Stream positioning - #region Stream positioning + public override bool IsEof => + _readerContext.IsEof().GetAwaiter().GetResult() && _bitsLeft == 0; //TODO should be async method - public override bool IsEof => BaseStream.Position >= BaseStream.Length && _bitsLeft == 0; + public virtual async Task SeekAsync(long position) => await _readerContext.SeekAsync(position); - public virtual Task SeekAsync(long position) - { - BaseStream.Seek(position, SeekOrigin.Begin); - return Task.CompletedTask; - } + public override long Pos => _readerContext.Position; - public override long Pos => BaseStream.Position; + public override long Size => ReadBytesFullAsync().GetAwaiter().GetResult().Length; //TODO should be async method - public override long Size => BaseStream.Length; + #endregion - #endregion + #region Integer types - #region Integer types + #region Signed - #region Signed + public async Task ReadS1Async() => (sbyte) await ReadU1Async(); - public async Task ReadS1Async() => await AsyncBinaryReader.ReadSByteAsync(); + #region Big-endian - #region Big-endian + public async Task ReadS2beAsync() => BitConverter.ToInt16(await ReadBytesNormalisedBigEndianAsync(2), 0); - public async Task ReadS2beAsync() => BitConverter.ToInt16(await ReadBytesNormalisedBigEndianAsync(2), 0); + public async Task ReadS4beAsync() => BitConverter.ToInt32(await ReadBytesNormalisedBigEndianAsync(4), 0); - public async Task ReadS4beAsync() => BitConverter.ToInt32(await ReadBytesNormalisedBigEndianAsync(4), 0); + public async Task ReadS8beAsync() => BitConverter.ToInt64(await ReadBytesNormalisedBigEndianAsync(8), 0); - public async Task ReadS8beAsync() => BitConverter.ToInt64(await ReadBytesNormalisedBigEndianAsync(8), 0); + #endregion - #endregion + #region Little-endian - #region Little-endian + public async Task ReadS2leAsync() => BitConverter.ToInt16(await ReadBytesNormalisedLittleEndianAsync(2), 0); - public async Task ReadS2leAsync() => BitConverter.ToInt16(await ReadBytesNormalisedLittleEndianAsync(2), 0); + public async Task ReadS4leAsync() => BitConverter.ToInt32(await ReadBytesNormalisedLittleEndianAsync(4), 0); - public async Task ReadS4leAsync() => BitConverter.ToInt32(await ReadBytesNormalisedLittleEndianAsync(4), 0); + public async Task ReadS8leAsync() => BitConverter.ToInt64(await ReadBytesNormalisedLittleEndianAsync(8), 0); - public async Task ReadS8leAsync() => BitConverter.ToInt64(await ReadBytesNormalisedLittleEndianAsync(8), 0); + #endregion - #endregion + #endregion - #endregion + #region Unsigned - #region Unsigned + public async Task ReadU1Async() => await _readerContext.ReadByteAsync(); - public async Task ReadU1Async() => await AsyncBinaryReader.ReadByteAsync(); + #region Big-endian - #region Big-endian + public async Task ReadU2beAsync() => BitConverter.ToUInt16(await ReadBytesNormalisedBigEndianAsync(2), 0); - public async Task ReadU2beAsync() => BitConverter.ToUInt16(await ReadBytesNormalisedBigEndianAsync(2), 0); + public async Task ReadU4beAsync() => BitConverter.ToUInt32(await ReadBytesNormalisedBigEndianAsync(4), 0); - public async Task ReadU4beAsync() => BitConverter.ToUInt32(await ReadBytesNormalisedBigEndianAsync(4), 0); + public async Task ReadU8beAsync() => BitConverter.ToUInt64(await ReadBytesNormalisedBigEndianAsync(8), 0); - public async Task ReadU8beAsync() => BitConverter.ToUInt64(await ReadBytesNormalisedBigEndianAsync(8), 0); + #endregion - #endregion + #region Little-endian - #region Little-endian + public async Task ReadU2leAsync() => + BitConverter.ToUInt16(await ReadBytesNormalisedLittleEndianAsync(2), 0); - public async Task ReadU2leAsync() => BitConverter.ToUInt16(await ReadBytesNormalisedLittleEndianAsync(2), 0); + public async Task ReadU4leAsync() => BitConverter.ToUInt32(await ReadBytesNormalisedLittleEndianAsync(4), 0); - public async Task ReadU4leAsync() => BitConverter.ToUInt32(await ReadBytesNormalisedLittleEndianAsync(4), 0); + public async Task ReadU8leAsync() => BitConverter.ToUInt64(await ReadBytesNormalisedLittleEndianAsync(8), 0); - public async Task ReadU8leAsync() => BitConverter.ToUInt64(await ReadBytesNormalisedLittleEndianAsync(8), 0); + #endregion - #endregion + #endregion - #endregion + #endregion - #endregion + #region Floating point types - #region Floating point types + #region Big-endian - #region Big-endian + public async Task ReadF4beAsync() => BitConverter.ToSingle(await ReadBytesNormalisedBigEndianAsync(4), 0); - public async Task ReadF4beAsync() => BitConverter.ToSingle(await ReadBytesNormalisedBigEndianAsync(4), 0); + public async Task ReadF8beAsync() => BitConverter.ToDouble(await ReadBytesNormalisedBigEndianAsync(8), 0); - public async Task ReadF8beAsync() => BitConverter.ToDouble(await ReadBytesNormalisedBigEndianAsync(8), 0); + #endregion - #endregion + #region Little-endian - #region Little-endian + public async Task ReadF4leAsync() => BitConverter.ToSingle(await ReadBytesNormalisedLittleEndianAsync(4), 0); - public async Task ReadF4leAsync() => BitConverter.ToSingle(await ReadBytesNormalisedLittleEndianAsync(4), 0); + public async Task ReadF8leAsync() => + BitConverter.ToDouble(await ReadBytesNormalisedLittleEndianAsync(8), 0); - public async Task ReadF8leAsync() => BitConverter.ToDouble(await ReadBytesNormalisedLittleEndianAsync(8), 0); + #endregion - #endregion + #endregion - #endregion + #region Unaligned bit values - #region Unaligned bit values + public override void AlignToByte() + { + _bits = 0; + _bitsLeft = 0; + } - public override void AlignToByte() + public async Task ReadBitsIntAsync(int n) + { + int bitsNeeded = n - _bitsLeft; + if (bitsNeeded > 0) + { + // 1 bit => 1 byte + // 8 bits => 1 byte + // 9 bits => 2 bytes + int bytesNeeded = (bitsNeeded - 1) / 8 + 1; + var buf = await ReadBytesAsync(bytesNeeded); + for (var i = 0; i < buf.Length; i++) { - _bits = 0; - _bitsLeft = 0; + _bits <<= 8; + _bits |= buf[i]; + _bitsLeft += 8; } + } + + // raw mask with required number of 1s, starting from lowest bit + ulong mask = GetMaskOnes(n); + // shift mask to align with highest bits available in "bits" + int shiftBits = _bitsLeft - n; + mask = mask << shiftBits; + // derive reading result + ulong res = (_bits & mask) >> shiftBits; + // clear top bits that we've just read => AND with 1s + _bitsLeft -= n; + mask = GetMaskOnes(_bitsLeft); + _bits &= mask; + + return res; + } - public async Task ReadBitsIntAsync(int n) + //Method ported from algorithm specified @ issue#155 + public async Task ReadBitsIntLeAsync(int n) + { + int bitsNeeded = n - _bitsLeft; + + if (bitsNeeded > 0) + { + // 1 bit => 1 byte + // 8 bits => 1 byte + // 9 bits => 2 bytes + int bytesNeeded = (bitsNeeded - 1) / 8 + 1; + var buf = await ReadBytesAsync(bytesNeeded); + for (var i = 0; i < buf.Length; i++) { - int bitsNeeded = n - _bitsLeft; - if (bitsNeeded > 0) - { - // 1 bit => 1 byte - // 8 bits => 1 byte - // 9 bits => 2 bytes - int bytesNeeded = ((bitsNeeded - 1) / 8) + 1; - byte[] buf = await ReadBytesAsync(bytesNeeded); - for (int i = 0; i < buf.Length; i++) - { - _bits <<= 8; - _bits |= buf[i]; - _bitsLeft += 8; - } - } - - // raw mask with required number of 1s, starting from lowest bit - ulong mask = GetMaskOnes(n); - // shift mask to align with highest bits available in "bits" - int shiftBits = _bitsLeft - n; - mask = mask << shiftBits; - // derive reading result - ulong res = (_bits & mask) >> shiftBits; - // clear top bits that we've just read => AND with 1s - _bitsLeft -= n; - mask = GetMaskOnes(_bitsLeft); - _bits &= mask; - - return res; + ulong v = (ulong) buf[i] << _bitsLeft; + _bits |= v; + _bitsLeft += 8; } + } - //Method ported from algorithm specified @ issue#155 - public async Task ReadBitsIntLeAsync(int n) - { - int bitsNeeded = n - _bitsLeft; - - if (bitsNeeded > 0) - { - // 1 bit => 1 byte - // 8 bits => 1 byte - // 9 bits => 2 bytes - int bytesNeeded = ((bitsNeeded - 1) / 8) + 1; - byte[] buf = await ReadBytesAsync(bytesNeeded); - for (int i = 0; i < buf.Length; i++) - { - ulong v = (ulong)((ulong)buf[i] << _bitsLeft); - _bits |= v; - _bitsLeft += 8; - } - } - - // raw mask with required number of 1s, starting from lowest bit - ulong mask = GetMaskOnes(n); - - // derive reading result - ulong res = (_bits & mask); - - // remove bottom bits that we've just read by shifting - _bits >>= n; - _bitsLeft -= n; - - return res; - } + // raw mask with required number of 1s, starting from lowest bit + ulong mask = GetMaskOnes(n); - #endregion + // derive reading result + ulong res = _bits & mask; - #region Byte arrays + // remove bottom bits that we've just read by shifting + _bits >>= n; + _bitsLeft -= n; - public async Task ReadBytesAsync(long count) - { - if (count < 0 || count > Int32.MaxValue) - throw new ArgumentOutOfRangeException("requested " + count + " bytes, while only non-negative int32 amount of bytes possible"); - byte[] bytes = await AsyncBinaryReader.ReadBytesAsync((int)count); - if (bytes.Length < count) - throw new EndOfStreamException("requested " + count + " bytes, but got only " + bytes.Length + " bytes"); - return bytes; - } + return res; + } - public async Task ReadBytesAsync(ulong count) - { - if (count > Int32.MaxValue) - throw new ArgumentOutOfRangeException("requested " + count + " bytes, while only non-negative int32 amount of bytes possible"); - byte[] bytes = await AsyncBinaryReader.ReadBytesAsync((int)count); - if (bytes.Length < (int)count) - throw new EndOfStreamException("requested " + count + " bytes, but got only " + bytes.Length + " bytes"); - return bytes; - } + #endregion - /// - /// Read bytes from the stream in little endian format and convert them to the endianness of the current platform - /// - /// The number of bytes to read - /// An array of bytes that matches the endianness of the current platform - protected async Task ReadBytesNormalisedLittleEndianAsync(int count) - { - byte[] bytes = await ReadBytesAsync(count); - if (!IsLittleEndian) Array.Reverse(bytes); - return bytes; - } + #region Byte arrays + + public async Task ReadBytesAsync(long count) => await _readerContext.ReadBytesAsync(count); + + public async Task ReadBytesAsync(ulong count) + { + if (count > int.MaxValue) + { + throw new ArgumentOutOfRangeException( + $"requested {count} bytes, while only non-negative int32 amount of bytes possible"); + } + + return await ReadBytesAsync((long) count); + } + + /// + /// Read bytes from the stream in little endian format and convert them to the endianness of the current platform + /// + /// The number of bytes to read + /// An array of bytes that matches the endianness of the current platform + protected async Task ReadBytesNormalisedLittleEndianAsync(int count) + { + var bytes = await ReadBytesAsync(count); + if (!IsLittleEndian) + { + Array.Reverse(bytes); + } + + return bytes; + } + + /// + /// Read bytes from the stream in big endian format and convert them to the endianness of the current platform + /// + /// The number of bytes to read + /// An array of bytes that matches the endianness of the current platform + protected async Task ReadBytesNormalisedBigEndianAsync(int count) + { + var bytes = await ReadBytesAsync(count); + if (IsLittleEndian) + { + Array.Reverse(bytes); + } + + return bytes; + } - /// - /// Read bytes from the stream in big endian format and convert them to the endianness of the current platform - /// - /// The number of bytes to read - /// An array of bytes that matches the endianness of the current platform - protected async Task ReadBytesNormalisedBigEndianAsync(int count) + /// + /// Read all the remaining bytes from the stream until the end is reached + /// + /// + public virtual async Task ReadBytesFullAsync() => await _readerContext.ReadBytesFullAsync(); + + /// + /// Read a terminated string from the stream + /// + /// The string terminator value + /// True to include the terminator in the returned string + /// True to consume the terminator byte before returning + /// True to throw an error when the EOS was reached before the terminator + /// + public async Task ReadBytesTermAsync(byte terminator, + bool includeTerminator, + bool consumeTerminator, + bool eosError) + { + var bytes = new List(); + while (true) + { + if (IsEof) { - byte[] bytes = await ReadBytesAsync(count); - if (IsLittleEndian) Array.Reverse(bytes); - return bytes; + if (eosError) + { + throw new EndOfStreamException( + $"End of stream reached, but no terminator `{terminator}` found"); + } + + break; } - /// - /// Read all the remaining bytes from the stream until the end is reached - /// - /// - /// //TODO Handle asynchronously, BaseStream.Length is troublesome - public virtual async Task ReadBytesFullAsync() => await ReadBytesAsync(BaseStream.Length - BaseStream.Position); - - /// - /// Read a terminated string from the stream - /// - /// The string terminator value - /// True to include the terminator in the returned string - /// True to consume the terminator byte before returning - /// True to throw an error when the EOS was reached before the terminator - /// - public async Task ReadBytesTermAsync(byte terminator, bool includeTerminator, bool consumeTerminator, bool eosError) + byte b = await ReadU1Async(); + if (b == terminator) { - List bytes = new List(); - while (true) - { - if (IsEof) - { - if (eosError) throw new EndOfStreamException( - $"End of stream reached, but no terminator `{terminator}` found"); - break; - } - - byte b = await AsyncBinaryReader.ReadByteAsync(); - if (b == terminator) - { - if (includeTerminator) bytes.Add(b); - if (!consumeTerminator) await SeekAsync(Pos - 1); - break; - } - bytes.Add(b); - } - return bytes.ToArray(); + if (includeTerminator) + { + bytes.Add(b); + } + + if (!consumeTerminator) + { + await SeekAsync(Pos - 1); + } + + break; } - /// - /// Read a specific set of bytes and assert that they are the same as an expected result - /// - /// The expected result - /// - public async Task EnsureFixedContentsAsync(byte[] expected) + bytes.Add(b); + } + + return bytes.ToArray(); + } + + /// + /// Read a specific set of bytes and assert that they are the same as an expected result + /// + /// The expected result + /// + public async Task EnsureFixedContentsAsync(byte[] expected) + { + var bytes = await ReadBytesAsync(expected.Length); + + if (bytes.Length != expected.Length) //TODO Is this necessary? + { + throw new Exception( + $"Expected bytes: {Convert.ToBase64String(expected)} ({expected.Length} bytes), Instead got: {Convert.ToBase64String(bytes)} ({bytes.Length} bytes)"); + } + + for (var i = 0; i < bytes.Length; i++) + { + if (bytes[i] != expected[i]) { - byte[] bytes = await ReadBytesAsync(expected.Length); - - if (bytes.Length != expected.Length) //TODO Is this necessary? - { - throw new Exception( - $"Expected bytes: {Convert.ToBase64String(expected)} ({expected.Length} bytes), Instead got: {Convert.ToBase64String(bytes)} ({bytes.Length} bytes)"); - } - for (int i = 0; i < bytes.Length; i++) - { - if (bytes[i] != expected[i]) - { - throw new Exception( - $"Expected bytes: {Convert.ToBase64String(expected)} ({expected.Length} bytes), Instead got: {Convert.ToBase64String(bytes)} ({bytes.Length} bytes)"); - } - } - - return bytes; + throw new Exception( + $"Expected bytes: {Convert.ToBase64String(expected)} ({expected.Length} bytes), Instead got: {Convert.ToBase64String(bytes)} ({bytes.Length} bytes)"); } + } - #endregion + return bytes; } -} + + #endregion + } +} \ No newline at end of file diff --git a/Kaitai.Struct.Runtime.Async/ReaderContext/PipeReaderContext.cs b/Kaitai.Struct.Runtime.Async/ReaderContext/PipeReaderContext.cs new file mode 100644 index 0000000..5522c73 --- /dev/null +++ b/Kaitai.Struct.Runtime.Async/ReaderContext/PipeReaderContext.cs @@ -0,0 +1,148 @@ +using System; +using System.Buffers; +using System.IO; +using System.IO.Pipelines; +using System.Threading.Tasks; + +namespace Kaitai.Async +{ + internal class PipeReaderContext : IReaderContext + { + private readonly PipeReader _pipeReader; + private ReadResult _readResult; + + public PipeReaderContext(PipeReader pipeReader) + { + _pipeReader = pipeReader; + } + + private long RemainingBytesInReadResult => _readResult.Buffer.Length - Position; + + public long Position { get; private set; } + + public async ValueTask GetSize() + { + await FillReadResultBufferToTheEnd(); + + return _readResult.Buffer.Length; + } + + public async ValueTask IsEof() + { + if (_readResult.Equals(default(ReadResult)) || + Position >= _readResult.Buffer.Length && !_readResult.IsCompleted) + { + _pipeReader.AdvanceTo(_readResult.Buffer.Start, _readResult.Buffer.GetPosition(Position)); + _readResult = await _pipeReader.ReadAsync(); + } + + return Position >= _readResult.Buffer.Length && _readResult.IsCompleted; + } + + + public async ValueTask SeekAsync(long position) + { + if (position <= Position) + { + Position = position; + } + else + { + while (_readResult.Buffer.Length < position && !_readResult.IsCompleted) + { + _pipeReader.AdvanceTo(_readResult.Buffer.Start, _readResult.Buffer.End); + _readResult = await _pipeReader.ReadAsync(); + } + + if (_readResult.Buffer.Length <= position) + { + Position = position; + return; + } + + if (_readResult.IsCompleted) + { + throw new EndOfStreamException( + $"requested {position} bytes, but got only {RemainingBytesInReadResult} bytes"); + } + } + } + + public async ValueTask ReadByteAsync() + { + var value = byte.MinValue; + while (!TryReadByte(out value) && !_readResult.IsCompleted) + { + _pipeReader.AdvanceTo(_readResult.Buffer.Start, _readResult.Buffer.GetPosition(Position)); + _readResult = await _pipeReader.ReadAsync(); + } + + Position += 1; + return value; + + bool TryReadByte(out byte readValue) + { + var sequenceReader = new SequenceReader(_readResult.Buffer); + sequenceReader.Advance(Position); + return sequenceReader.TryRead(out readValue); + } + } + + public async ValueTask ReadBytesAsync(long count) + { + if (count < 0 || count > int.MaxValue) + { + throw new ArgumentOutOfRangeException( + $"requested {count} bytes, while only non-negative int32 amount of bytes possible"); + } + + byte[] value = null; + + while (!TryRead(out value, count)) + { + if (_readResult.IsCompleted) + { + throw new EndOfStreamException( + $"requested {count} bytes, but got only {RemainingBytesInReadResult} bytes"); + } + + _pipeReader.AdvanceTo(_readResult.Buffer.Start, _readResult.Buffer.GetPosition(Position)); + _readResult = await _pipeReader.ReadAsync(); + } + + Position += count; + return value; + + bool TryRead(out byte[] readBytes, long readBytesCount) + { + if (RemainingBytesInReadResult < readBytesCount) + { + readBytes = null; + return false; + } + + readBytes = _readResult.Buffer.Slice(Position, readBytesCount).ToArray(); + return true; + } + } + + public virtual async ValueTask ReadBytesFullAsync() + { + await FillReadResultBufferToTheEnd(); + + _pipeReader.AdvanceTo(_readResult.Buffer.Start, _readResult.Buffer.End); + var value = _readResult.Buffer.Slice(Position, _readResult.Buffer.End).ToArray(); + Position += value.Length; + return value; + } + + private async Task FillReadResultBufferToTheEnd() + { + while (!_readResult.IsCompleted) + { + _pipeReader.AdvanceTo(_readResult.Buffer.Start, _readResult.Buffer.End); + _readResult = await _pipeReader.ReadAsync(); + } + } + } +} \ No newline at end of file diff --git a/Kaitai.Struct.Runtime.Async/ReaderContext/StreamReaderContext.cs b/Kaitai.Struct.Runtime.Async/ReaderContext/StreamReaderContext.cs new file mode 100644 index 0000000..2a0cd05 --- /dev/null +++ b/Kaitai.Struct.Runtime.Async/ReaderContext/StreamReaderContext.cs @@ -0,0 +1,53 @@ +using System; +using System.IO; +using System.Threading.Tasks; +using Overby.Extensions.AsyncBinaryReaderWriter; + +namespace Kaitai.Async +{ + internal class StreamReaderContext : IReaderContext + { + private readonly Stream _baseStream; + + public StreamReaderContext(Stream stream) + { + _baseStream = stream; + AsyncBinaryReader = new AsyncBinaryReader(_baseStream); + } + + protected AsyncBinaryReader AsyncBinaryReader { get; } + public long Position => _baseStream.Position; + public ValueTask GetSize() => new ValueTask(_baseStream.Length); + + public ValueTask IsEof() => + new ValueTask(_baseStream.Position >= _baseStream.Length); + + public ValueTask SeekAsync(long position) + { + _baseStream.Seek(position, SeekOrigin.Begin); + return new ValueTask(); + } + + public async ValueTask ReadByteAsync() => (byte) await AsyncBinaryReader.ReadSByteAsync(); + + public async ValueTask ReadBytesAsync(long count) + { + if (count < 0 || count > int.MaxValue) + { + throw new ArgumentOutOfRangeException( + $"requested {count} bytes, while only non-negative int32 amount of bytes possible"); + } + + var bytes = await AsyncBinaryReader.ReadBytesAsync((int) count); + if (bytes.Length < count) + { + throw new EndOfStreamException($"requested {count} bytes, but got only {bytes.Length} bytes"); + } + + return bytes; + } + + public async ValueTask ReadBytesFullAsync() => + await ReadBytesAsync(_baseStream.Length - _baseStream.Position); + } +} \ No newline at end of file From e389f4bf6f70d90f10d8ad54225fc855fcf6fb05 Mon Sep 17 00:00:00 2001 From: Jan Pluskal Date: Wed, 29 Jan 2020 14:51:08 +0100 Subject: [PATCH 16/34] Ensure that ReadResult is not default before first read. --- .../ReaderContext/PipeReaderContext.cs | 23 ++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/Kaitai.Struct.Runtime.Async/ReaderContext/PipeReaderContext.cs b/Kaitai.Struct.Runtime.Async/ReaderContext/PipeReaderContext.cs index 5522c73..86c2e1e 100644 --- a/Kaitai.Struct.Runtime.Async/ReaderContext/PipeReaderContext.cs +++ b/Kaitai.Struct.Runtime.Async/ReaderContext/PipeReaderContext.cs @@ -29,8 +29,9 @@ public async ValueTask GetSize() public async ValueTask IsEof() { - if (_readResult.Equals(default(ReadResult)) || - Position >= _readResult.Buffer.Length && !_readResult.IsCompleted) + await EnsureReadResultIsNotDefault(); + + if (Position >= _readResult.Buffer.Length && !_readResult.IsCompleted) { _pipeReader.AdvanceTo(_readResult.Buffer.Start, _readResult.Buffer.GetPosition(Position)); _readResult = await _pipeReader.ReadAsync(); @@ -48,6 +49,8 @@ public async ValueTask SeekAsync(long position) } else { + await EnsureReadResultIsNotDefault(); + while (_readResult.Buffer.Length < position && !_readResult.IsCompleted) { _pipeReader.AdvanceTo(_readResult.Buffer.Start, _readResult.Buffer.End); @@ -70,6 +73,8 @@ public async ValueTask SeekAsync(long position) public async ValueTask ReadByteAsync() { + await EnsureReadResultIsNotDefault(); + var value = byte.MinValue; while (!TryReadByte(out value) && !_readResult.IsCompleted) { @@ -96,6 +101,8 @@ public async ValueTask ReadBytesAsync(long count) $"requested {count} bytes, while only non-negative int32 amount of bytes possible"); } + await EnsureReadResultIsNotDefault(); + byte[] value = null; while (!TryRead(out value, count)) @@ -136,13 +143,23 @@ public virtual async ValueTask ReadBytesFullAsync() return value; } - private async Task FillReadResultBufferToTheEnd() + private async ValueTask FillReadResultBufferToTheEnd() { + await EnsureReadResultIsNotDefault(); + while (!_readResult.IsCompleted) { _pipeReader.AdvanceTo(_readResult.Buffer.Start, _readResult.Buffer.End); _readResult = await _pipeReader.ReadAsync(); } } + + private async ValueTask EnsureReadResultIsNotDefault() + { + if (_readResult.Equals(default(ReadResult))) + { + _readResult = await _pipeReader.ReadAsync(); + } + } } } \ No newline at end of file From 885971b1bffe71292634bd39cea870a055610136 Mon Sep 17 00:00:00 2001 From: Jan Pluskal Date: Thu, 30 Jan 2020 10:17:36 +0100 Subject: [PATCH 17/34] Introduce GetSizeAsync, IsEofAsync. Fix KaitaiAsyncStream Size delegation to _readerContext. --- .../KaitaiAsyncStreamBaseTests.cs | 18 +- .../Interface/IKaitaiAsyncStream.cs | 329 +++++++++--------- .../Interface/IReaderContext.cs | 4 +- .../KaitaiAsyncStream.cs | 8 +- .../ReaderContext/PipeReaderContext.cs | 4 +- .../ReaderContext/StreamReaderContext.cs | 4 +- 6 files changed, 197 insertions(+), 170 deletions(-) diff --git a/Kaitai.Struct.Runtime.Async.Tests/KaitaiAsyncStreamBaseTests.cs b/Kaitai.Struct.Runtime.Async.Tests/KaitaiAsyncStreamBaseTests.cs index 1d61599..0acf91a 100644 --- a/Kaitai.Struct.Runtime.Async.Tests/KaitaiAsyncStreamBaseTests.cs +++ b/Kaitai.Struct.Runtime.Async.Tests/KaitaiAsyncStreamBaseTests.cs @@ -1,4 +1,5 @@ using System.IO; +using System.IO.Pipelines; using System.Threading.Tasks; using Kaitai.Async; using Xunit; @@ -6,14 +7,14 @@ namespace Kaitai.Struct.Runtime.Async.Tests { public class StreamKaitaiAsyncStreamBaseTests : KaitaiAsyncStreamBaseTests - { + { protected override KaitaiAsyncStream Create(byte[] data) => new KaitaiAsyncStream(data); } public class PipeReaderKaitaiAsyncStreamBaseTests : KaitaiAsyncStreamBaseTests - { + { protected override KaitaiAsyncStream Create(byte[] data) => - new KaitaiAsyncStream(System.IO.Pipelines.PipeReader.Create(new MemoryStream(data))); + new KaitaiAsyncStream(PipeReader.Create(new MemoryStream(data))); } public abstract class KaitaiAsyncStreamBaseTests @@ -34,8 +35,8 @@ public abstract class KaitaiAsyncStreamBaseTests public async Task Eof_Test(bool shouldBeEof, int streamSize, int readBitsAmount) { var kaitaiStreamSUT = Create(new byte[streamSize]); - await kaitaiStreamSUT.ReadBitsIntAsync(readBitsAmount); + long positionBeforeIsEof = kaitaiStreamSUT.Pos; if (shouldBeEof) { @@ -45,6 +46,8 @@ public async Task Eof_Test(bool shouldBeEof, int streamSize, int readBitsAmount) { Assert.False(kaitaiStreamSUT.IsEof); } + + Assert.Equal(positionBeforeIsEof, kaitaiStreamSUT.Pos); } [Theory] @@ -74,11 +77,18 @@ public async Task Pos_BySeek_Test(int expectedPos, int position) [Theory] [InlineData(0)] [InlineData(1)] + [InlineData(100)] + [InlineData(1_000)] + [InlineData(10_000)] + [InlineData(100_000)] + [InlineData(1_000_000)] public void Size_Test(int streamSize) { var kaitaiStreamSUT = Create(new byte[streamSize]); + long positionBeforeIsEof = kaitaiStreamSUT.Pos; Assert.Equal(streamSize, kaitaiStreamSUT.Size); + Assert.Equal(positionBeforeIsEof, kaitaiStreamSUT.Pos); } [Fact] diff --git a/Kaitai.Struct.Runtime.Async/Interface/IKaitaiAsyncStream.cs b/Kaitai.Struct.Runtime.Async/Interface/IKaitaiAsyncStream.cs index 9898416..4df5abb 100644 --- a/Kaitai.Struct.Runtime.Async/Interface/IKaitaiAsyncStream.cs +++ b/Kaitai.Struct.Runtime.Async/Interface/IKaitaiAsyncStream.cs @@ -2,161 +2,174 @@ namespace Kaitai.Async { - public interface IKaitaiAsyncStream : IKaitaiStreamBase - { - /// - /// Seek to a specific position from the beginning of the stream - /// - /// The position to seek to - Task SeekAsync(long position); - - /// - /// Read a signed byte from the stream - /// - /// - Task ReadS1Async(); - - /// - /// Read a signed short from the stream (big endian) - /// - /// - Task ReadS2beAsync(); - - /// - /// Read a signed int from the stream (big endian) - /// - /// - Task ReadS4beAsync(); - - /// - /// Read a signed long from the stream (big endian) - /// - /// - Task ReadS8beAsync(); - - /// - /// Read a signed short from the stream (little endian) - /// - /// - Task ReadS2leAsync(); - - /// - /// Read a signed int from the stream (little endian) - /// - /// - Task ReadS4leAsync(); - - /// - /// Read a signed long from the stream (little endian) - /// - /// - Task ReadS8leAsync(); - - /// - /// Read an unsigned byte from the stream - /// - /// - Task ReadU1Async(); - - /// - /// Read an unsigned short from the stream (big endian) - /// - /// - Task ReadU2beAsync(); - - /// - /// Read an unsigned int from the stream (big endian) - /// - /// - Task ReadU4beAsync(); - - /// - /// Read an unsigned long from the stream (big endian) - /// - /// - Task ReadU8beAsync(); - - /// - /// Read an unsigned short from the stream (little endian) - /// - /// - Task ReadU2leAsync(); - - /// - /// Read an unsigned int from the stream (little endian) - /// - /// - Task ReadU4leAsync(); - - /// - /// Read an unsigned long from the stream (little endian) - /// - /// - Task ReadU8leAsync(); - - /// - /// Read a single-precision floating point value from the stream (big endian) - /// - /// - Task ReadF4beAsync(); - - /// - /// Read a double-precision floating point value from the stream (big endian) - /// - /// - Task ReadF8beAsync(); - - /// - /// Read a single-precision floating point value from the stream (little endian) - /// - /// - Task ReadF4leAsync(); - - /// - /// Read a double-precision floating point value from the stream (little endian) - /// - /// - Task ReadF8leAsync(); - - Task ReadBitsIntAsync(int n); - - Task ReadBitsIntLeAsync(int n); - - /// - /// Read a fixed number of bytes from the stream - /// - /// The number of bytes to read - /// - Task ReadBytesAsync(long count); - - /// - /// Read a fixed number of bytes from the stream - /// - /// The number of bytes to read - /// - Task ReadBytesAsync(ulong count); - - /// - /// Read all the remaining bytes from the stream until the end is reached - /// - /// - Task ReadBytesFullAsync(); - - /// - /// Read a terminated string from the stream - /// - /// The string terminator value - /// True to include the terminator in the returned string - /// True to consume the terminator byte before returning - /// True to throw an error when the EOS was reached before the terminator - /// - Task ReadBytesTermAsync(byte terminator, bool includeTerminator, bool consumeTerminator, bool eosError); - - /// - /// Read a specific set of bytes and assert that they are the same as an expected result - /// - /// The expected result - /// - Task EnsureFixedContentsAsync(byte[] expected); - } -} + public interface IKaitaiAsyncStream : IKaitaiStreamBase + { + /// + /// Check if the stream position is at the end of the stream + /// + ValueTask IsEofAsync(); + + /// + /// Get the total length of the stream (ie. file size) + /// + ValueTask GetSizeAsync(); + + /// + /// Seek to a specific position from the beginning of the stream + /// + /// The position to seek to + Task SeekAsync(long position); + + /// + /// Read a signed byte from the stream + /// + /// + Task ReadS1Async(); + + /// + /// Read a signed short from the stream (big endian) + /// + /// + Task ReadS2beAsync(); + + /// + /// Read a signed int from the stream (big endian) + /// + /// + Task ReadS4beAsync(); + + /// + /// Read a signed long from the stream (big endian) + /// + /// + Task ReadS8beAsync(); + + /// + /// Read a signed short from the stream (little endian) + /// + /// + Task ReadS2leAsync(); + + /// + /// Read a signed int from the stream (little endian) + /// + /// + Task ReadS4leAsync(); + + /// + /// Read a signed long from the stream (little endian) + /// + /// + Task ReadS8leAsync(); + + /// + /// Read an unsigned byte from the stream + /// + /// + Task ReadU1Async(); + + /// + /// Read an unsigned short from the stream (big endian) + /// + /// + Task ReadU2beAsync(); + + /// + /// Read an unsigned int from the stream (big endian) + /// + /// + Task ReadU4beAsync(); + + /// + /// Read an unsigned long from the stream (big endian) + /// + /// + Task ReadU8beAsync(); + + /// + /// Read an unsigned short from the stream (little endian) + /// + /// + Task ReadU2leAsync(); + + /// + /// Read an unsigned int from the stream (little endian) + /// + /// + Task ReadU4leAsync(); + + /// + /// Read an unsigned long from the stream (little endian) + /// + /// + Task ReadU8leAsync(); + + /// + /// Read a single-precision floating point value from the stream (big endian) + /// + /// + Task ReadF4beAsync(); + + /// + /// Read a double-precision floating point value from the stream (big endian) + /// + /// + Task ReadF8beAsync(); + + /// + /// Read a single-precision floating point value from the stream (little endian) + /// + /// + Task ReadF4leAsync(); + + /// + /// Read a double-precision floating point value from the stream (little endian) + /// + /// + Task ReadF8leAsync(); + + Task ReadBitsIntAsync(int n); + + Task ReadBitsIntLeAsync(int n); + + /// + /// Read a fixed number of bytes from the stream + /// + /// The number of bytes to read + /// + Task ReadBytesAsync(long count); + + /// + /// Read a fixed number of bytes from the stream + /// + /// The number of bytes to read + /// + Task ReadBytesAsync(ulong count); + + /// + /// Read all the remaining bytes from the stream until the end is reached + /// + /// + Task ReadBytesFullAsync(); + + /// + /// Read a terminated string from the stream + /// + /// The string terminator value + /// True to include the terminator in the returned string + /// True to consume the terminator byte before returning + /// True to throw an error when the EOS was reached before the terminator + /// + Task ReadBytesTermAsync(byte terminator, + bool includeTerminator, + bool consumeTerminator, + bool eosError); + + /// + /// Read a specific set of bytes and assert that they are the same as an expected result + /// + /// The expected result + /// + Task EnsureFixedContentsAsync(byte[] expected); + } +} \ No newline at end of file diff --git a/Kaitai.Struct.Runtime.Async/Interface/IReaderContext.cs b/Kaitai.Struct.Runtime.Async/Interface/IReaderContext.cs index 1f954d6..4eab4fb 100644 --- a/Kaitai.Struct.Runtime.Async/Interface/IReaderContext.cs +++ b/Kaitai.Struct.Runtime.Async/Interface/IReaderContext.cs @@ -5,8 +5,8 @@ namespace Kaitai.Async public interface IReaderContext { long Position { get; } - ValueTask GetSize(); - ValueTask IsEof(); + ValueTask GetSizeAsync(); + ValueTask IsEofAsync(); ValueTask SeekAsync(long position); ValueTask ReadByteAsync(); ValueTask ReadBytesAsync(long count); diff --git a/Kaitai.Struct.Runtime.Async/KaitaiAsyncStream.cs b/Kaitai.Struct.Runtime.Async/KaitaiAsyncStream.cs index 804a11d..d6e6f5b 100644 --- a/Kaitai.Struct.Runtime.Async/KaitaiAsyncStream.cs +++ b/Kaitai.Struct.Runtime.Async/KaitaiAsyncStream.cs @@ -51,13 +51,17 @@ public KaitaiAsyncStream(byte[] bytes) : this(new MemoryStream(bytes)) #region Stream positioning public override bool IsEof => - _readerContext.IsEof().GetAwaiter().GetResult() && _bitsLeft == 0; //TODO should be async method + _readerContext.IsEofAsync().GetAwaiter().GetResult() && _bitsLeft == 0; + + public ValueTask IsEofAsync() => _readerContext.IsEofAsync(); + + public ValueTask GetSizeAsync() => _readerContext.GetSizeAsync(); public virtual async Task SeekAsync(long position) => await _readerContext.SeekAsync(position); public override long Pos => _readerContext.Position; - public override long Size => ReadBytesFullAsync().GetAwaiter().GetResult().Length; //TODO should be async method + public override long Size => _readerContext.GetSizeAsync().GetAwaiter().GetResult(); #endregion diff --git a/Kaitai.Struct.Runtime.Async/ReaderContext/PipeReaderContext.cs b/Kaitai.Struct.Runtime.Async/ReaderContext/PipeReaderContext.cs index 86c2e1e..e6077ee 100644 --- a/Kaitai.Struct.Runtime.Async/ReaderContext/PipeReaderContext.cs +++ b/Kaitai.Struct.Runtime.Async/ReaderContext/PipeReaderContext.cs @@ -20,14 +20,14 @@ public PipeReaderContext(PipeReader pipeReader) public long Position { get; private set; } - public async ValueTask GetSize() + public async ValueTask GetSizeAsync() { await FillReadResultBufferToTheEnd(); return _readResult.Buffer.Length; } - public async ValueTask IsEof() + public async ValueTask IsEofAsync() { await EnsureReadResultIsNotDefault(); diff --git a/Kaitai.Struct.Runtime.Async/ReaderContext/StreamReaderContext.cs b/Kaitai.Struct.Runtime.Async/ReaderContext/StreamReaderContext.cs index 2a0cd05..60465b2 100644 --- a/Kaitai.Struct.Runtime.Async/ReaderContext/StreamReaderContext.cs +++ b/Kaitai.Struct.Runtime.Async/ReaderContext/StreamReaderContext.cs @@ -17,9 +17,9 @@ public StreamReaderContext(Stream stream) protected AsyncBinaryReader AsyncBinaryReader { get; } public long Position => _baseStream.Position; - public ValueTask GetSize() => new ValueTask(_baseStream.Length); + public ValueTask GetSizeAsync() => new ValueTask(_baseStream.Length); - public ValueTask IsEof() => + public ValueTask IsEofAsync() => new ValueTask(_baseStream.Position >= _baseStream.Length); public ValueTask SeekAsync(long position) From c071b833058acd0d6cc1dce97ddbdafb9061ddc9 Mon Sep 17 00:00:00 2001 From: Jan Pluskal Date: Tue, 11 Feb 2020 10:08:26 +0100 Subject: [PATCH 18/34] IsEofAsync should check the _bitsLeft as well. --- Kaitai.Struct.Runtime.Async/KaitaiAsyncStream.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Kaitai.Struct.Runtime.Async/KaitaiAsyncStream.cs b/Kaitai.Struct.Runtime.Async/KaitaiAsyncStream.cs index d6e6f5b..260e2f3 100644 --- a/Kaitai.Struct.Runtime.Async/KaitaiAsyncStream.cs +++ b/Kaitai.Struct.Runtime.Async/KaitaiAsyncStream.cs @@ -53,7 +53,7 @@ public KaitaiAsyncStream(byte[] bytes) : this(new MemoryStream(bytes)) public override bool IsEof => _readerContext.IsEofAsync().GetAwaiter().GetResult() && _bitsLeft == 0; - public ValueTask IsEofAsync() => _readerContext.IsEofAsync(); + public async ValueTask IsEofAsync() => await _readerContext.IsEofAsync() && _bitsLeft == 0; public ValueTask GetSizeAsync() => _readerContext.GetSizeAsync(); From 628f31dcd38093954168443886ef89104ef9177a Mon Sep 17 00:00:00 2001 From: Jan Pluskal Date: Fri, 12 Jun 2020 12:45:41 +0200 Subject: [PATCH 19/34] Add Seek with ulong overrides. --- Kaitai.Struct.Runtime.Async/KaitaiAsyncStream.cs | 1 + Kaitai.Struct.Runtime/KaitaiStream.cs | 1 + 2 files changed, 2 insertions(+) diff --git a/Kaitai.Struct.Runtime.Async/KaitaiAsyncStream.cs b/Kaitai.Struct.Runtime.Async/KaitaiAsyncStream.cs index 260e2f3..85ab7f0 100644 --- a/Kaitai.Struct.Runtime.Async/KaitaiAsyncStream.cs +++ b/Kaitai.Struct.Runtime.Async/KaitaiAsyncStream.cs @@ -58,6 +58,7 @@ public KaitaiAsyncStream(byte[] bytes) : this(new MemoryStream(bytes)) public ValueTask GetSizeAsync() => _readerContext.GetSizeAsync(); public virtual async Task SeekAsync(long position) => await _readerContext.SeekAsync(position); + public virtual async Task SeekAsync(ulong position) => await SeekAsync((long)position); public override long Pos => _readerContext.Position; diff --git a/Kaitai.Struct.Runtime/KaitaiStream.cs b/Kaitai.Struct.Runtime/KaitaiStream.cs index dbb5263..35f1954 100644 --- a/Kaitai.Struct.Runtime/KaitaiStream.cs +++ b/Kaitai.Struct.Runtime/KaitaiStream.cs @@ -49,6 +49,7 @@ protected BinaryReader BinaryReader public override bool IsEof => BaseStream.Position >= BaseStream.Length && BitsLeft == 0; public void Seek(long position) => BaseStream.Seek(position, SeekOrigin.Begin); + public void Seek(ulong position) => Seek((long)position); public override long Pos => BaseStream.Position; From 7c10982b888756610ee3fc74330ae94a91e96537 Mon Sep 17 00:00:00 2001 From: Jan Pluskal Date: Fri, 3 Jul 2020 15:04:54 +0200 Subject: [PATCH 20/34] Add ImplicitNullable type. --- Kaitai.Struct.Runtime/ImplicitNullable.cs | 26 +++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 Kaitai.Struct.Runtime/ImplicitNullable.cs diff --git a/Kaitai.Struct.Runtime/ImplicitNullable.cs b/Kaitai.Struct.Runtime/ImplicitNullable.cs new file mode 100644 index 0000000..e295856 --- /dev/null +++ b/Kaitai.Struct.Runtime/ImplicitNullable.cs @@ -0,0 +1,26 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Kaitai +{ + public struct ImplicitNullable where T : struct + { + public bool HasValue { get { return this._value.HasValue; } } + public T Value { get { return this._value.Value; } } + + public ImplicitNullable(T value) : this() { this._value = value; } + public ImplicitNullable(T? value) : this() { this._value = value; } + + public static implicit operator ImplicitNullable(T value) { return new ImplicitNullable(value); } + public static implicit operator ImplicitNullable(T? value) { return new ImplicitNullable(value); } + + public static implicit operator T(ImplicitNullable value) { return value._value ?? default(T); } + public static implicit operator T?(ImplicitNullable value) { return value._value; } + + private T? _value { get; set; } + + // Should define other Nullable members, especially + // Equals and GetHashCode to avoid boxing + } +} From 5c1be973272b45c381288191c51a51f6866100c8 Mon Sep 17 00:00:00 2001 From: Jan Pluskal Date: Mon, 13 Jul 2020 12:37:13 +0000 Subject: [PATCH 21/34] Resolve "ImplicitNullable fails during serialization." --- Kaitai.Struct.Runtime/ImplicitNullable.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Kaitai.Struct.Runtime/ImplicitNullable.cs b/Kaitai.Struct.Runtime/ImplicitNullable.cs index e295856..0b8f235 100644 --- a/Kaitai.Struct.Runtime/ImplicitNullable.cs +++ b/Kaitai.Struct.Runtime/ImplicitNullable.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.Text; @@ -7,7 +7,7 @@ namespace Kaitai public struct ImplicitNullable where T : struct { public bool HasValue { get { return this._value.HasValue; } } - public T Value { get { return this._value.Value; } } + public T Value { get { return this._value ?? default; } } public ImplicitNullable(T value) : this() { this._value = value; } public ImplicitNullable(T? value) : this() { this._value = value; } From 80da927370db8936dc76c8a722160171f6a13169 Mon Sep 17 00:00:00 2001 From: Jan Pluskal Date: Fri, 17 Jul 2020 10:37:48 +0200 Subject: [PATCH 22/34] Allow extensibility in end-user code. --- .../KaitaiAsyncStream.cs | 26 +++---- .../ReaderContext/PipeReaderContext.cs | 72 +++++++++---------- .../ReaderContext/StreamReaderContext.cs | 16 ++--- 3 files changed, 57 insertions(+), 57 deletions(-) diff --git a/Kaitai.Struct.Runtime.Async/KaitaiAsyncStream.cs b/Kaitai.Struct.Runtime.Async/KaitaiAsyncStream.cs index 85ab7f0..560c0ea 100644 --- a/Kaitai.Struct.Runtime.Async/KaitaiAsyncStream.cs +++ b/Kaitai.Struct.Runtime.Async/KaitaiAsyncStream.cs @@ -8,7 +8,7 @@ namespace Kaitai.Async { public class KaitaiAsyncStream : KaitaiStreamBase, IKaitaiAsyncStream { - private readonly IReaderContext _readerContext; + protected readonly IReaderContext ReaderContext; private ulong _bits; private int _bitsLeft; @@ -16,17 +16,17 @@ public class KaitaiAsyncStream : KaitaiStreamBase, IKaitaiAsyncStream public KaitaiAsyncStream(IReaderContext readerContext) { - _readerContext = readerContext; + ReaderContext = readerContext; } public KaitaiAsyncStream(PipeReader pipeReader) { - _readerContext = new PipeReaderContext(pipeReader); + ReaderContext = new PipeReaderContext(pipeReader); } public KaitaiAsyncStream(Stream stream) { - _readerContext = new StreamReaderContext(stream); + ReaderContext = new StreamReaderContext(stream); } /// @@ -51,18 +51,18 @@ public KaitaiAsyncStream(byte[] bytes) : this(new MemoryStream(bytes)) #region Stream positioning public override bool IsEof => - _readerContext.IsEofAsync().GetAwaiter().GetResult() && _bitsLeft == 0; + ReaderContext.IsEofAsync().GetAwaiter().GetResult() && _bitsLeft == 0; - public async ValueTask IsEofAsync() => await _readerContext.IsEofAsync() && _bitsLeft == 0; + public async ValueTask IsEofAsync() => await ReaderContext.IsEofAsync() && _bitsLeft == 0; - public ValueTask GetSizeAsync() => _readerContext.GetSizeAsync(); + public ValueTask GetSizeAsync() => ReaderContext.GetSizeAsync(); - public virtual async Task SeekAsync(long position) => await _readerContext.SeekAsync(position); + public virtual async Task SeekAsync(long position) => await ReaderContext.SeekAsync(position); public virtual async Task SeekAsync(ulong position) => await SeekAsync((long)position); - public override long Pos => _readerContext.Position; + public override long Pos => ReaderContext.Position; - public override long Size => _readerContext.GetSizeAsync().GetAwaiter().GetResult(); + public override long Size => ReaderContext.GetSizeAsync().GetAwaiter().GetResult(); #endregion @@ -96,7 +96,7 @@ public KaitaiAsyncStream(byte[] bytes) : this(new MemoryStream(bytes)) #region Unsigned - public async Task ReadU1Async() => await _readerContext.ReadByteAsync(); + public async Task ReadU1Async() => await ReaderContext.ReadByteAsync(); #region Big-endian @@ -222,7 +222,7 @@ public async Task ReadBitsIntLeAsync(int n) #region Byte arrays - public async Task ReadBytesAsync(long count) => await _readerContext.ReadBytesAsync(count); + public async Task ReadBytesAsync(long count) => await ReaderContext.ReadBytesAsync(count); public async Task ReadBytesAsync(ulong count) { @@ -271,7 +271,7 @@ protected async Task ReadBytesNormalisedBigEndianAsync(int count) /// Read all the remaining bytes from the stream until the end is reached /// /// - public virtual async Task ReadBytesFullAsync() => await _readerContext.ReadBytesFullAsync(); + public virtual async Task ReadBytesFullAsync() => await ReaderContext.ReadBytesFullAsync(); /// /// Read a terminated string from the stream diff --git a/Kaitai.Struct.Runtime.Async/ReaderContext/PipeReaderContext.cs b/Kaitai.Struct.Runtime.Async/ReaderContext/PipeReaderContext.cs index e6077ee..e4731b4 100644 --- a/Kaitai.Struct.Runtime.Async/ReaderContext/PipeReaderContext.cs +++ b/Kaitai.Struct.Runtime.Async/ReaderContext/PipeReaderContext.cs @@ -6,42 +6,42 @@ namespace Kaitai.Async { - internal class PipeReaderContext : IReaderContext + public class PipeReaderContext : IReaderContext { - private readonly PipeReader _pipeReader; - private ReadResult _readResult; + protected readonly PipeReader PipeReader; + protected ReadResult ReadResult; public PipeReaderContext(PipeReader pipeReader) { - _pipeReader = pipeReader; + PipeReader = pipeReader; } - private long RemainingBytesInReadResult => _readResult.Buffer.Length - Position; + protected long RemainingBytesInReadResult => ReadResult.Buffer.Length - Position; - public long Position { get; private set; } + public long Position { get; protected set; } - public async ValueTask GetSizeAsync() + public virtual async ValueTask GetSizeAsync() { await FillReadResultBufferToTheEnd(); - return _readResult.Buffer.Length; + return ReadResult.Buffer.Length; } - public async ValueTask IsEofAsync() + public virtual async ValueTask IsEofAsync() { await EnsureReadResultIsNotDefault(); - if (Position >= _readResult.Buffer.Length && !_readResult.IsCompleted) + if (Position >= ReadResult.Buffer.Length && !ReadResult.IsCompleted) { - _pipeReader.AdvanceTo(_readResult.Buffer.Start, _readResult.Buffer.GetPosition(Position)); - _readResult = await _pipeReader.ReadAsync(); + PipeReader.AdvanceTo(ReadResult.Buffer.Start, ReadResult.Buffer.GetPosition(Position)); + ReadResult = await PipeReader.ReadAsync(); } - return Position >= _readResult.Buffer.Length && _readResult.IsCompleted; + return Position >= ReadResult.Buffer.Length && ReadResult.IsCompleted; } - public async ValueTask SeekAsync(long position) + public virtual async ValueTask SeekAsync(long position) { if (position <= Position) { @@ -51,19 +51,19 @@ public async ValueTask SeekAsync(long position) { await EnsureReadResultIsNotDefault(); - while (_readResult.Buffer.Length < position && !_readResult.IsCompleted) + while (ReadResult.Buffer.Length < position && !ReadResult.IsCompleted) { - _pipeReader.AdvanceTo(_readResult.Buffer.Start, _readResult.Buffer.End); - _readResult = await _pipeReader.ReadAsync(); + PipeReader.AdvanceTo(ReadResult.Buffer.Start, ReadResult.Buffer.End); + ReadResult = await PipeReader.ReadAsync(); } - if (_readResult.Buffer.Length <= position) + if (ReadResult.Buffer.Length <= position) { Position = position; return; } - if (_readResult.IsCompleted) + if (ReadResult.IsCompleted) { throw new EndOfStreamException( $"requested {position} bytes, but got only {RemainingBytesInReadResult} bytes"); @@ -71,15 +71,15 @@ public async ValueTask SeekAsync(long position) } } - public async ValueTask ReadByteAsync() + public virtual async ValueTask ReadByteAsync() { await EnsureReadResultIsNotDefault(); var value = byte.MinValue; - while (!TryReadByte(out value) && !_readResult.IsCompleted) + while (!TryReadByte(out value) && !ReadResult.IsCompleted) { - _pipeReader.AdvanceTo(_readResult.Buffer.Start, _readResult.Buffer.GetPosition(Position)); - _readResult = await _pipeReader.ReadAsync(); + PipeReader.AdvanceTo(ReadResult.Buffer.Start, ReadResult.Buffer.GetPosition(Position)); + ReadResult = await PipeReader.ReadAsync(); } Position += 1; @@ -87,13 +87,13 @@ public async ValueTask ReadByteAsync() bool TryReadByte(out byte readValue) { - var sequenceReader = new SequenceReader(_readResult.Buffer); + var sequenceReader = new SequenceReader(ReadResult.Buffer); sequenceReader.Advance(Position); return sequenceReader.TryRead(out readValue); } } - public async ValueTask ReadBytesAsync(long count) + public virtual async ValueTask ReadBytesAsync(long count) { if (count < 0 || count > int.MaxValue) { @@ -107,14 +107,14 @@ public async ValueTask ReadBytesAsync(long count) while (!TryRead(out value, count)) { - if (_readResult.IsCompleted) + if (ReadResult.IsCompleted) { throw new EndOfStreamException( $"requested {count} bytes, but got only {RemainingBytesInReadResult} bytes"); } - _pipeReader.AdvanceTo(_readResult.Buffer.Start, _readResult.Buffer.GetPosition(Position)); - _readResult = await _pipeReader.ReadAsync(); + PipeReader.AdvanceTo(ReadResult.Buffer.Start, ReadResult.Buffer.GetPosition(Position)); + ReadResult = await PipeReader.ReadAsync(); } Position += count; @@ -128,7 +128,7 @@ bool TryRead(out byte[] readBytes, long readBytesCount) return false; } - readBytes = _readResult.Buffer.Slice(Position, readBytesCount).ToArray(); + readBytes = ReadResult.Buffer.Slice(Position, readBytesCount).ToArray(); return true; } } @@ -137,8 +137,8 @@ public virtual async ValueTask ReadBytesFullAsync() { await FillReadResultBufferToTheEnd(); - _pipeReader.AdvanceTo(_readResult.Buffer.Start, _readResult.Buffer.End); - var value = _readResult.Buffer.Slice(Position, _readResult.Buffer.End).ToArray(); + PipeReader.AdvanceTo(ReadResult.Buffer.Start, ReadResult.Buffer.End); + var value = ReadResult.Buffer.Slice(Position, ReadResult.Buffer.End).ToArray(); Position += value.Length; return value; } @@ -147,18 +147,18 @@ private async ValueTask FillReadResultBufferToTheEnd() { await EnsureReadResultIsNotDefault(); - while (!_readResult.IsCompleted) + while (!ReadResult.IsCompleted) { - _pipeReader.AdvanceTo(_readResult.Buffer.Start, _readResult.Buffer.End); - _readResult = await _pipeReader.ReadAsync(); + PipeReader.AdvanceTo(ReadResult.Buffer.Start, ReadResult.Buffer.End); + ReadResult = await PipeReader.ReadAsync(); } } private async ValueTask EnsureReadResultIsNotDefault() { - if (_readResult.Equals(default(ReadResult))) + if (ReadResult.Equals(default(ReadResult))) { - _readResult = await _pipeReader.ReadAsync(); + ReadResult = await PipeReader.ReadAsync(); } } } diff --git a/Kaitai.Struct.Runtime.Async/ReaderContext/StreamReaderContext.cs b/Kaitai.Struct.Runtime.Async/ReaderContext/StreamReaderContext.cs index 60465b2..541680a 100644 --- a/Kaitai.Struct.Runtime.Async/ReaderContext/StreamReaderContext.cs +++ b/Kaitai.Struct.Runtime.Async/ReaderContext/StreamReaderContext.cs @@ -5,9 +5,10 @@ namespace Kaitai.Async { - internal class StreamReaderContext : IReaderContext + public class StreamReaderContext : IReaderContext { private readonly Stream _baseStream; + protected readonly AsyncBinaryReader AsyncBinaryReader; public StreamReaderContext(Stream stream) { @@ -15,22 +16,21 @@ public StreamReaderContext(Stream stream) AsyncBinaryReader = new AsyncBinaryReader(_baseStream); } - protected AsyncBinaryReader AsyncBinaryReader { get; } public long Position => _baseStream.Position; - public ValueTask GetSizeAsync() => new ValueTask(_baseStream.Length); + public virtual ValueTask GetSizeAsync() => new ValueTask(_baseStream.Length); - public ValueTask IsEofAsync() => + public virtual ValueTask IsEofAsync() => new ValueTask(_baseStream.Position >= _baseStream.Length); - public ValueTask SeekAsync(long position) + public virtual ValueTask SeekAsync(long position) { _baseStream.Seek(position, SeekOrigin.Begin); return new ValueTask(); } - public async ValueTask ReadByteAsync() => (byte) await AsyncBinaryReader.ReadSByteAsync(); + public virtual async ValueTask ReadByteAsync() => (byte) await AsyncBinaryReader.ReadSByteAsync(); - public async ValueTask ReadBytesAsync(long count) + public virtual async ValueTask ReadBytesAsync(long count) { if (count < 0 || count > int.MaxValue) { @@ -47,7 +47,7 @@ public async ValueTask ReadBytesAsync(long count) return bytes; } - public async ValueTask ReadBytesFullAsync() => + public virtual async ValueTask ReadBytesFullAsync() => await ReadBytesAsync(_baseStream.Length - _baseStream.Position); } } \ No newline at end of file From 2547b572c76689e3fdd5adc5c92cb297892ff788 Mon Sep 17 00:00:00 2001 From: Jan Pluskal Date: Thu, 1 Oct 2020 11:26:47 +0200 Subject: [PATCH 23/34] Add CancellationToken support. --- .../CancelableTestsBase.cs | 54 +++++ .../KaitaiAsyncStreamBaseTests.cs | 77 ++++++-- .../ReadBytesAsyncTests.cs | 187 +++++++++++------- .../ReadDecimalAsyncTests.cs | 44 ++++- .../ReadSignedAsyncTests.cs | 50 ++++- .../ReadUnSignedAsyncTests.cs | 57 +++++- .../Interface/IKaitaiAsyncStream.cs | 64 +++--- .../Interface/IReaderContext.cs | 15 +- .../KaitaiAsyncStream.cs | 88 +++++---- .../ReaderContext/PipeReaderContext.cs | 43 ++-- .../ReaderContext/StreamReaderContext.cs | 43 +++- 11 files changed, 500 insertions(+), 222 deletions(-) create mode 100644 Kaitai.Struct.Runtime.Async.Tests/CancelableTestsBase.cs diff --git a/Kaitai.Struct.Runtime.Async.Tests/CancelableTestsBase.cs b/Kaitai.Struct.Runtime.Async.Tests/CancelableTestsBase.cs new file mode 100644 index 0000000..8904346 --- /dev/null +++ b/Kaitai.Struct.Runtime.Async.Tests/CancelableTestsBase.cs @@ -0,0 +1,54 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Xunit; + +namespace Kaitai.Struct.Runtime.Async.Tests +{ + public abstract class CancelableTestsBase + { + protected readonly CancellationToken CancellationToken; + + protected CancelableTestsBase(bool isTestingCancellation) + { + CancellationToken = new CancellationToken(isTestingCancellation); + } + + protected async Task Evaluate(Func assertFunc) + { + if (CancellationToken.IsCancellationRequested) + { + await Assert.ThrowsAsync(assertFunc); + } + else + { + await assertFunc(); + } + } + + protected async Task EvaluateMaybeCancelled(Func assertFunc) + { + try + { + await assertFunc(); + } + catch (TaskCanceledException) + { + } + } + + protected async Task Evaluate(Func assertFunc) where TExpectedException : Exception + { + try + { + await assertFunc(); + } + catch (TaskCanceledException) + { + } + catch (TExpectedException) + { + } + } + } +} \ No newline at end of file diff --git a/Kaitai.Struct.Runtime.Async.Tests/KaitaiAsyncStreamBaseTests.cs b/Kaitai.Struct.Runtime.Async.Tests/KaitaiAsyncStreamBaseTests.cs index 0acf91a..b1b5d77 100644 --- a/Kaitai.Struct.Runtime.Async.Tests/KaitaiAsyncStreamBaseTests.cs +++ b/Kaitai.Struct.Runtime.Async.Tests/KaitaiAsyncStreamBaseTests.cs @@ -1,4 +1,5 @@ -using System.IO; +using System; +using System.IO; using System.IO.Pipelines; using System.Threading.Tasks; using Kaitai.Async; @@ -8,17 +9,48 @@ namespace Kaitai.Struct.Runtime.Async.Tests { public class StreamKaitaiAsyncStreamBaseTests : KaitaiAsyncStreamBaseTests { + public StreamKaitaiAsyncStreamBaseTests() : base(false) + { + } + protected override KaitaiAsyncStream Create(byte[] data) => new KaitaiAsyncStream(data); } public class PipeReaderKaitaiAsyncStreamBaseTests : KaitaiAsyncStreamBaseTests { + public PipeReaderKaitaiAsyncStreamBaseTests() : base(false) + { + } + protected override KaitaiAsyncStream Create(byte[] data) => new KaitaiAsyncStream(PipeReader.Create(new MemoryStream(data))); } - public abstract class KaitaiAsyncStreamBaseTests + public class StreamKaitaiAsyncStreamBaseCancelledTests : KaitaiAsyncStreamBaseTests + { + public StreamKaitaiAsyncStreamBaseCancelledTests() : base(true) + { + } + + protected override KaitaiAsyncStream Create(byte[] data) => new KaitaiAsyncStream(data); + } + + public class PipeReaderKaitaiAsyncStreamBaseCancelledTests : KaitaiAsyncStreamBaseTests { + public PipeReaderKaitaiAsyncStreamBaseCancelledTests() : base(true) + { + } + + protected override KaitaiAsyncStream Create(byte[] data) => + new KaitaiAsyncStream(PipeReader.Create(new MemoryStream(data))); + } + + public abstract class KaitaiAsyncStreamBaseTests : CancelableTestsBase + { + protected KaitaiAsyncStreamBaseTests(bool isTestingCancellation) : base(isTestingCancellation) + { + } + protected abstract KaitaiAsyncStream Create(byte[] data); [Theory] @@ -35,19 +67,22 @@ public abstract class KaitaiAsyncStreamBaseTests public async Task Eof_Test(bool shouldBeEof, int streamSize, int readBitsAmount) { var kaitaiStreamSUT = Create(new byte[streamSize]); - await kaitaiStreamSUT.ReadBitsIntAsync(readBitsAmount); - long positionBeforeIsEof = kaitaiStreamSUT.Pos; - - if (shouldBeEof) + await EvaluateMaybeCancelled(async () => { - Assert.True(kaitaiStreamSUT.IsEof); - } - else - { - Assert.False(kaitaiStreamSUT.IsEof); - } - - Assert.Equal(positionBeforeIsEof, kaitaiStreamSUT.Pos); + await kaitaiStreamSUT.ReadBitsIntAsync(readBitsAmount); + long positionBeforeIsEof = kaitaiStreamSUT.Pos; + + if (shouldBeEof) + { + Assert.True(kaitaiStreamSUT.IsEof); + } + else + { + Assert.False(kaitaiStreamSUT.IsEof); + } + + Assert.Equal(positionBeforeIsEof, kaitaiStreamSUT.Pos); + }); } [Theory] @@ -57,9 +92,12 @@ public async Task Pos_ByRead_Test(int expectedPos, int readBitsAmount) { var kaitaiStreamSUT = Create(new byte[1]); - await kaitaiStreamSUT.ReadBytesAsync(readBitsAmount); + await EvaluateMaybeCancelled(async () => + { + await kaitaiStreamSUT.ReadBytesAsync(readBitsAmount); - Assert.Equal(expectedPos, kaitaiStreamSUT.Pos); + Assert.Equal(expectedPos, kaitaiStreamSUT.Pos); + }); } [Theory] @@ -69,9 +107,12 @@ public async Task Pos_BySeek_Test(int expectedPos, int position) { var kaitaiStreamSUT = Create(new byte[1]); - await kaitaiStreamSUT.SeekAsync(position); + await EvaluateMaybeCancelled(async () => + { + await kaitaiStreamSUT.SeekAsync(position); - Assert.Equal(expectedPos, kaitaiStreamSUT.Pos); + Assert.Equal(expectedPos, kaitaiStreamSUT.Pos); + }); } [Theory] diff --git a/Kaitai.Struct.Runtime.Async.Tests/ReadBytesAsyncTests.cs b/Kaitai.Struct.Runtime.Async.Tests/ReadBytesAsyncTests.cs index 1de942b..6b903c2 100644 --- a/Kaitai.Struct.Runtime.Async.Tests/ReadBytesAsyncTests.cs +++ b/Kaitai.Struct.Runtime.Async.Tests/ReadBytesAsyncTests.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.IO; +using System.IO.Pipelines; using System.Linq; using System.Text; using System.Threading.Tasks; @@ -11,19 +12,48 @@ namespace Kaitai.Struct.Runtime.Async.Tests { public class StreamReadBytesAsyncTests : ReadBytesAsyncTests { + public StreamReadBytesAsyncTests() : base(false) + { + } + protected override KaitaiAsyncStream Create(byte[] data) => new KaitaiAsyncStream(data); } public class PipeReaderReadBytesAsyncTests : ReadBytesAsyncTests { + public PipeReaderReadBytesAsyncTests() : base(false) + { + } + protected override KaitaiAsyncStream Create(byte[] data) => - new KaitaiAsyncStream(System.IO.Pipelines.PipeReader.Create(new MemoryStream(data))); + new KaitaiAsyncStream(PipeReader.Create(new MemoryStream(data))); } - public abstract class ReadBytesAsyncTests + public class StreamReadBytesAsyncCancelledTests : ReadBytesAsyncTests { - protected abstract KaitaiAsyncStream Create(byte[] data); - + public StreamReadBytesAsyncCancelledTests() : base(true) + { + } + + protected override KaitaiAsyncStream Create(byte[] data) => new KaitaiAsyncStream(data); + } + + public class PipeReaderReadBytesAsyncCancelledTests : ReadBytesAsyncTests + { + public PipeReaderReadBytesAsyncCancelledTests() : base(true) + { + } + + protected override KaitaiAsyncStream Create(byte[] data) => + new KaitaiAsyncStream(PipeReader.Create(new MemoryStream(data))); + } + + public abstract class ReadBytesAsyncTests : CancelableTestsBase + { + protected ReadBytesAsyncTests(bool isTestingCancellation) : base(isTestingCancellation) + { + } + public static IEnumerable BytesData => new List<(byte[] streamContent, int bytesCount)> { @@ -33,6 +63,39 @@ public abstract class ReadBytesAsyncTests (new byte[] {0b_1101_0101, 0b_1101_0101}, 2) }.Select(t => new object[] {t.streamContent, t.bytesCount}); + public static IEnumerable StringData => + new List + { + "", + "ABC", + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + }.Select(t => new[] {Encoding.ASCII.GetBytes(t)}); + + public static IEnumerable StringWithTerminatorsData => + new List<(string streamContent, string expected, char terminator, bool isPresent, bool shouldInclude)> + { + ("", "", '\0', false, false), + ("", "", '\0', false, true), + + ("ABC", "ABC", '\0', false, false), + ("ABC", "ABC", '\0', false, true), + + ("ABC", "", 'A', true, false), + ("ABC", "A", 'A', true, true), + + ("ABC", "A", 'B', true, false), + ("ABC", "AB", 'B', true, true), + + ("ABC", "AB", 'C', true, false), + ("ABC", "ABC", 'C', true, true) + }.Select(t => new[] + { + Encoding.ASCII.GetBytes(t.streamContent), Encoding.ASCII.GetBytes(t.expected), (object) (byte) t.terminator, + t.isPresent, t.shouldInclude + }); + + protected abstract KaitaiAsyncStream Create(byte[] data); + [Theory] [MemberData(nameof(BytesData))] @@ -40,7 +103,9 @@ public async Task ReadBytesAsync_long_Test(byte[] streamContent, long bytesCount { var kaitaiStreamSUT = Create(streamContent); - Assert.Equal(streamContent.Take((int) bytesCount), await kaitaiStreamSUT.ReadBytesAsync(bytesCount)); + await Evaluate(async () => + Assert.Equal(streamContent.Take((int) bytesCount), + await kaitaiStreamSUT.ReadBytesAsync(bytesCount, CancellationToken))); } [Theory] @@ -49,17 +114,11 @@ public async Task ReadBytesAsync_ulong_Test(byte[] streamContent, ulong bytesCou { var kaitaiStreamSUT = Create(streamContent); - Assert.Equal(streamContent.Take((int) bytesCount), await kaitaiStreamSUT.ReadBytesAsync(bytesCount)); + await Evaluate(async () => + Assert.Equal(streamContent.Take((int) bytesCount), + await kaitaiStreamSUT.ReadBytesAsync(bytesCount, CancellationToken))); } - public static IEnumerable StringData => - new List - { - "", - "ABC", - "ABCDEFGHIJKLMNOPQRSTUVWXYZ" - }.Select(t => new[] {Encoding.ASCII.GetBytes(t)}); - [Theory] [MemberData(nameof(StringData))] @@ -67,7 +126,8 @@ public async Task ReadBytesFullAsync_Test(byte[] streamContent) { var kaitaiStreamSUT = Create(streamContent); - Assert.Equal(streamContent, await kaitaiStreamSUT.ReadBytesFullAsync()); + await Evaluate(async () => + Assert.Equal(streamContent, await kaitaiStreamSUT.ReadBytesFullAsync(CancellationToken))); } [Theory] @@ -76,7 +136,8 @@ public async Task EnsureFixedContentsAsync_Test(byte[] streamContent) { var kaitaiStreamSUT = Create(streamContent); - Assert.Equal(streamContent, await kaitaiStreamSUT.EnsureFixedContentsAsync(streamContent)); + await Evaluate(async () => + Assert.Equal(streamContent, await kaitaiStreamSUT.EnsureFixedContentsAsync(streamContent, CancellationToken))); } [Theory] @@ -93,32 +154,10 @@ public async Task EnsureFixedContentsAsync_ThrowsIfByteIsChanged(byte[] streamCo var expected = streamContent.ToArray(); expected[0] = (byte) ~expected[0]; - await Assert.ThrowsAsync(async () => await kaitaiStreamSUT.EnsureFixedContentsAsync(expected)); + await Evaluate(async () => + await kaitaiStreamSUT.EnsureFixedContentsAsync(expected, CancellationToken)); } - public static IEnumerable StringWithTerminatorsData => - new List<(string streamContent, string expected, char terminator, bool isPresent, bool shouldInclude)> - { - ("", "", '\0', false, false), - ("", "", '\0', false, true), - - ("ABC", "ABC", '\0', false, false), - ("ABC", "ABC", '\0', false, true), - - ("ABC", "", 'A', true, false), - ("ABC", "A", 'A', true, true), - - ("ABC", "A", 'B', true, false), - ("ABC", "AB", 'B', true, true), - - ("ABC", "AB", 'C', true, false), - ("ABC", "ABC", 'C', true, true) - }.Select(t => new[] - { - Encoding.ASCII.GetBytes(t.streamContent), Encoding.ASCII.GetBytes(t.expected), (object) (byte) t.terminator, - t.isPresent, t.shouldInclude - }); - [Theory] [MemberData(nameof(StringWithTerminatorsData))] public async Task ReadBytesTermAsync(byte[] streamContent, @@ -129,7 +168,8 @@ public async Task ReadBytesTermAsync(byte[] streamContent, { var kaitaiStreamSUT = Create(streamContent); - Assert.Equal(expected, await kaitaiStreamSUT.ReadBytesTermAsync(terminator, shouldInclude, false, false)); + await Evaluate(async () => Assert.Equal(expected, + await kaitaiStreamSUT.ReadBytesTermAsync(terminator, shouldInclude, false, false, CancellationToken))); } [Theory] @@ -147,8 +187,8 @@ public async Task ReadBytesTermAsync_ThrowsIsTerminatorNotPresent(byte[] streamC return; } - await Assert.ThrowsAsync(async () => - await kaitaiStreamSUT.ReadBytesTermAsync(terminator, shouldInclude, false, true)); + await Evaluate(async () => + await kaitaiStreamSUT.ReadBytesTermAsync(terminator, shouldInclude, false, true, CancellationToken)); } [Theory] @@ -162,17 +202,20 @@ public async Task ReadBytesTermAsync_ShouldNotConsumeTerminator(byte[] streamCon //Arrange var kaitaiStreamSUT = Create(streamContent); - //Act - await kaitaiStreamSUT.ReadBytesTermAsync(terminator, shouldInclude, false, false); - - //Assert - int amountToConsume = expected.Length; - if (expected.Length > 0 && shouldInclude && terminatorIsPresent) + await Evaluate(async () => { - amountToConsume--; - } + //Act + await kaitaiStreamSUT.ReadBytesTermAsync(terminator, shouldInclude, false, false, CancellationToken); - Assert.Equal(amountToConsume, kaitaiStreamSUT.Pos); + //Assert + int amountToConsume = expected.Length; + if (expected.Length > 0 && shouldInclude && terminatorIsPresent) + { + amountToConsume--; + } + + Assert.Equal(amountToConsume, kaitaiStreamSUT.Pos); + }); } [Theory] @@ -186,17 +229,20 @@ public async Task ReadBytesTermAsync_ShouldConsumeTerminator(byte[] streamConten //Arrange var kaitaiStreamSUT = Create(streamContent); - //Act - await kaitaiStreamSUT.ReadBytesTermAsync(terminator, shouldInclude, true, false); - - //Assert - int amountToConsume = expected.Length; - if (!shouldInclude && terminatorIsPresent) + await Evaluate(async () => { - amountToConsume++; - } + //Act + await kaitaiStreamSUT.ReadBytesTermAsync(terminator, shouldInclude, true, false, CancellationToken); + + //Assert + int amountToConsume = expected.Length; + if (!shouldInclude && terminatorIsPresent) + { + amountToConsume++; + } - Assert.Equal(amountToConsume, kaitaiStreamSUT.Pos); + Assert.Equal(amountToConsume, kaitaiStreamSUT.Pos); + }); } [Fact] @@ -204,8 +250,7 @@ public async Task ReadBytesAsyncLong_LargerThanBufferInvoke_ThrowsArgumentOutOfR { var kaitaiStreamSUT = Create(new byte[0]); - await Assert.ThrowsAsync(async () => - await kaitaiStreamSUT.ReadBytesAsync(1)); + await Evaluate(async () => await kaitaiStreamSUT.ReadBytesAsync(1, CancellationToken)); } [Fact] @@ -213,8 +258,8 @@ public async Task ReadBytesAsyncLong_LargerThanInt32Invoke_ThrowsArgumentOutOfRa { var kaitaiStreamSUT = Create(new byte[0]); - await Assert.ThrowsAsync(async () => - await kaitaiStreamSUT.ReadBytesAsync((long) int.MaxValue + 1)); + await Evaluate(async () => + await kaitaiStreamSUT.ReadBytesAsync((long) int.MaxValue + 1, CancellationToken)); } [Fact] @@ -222,8 +267,8 @@ public async Task ReadBytesAsyncLong_NegativeInvoke_ThrowsArgumentOutOfRangeExce { var kaitaiStreamSUT = Create(new byte[0]); - await Assert.ThrowsAsync(async () => - await kaitaiStreamSUT.ReadBytesAsync(-1)); + await Evaluate(async () => + await kaitaiStreamSUT.ReadBytesAsync(-1, CancellationToken)); } [Fact] @@ -231,8 +276,8 @@ public async Task ReadBytesAsyncULong_LargerThanBufferInvoke_ThrowsArgumentOutOf { var kaitaiStreamSUT = Create(new byte[0]); - await Assert.ThrowsAsync(async () => - await kaitaiStreamSUT.ReadBytesAsync((ulong) 1)); + await Evaluate(async () => + await kaitaiStreamSUT.ReadBytesAsync((ulong) 1, CancellationToken)); } [Fact] @@ -240,8 +285,8 @@ public async Task ReadBytesAsyncULong_LargerThanInt32Invoke_ThrowsArgumentOutOfR { var kaitaiStreamSUT = Create(new byte[0]); - await Assert.ThrowsAsync(async () => - await kaitaiStreamSUT.ReadBytesAsync((ulong) int.MaxValue + 1)); + await Evaluate(async () => + await kaitaiStreamSUT.ReadBytesAsync((ulong) int.MaxValue + 1, CancellationToken)); } } } \ No newline at end of file diff --git a/Kaitai.Struct.Runtime.Async.Tests/ReadDecimalAsyncTests.cs b/Kaitai.Struct.Runtime.Async.Tests/ReadDecimalAsyncTests.cs index dc4f652..3891dea 100644 --- a/Kaitai.Struct.Runtime.Async.Tests/ReadDecimalAsyncTests.cs +++ b/Kaitai.Struct.Runtime.Async.Tests/ReadDecimalAsyncTests.cs @@ -1,4 +1,5 @@ using System.IO; +using System.IO.Pipelines; using System.Linq; using System.Threading.Tasks; using Kaitai.Async; @@ -8,17 +9,48 @@ namespace Kaitai.Struct.Runtime.Async.Tests { public class StreamReadDecimalAsyncTests : ReadDecimalAsyncTests { + public StreamReadDecimalAsyncTests() : base(false) + { + } + protected override KaitaiAsyncStream Create(byte[] data) => new KaitaiAsyncStream(data); } public class PipeReaderReadDecimalAsyncTests : ReadDecimalAsyncTests { + public PipeReaderReadDecimalAsyncTests() : base(false) + { + } + protected override KaitaiAsyncStream Create(byte[] data) => - new KaitaiAsyncStream(System.IO.Pipelines.PipeReader.Create(new MemoryStream(data))); + new KaitaiAsyncStream(PipeReader.Create(new MemoryStream(data))); } - public abstract class ReadDecimalAsyncTests + public class StreamReadDecimalAsyncCancelledTests : ReadDecimalAsyncTests { + public StreamReadDecimalAsyncCancelledTests() : base(true) + { + } + + protected override KaitaiAsyncStream Create(byte[] data) => new KaitaiAsyncStream(data); + } + + public class PipeReaderReadDecimalAsyncCancelledTests : ReadDecimalAsyncTests + { + public PipeReaderReadDecimalAsyncCancelledTests() : base(true) + { + } + + protected override KaitaiAsyncStream Create(byte[] data) => + new KaitaiAsyncStream(PipeReader.Create(new MemoryStream(data))); + } + + public abstract class ReadDecimalAsyncTests : CancelableTestsBase + { + protected ReadDecimalAsyncTests(bool isTestingCancellation) : base(isTestingCancellation) + { + } + protected abstract KaitaiAsyncStream Create(byte[] data); [Theory] @@ -27,7 +59,7 @@ public async Task ReadF4beAsync_Test(float expected, byte[] streamContent) { var kaitaiStreamSUT = Create(streamContent.Reverse().ToArray()); - Assert.Equal(expected, await kaitaiStreamSUT.ReadF4beAsync()); + await Evaluate(async () => Assert.Equal(expected, await kaitaiStreamSUT.ReadF4beAsync(CancellationToken))); } [Theory] @@ -36,7 +68,7 @@ public async Task ReadF4leAsync_Test(float expected, byte[] streamContent) { var kaitaiStreamSUT = Create(streamContent); - Assert.Equal(expected, await kaitaiStreamSUT.ReadF4leAsync()); + await Evaluate(async () => Assert.Equal(expected, await kaitaiStreamSUT.ReadF4leAsync(CancellationToken))); } [Theory] @@ -45,7 +77,7 @@ public async Task ReadF8beAsync_Test(double expected, byte[] streamContent) { var kaitaiStreamSUT = Create(streamContent.Reverse().ToArray()); - Assert.Equal(expected, await kaitaiStreamSUT.ReadF8beAsync()); + await Evaluate(async () => Assert.Equal(expected, await kaitaiStreamSUT.ReadF8beAsync(CancellationToken))); } [Theory] @@ -54,7 +86,7 @@ public async Task ReadF8leAsync_Test(double expected, byte[] streamContent) { var kaitaiStreamSUT = Create(streamContent); - Assert.Equal(expected, await kaitaiStreamSUT.ReadF8leAsync()); + await Evaluate(async () => Assert.Equal(expected, await kaitaiStreamSUT.ReadF8leAsync(CancellationToken))); } } } \ No newline at end of file diff --git a/Kaitai.Struct.Runtime.Async.Tests/ReadSignedAsyncTests.cs b/Kaitai.Struct.Runtime.Async.Tests/ReadSignedAsyncTests.cs index 64ba08b..b31f4c9 100644 --- a/Kaitai.Struct.Runtime.Async.Tests/ReadSignedAsyncTests.cs +++ b/Kaitai.Struct.Runtime.Async.Tests/ReadSignedAsyncTests.cs @@ -1,4 +1,5 @@ using System.IO; +using System.IO.Pipelines; using System.Linq; using System.Threading.Tasks; using Kaitai.Async; @@ -8,17 +9,48 @@ namespace Kaitai.Struct.Runtime.Async.Tests { public class StreamReadSignedAsyncTests : ReadSignedAsyncTests { + public StreamReadSignedAsyncTests() : base(false) + { + } + protected override KaitaiAsyncStream Create(byte[] data) => new KaitaiAsyncStream(data); } public class PipeReaderReadSignedAsyncTests : ReadSignedAsyncTests { + public PipeReaderReadSignedAsyncTests() : base(false) + { + } + protected override KaitaiAsyncStream Create(byte[] data) => - new KaitaiAsyncStream(System.IO.Pipelines.PipeReader.Create(new MemoryStream(data))); + new KaitaiAsyncStream(PipeReader.Create(new MemoryStream(data))); } - public abstract class ReadSignedAsyncTests + public class StreamReadSignedAsyncCancelledTests : ReadSignedAsyncTests { + public StreamReadSignedAsyncCancelledTests() : base(true) + { + } + + protected override KaitaiAsyncStream Create(byte[] data) => new KaitaiAsyncStream(data); + } + + public class PipeReaderReadSignedAsyncCancelledTests : ReadSignedAsyncTests + { + public PipeReaderReadSignedAsyncCancelledTests() : base(true) + { + } + + protected override KaitaiAsyncStream Create(byte[] data) => + new KaitaiAsyncStream(PipeReader.Create(new MemoryStream(data))); + } + + public abstract class ReadSignedAsyncTests : CancelableTestsBase + { + protected ReadSignedAsyncTests(bool isTestingCancellation) : base(isTestingCancellation) + { + } + protected abstract KaitaiAsyncStream Create(byte[] data); [Theory] @@ -27,7 +59,7 @@ public async Task ReadS1Async_Test(sbyte expected, byte[] streamContent) { var kaitaiStreamSUT = Create(streamContent); - Assert.Equal(expected, await kaitaiStreamSUT.ReadS1Async()); + await Evaluate(async ()=>Assert.Equal(expected, await kaitaiStreamSUT.ReadS1Async(CancellationToken))); } [Theory] @@ -36,7 +68,7 @@ public async Task ReadS2beAsync_Test(short expected, byte[] streamContent) { var kaitaiStreamSUT = Create(streamContent); - Assert.Equal(expected, await kaitaiStreamSUT.ReadS2beAsync()); + await Evaluate(async () => Assert.Equal(expected, await kaitaiStreamSUT.ReadS2beAsync(CancellationToken))); } [Theory] @@ -45,7 +77,7 @@ public async Task ReadS4beAsync_Test(int expected, byte[] streamContent) { var kaitaiStreamSUT = Create(streamContent); - Assert.Equal(expected, await kaitaiStreamSUT.ReadS4beAsync()); + await Evaluate(async () => Assert.Equal(expected, await kaitaiStreamSUT.ReadS4beAsync(CancellationToken))); } [Theory] @@ -54,7 +86,7 @@ public async Task ReadS8beAsync_Test(long expected, byte[] streamContent) { var kaitaiStreamSUT = Create(streamContent); - Assert.Equal(expected, await kaitaiStreamSUT.ReadS8beAsync()); + await Evaluate(async () => Assert.Equal(expected, await kaitaiStreamSUT.ReadS8beAsync(CancellationToken))); } [Theory] @@ -63,7 +95,7 @@ public async Task ReadS2leAsync_Test(short expected, byte[] streamContent) { var kaitaiStreamSUT = Create(streamContent.Reverse().ToArray()); - Assert.Equal(expected, await kaitaiStreamSUT.ReadS2leAsync()); + await Evaluate(async () => Assert.Equal(expected, await kaitaiStreamSUT.ReadS2leAsync(CancellationToken))); } [Theory] @@ -72,7 +104,7 @@ public async Task ReadS4leAsync_Test(int expected, byte[] streamContent) { var kaitaiStreamSUT = Create(streamContent.Reverse().ToArray()); - Assert.Equal(expected, await kaitaiStreamSUT.ReadS4leAsync()); + await Evaluate(async () => Assert.Equal(expected, await kaitaiStreamSUT.ReadS4leAsync(CancellationToken))); } [Theory] @@ -81,7 +113,7 @@ public async Task ReadS8leAsync_Test(long expected, byte[] streamContent) { var kaitaiStreamSUT = Create(streamContent.Reverse().ToArray()); - Assert.Equal(expected, await kaitaiStreamSUT.ReadS8leAsync()); + await Evaluate(async () => Assert.Equal(expected, await kaitaiStreamSUT.ReadS8leAsync(CancellationToken))); } } } \ No newline at end of file diff --git a/Kaitai.Struct.Runtime.Async.Tests/ReadUnSignedAsyncTests.cs b/Kaitai.Struct.Runtime.Async.Tests/ReadUnSignedAsyncTests.cs index e593961..2c0c441 100644 --- a/Kaitai.Struct.Runtime.Async.Tests/ReadUnSignedAsyncTests.cs +++ b/Kaitai.Struct.Runtime.Async.Tests/ReadUnSignedAsyncTests.cs @@ -1,4 +1,5 @@ using System.IO; +using System.IO.Pipelines; using System.Linq; using System.Threading.Tasks; using Kaitai.Async; @@ -8,17 +9,49 @@ namespace Kaitai.Struct.Runtime.Async.Tests { public class StreamReadUnSignedAsyncTests : ReadUnSignedAsyncTests { + public StreamReadUnSignedAsyncTests() : base(false) + { + } + + protected override KaitaiAsyncStream Create(byte[] data) => new KaitaiAsyncStream(data); + } + + public class StreamReadUnSignedAsyncCancelledTests : ReadUnSignedAsyncTests + { + public StreamReadUnSignedAsyncCancelledTests() : base(true) + { + } + protected override KaitaiAsyncStream Create(byte[] data) => new KaitaiAsyncStream(data); } public class PipeReaderReadUnSignedAsyncTests : ReadUnSignedAsyncTests { + public PipeReaderReadUnSignedAsyncTests() : base(false) + { + } + + protected override KaitaiAsyncStream Create(byte[] data) => + new KaitaiAsyncStream(PipeReader.Create(new MemoryStream(data))); + } + + public class PipeReaderReadUnSignedAsyncCancelledTests : ReadUnSignedAsyncTests + { + public PipeReaderReadUnSignedAsyncCancelledTests() : base(true) + { + } + protected override KaitaiAsyncStream Create(byte[] data) => - new KaitaiAsyncStream(System.IO.Pipelines.PipeReader.Create(new MemoryStream(data))); + new KaitaiAsyncStream(PipeReader.Create(new MemoryStream(data))); } - public abstract class ReadUnSignedAsyncTests + + public abstract class ReadUnSignedAsyncTests : CancelableTestsBase { + protected ReadUnSignedAsyncTests(bool isTestingCancellation) : base(isTestingCancellation) + { + } + protected abstract KaitaiAsyncStream Create(byte[] data); [Theory] @@ -27,7 +60,7 @@ public async Task ReadU1Async_Test( /*u*/ sbyte expected, byte[] streamContent) { var kaitaiStreamSUT = Create(streamContent); - Assert.Equal((byte) expected, await kaitaiStreamSUT.ReadU1Async()); + await Evaluate(async () => Assert.Equal((byte) expected, await kaitaiStreamSUT.ReadU1Async(CancellationToken))); } [Theory] @@ -36,7 +69,8 @@ public async Task ReadU2beAsync_Test( /*u*/ short expected, byte[] streamContent { var kaitaiStreamSUT = Create(streamContent); - Assert.Equal((ushort) expected, await kaitaiStreamSUT.ReadU2beAsync()); + await Evaluate(async () => + Assert.Equal((ushort) expected, await kaitaiStreamSUT.ReadU2beAsync(CancellationToken))); } [Theory] @@ -45,7 +79,8 @@ public async Task ReadU4beAsync_Test( /*u*/ int expected, byte[] streamContent) { var kaitaiStreamSUT = Create(streamContent); - Assert.Equal((uint) expected, await kaitaiStreamSUT.ReadU4beAsync()); + await Evaluate(async () => + Assert.Equal((uint) expected, await kaitaiStreamSUT.ReadU4beAsync(CancellationToken))); } [Theory] @@ -54,7 +89,8 @@ public async Task ReadU8beAsync_Test( /*u*/ long expected, byte[] streamContent) { var kaitaiStreamSUT = Create(streamContent); - Assert.Equal((ulong) expected, await kaitaiStreamSUT.ReadU8beAsync()); + await Evaluate( + async () => Assert.Equal((ulong) expected, await kaitaiStreamSUT.ReadU8beAsync(CancellationToken))); } [Theory] @@ -63,7 +99,8 @@ public async Task ReadU2leAsync_Test( /*u*/ short expected, byte[] streamContent { var kaitaiStreamSUT = Create(streamContent.Reverse().ToArray()); - Assert.Equal((ushort) expected, await kaitaiStreamSUT.ReadU2leAsync()); + await Evaluate(async () => + Assert.Equal((ushort) expected, await kaitaiStreamSUT.ReadU2leAsync(CancellationToken))); } [Theory] @@ -72,7 +109,8 @@ public async Task ReadU4leAsync_Test( /*u*/ int expected, byte[] streamContent) { var kaitaiStreamSUT = Create(streamContent.Reverse().ToArray()); - Assert.Equal((uint) expected, await kaitaiStreamSUT.ReadU4leAsync()); + await Evaluate(async () => + Assert.Equal((uint) expected, await kaitaiStreamSUT.ReadU4leAsync(CancellationToken))); } [Theory] @@ -81,7 +119,8 @@ public async Task ReadU8leAsync_Test( /*u*/ long expected, byte[] streamContent) { var kaitaiStreamSUT = Create(streamContent.Reverse().ToArray()); - Assert.Equal((ulong) expected, await kaitaiStreamSUT.ReadU8leAsync()); + await Evaluate( + async () => Assert.Equal((ulong) expected, await kaitaiStreamSUT.ReadU8leAsync(CancellationToken))); } } } \ No newline at end of file diff --git a/Kaitai.Struct.Runtime.Async/Interface/IKaitaiAsyncStream.cs b/Kaitai.Struct.Runtime.Async/Interface/IKaitaiAsyncStream.cs index 4df5abb..5b58aa7 100644 --- a/Kaitai.Struct.Runtime.Async/Interface/IKaitaiAsyncStream.cs +++ b/Kaitai.Struct.Runtime.Async/Interface/IKaitaiAsyncStream.cs @@ -1,4 +1,5 @@ -using System.Threading.Tasks; +using System.Threading; +using System.Threading.Tasks; namespace Kaitai.Async { @@ -7,150 +8,153 @@ public interface IKaitaiAsyncStream : IKaitaiStreamBase /// /// Check if the stream position is at the end of the stream /// - ValueTask IsEofAsync(); + ValueTask IsEofAsync(CancellationToken cancellationToken = default); /// /// Get the total length of the stream (ie. file size) /// - ValueTask GetSizeAsync(); + ValueTask GetSizeAsync(CancellationToken cancellationToken = default); /// /// Seek to a specific position from the beginning of the stream /// /// The position to seek to - Task SeekAsync(long position); + /// + Task SeekAsync(long position, CancellationToken cancellationToken = default); /// /// Read a signed byte from the stream /// /// - Task ReadS1Async(); + Task ReadS1Async(CancellationToken cancellationToken = default); /// /// Read a signed short from the stream (big endian) /// /// - Task ReadS2beAsync(); + Task ReadS2beAsync(CancellationToken cancellationToken = default); /// /// Read a signed int from the stream (big endian) /// /// - Task ReadS4beAsync(); + Task ReadS4beAsync(CancellationToken cancellationToken = default); /// /// Read a signed long from the stream (big endian) /// /// - Task ReadS8beAsync(); + Task ReadS8beAsync(CancellationToken cancellationToken = default); /// /// Read a signed short from the stream (little endian) /// /// - Task ReadS2leAsync(); + Task ReadS2leAsync(CancellationToken cancellationToken = default); /// /// Read a signed int from the stream (little endian) /// /// - Task ReadS4leAsync(); + Task ReadS4leAsync(CancellationToken cancellationToken = default); /// /// Read a signed long from the stream (little endian) /// /// - Task ReadS8leAsync(); + Task ReadS8leAsync(CancellationToken cancellationToken = default); /// /// Read an unsigned byte from the stream /// /// - Task ReadU1Async(); + Task ReadU1Async(CancellationToken cancellationToken = default); /// /// Read an unsigned short from the stream (big endian) /// /// - Task ReadU2beAsync(); + Task ReadU2beAsync(CancellationToken cancellationToken = default); /// /// Read an unsigned int from the stream (big endian) /// /// - Task ReadU4beAsync(); + Task ReadU4beAsync(CancellationToken cancellationToken = default); /// /// Read an unsigned long from the stream (big endian) /// /// - Task ReadU8beAsync(); + Task ReadU8beAsync(CancellationToken cancellationToken = default); /// /// Read an unsigned short from the stream (little endian) /// /// - Task ReadU2leAsync(); + Task ReadU2leAsync(CancellationToken cancellationToken = default); /// /// Read an unsigned int from the stream (little endian) /// /// - Task ReadU4leAsync(); + Task ReadU4leAsync(CancellationToken cancellationToken = default); /// /// Read an unsigned long from the stream (little endian) /// /// - Task ReadU8leAsync(); + Task ReadU8leAsync(CancellationToken cancellationToken = default); /// /// Read a single-precision floating point value from the stream (big endian) /// /// - Task ReadF4beAsync(); + Task ReadF4beAsync(CancellationToken cancellationToken = default); /// /// Read a double-precision floating point value from the stream (big endian) /// /// - Task ReadF8beAsync(); + Task ReadF8beAsync(CancellationToken cancellationToken = default); /// /// Read a single-precision floating point value from the stream (little endian) /// /// - Task ReadF4leAsync(); + Task ReadF4leAsync(CancellationToken cancellationToken = default); /// /// Read a double-precision floating point value from the stream (little endian) /// /// - Task ReadF8leAsync(); + Task ReadF8leAsync(CancellationToken cancellationToken = default); - Task ReadBitsIntAsync(int n); + Task ReadBitsIntAsync(int n, CancellationToken cancellationToken = default); - Task ReadBitsIntLeAsync(int n); + Task ReadBitsIntLeAsync(int n, CancellationToken cancellationToken = default); /// /// Read a fixed number of bytes from the stream /// /// The number of bytes to read + /// /// - Task ReadBytesAsync(long count); + Task ReadBytesAsync(long count, CancellationToken cancellationToken = default); /// /// Read a fixed number of bytes from the stream /// /// The number of bytes to read + /// /// - Task ReadBytesAsync(ulong count); + Task ReadBytesAsync(ulong count, CancellationToken cancellationToken = default); /// /// Read all the remaining bytes from the stream until the end is reached /// /// - Task ReadBytesFullAsync(); + Task ReadBytesFullAsync(CancellationToken cancellationToken = default); /// /// Read a terminated string from the stream @@ -163,13 +167,15 @@ public interface IKaitaiAsyncStream : IKaitaiStreamBase Task ReadBytesTermAsync(byte terminator, bool includeTerminator, bool consumeTerminator, - bool eosError); + bool eosError, + CancellationToken cancellationToken = default); /// /// Read a specific set of bytes and assert that they are the same as an expected result /// /// The expected result + /// /// - Task EnsureFixedContentsAsync(byte[] expected); + Task EnsureFixedContentsAsync(byte[] expected, CancellationToken cancellationToken = default); } } \ No newline at end of file diff --git a/Kaitai.Struct.Runtime.Async/Interface/IReaderContext.cs b/Kaitai.Struct.Runtime.Async/Interface/IReaderContext.cs index 4eab4fb..d497968 100644 --- a/Kaitai.Struct.Runtime.Async/Interface/IReaderContext.cs +++ b/Kaitai.Struct.Runtime.Async/Interface/IReaderContext.cs @@ -1,15 +1,16 @@ -using System.Threading.Tasks; +using System.Threading; +using System.Threading.Tasks; namespace Kaitai.Async { public interface IReaderContext { long Position { get; } - ValueTask GetSizeAsync(); - ValueTask IsEofAsync(); - ValueTask SeekAsync(long position); - ValueTask ReadByteAsync(); - ValueTask ReadBytesAsync(long count); - ValueTask ReadBytesFullAsync(); + ValueTask GetSizeAsync(CancellationToken cancellationToken = default); + ValueTask IsEofAsync(CancellationToken cancellationToken = default); + ValueTask SeekAsync(long position, CancellationToken cancellationToken = default); + ValueTask ReadByteAsync(CancellationToken cancellationToken = default); + ValueTask ReadBytesAsync(long count, CancellationToken cancellationToken = default); + ValueTask ReadBytesFullAsync(CancellationToken cancellationToken = default); } } \ No newline at end of file diff --git a/Kaitai.Struct.Runtime.Async/KaitaiAsyncStream.cs b/Kaitai.Struct.Runtime.Async/KaitaiAsyncStream.cs index 560c0ea..193acc5 100644 --- a/Kaitai.Struct.Runtime.Async/KaitaiAsyncStream.cs +++ b/Kaitai.Struct.Runtime.Async/KaitaiAsyncStream.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.IO; using System.IO.Pipelines; +using System.Threading; using System.Threading.Tasks; namespace Kaitai.Async @@ -53,12 +54,12 @@ public KaitaiAsyncStream(byte[] bytes) : this(new MemoryStream(bytes)) public override bool IsEof => ReaderContext.IsEofAsync().GetAwaiter().GetResult() && _bitsLeft == 0; - public async ValueTask IsEofAsync() => await ReaderContext.IsEofAsync() && _bitsLeft == 0; + public async ValueTask IsEofAsync(CancellationToken cancellationToken = default) => await ReaderContext.IsEofAsync(cancellationToken) && _bitsLeft == 0; - public ValueTask GetSizeAsync() => ReaderContext.GetSizeAsync(); + public ValueTask GetSizeAsync(CancellationToken cancellationToken = default) => ReaderContext.GetSizeAsync(cancellationToken); - public virtual async Task SeekAsync(long position) => await ReaderContext.SeekAsync(position); - public virtual async Task SeekAsync(ulong position) => await SeekAsync((long)position); + public virtual async Task SeekAsync(long position, CancellationToken cancellationToken = default) => await ReaderContext.SeekAsync(position, cancellationToken); + public virtual async Task SeekAsync(ulong position, CancellationToken cancellationToken = default) => await SeekAsync((long)position, cancellationToken); public override long Pos => ReaderContext.Position; @@ -70,25 +71,25 @@ public KaitaiAsyncStream(byte[] bytes) : this(new MemoryStream(bytes)) #region Signed - public async Task ReadS1Async() => (sbyte) await ReadU1Async(); + public async Task ReadS1Async(CancellationToken cancellationToken = default) => (sbyte) await ReadU1Async(cancellationToken); #region Big-endian - public async Task ReadS2beAsync() => BitConverter.ToInt16(await ReadBytesNormalisedBigEndianAsync(2), 0); + public async Task ReadS2beAsync(CancellationToken cancellationToken = default) => BitConverter.ToInt16(await ReadBytesNormalisedBigEndianAsync(2, cancellationToken), 0); - public async Task ReadS4beAsync() => BitConverter.ToInt32(await ReadBytesNormalisedBigEndianAsync(4), 0); + public async Task ReadS4beAsync(CancellationToken cancellationToken = default) => BitConverter.ToInt32(await ReadBytesNormalisedBigEndianAsync(4, cancellationToken), 0); - public async Task ReadS8beAsync() => BitConverter.ToInt64(await ReadBytesNormalisedBigEndianAsync(8), 0); + public async Task ReadS8beAsync(CancellationToken cancellationToken = default) => BitConverter.ToInt64(await ReadBytesNormalisedBigEndianAsync(8, cancellationToken), 0); #endregion #region Little-endian - public async Task ReadS2leAsync() => BitConverter.ToInt16(await ReadBytesNormalisedLittleEndianAsync(2), 0); + public async Task ReadS2leAsync(CancellationToken cancellationToken = default) => BitConverter.ToInt16(await ReadBytesNormalisedLittleEndianAsync(2, cancellationToken), 0); - public async Task ReadS4leAsync() => BitConverter.ToInt32(await ReadBytesNormalisedLittleEndianAsync(4), 0); + public async Task ReadS4leAsync(CancellationToken cancellationToken = default) => BitConverter.ToInt32(await ReadBytesNormalisedLittleEndianAsync(4, cancellationToken), 0); - public async Task ReadS8leAsync() => BitConverter.ToInt64(await ReadBytesNormalisedLittleEndianAsync(8), 0); + public async Task ReadS8leAsync(CancellationToken cancellationToken = default) => BitConverter.ToInt64(await ReadBytesNormalisedLittleEndianAsync(8, cancellationToken), 0); #endregion @@ -96,26 +97,26 @@ public KaitaiAsyncStream(byte[] bytes) : this(new MemoryStream(bytes)) #region Unsigned - public async Task ReadU1Async() => await ReaderContext.ReadByteAsync(); + public async Task ReadU1Async(CancellationToken cancellationToken = default) => await ReaderContext.ReadByteAsync(cancellationToken); #region Big-endian - public async Task ReadU2beAsync() => BitConverter.ToUInt16(await ReadBytesNormalisedBigEndianAsync(2), 0); + public async Task ReadU2beAsync(CancellationToken cancellationToken = default) => BitConverter.ToUInt16(await ReadBytesNormalisedBigEndianAsync(2, cancellationToken), 0); - public async Task ReadU4beAsync() => BitConverter.ToUInt32(await ReadBytesNormalisedBigEndianAsync(4), 0); + public async Task ReadU4beAsync(CancellationToken cancellationToken = default) => BitConverter.ToUInt32(await ReadBytesNormalisedBigEndianAsync(4, cancellationToken), 0); - public async Task ReadU8beAsync() => BitConverter.ToUInt64(await ReadBytesNormalisedBigEndianAsync(8), 0); + public async Task ReadU8beAsync(CancellationToken cancellationToken = default) => BitConverter.ToUInt64(await ReadBytesNormalisedBigEndianAsync(8, cancellationToken), 0); #endregion #region Little-endian - public async Task ReadU2leAsync() => - BitConverter.ToUInt16(await ReadBytesNormalisedLittleEndianAsync(2), 0); + public async Task ReadU2leAsync(CancellationToken cancellationToken = default) => + BitConverter.ToUInt16(await ReadBytesNormalisedLittleEndianAsync(2, cancellationToken), 0); - public async Task ReadU4leAsync() => BitConverter.ToUInt32(await ReadBytesNormalisedLittleEndianAsync(4), 0); + public async Task ReadU4leAsync(CancellationToken cancellationToken = default) => BitConverter.ToUInt32(await ReadBytesNormalisedLittleEndianAsync(4, cancellationToken), 0); - public async Task ReadU8leAsync() => BitConverter.ToUInt64(await ReadBytesNormalisedLittleEndianAsync(8), 0); + public async Task ReadU8leAsync(CancellationToken cancellationToken = default) => BitConverter.ToUInt64(await ReadBytesNormalisedLittleEndianAsync(8, cancellationToken), 0); #endregion @@ -127,18 +128,18 @@ public async Task ReadU2leAsync() => #region Big-endian - public async Task ReadF4beAsync() => BitConverter.ToSingle(await ReadBytesNormalisedBigEndianAsync(4), 0); + public async Task ReadF4beAsync(CancellationToken cancellationToken = default) => BitConverter.ToSingle(await ReadBytesNormalisedBigEndianAsync(4, cancellationToken), 0); - public async Task ReadF8beAsync() => BitConverter.ToDouble(await ReadBytesNormalisedBigEndianAsync(8), 0); + public async Task ReadF8beAsync(CancellationToken cancellationToken = default) => BitConverter.ToDouble(await ReadBytesNormalisedBigEndianAsync(8, cancellationToken), 0); #endregion #region Little-endian - public async Task ReadF4leAsync() => BitConverter.ToSingle(await ReadBytesNormalisedLittleEndianAsync(4), 0); + public async Task ReadF4leAsync(CancellationToken cancellationToken = default) => BitConverter.ToSingle(await ReadBytesNormalisedLittleEndianAsync(4, cancellationToken), 0); - public async Task ReadF8leAsync() => - BitConverter.ToDouble(await ReadBytesNormalisedLittleEndianAsync(8), 0); + public async Task ReadF8leAsync(CancellationToken cancellationToken = default) => + BitConverter.ToDouble(await ReadBytesNormalisedLittleEndianAsync(8, cancellationToken), 0); #endregion @@ -152,7 +153,7 @@ public override void AlignToByte() _bitsLeft = 0; } - public async Task ReadBitsIntAsync(int n) + public async Task ReadBitsIntAsync(int n, CancellationToken cancellationToken = default) { int bitsNeeded = n - _bitsLeft; if (bitsNeeded > 0) @@ -161,7 +162,7 @@ public async Task ReadBitsIntAsync(int n) // 8 bits => 1 byte // 9 bits => 2 bytes int bytesNeeded = (bitsNeeded - 1) / 8 + 1; - var buf = await ReadBytesAsync(bytesNeeded); + var buf = await ReadBytesAsync(bytesNeeded, cancellationToken); for (var i = 0; i < buf.Length; i++) { _bits <<= 8; @@ -186,7 +187,7 @@ public async Task ReadBitsIntAsync(int n) } //Method ported from algorithm specified @ issue#155 - public async Task ReadBitsIntLeAsync(int n) + public async Task ReadBitsIntLeAsync(int n, CancellationToken cancellationToken = default) { int bitsNeeded = n - _bitsLeft; @@ -196,7 +197,7 @@ public async Task ReadBitsIntLeAsync(int n) // 8 bits => 1 byte // 9 bits => 2 bytes int bytesNeeded = (bitsNeeded - 1) / 8 + 1; - var buf = await ReadBytesAsync(bytesNeeded); + var buf = await ReadBytesAsync(bytesNeeded, cancellationToken); for (var i = 0; i < buf.Length; i++) { ulong v = (ulong) buf[i] << _bitsLeft; @@ -222,9 +223,9 @@ public async Task ReadBitsIntLeAsync(int n) #region Byte arrays - public async Task ReadBytesAsync(long count) => await ReaderContext.ReadBytesAsync(count); + public async Task ReadBytesAsync(long count, CancellationToken cancellationToken = default) => await ReaderContext.ReadBytesAsync(count, cancellationToken); - public async Task ReadBytesAsync(ulong count) + public async Task ReadBytesAsync(ulong count, CancellationToken cancellationToken = default) { if (count > int.MaxValue) { @@ -232,7 +233,7 @@ public async Task ReadBytesAsync(ulong count) $"requested {count} bytes, while only non-negative int32 amount of bytes possible"); } - return await ReadBytesAsync((long) count); + return await ReadBytesAsync((long) count, cancellationToken); } /// @@ -240,9 +241,9 @@ public async Task ReadBytesAsync(ulong count) /// /// The number of bytes to read /// An array of bytes that matches the endianness of the current platform - protected async Task ReadBytesNormalisedLittleEndianAsync(int count) + protected async Task ReadBytesNormalisedLittleEndianAsync(int count, CancellationToken cancellationToken = default) { - var bytes = await ReadBytesAsync(count); + var bytes = await ReadBytesAsync(count, cancellationToken); if (!IsLittleEndian) { Array.Reverse(bytes); @@ -256,9 +257,9 @@ protected async Task ReadBytesNormalisedLittleEndianAsync(int count) /// /// The number of bytes to read /// An array of bytes that matches the endianness of the current platform - protected async Task ReadBytesNormalisedBigEndianAsync(int count) + protected async Task ReadBytesNormalisedBigEndianAsync(int count, CancellationToken cancellationToken = default) { - var bytes = await ReadBytesAsync(count); + var bytes = await ReadBytesAsync(count, cancellationToken); if (IsLittleEndian) { Array.Reverse(bytes); @@ -271,7 +272,7 @@ protected async Task ReadBytesNormalisedBigEndianAsync(int count) /// Read all the remaining bytes from the stream until the end is reached /// /// - public virtual async Task ReadBytesFullAsync() => await ReaderContext.ReadBytesFullAsync(); + public virtual async Task ReadBytesFullAsync(CancellationToken cancellationToken = default) => await ReaderContext.ReadBytesFullAsync(cancellationToken); /// /// Read a terminated string from the stream @@ -280,16 +281,18 @@ protected async Task ReadBytesNormalisedBigEndianAsync(int count) /// True to include the terminator in the returned string /// True to consume the terminator byte before returning /// True to throw an error when the EOS was reached before the terminator + /// /// public async Task ReadBytesTermAsync(byte terminator, bool includeTerminator, bool consumeTerminator, - bool eosError) + bool eosError, + CancellationToken cancellationToken = default) { var bytes = new List(); while (true) { - if (IsEof) + if (await IsEofAsync(cancellationToken)) { if (eosError) { @@ -300,7 +303,7 @@ public async Task ReadBytesTermAsync(byte terminator, break; } - byte b = await ReadU1Async(); + byte b = await ReadU1Async(cancellationToken); if (b == terminator) { if (includeTerminator) @@ -310,7 +313,7 @@ public async Task ReadBytesTermAsync(byte terminator, if (!consumeTerminator) { - await SeekAsync(Pos - 1); + await SeekAsync(Pos - 1, cancellationToken); } break; @@ -326,10 +329,11 @@ public async Task ReadBytesTermAsync(byte terminator, /// Read a specific set of bytes and assert that they are the same as an expected result /// /// The expected result + /// /// - public async Task EnsureFixedContentsAsync(byte[] expected) + public async Task EnsureFixedContentsAsync(byte[] expected, CancellationToken cancellationToken = default) { - var bytes = await ReadBytesAsync(expected.Length); + var bytes = await ReadBytesAsync(expected.Length, cancellationToken); if (bytes.Length != expected.Length) //TODO Is this necessary? { diff --git a/Kaitai.Struct.Runtime.Async/ReaderContext/PipeReaderContext.cs b/Kaitai.Struct.Runtime.Async/ReaderContext/PipeReaderContext.cs index e4731b4..97f832f 100644 --- a/Kaitai.Struct.Runtime.Async/ReaderContext/PipeReaderContext.cs +++ b/Kaitai.Struct.Runtime.Async/ReaderContext/PipeReaderContext.cs @@ -2,6 +2,7 @@ using System.Buffers; using System.IO; using System.IO.Pipelines; +using System.Threading; using System.Threading.Tasks; namespace Kaitai.Async @@ -20,28 +21,28 @@ public PipeReaderContext(PipeReader pipeReader) public long Position { get; protected set; } - public virtual async ValueTask GetSizeAsync() + public virtual async ValueTask GetSizeAsync(CancellationToken cancellationToken = default) { - await FillReadResultBufferToTheEnd(); + await FillReadResultBufferToTheEndAsync(cancellationToken); return ReadResult.Buffer.Length; } - public virtual async ValueTask IsEofAsync() + public virtual async ValueTask IsEofAsync(CancellationToken cancellationToken = default) { - await EnsureReadResultIsNotDefault(); + await EnsureReadResultIsNotDefaultAsync(cancellationToken); if (Position >= ReadResult.Buffer.Length && !ReadResult.IsCompleted) { PipeReader.AdvanceTo(ReadResult.Buffer.Start, ReadResult.Buffer.GetPosition(Position)); - ReadResult = await PipeReader.ReadAsync(); + ReadResult = await PipeReader.ReadAsync(cancellationToken); } return Position >= ReadResult.Buffer.Length && ReadResult.IsCompleted; } - public virtual async ValueTask SeekAsync(long position) + public virtual async ValueTask SeekAsync(long position, CancellationToken cancellationToken = default) { if (position <= Position) { @@ -49,12 +50,12 @@ public virtual async ValueTask SeekAsync(long position) } else { - await EnsureReadResultIsNotDefault(); + await EnsureReadResultIsNotDefaultAsync(cancellationToken); while (ReadResult.Buffer.Length < position && !ReadResult.IsCompleted) { PipeReader.AdvanceTo(ReadResult.Buffer.Start, ReadResult.Buffer.End); - ReadResult = await PipeReader.ReadAsync(); + ReadResult = await PipeReader.ReadAsync(cancellationToken); } if (ReadResult.Buffer.Length <= position) @@ -71,15 +72,15 @@ public virtual async ValueTask SeekAsync(long position) } } - public virtual async ValueTask ReadByteAsync() + public virtual async ValueTask ReadByteAsync(CancellationToken cancellationToken = default) { - await EnsureReadResultIsNotDefault(); + await EnsureReadResultIsNotDefaultAsync(cancellationToken); var value = byte.MinValue; while (!TryReadByte(out value) && !ReadResult.IsCompleted) { PipeReader.AdvanceTo(ReadResult.Buffer.Start, ReadResult.Buffer.GetPosition(Position)); - ReadResult = await PipeReader.ReadAsync(); + ReadResult = await PipeReader.ReadAsync(cancellationToken); } Position += 1; @@ -93,7 +94,7 @@ bool TryReadByte(out byte readValue) } } - public virtual async ValueTask ReadBytesAsync(long count) + public virtual async ValueTask ReadBytesAsync(long count, CancellationToken cancellationToken = default) { if (count < 0 || count > int.MaxValue) { @@ -101,7 +102,7 @@ public virtual async ValueTask ReadBytesAsync(long count) $"requested {count} bytes, while only non-negative int32 amount of bytes possible"); } - await EnsureReadResultIsNotDefault(); + await EnsureReadResultIsNotDefaultAsync(cancellationToken); byte[] value = null; @@ -114,7 +115,7 @@ public virtual async ValueTask ReadBytesAsync(long count) } PipeReader.AdvanceTo(ReadResult.Buffer.Start, ReadResult.Buffer.GetPosition(Position)); - ReadResult = await PipeReader.ReadAsync(); + ReadResult = await PipeReader.ReadAsync(cancellationToken); } Position += count; @@ -133,9 +134,9 @@ bool TryRead(out byte[] readBytes, long readBytesCount) } } - public virtual async ValueTask ReadBytesFullAsync() + public virtual async ValueTask ReadBytesFullAsync(CancellationToken cancellationToken = default) { - await FillReadResultBufferToTheEnd(); + await FillReadResultBufferToTheEndAsync(cancellationToken); PipeReader.AdvanceTo(ReadResult.Buffer.Start, ReadResult.Buffer.End); var value = ReadResult.Buffer.Slice(Position, ReadResult.Buffer.End).ToArray(); @@ -143,22 +144,22 @@ public virtual async ValueTask ReadBytesFullAsync() return value; } - private async ValueTask FillReadResultBufferToTheEnd() + private async ValueTask FillReadResultBufferToTheEndAsync(CancellationToken cancellationToken = default) { - await EnsureReadResultIsNotDefault(); + await EnsureReadResultIsNotDefaultAsync(cancellationToken); while (!ReadResult.IsCompleted) { PipeReader.AdvanceTo(ReadResult.Buffer.Start, ReadResult.Buffer.End); - ReadResult = await PipeReader.ReadAsync(); + ReadResult = await PipeReader.ReadAsync(cancellationToken); } } - private async ValueTask EnsureReadResultIsNotDefault() + private async ValueTask EnsureReadResultIsNotDefaultAsync(CancellationToken cancellationToken = default) { if (ReadResult.Equals(default(ReadResult))) { - ReadResult = await PipeReader.ReadAsync(); + ReadResult = await PipeReader.ReadAsync(cancellationToken); } } } diff --git a/Kaitai.Struct.Runtime.Async/ReaderContext/StreamReaderContext.cs b/Kaitai.Struct.Runtime.Async/ReaderContext/StreamReaderContext.cs index 541680a..6eeddf5 100644 --- a/Kaitai.Struct.Runtime.Async/ReaderContext/StreamReaderContext.cs +++ b/Kaitai.Struct.Runtime.Async/ReaderContext/StreamReaderContext.cs @@ -1,5 +1,6 @@ using System; using System.IO; +using System.Threading; using System.Threading.Tasks; using Overby.Extensions.AsyncBinaryReaderWriter; @@ -17,20 +18,32 @@ public StreamReaderContext(Stream stream) } public long Position => _baseStream.Position; - public virtual ValueTask GetSizeAsync() => new ValueTask(_baseStream.Length); - public virtual ValueTask IsEofAsync() => - new ValueTask(_baseStream.Position >= _baseStream.Length); + public virtual async ValueTask GetSizeAsync(CancellationToken cancellationToken = default) + { + await CheckIsCancellationRequested(cancellationToken); + + return _baseStream.Length; + } + + public virtual async ValueTask IsEofAsync(CancellationToken cancellationToken = default) + { + await CheckIsCancellationRequested(cancellationToken); + + return _baseStream.Position >= _baseStream.Length; + } - public virtual ValueTask SeekAsync(long position) + public virtual async ValueTask SeekAsync(long position, CancellationToken cancellationToken = default) { + await CheckIsCancellationRequested(cancellationToken); + _baseStream.Seek(position, SeekOrigin.Begin); - return new ValueTask(); } - public virtual async ValueTask ReadByteAsync() => (byte) await AsyncBinaryReader.ReadSByteAsync(); + public virtual async ValueTask ReadByteAsync(CancellationToken cancellationToken = default) => + (byte) await AsyncBinaryReader.ReadSByteAsync(cancellationToken); - public virtual async ValueTask ReadBytesAsync(long count) + public virtual async ValueTask ReadBytesAsync(long count, CancellationToken cancellationToken = default) { if (count < 0 || count > int.MaxValue) { @@ -38,7 +51,9 @@ public virtual async ValueTask ReadBytesAsync(long count) $"requested {count} bytes, while only non-negative int32 amount of bytes possible"); } - var bytes = await AsyncBinaryReader.ReadBytesAsync((int) count); + await CheckIsCancellationRequested(cancellationToken); + + var bytes = await AsyncBinaryReader.ReadBytesAsync((int) count, cancellationToken); if (bytes.Length < count) { throw new EndOfStreamException($"requested {count} bytes, but got only {bytes.Length} bytes"); @@ -47,7 +62,15 @@ public virtual async ValueTask ReadBytesAsync(long count) return bytes; } - public virtual async ValueTask ReadBytesFullAsync() => - await ReadBytesAsync(_baseStream.Length - _baseStream.Position); + public virtual async ValueTask ReadBytesFullAsync(CancellationToken cancellationToken = default) => + await ReadBytesAsync(_baseStream.Length - _baseStream.Position, cancellationToken); + + private static async Task CheckIsCancellationRequested(CancellationToken cancellationToken) + { + if (cancellationToken.IsCancellationRequested) + { + await Task.FromCanceled(cancellationToken); + } + } } } \ No newline at end of file From c36377d2e51392e9ef06d0883398a62c96ae54d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vojt=C4=9Bch=20Ve=C4=8De=C5=99a?= Date: Tue, 8 Dec 2020 18:36:00 +0100 Subject: [PATCH 24/34] Fixing forward seek after read to end followed by position reset on PipeReader --- .../KaitaiAsyncStreamBaseTests.cs | 19 +++++++++++++++++++ .../ReaderContext/PipeReaderContext.cs | 2 +- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/Kaitai.Struct.Runtime.Async.Tests/KaitaiAsyncStreamBaseTests.cs b/Kaitai.Struct.Runtime.Async.Tests/KaitaiAsyncStreamBaseTests.cs index b1b5d77..6af533a 100644 --- a/Kaitai.Struct.Runtime.Async.Tests/KaitaiAsyncStreamBaseTests.cs +++ b/Kaitai.Struct.Runtime.Async.Tests/KaitaiAsyncStreamBaseTests.cs @@ -1,6 +1,7 @@ using System; using System.IO; using System.IO.Pipelines; +using System.Threading; using System.Threading.Tasks; using Kaitai.Async; using Xunit; @@ -115,6 +116,24 @@ await EvaluateMaybeCancelled(async () => }); } + [Fact] + public async Task ForwardSeek_AfterReadToEndAndBackwardSeek_Test() + { + const int toRead = 1; + + var kaitaiStreamSUT = Create(new byte[2]); + + await EvaluateMaybeCancelled(async () => + { + // Simulates kaitai compiler generated code for multiple fields defined as `instances` + await kaitaiStreamSUT.ReadBytesFullAsync(CancellationToken.None); + await kaitaiStreamSUT.SeekAsync(0); + await kaitaiStreamSUT.SeekAsync(toRead); + + Assert.Equal(toRead, kaitaiStreamSUT.Pos); + }); + } + [Theory] [InlineData(0)] [InlineData(1)] diff --git a/Kaitai.Struct.Runtime.Async/ReaderContext/PipeReaderContext.cs b/Kaitai.Struct.Runtime.Async/ReaderContext/PipeReaderContext.cs index 97f832f..76b947c 100644 --- a/Kaitai.Struct.Runtime.Async/ReaderContext/PipeReaderContext.cs +++ b/Kaitai.Struct.Runtime.Async/ReaderContext/PipeReaderContext.cs @@ -58,7 +58,7 @@ public virtual async ValueTask SeekAsync(long position, CancellationToken cancel ReadResult = await PipeReader.ReadAsync(cancellationToken); } - if (ReadResult.Buffer.Length <= position) + if (ReadResult.Buffer.Length >= position) { Position = position; return; From 99eb42003ae91d0204667662a9449d7122806a40 Mon Sep 17 00:00:00 2001 From: Jan Pluskal Date: Fri, 11 Dec 2020 14:50:21 +0100 Subject: [PATCH 25/34] Fix AdvanceTo that shoudl report buffer end as examined. --- .../PipeReaderContextTests.cs | 76 +++++++++++++++++++ .../ReaderContext/PipeReaderContext.cs | 6 +- 2 files changed, 79 insertions(+), 3 deletions(-) create mode 100644 Kaitai.Struct.Runtime.Async.Tests/PipeReaderContextTests.cs diff --git a/Kaitai.Struct.Runtime.Async.Tests/PipeReaderContextTests.cs b/Kaitai.Struct.Runtime.Async.Tests/PipeReaderContextTests.cs new file mode 100644 index 0000000..a28d9bd --- /dev/null +++ b/Kaitai.Struct.Runtime.Async.Tests/PipeReaderContextTests.cs @@ -0,0 +1,76 @@ +using System; +using System.IO; +using System.IO.Pipelines; +using System.Threading; +using System.Threading.Tasks; +using Kaitai.Async; +using Xunit; + +namespace Kaitai.Struct.Runtime.Async.Tests +{ + public class PipeReaderContextTests + { + private readonly PipeReaderContext _pipeReaderContextSUT; + private readonly PipeWriter _pipeWriter; + + public PipeReaderContextTests() + { + var pipe = new Pipe(new PipeOptions(minimumSegmentSize: 2)); + _pipeWriter = pipe.Writer; + _pipeReaderContextSUT = new PipeReaderContext(pipe.Reader); + } + + [Fact] + public async Task ReadByteAsync_RequestingMoreData_ShouldAwaitOnRead() + { + //Arrange + await _pipeWriter.WriteAsync(new byte[1]); + + await _pipeReaderContextSUT.ReadByteAsync(CancellationToken.None); + using var cts = new CancellationTokenSource(100); + + //Act & assert + await Assert.ThrowsAsync( + async () => await _pipeReaderContextSUT.ReadByteAsync(cts.Token)); + } + + [Fact] + public async Task ReadBytesAsync_RequestingMoreData_ShouldAwaitOnRead() + { + //Arrange + await _pipeWriter.WriteAsync(new byte[2]); + + await _pipeReaderContextSUT.ReadBytesAsync(1, CancellationToken.None); + using var cts = new CancellationTokenSource(100); + + //Act & assert + await Assert.ThrowsAsync( + async () => await _pipeReaderContextSUT.ReadBytesAsync(2, cts.Token)); + } + + [Fact] + public async Task SeekAsync_PassEof_Completed_ShouldThrow() + { + //Arrange + await _pipeWriter.WriteAsync(new byte[1]); + await _pipeWriter.CompleteAsync(); + using var cts = new CancellationTokenSource(100); + + //Act & assert + await Assert.ThrowsAsync( + async () => await _pipeReaderContextSUT.SeekAsync(2, cts.Token)); + } + + [Fact] + public async Task SeekAsync_PassEof_NotCompleted_ShouldThrow() + { + //Arrange + await _pipeWriter.WriteAsync(new byte[1]); + using var cts = new CancellationTokenSource(100); + + //Act & assert + await Assert.ThrowsAsync( + async () => await _pipeReaderContextSUT.SeekAsync(2, cts.Token)); + } + } +} \ No newline at end of file diff --git a/Kaitai.Struct.Runtime.Async/ReaderContext/PipeReaderContext.cs b/Kaitai.Struct.Runtime.Async/ReaderContext/PipeReaderContext.cs index 76b947c..600f1aa 100644 --- a/Kaitai.Struct.Runtime.Async/ReaderContext/PipeReaderContext.cs +++ b/Kaitai.Struct.Runtime.Async/ReaderContext/PipeReaderContext.cs @@ -16,7 +16,7 @@ public PipeReaderContext(PipeReader pipeReader) { PipeReader = pipeReader; } - + protected long RemainingBytesInReadResult => ReadResult.Buffer.Length - Position; public long Position { get; protected set; } @@ -79,7 +79,7 @@ public virtual async ValueTask ReadByteAsync(CancellationToken cancellatio var value = byte.MinValue; while (!TryReadByte(out value) && !ReadResult.IsCompleted) { - PipeReader.AdvanceTo(ReadResult.Buffer.Start, ReadResult.Buffer.GetPosition(Position)); + PipeReader.AdvanceTo(ReadResult.Buffer.Start, ReadResult.Buffer.End); ReadResult = await PipeReader.ReadAsync(cancellationToken); } @@ -114,7 +114,7 @@ public virtual async ValueTask ReadBytesAsync(long count, CancellationTo $"requested {count} bytes, but got only {RemainingBytesInReadResult} bytes"); } - PipeReader.AdvanceTo(ReadResult.Buffer.Start, ReadResult.Buffer.GetPosition(Position)); + PipeReader.AdvanceTo(ReadResult.Buffer.Start, ReadResult.Buffer.End); ReadResult = await PipeReader.ReadAsync(cancellationToken); } From 87e6ec62f6413fde4763ad16ba28aee73c70844a Mon Sep 17 00:00:00 2001 From: Jan Pluskal Date: Fri, 12 Mar 2021 16:09:12 +0100 Subject: [PATCH 26/34] Add Validation* errors as in Java - generalmimon committed on Mar 21, 2020 --- .../Exception/ValidationGreaterThanError.cs | 14 ++++++++++++++ .../Exception/ValidationLessThanError.cs | 14 ++++++++++++++ .../Exception/ValidationNotAnyOfError.cs | 14 ++++++++++++++ 3 files changed, 42 insertions(+) create mode 100644 Kaitai.Struct.Runtime/Exception/ValidationGreaterThanError.cs create mode 100644 Kaitai.Struct.Runtime/Exception/ValidationLessThanError.cs create mode 100644 Kaitai.Struct.Runtime/Exception/ValidationNotAnyOfError.cs diff --git a/Kaitai.Struct.Runtime/Exception/ValidationGreaterThanError.cs b/Kaitai.Struct.Runtime/Exception/ValidationGreaterThanError.cs new file mode 100644 index 0000000..e4e3e82 --- /dev/null +++ b/Kaitai.Struct.Runtime/Exception/ValidationGreaterThanError.cs @@ -0,0 +1,14 @@ +namespace Kaitai +{ + public class ValidationGreaterThanError : ValidationFailedError { + public ValidationGreaterThanError(long max, long actual, KaitaiStream io, string srcPath) + : base("not in range, max " + max + ", but got " + actual, io, srcPath) + { + this.max = max; + this.actual = actual; + } + + protected long max; + protected long actual; + } +} \ No newline at end of file diff --git a/Kaitai.Struct.Runtime/Exception/ValidationLessThanError.cs b/Kaitai.Struct.Runtime/Exception/ValidationLessThanError.cs new file mode 100644 index 0000000..7b3bb09 --- /dev/null +++ b/Kaitai.Struct.Runtime/Exception/ValidationLessThanError.cs @@ -0,0 +1,14 @@ +namespace Kaitai +{ + public class ValidationLessThanError : ValidationFailedError { + public ValidationLessThanError(long min, long actual, KaitaiStream io, string srcPath) + : base("not in range, min " + min + ", but got " + actual, io, srcPath) + { + this.min = min; + this.actual = actual; + } + + protected long min; + protected long actual; + } +} \ No newline at end of file diff --git a/Kaitai.Struct.Runtime/Exception/ValidationNotAnyOfError.cs b/Kaitai.Struct.Runtime/Exception/ValidationNotAnyOfError.cs new file mode 100644 index 0000000..894b4f2 --- /dev/null +++ b/Kaitai.Struct.Runtime/Exception/ValidationNotAnyOfError.cs @@ -0,0 +1,14 @@ +using System; + +namespace Kaitai +{ + public class ValidationNotAnyOfError : ValidationFailedError { + public ValidationNotAnyOfError(Object actual, KaitaiStream io, string srcPath) + : base("not any of the list, got " + actual, io, srcPath) + { + this.actual = actual; + } + + protected Object actual; + } +} From a290a186f85448d33ad96be36a409629a60e97e1 Mon Sep 17 00:00:00 2001 From: Jan Pluskal Date: Fri, 12 Mar 2021 16:10:24 +0100 Subject: [PATCH 27/34] Add UndecidedEndiannessError - generalmimon committed on Mar 21, 2020 --- .../Exception/UndecidedEndiannessError.cs | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 Kaitai.Struct.Runtime/Exception/UndecidedEndiannessError.cs diff --git a/Kaitai.Struct.Runtime/Exception/UndecidedEndiannessError.cs b/Kaitai.Struct.Runtime/Exception/UndecidedEndiannessError.cs new file mode 100644 index 0000000..b2ef825 --- /dev/null +++ b/Kaitai.Struct.Runtime/Exception/UndecidedEndiannessError.cs @@ -0,0 +1,24 @@ +using System; + +namespace Kaitai +{ + /// + /// Error that occurs when default endianness should be decided with a + /// switch, but nothing matches (although using endianness expression + /// implies that there should be some positive result). + /// + public class UndecidedEndiannessError : Exception { + public UndecidedEndiannessError() + : base("Unable to decide on endianness") + { + } + public UndecidedEndiannessError(string msg) + : base(msg) + { + } + public UndecidedEndiannessError(string msg, Exception inner) + : base(msg, inner) + { + } + } +} From a0055c6b5201d77e942d529233d8baafc3246343 Mon Sep 17 00:00:00 2001 From: Jan Pluskal Date: Fri, 12 Mar 2021 16:19:59 +0100 Subject: [PATCH 28/34] Implement StringReverse method - generalmimon committed on Mar 21, 2020 --- Kaitai.Struct.Runtime/KaitaiStreamBase.cs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/Kaitai.Struct.Runtime/KaitaiStreamBase.cs b/Kaitai.Struct.Runtime/KaitaiStreamBase.cs index c6487c3..91ac153 100644 --- a/Kaitai.Struct.Runtime/KaitaiStreamBase.cs +++ b/Kaitai.Struct.Runtime/KaitaiStreamBase.cs @@ -1,4 +1,6 @@ using System; +using System.Collections.Generic; +using System.Globalization; using System.IO; using System.IO.Compression; @@ -188,6 +190,22 @@ public static int ByteArrayCompare(byte[] a, byte[] b) return al - bl; } + + /// + /// Reverses the string, Unicode-aware. + /// + /// taken from here + public static string StringReverse(string s) + { + TextElementEnumerator enumerator = StringInfo.GetTextElementEnumerator(s); + + List elements = new List(); + while (enumerator.MoveNext()) + elements.Add(enumerator.GetTextElement()); + + elements.Reverse(); + return string.Concat(elements); + } protected static ulong GetMaskOnes(int n) => n == 64 ? 0xffffffffffffffffUL : (1UL << n) - 1; } From 6eb7a6850b4d228fabf63d4ff299fec34b29ac8b Mon Sep 17 00:00:00 2001 From: Jan Pluskal Date: Fri, 12 Mar 2021 16:25:27 +0100 Subject: [PATCH 29/34] Rename ReadBitsInt -> *Be, deprecate ReadBitsInt, Shorten ReadBitsInt deprecation warning message - generalmimon committed on Apr 26, 2020 --- .../Interface/IKaitaiAsyncStream.cs | 6 +++++- Kaitai.Struct.Runtime.Async/KaitaiAsyncStream.cs | 6 +++++- Kaitai.Struct.Runtime/Interface/IKaitaiStream.cs | 7 ++++++- Kaitai.Struct.Runtime/KaitaiStream.cs | 5 ++++- 4 files changed, 20 insertions(+), 4 deletions(-) diff --git a/Kaitai.Struct.Runtime.Async/Interface/IKaitaiAsyncStream.cs b/Kaitai.Struct.Runtime.Async/Interface/IKaitaiAsyncStream.cs index 5b58aa7..944794b 100644 --- a/Kaitai.Struct.Runtime.Async/Interface/IKaitaiAsyncStream.cs +++ b/Kaitai.Struct.Runtime.Async/Interface/IKaitaiAsyncStream.cs @@ -1,4 +1,5 @@ -using System.Threading; +using System; +using System.Threading; using System.Threading.Tasks; namespace Kaitai.Async @@ -130,7 +131,10 @@ public interface IKaitaiAsyncStream : IKaitaiStreamBase /// Task ReadF8leAsync(CancellationToken cancellationToken = default); + [Obsolete("use ReadBitsIntBe instead")] Task ReadBitsIntAsync(int n, CancellationToken cancellationToken = default); + + Task ReadBitsIntBeAsync(int n, CancellationToken cancellationToken = default); Task ReadBitsIntLeAsync(int n, CancellationToken cancellationToken = default); diff --git a/Kaitai.Struct.Runtime.Async/KaitaiAsyncStream.cs b/Kaitai.Struct.Runtime.Async/KaitaiAsyncStream.cs index 193acc5..2456d46 100644 --- a/Kaitai.Struct.Runtime.Async/KaitaiAsyncStream.cs +++ b/Kaitai.Struct.Runtime.Async/KaitaiAsyncStream.cs @@ -153,7 +153,7 @@ public override void AlignToByte() _bitsLeft = 0; } - public async Task ReadBitsIntAsync(int n, CancellationToken cancellationToken = default) + public async Task ReadBitsIntBeAsync(int n, CancellationToken cancellationToken = default) { int bitsNeeded = n - _bitsLeft; if (bitsNeeded > 0) @@ -186,6 +186,10 @@ public async Task ReadBitsIntAsync(int n, CancellationToken cancellationT return res; } + public Task ReadBitsIntAsync(int n, CancellationToken cancellationToken = default) => + ReadBitsIntBeAsync(n, cancellationToken); + + //Method ported from algorithm specified @ issue#155 public async Task ReadBitsIntLeAsync(int n, CancellationToken cancellationToken = default) { diff --git a/Kaitai.Struct.Runtime/Interface/IKaitaiStream.cs b/Kaitai.Struct.Runtime/Interface/IKaitaiStream.cs index 1f5df27..910e07c 100644 --- a/Kaitai.Struct.Runtime/Interface/IKaitaiStream.cs +++ b/Kaitai.Struct.Runtime/Interface/IKaitaiStream.cs @@ -1,4 +1,6 @@ -namespace Kaitai +using System; + +namespace Kaitai { public interface IKaitaiStream : IKaitaiStreamBase { @@ -116,7 +118,10 @@ public interface IKaitaiStream : IKaitaiStreamBase /// double ReadF8le(); + [Obsolete("use ReadBitsIntBe instead")] ulong ReadBitsInt(int n); + ulong ReadBitsIntBe(int n); + ulong ReadBitsIntLe(int n); diff --git a/Kaitai.Struct.Runtime/KaitaiStream.cs b/Kaitai.Struct.Runtime/KaitaiStream.cs index 35f1954..dfad995 100644 --- a/Kaitai.Struct.Runtime/KaitaiStream.cs +++ b/Kaitai.Struct.Runtime/KaitaiStream.cs @@ -141,7 +141,7 @@ public override void AlignToByte() BitsLeft = 0; } - public ulong ReadBitsInt(int n) + public ulong ReadBitsIntBe(int n) { int bitsNeeded = n - BitsLeft; if (bitsNeeded > 0) @@ -174,6 +174,9 @@ public ulong ReadBitsInt(int n) return res; } + public ulong ReadBitsInt(int n) => ReadBitsIntBe(n); + + //Method ported from algorithm specified @ issue#155 public ulong ReadBitsIntLe(int n) { From 8ae3adf42d89ce20fe0a65876748e463ea7cf222 Mon Sep 17 00:00:00 2001 From: Jan Pluskal Date: Fri, 12 Mar 2021 16:29:47 +0100 Subject: [PATCH 30/34] Simplify ReadBitsIntBe as in https://github.com/kaitai-io/kaitai_struct_java_runtime/commit/e1a30e4 - generalmimon committed on Apr 27, 2020 --- Kaitai.Struct.Runtime.Async/KaitaiAsyncStream.cs | 6 ++---- Kaitai.Struct.Runtime/KaitaiStream.cs | 6 ++---- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/Kaitai.Struct.Runtime.Async/KaitaiAsyncStream.cs b/Kaitai.Struct.Runtime.Async/KaitaiAsyncStream.cs index 2456d46..6813198 100644 --- a/Kaitai.Struct.Runtime.Async/KaitaiAsyncStream.cs +++ b/Kaitai.Struct.Runtime.Async/KaitaiAsyncStream.cs @@ -173,11 +173,9 @@ public async Task ReadBitsIntBeAsync(int n, CancellationToken cancellatio // raw mask with required number of 1s, starting from lowest bit ulong mask = GetMaskOnes(n); - // shift mask to align with highest bits available in "bits" + // shift "bits" to align the highest bits with the mask & derive reading result int shiftBits = _bitsLeft - n; - mask = mask << shiftBits; - // derive reading result - ulong res = (_bits & mask) >> shiftBits; + ulong res = (_bits >> shiftBits) & mask; // clear top bits that we've just read => AND with 1s _bitsLeft -= n; mask = GetMaskOnes(_bitsLeft); diff --git a/Kaitai.Struct.Runtime/KaitaiStream.cs b/Kaitai.Struct.Runtime/KaitaiStream.cs index dfad995..08c6712 100644 --- a/Kaitai.Struct.Runtime/KaitaiStream.cs +++ b/Kaitai.Struct.Runtime/KaitaiStream.cs @@ -161,11 +161,9 @@ public ulong ReadBitsIntBe(int n) // raw mask with required number of 1s, starting from lowest bit ulong mask = GetMaskOnes(n); - // shift mask to align with highest bits available in "bits" + // shift "bits" to align the highest bits with the mask & derive reading result int shiftBits = BitsLeft - n; - mask <<= shiftBits; - // derive reading result - ulong res = (Bits & mask) >> shiftBits; + ulong res = (Bits >> shiftBits) & mask; // clear top bits that we've just read => AND with 1s BitsLeft -= n; mask = GetMaskOnes(BitsLeft); From 11b940d8aa25348d90364f16688a0d8d212286c2 Mon Sep 17 00:00:00 2001 From: Jan Pluskal Date: Fri, 12 Mar 2021 16:30:50 +0100 Subject: [PATCH 31/34] Add ValidationExprError - generalmimon committed on May 21, 2020 --- .../Exception/ValidationExprError.cs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 Kaitai.Struct.Runtime/Exception/ValidationExprError.cs diff --git a/Kaitai.Struct.Runtime/Exception/ValidationExprError.cs b/Kaitai.Struct.Runtime/Exception/ValidationExprError.cs new file mode 100644 index 0000000..64b11ea --- /dev/null +++ b/Kaitai.Struct.Runtime/Exception/ValidationExprError.cs @@ -0,0 +1,14 @@ +using System; + +namespace Kaitai +{ + public class ValidationExprError : ValidationFailedError { + public ValidationExprError(Object actual, KaitaiStream io, string srcPath) + : base("not matching the expression, got " + actual, io, srcPath) + { + this.actual = actual; + } + + protected Object actual; + } +} From 94ee1eae8715e83398ad1e20ed72d6eb8e7ef09f Mon Sep 17 00:00:00 2001 From: Jan Pluskal Date: Fri, 12 Mar 2021 16:53:07 +0100 Subject: [PATCH 32/34] Add byte array support in validation errors - generalmimon committed on Jun 8, 2020 --- .../Exception/ValidationGreaterThanError.cs | 19 ++++++++++++++----- .../Exception/ValidationLessThanError.cs | 18 +++++++++++++----- 2 files changed, 27 insertions(+), 10 deletions(-) diff --git a/Kaitai.Struct.Runtime/Exception/ValidationGreaterThanError.cs b/Kaitai.Struct.Runtime/Exception/ValidationGreaterThanError.cs index e4e3e82..1db070a 100644 --- a/Kaitai.Struct.Runtime/Exception/ValidationGreaterThanError.cs +++ b/Kaitai.Struct.Runtime/Exception/ValidationGreaterThanError.cs @@ -1,14 +1,23 @@ -namespace Kaitai +using System; + +namespace Kaitai { public class ValidationGreaterThanError : ValidationFailedError { - public ValidationGreaterThanError(long max, long actual, KaitaiStream io, string srcPath) + public ValidationGreaterThanError(byte[] max, byte[] actual, KaitaiStream io, string srcPath) + : base("not in range, max " + ByteArrayToHex(max) + ", but got " + ByteArrayToHex(actual), io, srcPath) + { + this.max = max; + this.actual = actual; + } + + public ValidationGreaterThanError(Object max, Object actual, KaitaiStream io, string srcPath) : base("not in range, max " + max + ", but got " + actual, io, srcPath) { this.max = max; this.actual = actual; } - protected long max; - protected long actual; + protected Object max; + protected Object actual; } -} \ No newline at end of file +} diff --git a/Kaitai.Struct.Runtime/Exception/ValidationLessThanError.cs b/Kaitai.Struct.Runtime/Exception/ValidationLessThanError.cs index 7b3bb09..823d6b0 100644 --- a/Kaitai.Struct.Runtime/Exception/ValidationLessThanError.cs +++ b/Kaitai.Struct.Runtime/Exception/ValidationLessThanError.cs @@ -1,14 +1,22 @@ -namespace Kaitai +using System; + +namespace Kaitai { public class ValidationLessThanError : ValidationFailedError { - public ValidationLessThanError(long min, long actual, KaitaiStream io, string srcPath) + public ValidationLessThanError(byte[] min, byte[] actual, KaitaiStream io, string srcPath) + : base("not in range, min " + ByteArrayToHex(min) + ", but got " + ByteArrayToHex(actual), io, srcPath) + { + this.min = min; + this.actual = actual; + } + public ValidationLessThanError(Object min, Object actual, KaitaiStream io, string srcPath) : base("not in range, min " + min + ", but got " + actual, io, srcPath) { this.min = min; this.actual = actual; } - protected long min; - protected long actual; + protected Object min; + protected Object actual; } -} \ No newline at end of file +} From 3c824339549ded295e4aa1dc089db79cd4fa61cf Mon Sep 17 00:00:00 2001 From: Jan Pluskal Date: Fri, 12 Mar 2021 16:54:31 +0100 Subject: [PATCH 33/34] Mark EnsureFixedContents() as obsolete - generalmimon committed on Oct 3, 2020 --- Kaitai.Struct.Runtime.Async/Interface/IKaitaiAsyncStream.cs | 1 + Kaitai.Struct.Runtime/Interface/IKaitaiStream.cs | 1 + 2 files changed, 2 insertions(+) diff --git a/Kaitai.Struct.Runtime.Async/Interface/IKaitaiAsyncStream.cs b/Kaitai.Struct.Runtime.Async/Interface/IKaitaiAsyncStream.cs index 944794b..220d43c 100644 --- a/Kaitai.Struct.Runtime.Async/Interface/IKaitaiAsyncStream.cs +++ b/Kaitai.Struct.Runtime.Async/Interface/IKaitaiAsyncStream.cs @@ -180,6 +180,7 @@ Task ReadBytesTermAsync(byte terminator, /// The expected result /// /// + [Obsolete("use explicit \"if\" using ByteArrayCompare method instead")] Task EnsureFixedContentsAsync(byte[] expected, CancellationToken cancellationToken = default); } } \ No newline at end of file diff --git a/Kaitai.Struct.Runtime/Interface/IKaitaiStream.cs b/Kaitai.Struct.Runtime/Interface/IKaitaiStream.cs index 910e07c..984f5bb 100644 --- a/Kaitai.Struct.Runtime/Interface/IKaitaiStream.cs +++ b/Kaitai.Struct.Runtime/Interface/IKaitaiStream.cs @@ -160,6 +160,7 @@ public interface IKaitaiStream : IKaitaiStreamBase /// /// The expected result /// + [Obsolete("use explicit \"if\" using ByteArrayCompare method instead")] byte[] EnsureFixedContents(byte[] expected); } } From 94e1e2f97840cc1737035d8b219d8a20bd6701d9 Mon Sep 17 00:00:00 2001 From: Jan Pluskal Date: Thu, 18 Mar 2021 12:20:33 +0100 Subject: [PATCH 34/34] Exceptions should accept IKaitaiStreamBase --- Kaitai.Struct.Runtime/Exception/ValidationExprError.cs | 2 +- Kaitai.Struct.Runtime/Exception/ValidationGreaterThanError.cs | 4 ++-- Kaitai.Struct.Runtime/Exception/ValidationLessThanError.cs | 4 ++-- Kaitai.Struct.Runtime/Exception/ValidationNotAnyOfError.cs | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Kaitai.Struct.Runtime/Exception/ValidationExprError.cs b/Kaitai.Struct.Runtime/Exception/ValidationExprError.cs index 64b11ea..6d4030d 100644 --- a/Kaitai.Struct.Runtime/Exception/ValidationExprError.cs +++ b/Kaitai.Struct.Runtime/Exception/ValidationExprError.cs @@ -3,7 +3,7 @@ namespace Kaitai { public class ValidationExprError : ValidationFailedError { - public ValidationExprError(Object actual, KaitaiStream io, string srcPath) + public ValidationExprError(Object actual, IKaitaiStreamBase io, string srcPath) : base("not matching the expression, got " + actual, io, srcPath) { this.actual = actual; diff --git a/Kaitai.Struct.Runtime/Exception/ValidationGreaterThanError.cs b/Kaitai.Struct.Runtime/Exception/ValidationGreaterThanError.cs index 1db070a..71cb3ad 100644 --- a/Kaitai.Struct.Runtime/Exception/ValidationGreaterThanError.cs +++ b/Kaitai.Struct.Runtime/Exception/ValidationGreaterThanError.cs @@ -3,14 +3,14 @@ namespace Kaitai { public class ValidationGreaterThanError : ValidationFailedError { - public ValidationGreaterThanError(byte[] max, byte[] actual, KaitaiStream io, string srcPath) + public ValidationGreaterThanError(byte[] max, byte[] actual, IKaitaiStreamBase io, string srcPath) : base("not in range, max " + ByteArrayToHex(max) + ", but got " + ByteArrayToHex(actual), io, srcPath) { this.max = max; this.actual = actual; } - public ValidationGreaterThanError(Object max, Object actual, KaitaiStream io, string srcPath) + public ValidationGreaterThanError(Object max, Object actual, IKaitaiStreamBase io, string srcPath) : base("not in range, max " + max + ", but got " + actual, io, srcPath) { this.max = max; diff --git a/Kaitai.Struct.Runtime/Exception/ValidationLessThanError.cs b/Kaitai.Struct.Runtime/Exception/ValidationLessThanError.cs index 823d6b0..2b3383e 100644 --- a/Kaitai.Struct.Runtime/Exception/ValidationLessThanError.cs +++ b/Kaitai.Struct.Runtime/Exception/ValidationLessThanError.cs @@ -3,13 +3,13 @@ namespace Kaitai { public class ValidationLessThanError : ValidationFailedError { - public ValidationLessThanError(byte[] min, byte[] actual, KaitaiStream io, string srcPath) + public ValidationLessThanError(byte[] min, byte[] actual, IKaitaiStreamBase io, string srcPath) : base("not in range, min " + ByteArrayToHex(min) + ", but got " + ByteArrayToHex(actual), io, srcPath) { this.min = min; this.actual = actual; } - public ValidationLessThanError(Object min, Object actual, KaitaiStream io, string srcPath) + public ValidationLessThanError(Object min, Object actual, IKaitaiStreamBase io, string srcPath) : base("not in range, min " + min + ", but got " + actual, io, srcPath) { this.min = min; diff --git a/Kaitai.Struct.Runtime/Exception/ValidationNotAnyOfError.cs b/Kaitai.Struct.Runtime/Exception/ValidationNotAnyOfError.cs index 894b4f2..d8e0a4f 100644 --- a/Kaitai.Struct.Runtime/Exception/ValidationNotAnyOfError.cs +++ b/Kaitai.Struct.Runtime/Exception/ValidationNotAnyOfError.cs @@ -3,7 +3,7 @@ namespace Kaitai { public class ValidationNotAnyOfError : ValidationFailedError { - public ValidationNotAnyOfError(Object actual, KaitaiStream io, string srcPath) + public ValidationNotAnyOfError(Object actual, IKaitaiStreamBase io, string srcPath) : base("not any of the list, got " + actual, io, srcPath) { this.actual = actual;