Regex support in valid key

Working for python, cpp_stl_11. Have to be tested for other implementations (C#, Go, Java, JS, Ruby, Rust, PHP, Perl), not implemented on remaining. Can't work on cpp_stl_98 without other library.
kaitai-io · Apr 2, 2020 · 7120c8e · 7120c8e
1 parent a91920b
commit 7120c8e
Show file tree

Hide file tree

Showing 22 changed files with 146 additions and 1 deletion.
diff --git a/Test.php b/Test.php
@@ -0,0 +1,20 @@
+<?php
+// This is a generated file! Please edit source .ksy file and use kaitai-struct-compiler to rebuild
+
+namespace {
+    class Test extends \Kaitai\Struct\Struct {
+        public function __construct(\Kaitai\Struct\Stream $_io, \Kaitai\Struct\Struct $_parent = null, \Test $_root = null) {
+            parent::__construct($_io, $_parent, $_root);
+            $this->_read();
+        }
+
+        private function _read() {
+            $this->_m_name = \Kaitai\Struct\Stream::bytesToStr($this->_io->readBytes(2), "UTF-8");
+            if (!(preg_match("/\d\d/", $this->name()))) {
+                throw new \Kaitai\Struct\Error\ValidationRegexMatchError("\\d\\d", $this->name(), $this->_io(), "/seq/0");
+            }
+        }
+        protected $_m_name;
+        public function name() { return $this->_m_name; }
+    }
+}
diff --git a/shared/src/main/scala/io/kaitai/struct/datatype/KSError.scala b/shared/src/main/scala/io/kaitai/struct/datatype/KSError.scala
@@ -21,6 +21,7 @@ object KSError {
         case "ValidationLessThanError" => ValidationLessThanError
         case "ValidationGreaterThanError" => ValidationGreaterThanError
         case "ValidationNotAnyOfError" => ValidationNotAnyOfError
+        case "ValidationRegexMatchError" => ValidationRegexMatchError
       }
       excClass(dataType)
   }
@@ -64,6 +65,14 @@ case class ValidationNotAnyOfError(_dt: DataType) extends ValidationError(_dt) {
   def name = "ValidationNotAnyOfError"
 }
 
+/**
+  * Error to be thrown when validation fails with actual not matching regex
+  * @param _dt data type used in validation process
+  */
+case class ValidationRegexMatchError(_dt: DataType) extends ValidationError(_dt) {
+  def name = "ValidationRegexMatchError"
+}
+
 /**
   * Exception that is thrown when we can't decided on endianness
   * and thus can't proceed with parsing.

diff --git a/shared/src/main/scala/io/kaitai/struct/exprlang/Ast.scala b/shared/src/main/scala/io/kaitai/struct/exprlang/Ast.scala
@@ -91,6 +91,7 @@ object Ast {
     case class IfExp(condition: expr, ifTrue: expr, ifFalse: expr) extends expr
 //    case class Dict(keys: Seq[expr], values: Seq[expr]) extends expr
     case class Compare(left: expr, ops: cmpop, right: expr) extends expr
+    case class RegexMatch(str: expr, regex: String) extends expr
     case class Call(func: expr, args: Seq[expr]) extends expr
     case class IntNum(n: BigInt) extends expr
     case class FloatNum(n: BigDecimal) extends expr

diff --git a/shared/src/main/scala/io/kaitai/struct/format/ValidationSpec.scala b/shared/src/main/scala/io/kaitai/struct/format/ValidationSpec.scala
@@ -10,6 +10,7 @@ case class ValidationMax(max: Ast.expr) extends ValidationSpec
 case class ValidationRange(min: Ast.expr, max: Ast.expr) extends ValidationSpec
 case class ValidationAnyOf(values: List[Ast.expr]) extends ValidationSpec
 case class ValidationExpr(checkExpr: Ast.expr) extends ValidationSpec
+case class ValidationRegex(checkExpr: String) extends ValidationSpec
 
 object ValidationEq {
   val LEGAL_KEYS = Set("eq")
@@ -68,16 +69,37 @@ object ValidationExpr {
     }
 }
 
+object ValidationRegex {
+  val LEGAL_KEYS = Set("regex")
+
+  def fromMap(src: Map[String, Any], path: List[String]): Option[ValidationRegex] =
+    /*val regex = ParseUtils.getOptValueExpression(src, "regex", path)
+    regex match {
+      case Some(regexCt) => 
+        ParseUtils.ensureLegalKeys(src, LEGAL_KEYS, path)
+        Some((ValidationRegex(Ast.expr.Str(regexCt))))
+      case None => None
+    }
+  }*/
+    ParseUtils.getOptValueStr(src, "regex", path).map { case eqExpr =>
+      ParseUtils.ensureLegalKeys(src, LEGAL_KEYS, path)
+      ValidationRegex(eqExpr)
+    }
+}
+
+
 object ValidationSpec {
   val LEGAL_KEYS =
     ValidationEq.LEGAL_KEYS ++
     ValidationRange.LEGAL_KEYS ++
     ValidationAnyOf.LEGAL_KEYS ++
-    ValidationExpr.LEGAL_KEYS
+    ValidationExpr.LEGAL_KEYS ++
+    ValidationRegex.LEGAL_KEYS
 
   def fromYaml(src: Any, path: List[String]): ValidationSpec = {
     src match {
       case value: String =>
+
         fromString(value, path)
       case x: Boolean =>
         fromString(x.toString, path)
@@ -108,6 +130,9 @@ object ValidationSpec {
     val opt4 = ValidationExpr.fromMap(src, path)
     if (opt4.nonEmpty)
       return opt4.get
+    val opt5 = ValidationRegex.fromMap(src, path)
+    if (opt5.nonEmpty)
+      return opt5.get
 
     // No validation templates matched, check for any bogus keys
     ParseUtils.ensureLegalKeys(src, LEGAL_KEYS, path)

diff --git a/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala
@@ -992,6 +992,7 @@ class CppCompiler(
         case _: ValidationLessThanError => "validation_less_than_error"
         case _: ValidationGreaterThanError => "validation_greater_than_error"
         case _: ValidationNotAnyOfError => "validation_not_any_of_error"
+        case _: ValidationRegexMatchError => "validation_regex_match_error"
       }
       s"kaitai::$cppErrName<$cppType>"
   }

diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala
@@ -50,6 +50,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
     importList.add("std::default::Default")
     importList.add("kaitai_struct::KaitaiStream")
     importList.add("kaitai_struct::KaitaiStruct")
+    importList.add("regex::Regex")
 
     out.puts
   }

diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/ValidateOps.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/ValidateOps.scala
@@ -19,6 +19,19 @@ trait ValidateOps extends ExceptionNames {
       case ValidationRange(min, max) =>
         attrValidateExprCompare(attrId, attr, Ast.cmpop.GtE, min, ValidationLessThanError(attr.dataTypeComposite))
         attrValidateExprCompare(attrId, attr, Ast.cmpop.LtE, max, ValidationGreaterThanError(attr.dataTypeComposite))
+      case ValidationRegex(regex) =>
+        val regexMatch = Ast.expr.RegexMatch(Ast.expr.Name(attrId.toAstIdentifier), regex)
+        attrValidateExpr(attrId, attr.dataTypeComposite, 
+          checkExpr = regexMatch, 
+          errName = ksErrorName(ValidationRegexMatchError(attr.dataTypeComposite)),
+          errArgs = List(
+            Ast.expr.Str(regex),
+            Ast.expr.Name(attrId.toAstIdentifier),
+            Ast.expr.Name(IoIdentifier.toAstIdentifier),
+            Ast.expr.Str(attr.path.mkString("/", "/", ""))
+          )
+        )
+
       case ValidationAnyOf(values) =>
         val bigOrExpr = Ast.expr.BoolOp(
           Ast.boolop.Or,

diff --git a/shared/src/main/scala/io/kaitai/struct/translators/BaseTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/BaseTranslator.scala
@@ -102,6 +102,14 @@ abstract class BaseTranslator(val provider: TypeProvider)
           case (ltype, rtype) =>
             throw new TypeMismatchError(s"can't compare $ltype and $rtype")
         }
+      case Ast.expr.RegexMatch(str: Ast.expr, regex: String) => {
+        detectType(str) match {
+          case (_: StrType) =>
+            doRegexMatchOp(translate(str), doRegex(regex))
+          case _ =>
+            throw new TypeMismatchError(s"regex match need strings")
+        }
+      }
       case Ast.expr.BinOp(left: Ast.expr, op: Ast.operator, right: Ast.expr) =>
         (detectType(left), detectType(right), op) match {
           case (_: NumericType, _: NumericType, _) =>
@@ -179,6 +187,7 @@ abstract class BaseTranslator(val provider: TypeProvider)
 
   def doEnumByLabel(enumTypeAbs: List[String], label: String): String
   def doEnumById(enumTypeAbs: List[String], id: String): String
+  def doRegex(reg: String): String = reg
 
   // Predefined methods of various types
   def strConcat(left: Ast.expr, right: Ast.expr): String = s"${translate(left)} + ${translate(right)}"

diff --git a/shared/src/main/scala/io/kaitai/struct/translators/CSharpTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/CSharpTranslator.scala
@@ -78,6 +78,13 @@ class CSharpTranslator(provider: TypeProvider, importList: ImportList) extends B
     }
   }
 
+  override def doRegexMatchOp(str: String, regex: String): String = {
+    importList.add("System.Text.RegularExpressions")
+    s"new Regex(${regex}).IsMatch(${str})"
+  }
+
+  override def doRegex(reg: String) : String = doStringLiteral(reg)
+
   override def doBytesCompareOp(left: Ast.expr, op: Ast.cmpop, right: Ast.expr): String =
     s"(${CSharpCompiler.kstreamName}.ByteArrayCompare(${translate(left)}, ${translate(right)}) ${cmpOp(op)} 0)"
 

diff --git a/shared/src/main/scala/io/kaitai/struct/translators/CommonOps.scala b/shared/src/main/scala/io/kaitai/struct/translators/CommonOps.scala
@@ -64,4 +64,7 @@ trait CommonOps extends AbstractTranslator {
     case Ast.unaryop.Minus => "-"
     case Ast.unaryop.Not => "!"
   }
+
+  def doRegexMatchOp(str: String, regex: String): String
+
 }
diff --git a/shared/src/main/scala/io/kaitai/struct/translators/CppTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/CppTranslator.scala
@@ -158,6 +158,13 @@ class CppTranslator(provider: TypeProvider, importListSrc: CppImportList, import
     }
   }
 
+  override def doRegexMatchOp(str: String, regex: String): String = {
+    importListSrc.addSystem("regex")
+    s"std::regex_match(${str}, std::regex(${regex}))"
+  }
+
+  override def doRegex(reg: String): String = doStringLiteral(reg)
+
   override def arraySubscript(container: expr, idx: expr): String =
     s"${translate(container)}->at(${translate(idx)})"
   override def doIfExp(condition: expr, ifTrue: expr, ifFalse: expr): String =

diff --git a/shared/src/main/scala/io/kaitai/struct/translators/ExpressionValidator.scala b/shared/src/main/scala/io/kaitai/struct/translators/ExpressionValidator.scala
@@ -46,6 +46,8 @@ class ExpressionValidator(val provider: TypeProvider)
       case Ast.expr.Compare(left: Ast.expr, op: Ast.cmpop, right: Ast.expr) =>
         validate(left)
         validate(right)
+      case Ast.expr.RegexMatch(str: Ast.expr, regex: String) =>
+        validate(str)
       case Ast.expr.BinOp(left: Ast.expr, op: Ast.operator, right: Ast.expr) =>
         validate(left)
         validate(right)

diff --git a/shared/src/main/scala/io/kaitai/struct/translators/GoTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/GoTranslator.scala
@@ -253,6 +253,11 @@ class GoTranslator(out: StringLanguageOutputWriter, provider: TypeProvider, impo
     }
   }
 
+  override def doRegexMatchOp(str: String, regex: String): String = {
+    importList.add("regexp")
+    s"matched, _ := regexp.MatchString(`${regex}`, ${trStringLiteral(str)}); matched"
+  }
+
   override def doCast(value: Ast.expr, typeName: DataType): TranslatorResult = ???
 
   override def doArrayLiteral(t: DataType, value: Seq[Ast.expr]) =

diff --git a/shared/src/main/scala/io/kaitai/struct/translators/JavaScriptTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/JavaScriptTranslator.scala
@@ -74,6 +74,12 @@ class JavaScriptTranslator(provider: TypeProvider) extends BaseTranslator(provid
   override def enumToInt(v: expr, et: EnumType): String =
     translate(v)
 
+  override def doRegexMatchOp(str: String, regex: String): String = {
+    s"RegExp(${regex}).test(${str})"
+  }
+
+  override def doRegex(reg: String) : String = doStringLiteral(reg)
+
   /**
     * Converts a boolean (true or false) to integer (1 or 0, respectively) in
     * JavaScript. There are quite a few methods to so, this one is generally

diff --git a/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala
@@ -88,6 +88,13 @@ class JavaTranslator(provider: TypeProvider, importList: ImportList) extends Bas
     }
   }
 
+  override def doRegexMatchOp(str: String, regex: String): String = {
+    importList.add("java.util.regex.Pattern")
+    s"Pattern.matches(${regex}, ${str})"
+  }
+
+  override def doRegex(reg: String) : String = doStringLiteral(reg)
+
   override def arraySubscript(container: expr, idx: expr): String =
     s"${translate(container)}.get((int) ${translate(idx)})"
   override def doIfExp(condition: expr, ifTrue: expr, ifFalse: expr): String =

diff --git a/shared/src/main/scala/io/kaitai/struct/translators/LuaTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/LuaTranslator.scala
@@ -145,6 +145,8 @@ class LuaTranslator(provider: TypeProvider, importList: ImportList) extends Base
     case _ => super.unaryOp(op)
   }
 
+  override def doRegexMatchOp(str: String, regex: String): String = s"" //TODO
+
   /**
    * Converts byte array (Seq[Byte]) into decimal-escaped Lua-style literal
    * characters (i.e. like \255).

diff --git a/shared/src/main/scala/io/kaitai/struct/translators/NimTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/NimTranslator.scala
@@ -31,6 +31,8 @@ class NimTranslator(provider: TypeProvider, importList: ImportList) extends Base
 
   override def strConcat(left: Ast.expr, right: Ast.expr): String = s"${translate(left)} & ${translate(right)}"
 
+  override def doRegexMatchOp(str: String, regex: String): String = s"" //TODO
+
   // Members declared in io.kaitai.struct.translators.CommonMethods
 
   override def unaryOp(op: Ast.unaryop): String = op match {

diff --git a/shared/src/main/scala/io/kaitai/struct/translators/PHPTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/PHPTranslator.scala
@@ -57,6 +57,12 @@ class PHPTranslator(provider: TypeProvider, config: RuntimeConfig) extends BaseT
 
   override def doName(s: String) = s"${Utils.lowerCamelCase(s)}()"
 
+  override def doRegexMatchOp(str: String, regex: String): String = {
+    s"""preg_match(\"${regex}\", ${str})"""
+  }
+
+  override def doRegex(reg: String) : String = s"/" + reg + "/"
+
   override def doEnumByLabel(enumTypeAbs: List[String], label: String): String = {
     val enumClass = types2classAbs(enumTypeAbs)
     s"$enumClass::${Utils.upperUnderscoreCase(label)}"

diff --git a/shared/src/main/scala/io/kaitai/struct/translators/PerlTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/PerlTranslator.scala
@@ -73,6 +73,10 @@ class PerlTranslator(provider: TypeProvider, importList: ImportList) extends Bas
     }
   }
 
+  override def doRegexMatchOp(str: String, regex: String): String = {
+    s"${str} ~= m/${regex}/"
+  }
+
   override def doEnumByLabel(enumType: List[String], label: String): String = {
     val enumClass = PerlCompiler.types2class(enumType.init)
     val enumClassWithScope = if (enumClass.isEmpty) "" else s"$enumClass::"

diff --git a/shared/src/main/scala/io/kaitai/struct/translators/PythonTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/PythonTranslator.scala
@@ -72,6 +72,11 @@ class PythonTranslator(provider: TypeProvider, importList: ImportList) extends B
   override def doIfExp(condition: Ast.expr, ifTrue: Ast.expr, ifFalse: Ast.expr): String =
     s"(${translate(ifTrue)} if ${translate(condition)} else ${translate(ifFalse)})"
 
+  override def doRegexMatchOp(str: String, regex: String): String = {
+    importList.add("import re")
+    s"re.match('${regex}', ${str})"
+  }
+
   // Predefined methods of various types
   override def strToInt(s: Ast.expr, base: Ast.expr): String = {
     val baseStr = translate(base)

diff --git a/shared/src/main/scala/io/kaitai/struct/translators/RubyTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/RubyTranslator.scala
@@ -37,6 +37,12 @@ class RubyTranslator(provider: TypeProvider) extends BaseTranslator(provider)
     }
   }
 
+  override def doRegexMatchOp(str: String, regex: String): String = {
+    s"${str}.match ${regex}"
+  }
+
+  override def doRegex(regex: String) : String = s"/" + regex + "/"
+
   override def doEnumByLabel(enumTypeAbs: List[String], label: String): String =
     s":${enumTypeAbs.last}_$label"
   override def doEnumById(enumType: List[String], id: String): String =

diff --git a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala
@@ -48,6 +48,10 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) extends Base
 
   override def doName(s: String) = s
 
+  override def doRegexMatchOp(str: String, regex: String): String = {
+    s"""Regex::new(r\"${regex}\").unwrap().is_match(${str});"""
+  }
+
   override def doEnumByLabel(enumTypeAbs: List[String], label: String): String = {
     val enumClass = types2classAbs(enumTypeAbs)
     s"$enumClass::${Utils.upperUnderscoreCase(label)}"