Updated Changelog and added deleted pretrained models

CHANGELOG

@@ -1,6 +1,14 @@
 ========
 1.6.1
 ========
+---------------
+Overview
+---------------
+Hi! We're glad to announce new hotfix 1.6.1. Although changes seem modest or very specific, there is a lot going underground. First of all, we've worked hard with the community to understand S3-based clusters,
+which don't have a common fs.defaultFS configuration, which is the one we use to tell where is the cluster temp folder located in order to distribute word embeddings. We fixed two things here,
+on one side we fixed a bug pointing to the wrong filesystem. Second, we added a custom override setting in application.conf that allows manually setting where to put temp folders in cluster. This should help S3 users.
+Please share your feedback on this regard.
+On the other hand, we created a new annotator type internally. The CHUNK type allows better modulary in the communication between different annotators. Impact will be noticed implicitly and over time.
 
 ---------------
 New features

python/sparknlp/annotator.py

@@ -1059,6 +1059,11 @@ def __init__(self, java_model=None):
         else:
             super(AssertionLogRegModel, self).__init__(classname="com.johnsnowlabs.nlp.annotators.assertion.logreg.AssertionLogRegModel")
 
+    @staticmethod
+    def pretrained(name="as_fast_lg", language="en"):
+        from sparknlp.pretrained import ResourceDownloader
+        return ResourceDownloader.downloadModel(AssertionLogRegModel, name, language)
+
 
 class NerDLApproach(AnnotatorApproach, ApproachWithEmbeddings, NerApproach):
 
@@ -1190,3 +1195,8 @@ def __init__(self, java_model=None):
             super(JavaModel, self).__init__(java_model)
         else:
             super(AssertionDLModel, self).__init__(classname="com.johnsnowlabs.nlp.annotators.assertion.dl.AssertionDLModel")
+
+    @staticmethod
+    def pretrained(name="as_fast_dl", language="en"):
+        from sparknlp.pretrained import ResourceDownloader
+        return ResourceDownloader.downloadModel(AssertionDLModel, name, language)

src/main/scala/com/johnsnowlabs/nlp/annotator.scala

@@ -1,7 +1,8 @@
 package com.johnsnowlabs.nlp
 
 import com.johnsnowlabs.nlp.annotators.PretrainedLemmatizer
-import com.johnsnowlabs.nlp.annotators.assertion.dl.ReadsAssertionGraph
+import com.johnsnowlabs.nlp.annotators.assertion.dl.{PretrainedDLAssertionStatus, ReadsAssertionGraph}
+import com.johnsnowlabs.nlp.annotators.assertion.logreg.PretrainedAssertionLogRegModel
 import com.johnsnowlabs.nlp.annotators.ner.crf.PretrainedNerCrf
 import com.johnsnowlabs.nlp.annotators.ner.dl.{PretrainedNerDL, ReadsNERGraph, WithGraphResolver}
 import com.johnsnowlabs.nlp.annotators.pos.perceptron.PretrainedPerceptronModel
@@ -47,7 +48,7 @@ object annotator {
   type AssertionLogRegApproach = com.johnsnowlabs.nlp.annotators.assertion.logreg.AssertionLogRegApproach
   object AssertionLogRegApproach extends DefaultParamsReadable[AssertionLogRegApproach]
   type AssertionLogRegModel = com.johnsnowlabs.nlp.annotators.assertion.logreg.AssertionLogRegModel
-  object AssertionLogRegModel extends EmbeddingsReadable[AssertionLogRegModel]
+  object AssertionLogRegModel extends EmbeddingsReadable[AssertionLogRegModel] with PretrainedAssertionLogRegModel
 
   type NerCrfApproach = com.johnsnowlabs.nlp.annotators.ner.crf.NerCrfApproach
   object NerCrfApproach extends DefaultParamsReadable[NerCrfApproach]
@@ -95,6 +96,6 @@ object annotator {
   type AssertionDLApproach = com.johnsnowlabs.nlp.annotators.assertion.dl.AssertionDLApproach
   object AssertionDLApproach extends DefaultParamsReadable[AssertionDLApproach]
   type AssertionDLModel = com.johnsnowlabs.nlp.annotators.assertion.dl.AssertionDLModel
-  object AssertionDLModel extends EmbeddingsReadable[AssertionDLModel] with ReadsAssertionGraph
+  object AssertionDLModel extends EmbeddingsReadable[AssertionDLModel] with ReadsAssertionGraph with PretrainedDLAssertionStatus
 
 }

src/main/scala/com/johnsnowlabs/nlp/annotators/assertion/dl/AssertionDLModel.scala

@@ -6,6 +6,7 @@ import com.johnsnowlabs.nlp.annotators.ner.Verbose
 import com.johnsnowlabs.nlp.serialization.StructFeature
 import com.johnsnowlabs.nlp._
 import com.johnsnowlabs.nlp.embeddings.EmbeddingsReadable
+import com.johnsnowlabs.nlp.pretrained.ResourceDownloader
 import org.apache.spark.ml.param.{IntParam, ParamMap}
 import org.apache.spark.ml.util.Identifiable
 import org.apache.spark.sql._
@@ -117,4 +118,9 @@ trait ReadsAssertionGraph extends ParamsAndFeaturesReadable[AssertionDLModel] wi
   addReader(readAssertionGraph)
 }
 
-object AssertionDLModel extends EmbeddingsReadable[AssertionDLModel] with ReadsAssertionGraph
+trait PretrainedDLAssertionStatus {
+  def pretrained(name: String = "as_fast_dl", language: Option[String] = Some("en"), folder: String = ResourceDownloader.publicLoc): AssertionDLModel =
+    ResourceDownloader.downloadModel(AssertionDLModel, name, language, folder)
+}
+
+object AssertionDLModel extends EmbeddingsReadable[AssertionDLModel] with ReadsAssertionGraph with PretrainedDLAssertionStatus

src/main/scala/com/johnsnowlabs/nlp/annotators/assertion/logreg/AssertionLogRegModel.scala

@@ -3,6 +3,7 @@ package com.johnsnowlabs.nlp.annotators.assertion.logreg
 import com.johnsnowlabs.nlp.AnnotatorType._
 import com.johnsnowlabs.nlp._
 import com.johnsnowlabs.nlp.embeddings.{EmbeddingsReadable, WordEmbeddings}
+import com.johnsnowlabs.nlp.pretrained.ResourceDownloader
 import com.johnsnowlabs.nlp.serialization.{MapFeature, StructFeature}
 import org.apache.spark.ml.classification.LogisticRegressionModel
 import org.apache.spark.ml.util.Identifiable
@@ -97,4 +98,10 @@ class AssertionLogRegModel(override val uid: String) extends RawAnnotator[Assert
   override def copy(extra: ParamMap): AssertionLogRegModel = defaultCopy(extra)
 }
 
-object AssertionLogRegModel extends EmbeddingsReadable[AssertionLogRegModel]
+trait PretrainedAssertionLogRegModel {
+  def pretrained(name: String = "as_fast_lg", language: Option[String] = Some("en"), remoteLoc: String = ResourceDownloader.publicLoc): AssertionLogRegModel =
+    ResourceDownloader.downloadModel(AssertionLogRegModel, name, language, remoteLoc)
+}
+
+
+object AssertionLogRegModel extends EmbeddingsReadable[AssertionLogRegModel] with PretrainedAssertionLogRegModel

src/main/scala/com/johnsnowlabs/nlp/base.scala

@@ -8,6 +8,9 @@ object base {
   type DocumentAssembler = com.johnsnowlabs.nlp.DocumentAssembler
   object DocumentAssembler extends DefaultParamsReadable[DocumentAssembler]
 
+  type ChunkAssembler = com.johnsnowlabs.nlp.ChunkAssembler
+  object ChunkAssembler extends DefaultParamsReadable[ChunkAssembler]
+
   type TokenAssembler = com.johnsnowlabs.nlp.TokenAssembler
   object TokenAssembler extends DefaultParamsReadable[TokenAssembler]