Merge pull request #46 from BrikerMan/develop

fix missing sequence_labeling_tokenize_add_bos_eos
BrikerMan · Mar 5, 2019 · 6a75f15 · 6a75f15
2 parents 647c8c4 + 06a883c
commit 6a75f15
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 1 deletion.
diff --git a/kashgari/macros.py b/kashgari/macros.py
@@ -39,6 +39,7 @@
 class _Config(object):
     def __init__(self):
         self.use_CuDNN_cell = False
+        self.sequence_labeling_tokenize_add_bos_eos = False
 
 
 config = _Config()

diff --git a/kashgari/tasks/seq_labeling/base_model.py b/kashgari/tasks/seq_labeling/base_model.py
@@ -92,7 +92,10 @@ def convert_labels_to_idx(self,
         def tokenize_tokens(seq: List[str]):
             tokens = [self._label2idx[i] for i in seq]
             if add_eos_bos:
-                tokens = [self._label2idx[k.BOS]] + tokens + [self._label2idx[k.EOS]]
+                if k.config.sequence_labeling_tokenize_add_bos_eos:
+                    tokens = [self._label2idx[k.BOS]] + tokens + [self._label2idx[k.EOS]]
+                else:
+                    tokens = [self._label2idx[k.NO_TAG]] + tokens + [self._label2idx[k.NO_TAG]]
             return tokens
 
         if isinstance(label[0], str):