Vue mode, derived from mmm-mode, fails with latest version

[garyo]

I'm using Emacs trunk build from today, and most recent mmm-mode and vue-mode. I have a Vue-mode file that causes mmm-mode to get a CL assertion failure when it syntax-parses the buffer.

Debugger entered--Lisp error: (cl-assertion-failed ((>= (cadr sgml--syntax-propertize-ppss) 0) nil))
  cl--assertion-failed((>= (cadr sgml--syntax-propertize-ppss) 0))
  (or (>= (car (cdr sgml--syntax-propertize-ppss)) 0) (cl--assertion-failed '(>= (cadr sgml--syntax-propertize-ppss) 0)))
  (progn (or (>= (car (cdr sgml--syntax-propertize-ppss)) 0) (cl--assertion-failed '(>= (cadr sgml--syntax-propertize-ppss) 0))) nil)
  sgml-syntax-propertize(578 603)
  funcall(sgml-syntax-propertize 578 603)
  (cond (func (funcall func beg end)) (font-lock-syntactic-keywords (let ((syntax-propertize-function nil)) (font-lock-fontify-syntactic-keywords-region beg end))))
  (save-restriction (if mmm-current-overlay (progn (narrow-to-region (overlay-start mmm-current-overlay) (overlay-end mmm-current-overlay)))) (cond (func (funcall func beg end)) (font-lock-syntactic-keywords (let ((syntax-propertize-function nil)) (font-lock-fontify-syntactic-keywords-region beg end)))) (run-hook-with-args 'mmm-after-syntax-propertize-functions mmm-current-overlay mode beg end))
  (let* ((mode (car elt)) (func (get mode 'mmm-syntax-propertize-function)) (beg (car (cdr elt))) (end (nth 2 elt)) (ovl (nth 3 elt)) syntax-ppss-cache syntax-ppss-last) (goto-char beg) (mmm-set-current-pair mode ovl) (mmm-set-local-variables mode mmm-current-overlay) (save-restriction (if mmm-current-overlay (progn (narrow-to-region (overlay-start mmm-current-overlay) (overlay-end mmm-current-overlay)))) (cond (func (funcall func beg end)) (font-lock-syntactic-keywords (let ((syntax-propertize-function nil)) (font-lock-fontify-syntactic-keywords-region beg end)))) (run-hook-with-args 'mmm-after-syntax-propertize-functions mmm-current-overlay mode beg end)))
  (lambda (elt) (let* ((mode (car elt)) (func (get mode 'mmm-syntax-propertize-function)) (beg (car (cdr elt))) (end (nth 2 elt)) (ovl (nth 3 elt)) syntax-ppss-cache syntax-ppss-last) (goto-char beg) (mmm-set-current-pair mode ovl) (mmm-set-local-variables mode mmm-current-overlay) (save-restriction (if mmm-current-overlay (progn (narrow-to-region (overlay-start mmm-current-overlay) (overlay-end mmm-current-overlay)))) (cond (func (funcall func beg end)) (font-lock-syntactic-keywords (let (...) (font-lock-fontify-syntactic-keywords-region beg end)))) (run-hook-with-args 'mmm-after-syntax-propertize-functions mmm-current-overlay mode beg end))))((vue-mode 578 603 nil))
  mapc((lambda (elt) (let* ((mode (car elt)) (func (get mode 'mmm-syntax-propertize-function)) (beg (car (cdr elt))) (end (nth 2 elt)) (ovl (nth 3 elt)) syntax-ppss-cache syntax-ppss-last) (goto-char beg) (mmm-set-current-pair mode ovl) (mmm-set-local-variables mode mmm-current-overlay) (save-restriction (if mmm-current-overlay (progn (narrow-to-region (overlay-start mmm-current-overlay) (overlay-end mmm-current-overlay)))) (cond (func (funcall func beg end)) (font-lock-syntactic-keywords (let (...) (font-lock-fontify-syntactic-keywords-region beg end)))) (run-hook-with-args 'mmm-after-syntax-propertize-functions mmm-current-overlay mode beg end)))) ((vue-mode 1 12 nil) (vue-html-mode 12 94 #<overlay from 12 to 94 in foo.vue</tmp>>) (vue-mode 94 125 nil) (nil 125 578 #<overlay from 125 to 578 in foo.vue</tmp>>) (vue-mode 578 603 nil) (css-mode 603 862 #<overlay from 603 to 862 in foo.vue</tmp>>) (vue-mode 862 871 nil)))
  (unwind-protect (mapc #'(lambda (elt) (let* ((mode (car elt)) (func (get mode ...)) (beg (car ...)) (end (nth 2 elt)) (ovl (nth 3 elt)) syntax-ppss-cache syntax-ppss-last) (goto-char beg) (mmm-set-current-pair mode ovl) (mmm-set-local-variables mode mmm-current-overlay) (save-restriction (if mmm-current-overlay (progn ...)) (cond (func ...) (font-lock-syntactic-keywords ...)) (run-hook-with-args 'mmm-after-syntax-propertize-functions mmm-current-overlay mode beg end)))) (mmm-regions-in start stop)) (mmm-set-current-pair saved-mode saved-ovl) (mmm-set-local-variables (or saved-mode mmm-primary-mode) saved-ovl))
  (let ((saved-mode mmm-current-submode) (saved-ovl mmm-current-overlay)) (mmm-save-changed-local-variables mmm-current-submode mmm-current-overlay) (unwind-protect (mapc #'(lambda (elt) (let* ((mode ...) (func ...) (beg ...) (end ...) (ovl ...) syntax-ppss-cache syntax-ppss-last) (goto-char beg) (mmm-set-current-pair mode ovl) (mmm-set-local-variables mode mmm-current-overlay) (save-restriction (if mmm-current-overlay ...) (cond ... ...) (run-hook-with-args ... mmm-current-overlay mode beg end)))) (mmm-regions-in start stop)) (mmm-set-current-pair saved-mode saved-ovl) (mmm-set-local-variables (or saved-mode mmm-primary-mode) saved-ovl)))
  mmm-syntax-propertize-function(1 871)
  syntax-propertize(871)
  mmm-apply-all()
  mmm-parse-buffer()
  vue-mode-reparse()
  funcall-interactively(vue-mode-reparse)
  call-interactively(vue-mode-reparse nil nil)
  command-execute(vue-mode-reparse)

At the point of failure, (cadr sgml--syntax-propertize-ppss) is -3. But the typescript syntax in that section of the file is OK, there are no mismatched parens.

To repro, use this emacs startup file (emacs -Q -l mmm-bug-test.el):

(defvar bootstrap-version)
(let ((bootstrap-file
       (expand-file-name "straight/repos/straight.el/bootstrap.el" user-emacs-directory))
      (bootstrap-version 5))
  (unless (file-exists-p bootstrap-file)
    (with-current-buffer
        (url-retrieve-synchronously
         "https://raw.githubusercontent.com/raxod502/straight.el/develop/install.el"
         'silent 'inhibit-cookies)
      (goto-char (point-max))
      (eval-print-last-sexp)))
  (load bootstrap-file nil 'nomessage))

(straight-use-package 'vue-mode)

The vue file for testing is this. Just load it, and if needed do C-c C-l to reparse.

<template>
  <div class="review">
    <v-container class="pt-0">
    </v-container>
  </div>
</template>
<script lang="ts">
@Component({
  components: {
    VuePerfectScrollbar,
  }
})
export default class Review extends Vue {

  sortedTasks(taskGroupIds: string[]) {
    const copy = [...taskGroupIds]
    return copy.sort((e1, e2) => {
      return this.taskGroupDates[e2].seconds - this.taskGroupDates[e1].seconds
    })
  }

  shot(shotId: string) {
    return shotStore?.shots.find(shot => shot.id === shotId)
  }

  async mounted() {
    await this.getPostDocs()
  }
}
</script>
<style scoped>
  .scroll-area {
    position: relative;
    margin: auto;
    height: 80vh;
  }
  .video-placeholder {
      width: 1066px;
      height: 100px;
      line-height: 100px;
      text-align: center;
      background-color: #8888aa;
      margin-bottom: 0;
  }
</style>

[garyo]

You may also need (straight-use-package 'typescript-mode), and point must be in the typescript section.

[garyo]

Here's a shorter test case file:

<template>
  <div class="review">
  </div>
</template>
<script lang="ts">
class Foo {
  foo() {
    return xyz(foo => true) // return xyz(true) is OK here
  }
}
</script>
<style scoped>
</style>

The key line is the body of function foo. If I remove that line, or even remove just the arrow function foo => true, it parses OK. But in normal typescript mode, that entire section parses just fine. So something about the arrow-function syntax?

[garyo]

It seems that while reparsing, it calls (syntax-ppss 131) where 131 is the < at the beginning of </script>. That ends up calling (parse-partial-sexp 1 131 nil nil ...oldstate...) but that range includes two different submode regions, so it ends up getting -1 for the paren level.

Aha, I think I see what's happening. It's not the parsing of the typescript submode regionb, it's the parsing of the part after that (from char 131 to 165). The part immediately after the typescript doesn't have an associated submode (it's just "vue"), so mmm-syntax-propertize-function (mmm-region.el:871) does not narrow the buffer when parsing it because mmm-current-overlay is nil. Then it just calls (sgml-syntax-propertize 131 165) but that ends up parsing the whole buffer up to 131, which causes the error.

And based on that, this patch works for me in this particular case:

modified   mmm-region.el
@@ -868,9 +868,10 @@ calls each respective submode's `syntax-propertize-function'."
                     (mmm-set-current-pair mode ovl)
                     (mmm-set-local-variables mode mmm-current-overlay)
                     (save-restriction
-                      (when mmm-current-overlay
+                      (if mmm-current-overlay
                         (narrow-to-region (overlay-start mmm-current-overlay)
-                                          (overlay-end mmm-current-overlay)))
+                                          (overlay-end mmm-current-overlay))
+			(narrow-to-region beg end))
                       (cond
                        (func
                         (funcall func beg end))

but someone with more experience will have to see if that's the proper solution or just a hack for my case. I suppose an alternative answer is that vue-mode should ensure that there is always a valid overlay for all chars of the file. But it seems best for mmm-mode to protect against this itself.

[dgutov]

Hi!

Sorry for the late reply.

It seems like the same problem html-erb-mode is more-or-less solving using mmm-after-syntax-propertize-functions: https://github.com/purcell/mmm-mode/blob/master/mmm-erb.el#L87

Curious it has only come up now. But vue-mode can probably copy html-erb-after-syntax-propertize wholesale, it's tiny.

As a result, that parse-partial-sexp calls you mention above won't take the > in arrow functions into account. Could you try that and see if that helps?

Narrowing in the "else" case, like you suggested, could work, but it's likely to bring other unwanted side-effects as well. Like when some particular tricky literal has a subregion inside, it wouldn't be propertized as a whole.

[lrpereira]

Hi all,
Is there a fix for this issue ?
I'm experiencing this same problem.
Cheers.

[garyo]

I'm using this, some of which may no longer be needed with newer versions of mmm-mode:

  ;; fix for Emacs27/mmm-mode bug (as of June 2019)
  ;; without this, TAB doesn't indent in the <script> section
  ;; remove once mmm-mode has a fix for this
  ;; (see https://github.com/purcell/mmm-mode/issues/99)
  (add-to-list 'mmm-save-local-variables '(syntax-ppss-table buffer))
  (add-to-list 'mmm-save-local-variables '(sgml--syntax-propertize-ppss))
  ;; Fix for mmm-mode bug https://github.com/purcell/mmm-mode/issues/100
  ;; (can remove once that's fixed & released)
  (add-to-list 'mmm-save-local-variables '(c-current-comment-prefix region))
  ;; Fix for mmm-mode bug #107 where M-x occur fails while fontifying in Vue mode
  (add-to-list 'mmm-save-local-variables '(typescript--quick-match-re-func region))
  (add-to-list 'mmm-save-local-variables '(typescript--quick-match-re region))

[lrpereira]

I'm using this, some of which may no longer be needed with newer versions of mmm-mode:

  ;; fix for Emacs27/mmm-mode bug (as of June 2019)
  ;; without this, TAB doesn't indent in the <script> section
  ;; remove once mmm-mode has a fix for this
  ;; (see https://github.com/purcell/mmm-mode/issues/99)
  (add-to-list 'mmm-save-local-variables '(syntax-ppss-table buffer))
  (add-to-list 'mmm-save-local-variables '(sgml--syntax-propertize-ppss))
  ;; Fix for mmm-mode bug https://github.com/purcell/mmm-mode/issues/100
  ;; (can remove once that's fixed & released)
  (add-to-list 'mmm-save-local-variables '(c-current-comment-prefix region))
  ;; Fix for mmm-mode bug #107 where M-x occur fails while fontifying in Vue mode
  (add-to-list 'mmm-save-local-variables '(typescript--quick-match-re-func region))
  (add-to-list 'mmm-save-local-variables '(typescript--quick-match-re region))

No luck, i keep getting this kind of errors :|
cl--assertion-failed: Assertion failed: (>= (cadr sgml--syntax-propertize-ppss) 0)

Are you compiling mmm-mode yourself ?
Running emacs 27.1. Using use-package.

[garyo]

I'm using straight.el so getting it "straight" from the source.

[dgutov]

@garyo If you're using mmm-mode from master, could you try removing those customizations from your init file and testing whether everything still works?

I mean all but one (the one with sgml--syntax-propertize-ppss), the rest should already be incorporated in the current master version.

[garyo]

@dgutov I just removed all of those (except `sgml--syntax-propertize-ppss) and things seem to be OK. Tested for the issues in #99, #100, and #107. Thanks as always!

[dgutov]

Very good.

Now, about that last line. It doesn't help in my testing, I still get that "assertion failed" with your example (only I tried it with html-mode and not vue-mode). The only thing that helped is to follow my original recommendation:

• Define a custom syntax-propertize-function,

(defun html-after-syntax-propertize (overlay _mode beg end)
  (when overlay
    (with-silent-modifications
      (funcall
       (syntax-propertize-rules ("<\\|>" (0 ".")))
       beg end))))

• Evaluate this in the buffer visiting the example file:

(add-hook 'mmm-after-syntax-propertize-functions #'html-after-syntax-propertize nil t)

• Make an edit to trigger syntax-propertize, which will trigger re-fontification, which now happens without the error.

So I think vue-mode can include a (renamed) copy of that function, and add the last line to the definition of vue-mode.

[dgutov]

Here's an alternative fix you could try, too:

diff --git a/mmm-region.el b/mmm-region.el
index a0c5b4f..7791e07 100644
--- a/mmm-region.el
+++ b/mmm-region.el
@@ -870,7 +870,10 @@ calls each respective submode's `syntax-propertize-function'."
                     (save-restriction
                       (when mmm-current-overlay
                         (narrow-to-region (overlay-start mmm-current-overlay)
-                                          (overlay-end mmm-current-overlay)))
+                                          (overlay-end mmm-current-overlay))
+                        (put-text-property
+                         (point-min) (point-max)
+                         'syntax-table (syntax-table)))
                       (cond
                        (func
                         (funcall func beg end))

I'm hesitant to apply it here, though (it feels pretty dirty on the high level). Wish overlays supported this property. Perhaps we should migrate off overlays first.

[garyo]

This problem went away for me for a while without applying your fix, @dgutov, but now it's back, and I had to apply your "alternative fix" above (put-text-property...) to my mmm-region.el and that helps. I didn't try the custom html-after-syntax-propertize.

[garyo]

I just wanted to see if you plan to apply your fix above. It definitely does help for me. (I accidentally lost my local version with this change and the errors came back.)

[rabdulatipoff]

Here's an alternative fix you could try

The issue's been temporarily fixed after applying this, though only until I called vue-mode-reparse on any of the affected buffers. Debugger output looks the same as in the original post, though I'll have to check on the latest emacs build (mine is 'master' branch as of Aug 5).

[sevillaarvin]

Here's an alternative fix you could try, too:

diff --git a/mmm-region.el b/mmm-region.el
index a0c5b4f..7791e07 100644
--- a/mmm-region.el
+++ b/mmm-region.el
@@ -870,7 +870,10 @@ calls each respective submode's `syntax-propertize-function'."
                     (save-restriction
                       (when mmm-current-overlay
                         (narrow-to-region (overlay-start mmm-current-overlay)
-                                          (overlay-end mmm-current-overlay)))
+                                          (overlay-end mmm-current-overlay))
+                        (put-text-property
+                         (point-min) (point-max)
+                         'syntax-table (syntax-table)))
                       (cond
                        (func
                         (funcall func beg end))

I'm hesitant to apply it here, though (it feels pretty dirty on the high level). Wish overlays supported this property. Perhaps we should migrate off overlays first.

Can confirm, the errors went away after I applied the fix.