Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Refactor generate source full tokens #312

Closed
wants to merge 6 commits into from
Closed

Refactor generate source full tokens #312

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
3f39a80
move closing tag check to scanLine
Jul 15, 2017
7250566
move old include code to scanLine
Jul 15, 2017
306f962
refactor parser, one full tag per scanline
Jul 20, 2017
0d01573
remove mode LITERAL / no longer needed
Jul 29, 2017
8bc8316
changed literal tags to be standalone (no longer affect the next tag)…
Aug 4, 2017
fdb09af
speed up, run regex for literals only if needed
Aug 4, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions docs/syntax.md
100644 → 100755
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@ Table of contents
- `-%>`: Removes trailing newline
- `_%>`: Removes all trailing whitespace
- Literal tags
- `<%%`: Insert `<%`
- `%%>`: Insert `%>`
- Including other files
- “Preprocessor” directive
- JavaScript `include()` function
Expand Down Expand Up @@ -400,7 +402,7 @@ To output literal `<%` or `%>`, use `<%%` or `%%>`, respectively. If a customize
the same syntax. E.g. use `<$$` to get `<$` if the delimiter is `$`.

In regards to all the other tags, the literal tags are special as they do not
need a closing tag to function.
have a closing tag.

However, think twice before you use these tags because `<` and `>` characters might
need to be escaped as `&lt;` and `&gt;`, respectively.
Expand All @@ -414,16 +416,14 @@ escape `<` or `>` at all.

```html
<pre>This is literal: <%%</pre>
<pre>This is literal too: <%% %></pre>
<pre>This is literal as well: %%></pre>
<pre>This is literal too: %%></pre>
```

##### HTML

```html
<pre>This is literal: <%</pre>
<pre>This is literal too: <% %></pre>
<pre>This is literal as well: %></pre>
<pre>This is literal too: %></pre>
```

## Including other files
Expand Down
290 changes: 140 additions & 150 deletions lib/ejs.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,28 @@ var _VERSION_STRING = require('../package.json').version;
var _DEFAULT_DELIMITER = '%';
var _DEFAULT_LOCALS_NAME = 'locals';
var _NAME = 'ejs';
var _REGEX_STRING = '(<%%|%%>|<%=|<%-|<%_|<%#|<%|%>|-%>|_%>)';

var _REGEX_ALL_BUT_TAGS = '(?:(?![-_%]?%>|<%)[^]|(<%%|%%>))';
/**
* Find next open tag, skip literals
* $1: any data before open tag
* $2: indicates presence of literal(s) (from _REGEX_ALL_BUT_TAGS)
* $3: open tag modifier (_=-#)
* $4: invalid/unmatched closing tag
*/
var _REGEX_STRING_OPEN = '^('+_REGEX_ALL_BUT_TAGS+'*)(?:<%(?!%)([_=#-])?|([_-]?%>))';
/**
* Find next close tag, skip literals
* $1: any data before close tag
* $2: indicates presence of literal(s) (from _REGEX_ALL_BUT_TAGS)
* $3: close tag modifier (_-)
*/
var _REGEX_STRING_CLOSE = '^('+_REGEX_ALL_BUT_TAGS+'*)([-_]?)%>';
// Replace literals
var _REGEX_STRING_LITERAL = '(<%)%|%(%>)';
var _REGEX_STRING_LITERAL_REPL = '$1$2';


var _OPTS = ['delimiter', 'scope', 'context', 'debug', 'compileDebug',
'client', '_with', 'rmWhitespace', 'strict', 'filename'];
// We don't allow 'cache' option to be passed in the data obj
Expand Down Expand Up @@ -477,24 +498,31 @@ function Template(text, opts) {
}

this.opts = options;

this.regex = this.createRegex();
this.createRegex();
this.modeMap = {
' ': Template.modes.EVAL,
'=': Template.modes.ESCAPED,
'-': Template.modes.RAW,
'_': Template.modes.EVAL,
'#': Template.modes.COMMENT,
};
}

Template.modes = {
EVAL: 'eval',
ESCAPED: 'escaped',
RAW: 'raw',
COMMENT: 'comment',
LITERAL: 'literal'
};

Template.prototype = {
createRegex: function () {
var str = _REGEX_STRING;
var delim = utils.escapeRegExpChars(this.opts.delimiter);
str = str.replace(/%/g, delim);
return new RegExp(str);

this.regexOpen = new RegExp(_REGEX_STRING_OPEN.replace(/%/g, delim));
this.regexJS = new RegExp(_REGEX_STRING_CLOSE.replace(/%/g, delim));
this.regexLiteral = new RegExp(_REGEX_STRING_LITERAL.replace(/%/g, delim), 'g');
this.regexLiteralReplace = _REGEX_STRING_LITERAL_REPL;
},

compile: function () {
Expand Down Expand Up @@ -597,89 +625,44 @@ Template.prototype = {
this.templateText =
this.templateText.replace(/[ \t]*<%_/gm, '<%_').replace(/_%>[ \t]*/gm, '_%>');

var self = this;
var matches = this.parseTemplateText();
var d = this.opts.delimiter;

if (matches && matches.length) {
matches.forEach(function (line, index) {
var opening;
var closing;
var include;
var includeOpts;
var includeObj;
var includeSrc;
// If this is an opening tag, check for closing tags
// FIXME: May end up with some false positives here
// Better to store modes as k/v with '<' + delimiter as key
// Then this can simply check against the map
if ( line.indexOf('<' + d) === 0 // If it is a tag
&& line.indexOf('<' + d + d) !== 0) { // and is not escaped
closing = matches[index + 2];
if (!(closing == d + '>' || closing == '-' + d + '>' || closing == '_' + d + '>')) {
throw new Error('Could not find matching close tag for "' + line + '".');
}
}
// HACK: backward-compat `include` preprocessor directives
if ((include = line.match(/^\s*include\s+(\S+)/))) {
opening = matches[index - 1];
// Must be in EVAL or RAW mode
if (opening && (opening == '<' + d || opening == '<' + d + '-' || opening == '<' + d + '_')) {
includeOpts = utils.shallowCopy({}, self.opts);
includeObj = includeSource(include[1], includeOpts);
if (self.opts.compileDebug) {
includeSrc =
' ; (function(){' + '\n'
+ ' var __line = 1' + '\n'
+ ' , __lines = ' + JSON.stringify(includeObj.template) + '\n'
+ ' , __filename = ' + JSON.stringify(includeObj.filename) + ';' + '\n'
+ ' try {' + '\n'
+ includeObj.source
+ ' } catch (e) {' + '\n'
+ ' rethrow(e, __lines, __filename, __line, escapeFn);' + '\n'
+ ' }' + '\n'
+ ' ; }).call(this)' + '\n';
}else{
includeSrc = ' ; (function(){' + '\n' + includeObj.source +
' ; }).call(this)' + '\n';
}
self.source += includeSrc;
self.dependencies.push(exports.resolveInclude(include[1],
includeOpts.filename));
return;
}
}
self.scanLine(line);
});
}

},

parseTemplateText: function () {
var str = this.templateText;
var pat = this.regex;
var result = pat.exec(str);
var arr = [];
var firstPos;
var patOpen = this.regexOpen;
var patJS = this.regexJS;
var patLit = this.regexLiteral;
var replLit = this.regexLiteralReplace;

var result = patOpen.exec(str);
while (result) {
firstPos = result.index;
if (result[4]) {
throw new Error('Found close tag "' + result[4] + '" without matching opening tag.');
}
str = str.slice(result[0].length);
var s = result[1];
if (result[2] !== undefined) {
s = s.replace(patLit, replLit);
}
this.scanTextLine(s);

if (firstPos !== 0) {
arr.push(str.substring(0, firstPos));
str = str.slice(firstPos);
var t = result[3]||' ';

var res2 = patJS.exec(str);
if (!res2) {
throw new Error('Could not find matching close tag for "<' + this.opts.delimiter + t + '".');
}
str = str.slice(res2[0].length);
var tag = res2[1];
if (res2[2] !== undefined) {
tag = tag.replace(patLit, replLit);
}
this.scanTagLine(t, tag,res2[3] || ' ');

arr.push(result[0]);
str = str.slice(result[0].length);
result = pat.exec(str);
result = patOpen.exec(str);
}

if (str) {
arr.push(str);
str = str.replace(patLit, replLit);
this.scanTextLine(str);
}

return arr;
},

_addOutput: function (line) {
Expand Down Expand Up @@ -714,90 +697,97 @@ Template.prototype = {
this.source += ' ; __append("' + line + '")' + '\n';
},

scanLine: function (line) {
scanTextLine: function (line) {
var newLineCount = (line.split('\n').length - 1);
this._addOutput(line);
if (this.opts.compileDebug && newLineCount) {
this.currentLine += newLineCount;
this.source += ' ; __line = ' + this.currentLine + '\n';
}
},

scanTagLine: function (openTag, line, closeTag) {
var self = this;
var d = this.opts.delimiter;
var newLineCount = 0;
var newLineCount = (line.split('\n').length - 1);

newLineCount = (line.split('\n').length - 1);
this.mode = this.modeMap[openTag];

switch (line) {
case '<' + d:
case '<' + d + '_':
this.mode = Template.modes.EVAL;
break;
case '<' + d + '=':
this.mode = Template.modes.ESCAPED;
break;
case '<' + d + '-':
this.mode = Template.modes.RAW;
// If '//' is found without a line break, add a line break.
switch (this.mode) {
case Template.modes.EVAL:
case Template.modes.RAW:
// HACK: backward-compat `include` preprocessor directives
var include = line.match(/^\s*include\s+(\S+)/);
if (include) {
var includeSrc;
var includeOpts = utils.shallowCopy({}, self.opts);
var includeObj = includeSource(include[1], includeOpts);
if (self.opts.compileDebug) {
includeSrc =
' ; (function(){' + '\n'
+ ' var __line = 1' + '\n'
+ ' , __lines = ' + JSON.stringify(includeObj.template) + '\n'
+ ' , __filename = ' + JSON.stringify(includeObj.filename) + ';' + '\n'
+ ' try {' + '\n'
+ includeObj.source
+ ' } catch (e) {' + '\n'
+ ' rethrow(e, __lines, __filename, __line, escapeFn);' + '\n'
+ ' }' + '\n'
+ ' ; }).call(this)' + '\n';
}else{
includeSrc = ' ; (function(){' + '\n' + includeObj.source +
' ; }).call(this)' + '\n';
}
self.source += includeSrc;
self.dependencies.push(exports.resolveInclude(include[1],
includeOpts.filename));
this.mode = null; // skip normal handling
break;
}
// END HACK include
if (line.lastIndexOf('//') > line.lastIndexOf('\n')) {
line += '\n';
}
break;
case '<' + d + '#':
this.mode = Template.modes.COMMENT;
case Template.modes.ESCAPED:
if (line.lastIndexOf('//') > line.lastIndexOf('\n')) {
line += '\n';
}
}

switch (this.mode) {
// Just executing code
case Template.modes.EVAL:
this.source += ' ; ' + line + '\n';
break;
case '<' + d + d:
this.mode = Template.modes.LITERAL;
this.source += ' ; __append("' + line.replace('<' + d + d, '<' + d) + '")' + '\n';
// Exec, esc, and output
case Template.modes.ESCAPED:
this.source += ' ; __append(escapeFn(' + stripSemi(line) + '))' + '\n';
break;
case d + d + '>':
this.mode = Template.modes.LITERAL;
this.source += ' ; __append("' + line.replace(d + d + '>', d + '>') + '")' + '\n';
// Exec and output
case Template.modes.RAW:
this.source += ' ; __append(' + stripSemi(line) + ')' + '\n';
break;
case d + '>':
case '-' + d + '>':
case '_' + d + '>':
if (this.mode == Template.modes.LITERAL) {
this._addOutput(line);
}

this.mode = null;
this.truncate = line.indexOf('-') === 0 || line.indexOf('_') === 0;
case Template.modes.COMMENT:
// Do nothing
break;
default:
// In script mode, depends on type of tag
if (this.mode) {
// If '//' is found without a line break, add a line break.
switch (this.mode) {
case Template.modes.EVAL:
case Template.modes.ESCAPED:
case Template.modes.RAW:
if (line.lastIndexOf('//') > line.lastIndexOf('\n')) {
line += '\n';
}
}
switch (this.mode) {
// Just executing code
case Template.modes.EVAL:
this.source += ' ; ' + line + '\n';
break;
// Exec, esc, and output
case Template.modes.ESCAPED:
this.source += ' ; __append(escapeFn(' + stripSemi(line) + '))' + '\n';
break;
// Exec and output
case Template.modes.RAW:
this.source += ' ; __append(' + stripSemi(line) + ')' + '\n';
break;
case Template.modes.COMMENT:
// Do nothing
break;
// Literal <%% mode, append as raw output
case Template.modes.LITERAL:
this._addOutput(line);
break;
}
}
// In string mode, just add the output
else {
this._addOutput(line);
}
}

if (self.opts.compileDebug && newLineCount) {
this.currentLine += newLineCount;
this.source += ' ; __line = ' + this.currentLine + '\n';
}
}


this.mode = null;
switch (closeTag) {
case '-':
case '_':
this.truncate = true;
}

},

};

/**
Expand Down
Loading