add commonmark spec and gfm spec test and compare with markedjs (#943)

* add commonmark spec and gfm spec test and compare with markedjs

* ignore .env

* fix: load lang multiple times

* add eslint disable

* add `math`, `emoji`, `frontMatter` options to marked

* fix: commonmark example 223

* fix: import markdown error

* fix: commonmark example 7

* fix: commonmark example 40

* update test result

* update changelog

* update loose test regexp

* remove unused comments
This commit is contained in:
Ran Luo 2019-04-17 21:29:00 +08:00 committed by Felix Häusler
parent ef9fe7566a
commit f4f5ba3e00
19 changed files with 8472 additions and 51 deletions

15
.github/CHANGELOG.md vendored
View File

@ -1,3 +1,18 @@
## [unrelease]
**:warning:Breaking Changes:**
**:cactus:Feature**
**:butterfly:Optimization**
- Rewrite `select all` when press `CtrlOrCmd + A` (#937)
**:beetle:Bug fix**
- Fixed some commonmark failed examples and add test case (#943)
- Fixed some bugs after press `backspace` (#934, #938)
### 0.14.0
This update **fixes a XSS security vulnerability** when exporting a document.

View File

@ -22,6 +22,7 @@
"pack:main": "cross-env NODE_ENV=production webpack --progress --colors --config .electron-vue/webpack.main.config.js",
"pack:renderer": "cross-env NODE_ENV=production webpack --progress --colors --config .electron-vue/webpack.renderer.config.js",
"test": "npm run unit && npm run e2e",
"test:specs": "node -r esm test/specs/commonMark/run.spec.js && node -r esm test/specs/gfm/run.spec.js",
"unit": "cross-env NODE_ENV=test karma start test/unit/karma.conf.js",
"preinstall": "node .electron-vue/preinstall.js",
"postinstall": "npm run rebuild && npm run lint:fix",
@ -200,6 +201,7 @@
"vuex": "^3.1.0"
},
"devDependencies": {
"@markedjs/html-differ": "^2.0.1",
"babel-core": "^6.26.3",
"babel-eslint": "^10.0.1",
"babel-loader": "^7.1.5",
@ -212,6 +214,7 @@
"cfonts": "^2.4.2",
"chai": "^4.2.0",
"chalk": "^2.4.2",
"cheerio": "^1.0.0-rc.3",
"copy-webpack-plugin": "^5.0.2",
"cross-env": "^5.2.0",
"css-loader": "^2.1.1",
@ -235,6 +238,7 @@
"eslint-plugin-promise": "^4.1.1",
"eslint-plugin-standard": "^4.0.0",
"eslint-plugin-vue": "^5.2.2",
"esm": "^3.2.22",
"file-loader": "^3.0.1",
"git-revision-webpack-plugin": "^3.0.3",
"html-webpack-plugin": "^3.2.0",
@ -248,9 +252,11 @@
"karma-spec-reporter": "0.0.32",
"karma-webpack": "^3.0.5",
"license-checker": "^25.0.1",
"marked": "^0.6.2",
"mini-css-extract-plugin": "^0.5.0",
"mocha": "^6.0.2",
"multispinner": "^0.2.1",
"node-fetch": "^2.3.0",
"node-loader": "^0.6.0",
"postcss-loader": "^3.0.0",
"postcss-preset-env": "^6.6.0",

View File

@ -27,7 +27,7 @@ export const block = {
+ ')',
def: /^ {0,3}\[(label)\]: *\n? *<?([^\s>]+)>?(?:(?: +\n? *| *\n *)(title))? *(?:\n+|$)/,
table: noop,
lheading: /^([^\n]+)\n *(=|-){2,} *(?:\n+|$)/,
lheading: /^([^\n]+)\n {0,3}(=|-){2,} *(?:\n+|$)/,
paragraph: /^([^\n]+(?:\n(?!hr|heading|lheading| {0,3}>|<\/?(?:tag)(?: +|\n|\/?>)|<(?:script|pre|style|!--))[^\n]+)*)/,
text: /^[^\n]+/,

View File

@ -34,7 +34,7 @@ function InlineLexer (links, options) {
*/
InlineLexer.prototype.output = function (src) {
const { disableInline } = this.options
const { disableInline, emoji, math } = this.options
if (disableInline) {
return escape(src)
}
@ -131,19 +131,23 @@ InlineLexer.prototype.output = function (src) {
}
// math
cap = this.rules.math.exec(src)
if (cap) {
src = src.substring(cap[0].length)
text = cap[1]
out += this.renderer.inlineMath(text)
if (math) {
cap = this.rules.math.exec(src)
if (cap) {
src = src.substring(cap[0].length)
text = cap[1]
out += this.renderer.inlineMath(text)
}
}
// emoji
cap = this.rules.emoji.exec(src)
if (cap) {
src = src.substring(cap[0].length)
text = cap[0]
out += this.renderer.emoji(text, cap[2])
if (emoji) {
cap = this.rules.emoji.exec(src)
if (cap) {
src = src.substring(cap[0].length)
text = cap[0]
out += this.renderer.emoji(text, cap[2])
}
}
// strong

View File

@ -30,6 +30,7 @@ function Lexer (opts) {
Lexer.prototype.lex = function (src) {
src = src
.replace(/\r\n|\r/g, '\n')
// replace `\t` to four space.
.replace(/\t/g, ' ')
.replace(/\u00a0/g, ' ')
.replace(/\u2424/g, '\n')
@ -42,6 +43,7 @@ Lexer.prototype.lex = function (src) {
*/
Lexer.prototype.token = function (src, top) {
const { frontMatter, math } = this.options
src = src.replace(/^ +$/gm, '')
let loose
@ -57,15 +59,17 @@ Lexer.prototype.token = function (src, top) {
// Only check front matter at the begining of a markdown file.
// Why "checkFrontmatter" and "top"? See note in "blockquote".
cap = this.rules.frontmatter.exec(src)
if (this.checkFrontmatter && top && cap) {
src = src.substring(cap[0].length)
this.tokens.push({
type: 'frontmatter',
text: cap[1]
})
if (frontMatter) {
cap = this.rules.frontmatter.exec(src)
if (this.checkFrontmatter && top && cap) {
src = src.substring(cap[0].length)
this.tokens.push({
type: 'frontmatter',
text: cap[1]
})
}
this.checkFrontmatter = false
}
this.checkFrontmatter = false
while (src) {
// newline
@ -80,29 +84,37 @@ Lexer.prototype.token = function (src, top) {
}
// code
// An indented code block cannot interrupt a paragraph.
cap = this.rules.code.exec(src)
if (cap) {
const lastToken = this.tokens[this.tokens.length - 1]
src = src.substring(cap[0].length)
cap = cap[0].replace(/^ {4}/gm, '')
this.tokens.push({
type: 'code',
codeBlockStyle: 'indented',
text: !this.options.pedantic
? rtrim(cap, '\n')
: cap
})
if (lastToken && lastToken.type === 'paragraph') {
lastToken.text += `\n${cap[0].trimRight()}`
} else {
cap = cap[0].replace(/^ {4}/gm, '')
this.tokens.push({
type: 'code',
codeBlockStyle: 'indented',
text: !this.options.pedantic
? rtrim(cap, '\n')
: cap
})
}
continue
}
// multiple line math
cap = this.rules.multiplemath.exec(src)
if (cap) {
src = src.substring(cap[0].length)
this.tokens.push({
type: 'multiplemath',
text: cap[1]
})
continue
if (math) {
cap = this.rules.multiplemath.exec(src)
if (cap) {
src = src.substring(cap[0].length)
this.tokens.push({
type: 'multiplemath',
text: cap[1]
})
continue
}
}
// fences (gfm)
@ -232,7 +244,7 @@ Lexer.prototype.token = function (src, top) {
// so it is seen as the next token.
space = item.length
let newBull
item = item.replace(/^ *([*+-]|\d+(?:\.|\))) */, function (m, p1) {
item = item.replace(/^ *([*+-]|\d+(?:\.|\))) {0,4}/, function (m, p1) {
// Get and remove list item bullet
newBull = p1 || bull
return ''
@ -301,8 +313,10 @@ Lexer.prototype.token = function (src, top) {
}
// Determine whether item is loose or not. If previous item is loose
// this item is also loose.
loose = next = next || /^ *([*+-]|\d{1,9}(?:\.|\)))( +\S+\n\n(?!\s*$)|\n\n(?!\s*$))/.test(itemWithBullet)
// A list is loose if any of its constituent list items are separated by blank lines,
// or if any of its constituent list items directly contain two block-level elements with a blank line between them.
// loose = next = next || /^ *([*+-]|\d{1,9}(?:\.|\)))( +\S+\n\n(?!\s*$)|\n\n(?!\s*$))/.test(itemWithBullet)
loose = next = next || /\n\n(?!\s*$)/.test(item)
// Check if previous line ends with a new line.
if (!loose && (i !== 0 || l > 1) && prevItem.length !== 0 && prevItem.charAt(prevItem.length - 1) === '\n') {
loose = next = true

View File

@ -2,6 +2,12 @@ export default {
baseUrl: null,
breaks: false,
gfm: true,
// TODO: We set weather to support `emoji`, `math`, `frontMatter` default value to `true`
// After we add user setting, we maybe set math and frontMatter default value to false.
// User need to enable them in the user setting.
emoji: true,
math: true,
frontMatter: true,
headerIds: true,
headerPrefix: '',
highlight: null,

View File

@ -105,7 +105,7 @@ Renderer.prototype.listitem = function (text, checked) {
// task list
return '<li class="task-list-item"><input type="checkbox"' +
(checked ? ' checked=""' : '') +
'disabled=""' +
' disabled=""' +
(this.options.xhtml ? ' /' : '') +
'> ' +
text +

View File

@ -31,6 +31,7 @@ const importRegister = ContentState => {
let block
let value
let parentList = [ rootState ]
const languageLoaded = new Set()
while ((token = tokens.shift())) {
switch (token.type) {
@ -102,7 +103,8 @@ const importRegister = ContentState => {
this.appendChild(codeBlock, codeLine)
})
const inputBlock = this.createBlock('span', lang)
if (lang) {
if (lang && !languageLoaded.has(lang)) {
languageLoaded.add(lang)
loadLanguage(lang)
.then(infoList => {
if (!Array.isArray(infoList)) return
@ -171,7 +173,20 @@ const importRegister = ContentState => {
this.appendChild(parentList[0], block)
break
}
case 'text':
case 'text': {
value = token.text
while (tokens[0].type === 'text') {
token = tokens.shift()
value += `\n${token.text}`
}
block = this.createBlock('p')
const lines = value.split(LINE_BREAKS_REG).map(line => this.createBlock('span', line))
for (const line of lines) {
this.appendChild(block, line)
}
this.appendChild(parentList[0], block)
break
}
case 'paragraph': {
value = token.text
block = this.createBlock('p')
@ -235,7 +250,7 @@ const importRegister = ContentState => {
break
}
}
languageLoaded.clear()
return rootState.children.length ? rootState.children : [this.createBlockP()]
}

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,275 @@
## Compare with `marked.js`
Marked.js failed examples count: 133
Mark Text failed examples count: 126
**Example7**
Mark Text success and marked.js fail
```markdown
Markdown content
- foo
Expected Html
<ul>
<li>
<pre><code> foo
</code></pre>
</li>
</ul>
Actural Html
<ul>
<li><pre><code class="indented-code-block">foo
</code></pre>
</li>
</ul>
marked.js html
<ul>
<li>foo</li>
</ul>
```
**Example40**
Mark Text success and marked.js fail
```markdown
Markdown content
foo
# bar
Expected Html
<p>foo
# bar</p>
Actural Html
<p>foo
# bar</p>
marked.js html
<p>foo</p>
<pre><code># bar</code></pre>
```
**Example57**
Mark Text success and marked.js fail
```markdown
Markdown content
Foo
---
Expected Html
<p>Foo
---</p>
Actural Html
<p>Foo
---</p>
marked.js html
<h2>Foo</h2>
```
**Example266**
Mark Text success and marked.js fail
```markdown
Markdown content
10) foo
- bar
Expected Html
<ol start="10">
<li>foo
<ul>
<li>bar</li>
</ul>
</li>
</ol>
Actural Html
<ol start="10">
<li>foo<ul>
<li>bar</li>
</ul>
</li>
</ol>
marked.js html
<p>10) foo
- bar</p>
```
**Example271**
Mark Text success and marked.js fail
```markdown
Markdown content
- foo
- bar
+ baz
Expected Html
<ul>
<li>foo</li>
<li>bar</li>
</ul>
<ul>
<li>baz</li>
</ul>
Actural Html
<ul>
<li>foo</li>
<li>bar</li>
</ul>
<ul>
<li>baz</li>
</ul>
marked.js html
<ul>
<li>foo</li>
<li>bar</li>
<li>baz</li>
</ul>
```
**Example272**
Mark Text success and marked.js fail
```markdown
Markdown content
1. foo
2. bar
3) baz
Expected Html
<ol>
<li>foo</li>
<li>bar</li>
</ol>
<ol start="3">
<li>baz</li>
</ol>
Actural Html
<ol>
<li>foo</li>
<li>bar</li>
</ol>
<ol start="3">
<li>baz</li>
</ol>
marked.js html
<ol>
<li>foo</li>
<li>bar
3) baz</li>
</ol>
```
**Example285**
Mark Text fail and marked.js success
```markdown
Markdown content
* a
*
* c
Expected Html
<ul>
<li>
<p>a</p>
</li>
<li></li>
<li>
<p>c</p>
</li>
</ul>
Actural Html
<ul>
<li><p>a</p>
</li>
<li><p></p>
</li>
<li><p>c</p>
</li>
</ul>
marked.js html
<ul>
<li><p>a</p>
</li>
<li></li>
<li><p>c</p>
</li>
</ul>
```
**Example310**
Mark Text success and marked.js fail
```markdown
Markdown content
\`\`\` foo\+bar
foo
\`\`\`
Expected Html
<pre><code class="language-foo+bar">foo
</code></pre>
Actural Html
<pre><code class="fenced-code-block language-foo\+bar">foo
</code></pre>
marked.js html
<pre><code class="language-foo\+bar">foo</code></pre>
```
**Example320**
Mark Text success and marked.js fail
```markdown
Markdown content
\`\`\` f&ouml;&ouml;
foo
\`\`\`
Expected Html
<pre><code class="language-föö">foo
</code></pre>
Actural Html
<pre><code class="fenced-code-block language-f&amp;ouml;&amp;ouml;">foo
</code></pre>
marked.js html
<pre><code class="language-f&amp;ouml;&amp;ouml;">foo</code></pre>
```
There are 9 examples are different with marked.js.

View File

@ -0,0 +1,100 @@
// This file is copy from marked and modified.
import { removeCustomClass } from '../help'
import { MT_MARKED_OPTIONS } from '../config'
const fetch = require('node-fetch')
const markedJs = require('marked')
const marked = require('../../../src/muya/lib/parser/marked/index.js').default
const HtmlDiffer = require('@markedjs/html-differ').HtmlDiffer
const fs = require('fs')
const path = require('path')
const options = { ignoreSelfClosingSlash: true, ignoreAttributes: ['id', 'class'] }
const htmlDiffer = new HtmlDiffer(options)
const getSpecs = async () => {
const version = await fetch('https://raw.githubusercontent.com/commonmark/commonmark.js/master/package.json')
.then(res => res.json())
.then(pkg => pkg.version.replace(/^(\d+\.\d+).*$/, '$1'))
return fetch(`https://spec.commonmark.org/${version}/spec.json`)
.then(res => res.json())
.then (specs => ({ specs, version }))
}
const getMarkedSpecs = async (version) => {
return fetch(`https://raw.githubusercontent.com/markedjs/marked/master/test/specs/commonmark/commonmark.${version}.json`)
.then(res => res.json())
}
export const writeResult = (version, specs, markedSpecs, type = 'commonmark') => {
let result = '## Test Result\n\n'
const totalCount = specs.length
const failedCount = specs.filter(s => s.shouldFail).length
result += `Total test ${totalCount} examples, and failed ${failedCount} examples:\n\n`
specs.filter(s => s.shouldFail)
.forEach(spec => {
const expectedHtml = spec.html
const acturalHtml = marked(spec.markdown, MT_MARKED_OPTIONS)
result += `**Example${spec.example}**\n\n`
result += '```markdown\n'
result += `Markdown content\n`
result += `${spec.markdown.replace(/`/g, '\\`')}\n`
result += `Expected Html\n`
result += `${expectedHtml}\n`
result += `Actural Html\n`
result += `${acturalHtml}\n`
result += '```\n\n'
})
const failedPath = type === 'commonmark' ? `./${type}.${version}.md` : `../gfm/${type}.${version}.md`
fs.writeFileSync(path.join(__dirname, failedPath), result)
// compare with markedjs
let compareResult = `## Compare with \`marked.js\`\n\n`
compareResult += `Marked.js failed examples count: ${markedSpecs.filter(s => s.shouldFail).length}\n`
compareResult += `Mark Text failed examples count: ${failedCount}\n\n`
let count = 0
specs.forEach((spec, i) => {
if (spec.shouldFail !== markedSpecs[i].shouldFail) {
count++
const acturalHtml = marked(spec.markdown, MT_MARKED_OPTIONS)
compareResult += `**Example${spec.example}**\n\n`
compareResult += `Mark Text ${spec.shouldFail ? 'fail' : 'success'} and marked.js ${markedSpecs[i].shouldFail ? 'fail' : 'success'}\n\n`
compareResult += '```markdown\n'
compareResult += `Markdown content\n`
compareResult += `${spec.markdown.replace(/`/g, '\\`')}\n`
compareResult += `Expected Html\n`
compareResult += `${spec.html}\n`
compareResult += `Actural Html\n`
compareResult += `${acturalHtml}\n`
compareResult += `marked.js html\n`
compareResult += `${markedJs(spec.markdown, { headerIds: false })}\n`
compareResult += '```\n\n'
}
})
compareResult += `There are ${count} examples are different with marked.js.`
const comparePath = type === 'commonmark' ? `./compare.marked.md` : `../gfm/compare.marked.md`
fs.writeFileSync(path.join(__dirname, comparePath), compareResult)
}
const diffAndGenerateResult = async () => {
const { specs, version } = await getSpecs()
const markedSpecs = await getMarkedSpecs(version)
specs.forEach(spec => {
const html = removeCustomClass(marked(spec.markdown, MT_MARKED_OPTIONS))
if (!htmlDiffer.isEqual(html, spec.html)) {
spec.shouldFail = true
}
})
fs.writeFileSync(path.join(__dirname, `./commonmark.${version}.json`), JSON.stringify(specs, null, 2) + '\n')
writeResult(version, specs, markedSpecs, 'commonmark')
}
try {
diffAndGenerateResult()
} catch (err) {
console.log(err)
}

6
test/specs/config.js Normal file
View File

@ -0,0 +1,6 @@
export const MT_MARKED_OPTIONS = {
headerIds: false,
emoji: false,
math: false,
frontMatter: false
}

View File

@ -0,0 +1,6 @@
## Compare with `marked.js`
Marked.js failed examples count: 1
Mark Text failed examples count: 1
There are 0 examples are different with marked.js.

View File

@ -0,0 +1,147 @@
[
{
"section": "Tables",
"html": "<table>\n<thead>\n<tr>\n<th>foo</th>\n<th>bar</th>\n</tr>\n</thead>\n<tbody>\n<tr>\n<td>baz</td>\n<td>bim</td>\n</tr>\n</tbody>\n</table>",
"markdown": "| foo | bar |\n| --- | --- |\n| baz | bim |",
"example": 198
},
{
"section": "Tables",
"html": "<table>\n<thead>\n<tr>\n<th align=\"center\">abc</th>\n<th align=\"right\">defghi</th>\n</tr>\n</thead>\n<tbody>\n<tr>\n<td align=\"center\">bar</td>\n<td align=\"right\">baz</td>\n</tr>\n</tbody>\n</table>",
"markdown": "| abc | defghi |\n:-: | -----------:\nbar | baz",
"example": 199
},
{
"section": "Tables",
"html": "<table>\n<thead>\n<tr>\n<th>f|oo</th>\n</tr>\n</thead>\n<tbody>\n<tr>\n<td>b <code>|</code> az</td>\n</tr>\n<tr>\n<td>b <strong>|</strong> im</td>\n</tr>\n</tbody>\n</table>",
"markdown": "| f\\|oo |\n| ------ |\n| b `\\|` az |\n| b **\\|** im |",
"example": 200
},
{
"section": "Tables",
"html": "<table>\n<thead>\n<tr>\n<th>abc</th>\n<th>def</th>\n</tr>\n</thead>\n<tbody>\n<tr>\n<td>bar</td>\n<td>baz</td>\n</tr>\n</tbody>\n</table>\n<blockquote>\n<p>bar</p>\n</blockquote>",
"markdown": "| abc | def |\n| --- | --- |\n| bar | baz |\n> bar",
"example": 201
},
{
"section": "Tables",
"html": "<table>\n<thead>\n<tr>\n<th>abc</th>\n<th>def</th>\n</tr>\n</thead>\n<tbody>\n<tr>\n<td>bar</td>\n<td>baz</td>\n</tr>\n<tr>\n<td>bar</td>\n<td></td>\n</tr>\n</tbody>\n</table>\n<p>bar</p>",
"markdown": "| abc | def |\n| --- | --- |\n| bar | baz |\nbar\n\nbar",
"example": 202
},
{
"section": "Tables",
"html": "<p>| abc | def |\n| --- |\n| bar |</p>",
"markdown": "| abc | def |\n| --- |\n| bar |",
"example": 203
},
{
"section": "Tables",
"html": "<table>\n<thead>\n<tr>\n<th>abc</th>\n<th>def</th>\n</tr>\n</thead>\n<tbody>\n<tr>\n<td>bar</td>\n<td></td>\n</tr>\n<tr>\n<td>bar</td>\n<td>baz</td>\n</tr>\n</tbody>\n</table>",
"markdown": "| abc | def |\n| --- | --- |\n| bar |\n| bar | baz | boo |",
"example": 204
},
{
"section": "Tables",
"html": "<table>\n<thead>\n<tr>\n<th>abc</th>\n<th>def</th>\n</tr>\n</thead>\n</table>",
"markdown": "| abc | def |\n| --- | --- |",
"example": 205
},
{
"section": "Task list items",
"html": "<ul>\n<li><input disabled=\"\" type=\"checkbox\"> foo</li>\n<li><input checked=\"\" disabled=\"\" type=\"checkbox\"> bar</li>\n</ul>",
"markdown": "- [ ] foo\n- [x] bar",
"example": 279
},
{
"section": "Task list items",
"html": "<ul>\n<li><input checked=\"\" disabled=\"\" type=\"checkbox\"> foo\n<ul>\n<li><input disabled=\"\" type=\"checkbox\"> bar</li>\n<li><input checked=\"\" disabled=\"\" type=\"checkbox\"> baz</li>\n</ul>\n</li>\n<li><input disabled=\"\" type=\"checkbox\"> bim</li>\n</ul>",
"markdown": "- [x] foo\n - [ ] bar\n - [x] baz\n- [ ] bim",
"example": 280
},
{
"section": "Strikethrough",
"html": "<p><del>Hi</del> Hello, world!</p>",
"markdown": "~~Hi~~ Hello, world!",
"example": 491
},
{
"section": "Strikethrough",
"html": "<p>This ~~has a</p>\n<p>new paragraph~~.</p>",
"markdown": "This ~~has a\n\nnew paragraph~~.",
"example": 492
},
{
"section": "Autolinks",
"html": "<p><a href=\"http://www.commonmark.org\">www.commonmark.org</a></p>",
"markdown": "www.commonmark.org",
"example": 621
},
{
"section": "Autolinks",
"html": "<p>Visit <a href=\"http://www.commonmark.org/help\">www.commonmark.org/help</a> for more information.</p>",
"markdown": "Visit www.commonmark.org/help for more information.",
"example": 622
},
{
"section": "Autolinks",
"html": "<p>Visit <a href=\"http://www.commonmark.org\">www.commonmark.org</a>.</p>\n<p>Visit <a href=\"http://www.commonmark.org/a.b\">www.commonmark.org/a.b</a>.</p>",
"markdown": "Visit www.commonmark.org.\n\nVisit www.commonmark.org/a.b.",
"example": 623
},
{
"section": "Autolinks",
"html": "<p><a href=\"http://www.google.com/search?q=Markup+(business)\">www.google.com/search?q=Markup+(business)</a></p>\n<p>(<a href=\"http://www.google.com/search?q=Markup+(business)\">www.google.com/search?q=Markup+(business)</a>)</p>",
"markdown": "www.google.com/search?q=Markup+(business)\n\n(www.google.com/search?q=Markup+(business))",
"example": 624
},
{
"section": "Autolinks",
"html": "<p><a href=\"http://www.google.com/search?q=(business))+ok\">www.google.com/search?q=(business))+ok</a></p>",
"markdown": "www.google.com/search?q=(business))+ok",
"example": 625
},
{
"section": "Autolinks",
"html": "<p><a href=\"http://www.google.com/search?q=commonmark&amp;hl=en\">www.google.com/search?q=commonmark&amp;hl=en</a></p>\n<p><a href=\"http://www.google.com/search?q=commonmark\">www.google.com/search?q=commonmark</a>&amp;hl;</p>",
"markdown": "www.google.com/search?q=commonmark&hl=en\n\nwww.google.com/search?q=commonmark&hl;",
"example": 626
},
{
"section": "Autolinks",
"html": "<p><a href=\"http://www.commonmark.org/he\">www.commonmark.org/he</a>&lt;lp</p>",
"markdown": "www.commonmark.org/he<lp",
"example": 627
},
{
"section": "Autolinks",
"html": "<p><a href=\"http://commonmark.org\">http://commonmark.org</a></p>\n<p>(Visit <a href=\"https://encrypted.google.com/search?q=Markup+(business)\">https://encrypted.google.com/search?q=Markup+(business)</a>)</p>\n<p>Anonymous FTP is available at <a href=\"ftp://foo.bar.baz\">ftp://foo.bar.baz</a>.</p>",
"markdown": "http://commonmark.org\n\n(Visit https://encrypted.google.com/search?q=Markup+(business))\n\nAnonymous FTP is available at ftp://foo.bar.baz.",
"example": 628
},
{
"section": "Autolinks",
"html": "<p><a href=\"mailto:foo@bar.baz\">foo@bar.baz</a></p>",
"markdown": "foo@bar.baz",
"example": 629
},
{
"section": "Autolinks",
"html": "<p>hello@mail+xyz.example isn't valid, but <a href=\"mailto:hello+xyz@mail.example\">hello+xyz@mail.example</a> is.</p>",
"markdown": "hello@mail+xyz.example isn't valid, but hello+xyz@mail.example is.",
"example": 630
},
{
"section": "Autolinks",
"html": "<p><a href=\"mailto:a.b-c_d@a.b\">a.b-c_d@a.b</a></p>\n<p><a href=\"mailto:a.b-c_d@a.b\">a.b-c_d@a.b</a>.</p>\n<p>a.b-c_d@a.b-</p>\n<p>a.b-c_d@a.b_</p>",
"markdown": "a.b-c_d@a.b\n\na.b-c_d@a.b.\n\na.b-c_d@a.b-\n\na.b-c_d@a.b_",
"example": 631
},
{
"section": "Disallowed Raw HTML",
"html": "<p><strong> &lt;title> &lt;style> <em></p>\n<blockquote>\n &lt;xmp> is disallowed. &lt;XMP> is also disallowed.\n</blockquote>",
"markdown": "<strong> <title> <style> <em>\n\n<blockquote>\n <xmp> is disallowed. <XMP> is also disallowed.\n</blockquote>",
"example": 653,
"shouldFail": true
}
]

View File

@ -0,0 +1,25 @@
## Test Result
Total test 24 examples, and failed 1 examples:
**Example653**
```markdown
Markdown content
<strong> <title> <style> <em>
<blockquote>
<xmp> is disallowed. <XMP> is also disallowed.
</blockquote>
Expected Html
<p><strong> &lt;title> &lt;style> <em></p>
<blockquote>
&lt;xmp> is disallowed. &lt;XMP> is also disallowed.
</blockquote>
Actural Html
<p><strong> <title> <style> <em></p>
<blockquote>
<xmp> is disallowed. <XMP> is also disallowed.
</blockquote>
```

View File

@ -0,0 +1,68 @@
// This file is copy from https://github.com/markedjs/marked/blob/master/test/specs/gfm/getSpecs.js
// And for custom use.
import { removeCustomClass } from '../help'
import { writeResult } from '../commonMark/run.spec'
import { MT_MARKED_OPTIONS } from '../config'
const fetch = require('node-fetch')
const cheerio = require('cheerio')
const marked = require('../../../src/muya/lib/parser/marked/index.js').default
const HtmlDiffer = require('@markedjs/html-differ').HtmlDiffer
const fs = require('fs')
const path = require('path')
const options = { ignoreSelfClosingSlash: true, ignoreAttributes: ['id', 'class'] }
const htmlDiffer = new HtmlDiffer(options)
const getSpecs = () => {
return fetch('https://github.github.com/gfm/')
.then(res => res.text())
.then(html => cheerio.load(html))
.then($ => {
const version = $('.version').text().match(/\d+\.\d+/)[0]
if (!version) {
throw new Error('No version found')
}
const specs = []
$('.extension').each((i, ext) => {
const section = $('.definition', ext).text().trim().replace(/^\d+\.\d+(.*?) \(extension\)[\s\S]*$/, '$1')
$('.example', ext).each((j, exa) => {
const example = +$(exa).attr('id').replace(/\D/g, '')
const markdown = $('.language-markdown', exa).text().trim()
const html = $('.language-html', exa).text().trim()
specs.push({
section,
html,
markdown,
example
})
})
})
return [version, specs]
})
}
const getMarkedSpecs = async (version) => {
return fetch(`https://raw.githubusercontent.com/markedjs/marked/master/test/specs/gfm/gfm.${version}.json`)
.then(res => res.json())
}
const diffAndGenerateResult = async () => {
const [version, specs] = await getSpecs()
const markedSpecs = await getMarkedSpecs(version)
specs.forEach(spec => {
const html = removeCustomClass(marked(spec.markdown, MT_MARKED_OPTIONS))
if (!htmlDiffer.isEqual(html, spec.html)) {
spec.shouldFail = true
}
})
fs.writeFileSync(path.resolve(__dirname, `./gfm.${version}.json`), JSON.stringify(specs, null, 2) + '\n')
writeResult(version, specs, markedSpecs, 'gfm')
}
try {
diffAndGenerateResult()
} catch (err) {
console.log(err)
}

14
test/specs/help.js Normal file
View File

@ -0,0 +1,14 @@
export const removeCustomClass = html => {
const customClass = ['indented-code-block', 'fenced-code-block', 'task-list-item']
customClass.forEach(className => {
if (html.indexOf(className) > -1) {
const REG_EXP = new RegExp(`class="${className}"`, 'g')
/* eslint-disable no-useless-escape */
const REG_EXP_SIMPLE = new RegExp(className + ` \*`, 'g')
/* eslint-enable no-useless-escape */
html = html.replace(REG_EXP, '')
.replace(REG_EXP_SIMPLE, '')
}
})
return html
}

100
yarn.lock
View File

@ -137,6 +137,19 @@
resolved "https://registry.yarnpkg.com/@hfelix/keyboardevent-from-electron-accelerator/-/keyboardevent-from-electron-accelerator-1.1.1.tgz#7e1d4fd913759c381b7919cc7faf4c0c641d457c"
integrity sha512-1eVkDSqoRQkF2FrPPia2EZ3310c0TvFKYvSuJbaxHpRKbI6eVHcVGKpmOSDli6Qdn3Bu0h7ozfgMZbAEBD+BLQ==
"@markedjs/html-differ@^2.0.1":
version "2.0.1"
resolved "https://registry.yarnpkg.com/@markedjs/html-differ/-/html-differ-2.0.1.tgz#22251369da9c376430662c963fc4f53946fd65f2"
integrity sha512-Sv1PMZ9RGt+MZqIk7p4wHiOWZo34p1s4x2AIIipmK4HeZK1Vn7Hlp+A7K73I3CQgLfvUu4ZyuU219LIal6sUrg==
dependencies:
chalk "^2.4.2"
coa "^2.0.2"
diff "1.3.2"
lodash "^4.17.11"
parse5 "1.5.1"
vow "^0.4.19"
vow-fs "^0.3.6"
"@types/clone@^0.1.30", "@types/clone@~0.1.30":
version "0.1.30"
resolved "http://registry.npm.taobao.org/@types/clone/download/@types/clone-0.1.30.tgz#e7365648c1b42136a59c7d5040637b3b5c83b614"
@ -147,6 +160,11 @@
resolved "https://registry.yarnpkg.com/@types/fast-json-stable-stringify/-/fast-json-stable-stringify-2.0.0.tgz#40363bb847cb86b2c2e1599f1398d11e8329c921"
integrity sha512-mky/O83TXmGY39P1H9YbUpjV6l6voRYlufqfFCvel8l1phuy8HRjdWc1rrPuN53ITBJlbyMSV6z3niOySO5pgQ==
"@types/node@*":
version "11.13.4"
resolved "https://registry.yarnpkg.com/@types/node/-/node-11.13.4.tgz#f83ec3c3e05b174b7241fadeb6688267fe5b22ca"
integrity sha512-+rabAZZ3Yn7tF/XPGHupKIL5EcAbrLxnTr/hgQICxbeuAfWtT0UZSfULE+ndusckBItcv4o6ZeOJplQikVcLvQ==
"@types/node@^10.12.18":
version "10.14.3"
resolved "https://registry.yarnpkg.com/@types/node/-/node-10.14.3.tgz#170a81168620d931cc3b83460be253cadd3028f1"
@ -2111,6 +2129,18 @@ check-types@^7.3.0:
resolved "https://registry.yarnpkg.com/check-types/-/check-types-7.4.0.tgz#0378ec1b9616ec71f774931a3c6516fad8c152f4"
integrity sha512-YbulWHdfP99UfZ73NcUDlNJhEIDgm9Doq9GhpyXbF+7Aegi3CVV7qqMCKTTqJxlvEvnQBp9IA+dxsGN6xK/nSg==
cheerio@^1.0.0-rc.3:
version "1.0.0-rc.3"
resolved "https://registry.yarnpkg.com/cheerio/-/cheerio-1.0.0-rc.3.tgz#094636d425b2e9c0f4eb91a46c05630c9a1a8bf6"
integrity sha512-0td5ijfUPuubwLUu0OBoe98gZj8C/AA+RW3v67GPlGOrvxWjZmBXiBCRU+I8VEiNyJzjth40POfHiz2RB3gImA==
dependencies:
css-select "~1.2.0"
dom-serializer "~0.1.1"
entities "~1.1.1"
htmlparser2 "^3.9.1"
lodash "^4.15.0"
parse5 "^3.0.1"
chokidar@^2.0.0, chokidar@^2.0.2, chokidar@^2.0.3:
version "2.1.2"
resolved "https://registry.yarnpkg.com/chokidar/-/chokidar-2.1.2.tgz#9c23ea40b01638439e0513864d362aeacc5ad058"
@ -2771,7 +2801,7 @@ css-select-base-adapter@^0.1.1:
resolved "https://registry.yarnpkg.com/css-select-base-adapter/-/css-select-base-adapter-0.1.1.tgz#3b2ff4972cc362ab88561507a95408a1432135d7"
integrity sha512-jQVeeRG70QI08vSTwf1jHxp74JoZsr2XSgETae8/xC8ovSnL2WF87GTLO86Sbwdt2lK4Umg4HnnwMO4YF3Ce7w==
css-select@^1.1.0:
css-select@^1.1.0, css-select@~1.2.0:
version "1.2.0"
resolved "https://registry.yarnpkg.com/css-select/-/css-select-1.2.0.tgz#2b3a110539c5355f1cd8d314623e870b121ec858"
integrity sha1-KzoRBTnFNV8c2NMUYj6HCxIeyFg=
@ -3477,6 +3507,11 @@ di@^0.0.1:
resolved "https://registry.yarnpkg.com/di/-/di-0.0.1.tgz#806649326ceaa7caa3306d75d985ea2748ba913c"
integrity sha1-gGZJMmzqp8qjMG112YXqJ0i6kTw=
diff@1.3.2:
version "1.3.2"
resolved "https://registry.yarnpkg.com/diff/-/diff-1.3.2.tgz#fd07a1f1f891519d9905a4c9a89dcf5a70b66037"
integrity sha1-/Qeh8fiRUZ2ZBaTJqJ3PWnC2YDc=
diff@3.5.0:
version "3.5.0"
resolved "https://registry.yarnpkg.com/diff/-/diff-3.5.0.tgz#800c0dd1e0a8bfbc95835c202ad220fe317e5a12"
@ -3564,7 +3599,7 @@ dom-serialize@^2.2.0:
extend "^3.0.0"
void-elements "^2.0.0"
dom-serializer@0:
dom-serializer@0, dom-serializer@~0.1.1:
version "0.1.1"
resolved "https://registry.yarnpkg.com/dom-serializer/-/dom-serializer-0.1.1.tgz#1ec4059e284babed36eec2941d4a970a189ce7c0"
integrity sha512-l0IU0pPzLWSHBcieZbpOKgkIn3ts3vAh7ZuFyXNwJxJXk/c4Gwj9xaTJwIDVQCXawWD0qb3IzMGH5rglQaO0XA==
@ -3959,7 +3994,7 @@ ent@~2.2.0:
resolved "https://registry.yarnpkg.com/ent/-/ent-2.2.0.tgz#e964219325a21d05f44466a2f686ed6ce5f5dd1d"
integrity sha1-6WQhkyWiHQX0RGai9obtbOX13R0=
entities@^1.1.1:
entities@^1.1.1, entities@~1.1.1:
version "1.1.2"
resolved "https://registry.yarnpkg.com/entities/-/entities-1.1.2.tgz#bdfa735299664dfafd34529ed4f8522a275fea56"
integrity sha512-f2LZMYl1Fzu7YSBKg+RoROelpOaNrcGmE9AZubeDfrCEia483oW4MI4VyFd5VNHIgQ/7qm1I0wUHK1eJnn2y2w==
@ -4257,6 +4292,11 @@ eslint@^5.16.0:
table "^5.2.3"
text-table "^0.2.0"
esm@^3.2.22:
version "3.2.22"
resolved "https://registry.yarnpkg.com/esm/-/esm-3.2.22.tgz#5062c2e22fee3ccfee4e8f20da768330da90d6e3"
integrity sha512-z8YG7U44L82j1XrdEJcqZOLUnjxco8pO453gKOlaMD1/md1n/5QrscAmYG+oKUspsmDLuBFZrpbxI6aQ67yRxA==
espree@^4.1.0:
version "4.1.0"
resolved "https://registry.yarnpkg.com/espree/-/espree-4.1.0.tgz#728d5451e0fd156c04384a7ad89ed51ff54eb25f"
@ -4965,7 +5005,7 @@ glob-parent@^3.1.0:
is-glob "^3.1.0"
path-dirname "^1.0.0"
glob@7.1.3, glob@^7.0.0, glob@^7.0.3, glob@^7.1.1, glob@^7.1.2, glob@^7.1.3, glob@~7.1.1:
glob@7.1.3, glob@^7.0.0, glob@^7.0.3, glob@^7.0.5, glob@^7.1.1, glob@^7.1.2, glob@^7.1.3, glob@~7.1.1:
version "7.1.3"
resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.3.tgz#3960832d3f1574108342dafd3a67b332c0969df1"
integrity sha512-vcfuiIxogLV4DlGBHIUOwI0IbrJ8HWPc4MU7HzviGeNho/UJDfi6B5p3sHeWIQ0KGIU0Jpxi5ZHxemQfLkkAwQ==
@ -5371,7 +5411,7 @@ html-webpack-plugin@^3.2.0:
toposort "^1.0.0"
util.promisify "1.0.0"
htmlparser2@^3.3.0, htmlparser2@^3.8.2, htmlparser2@^3.8.3:
htmlparser2@^3.3.0, htmlparser2@^3.8.2, htmlparser2@^3.8.3, htmlparser2@^3.9.1:
version "3.10.1"
resolved "https://registry.yarnpkg.com/htmlparser2/-/htmlparser2-3.10.1.tgz#bd679dc3f59897b6a34bb10749c855bb53a9392f"
integrity sha512-IgieNijUMbkDovyoKObU1DUhm1iwNYE/fuifEoEHfd1oZKZDaONBSkal7Y01shxsM49R4XaMdGez3WnF9UfiCQ==
@ -6736,7 +6776,7 @@ lodash.uniq@^4.5.0:
resolved "https://registry.yarnpkg.com/lodash.uniq/-/lodash.uniq-4.5.0.tgz#d0225373aeb652adc1bc82e4945339a842754773"
integrity sha1-0CJTc662Uq3BvILklFM5qEJ1R3M=
lodash@^4.0.0, lodash@^4.17.0, lodash@^4.17.10, lodash@^4.17.11, lodash@^4.17.3, lodash@^4.17.4, lodash@^4.17.5, lodash@^4.2.0, lodash@^4.3.0, lodash@^4.8.0, lodash@~4.17.10:
lodash@^4.0.0, lodash@^4.15.0, lodash@^4.17.0, lodash@^4.17.10, lodash@^4.17.11, lodash@^4.17.3, lodash@^4.17.4, lodash@^4.17.5, lodash@^4.2.0, lodash@^4.3.0, lodash@^4.8.0, lodash@~4.17.10:
version "4.17.11"
resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.11.tgz#b39ea6229ef607ecd89e2c8df12536891cac9b8d"
integrity sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg==
@ -6876,6 +6916,11 @@ map-visit@^1.0.0:
dependencies:
object-visit "^1.0.0"
marked@^0.6.2:
version "0.6.2"
resolved "https://registry.yarnpkg.com/marked/-/marked-0.6.2.tgz#c574be8b545a8b48641456ca1dbe0e37b6dccc1a"
integrity sha512-LqxwVH3P/rqKX4EKGz7+c2G9r98WeM/SW34ybhgNGhUQNKtf1GmmSkJ6cDGJ/t6tiyae49qRkpyTw2B9HOrgUA==
md5.js@^1.3.4:
version "1.3.5"
resolved "https://registry.yarnpkg.com/md5.js/-/md5.js-1.3.5.tgz#b5d07b8e3216e3e27cd728d72f70d1e6a342005f"
@ -7320,8 +7365,8 @@ node-environment-flags@1.0.4:
node-fetch@^2.3.0:
version "2.3.0"
resolved "http://registry.npm.taobao.org/node-fetch/download/node-fetch-2.3.0.tgz#1a1d940bbfb916a1d3e0219f037e89e71f8c5fa5"
integrity sha1-Gh2UC7+5FqHT4CGfA36J5x+MX6U=
resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.3.0.tgz#1a1d940bbfb916a1d3e0219f037e89e71f8c5fa5"
integrity sha512-MOd8pV3fxENbryESLgVIeaGKrdl+uaYhCSSVkjeOb/31/njTpcis5aWfdqgNlHIrKOLRbMnfPINPOML2CIFeXA==
node-forge@0.7.5:
version "0.7.5"
@ -7867,11 +7912,23 @@ parse-sel@^1.0.0:
dependencies:
browser-split "0.0.1"
parse5@1.5.1:
version "1.5.1"
resolved "https://registry.yarnpkg.com/parse5/-/parse5-1.5.1.tgz#9b7f3b0de32be78dc2401b17573ccaf0f6f59d94"
integrity sha1-m387DeMr543CQBsXVzzK8Pb1nZQ=
parse5@4.0.0:
version "4.0.0"
resolved "https://registry.yarnpkg.com/parse5/-/parse5-4.0.0.tgz#6d78656e3da8d78b4ec0b906f7c08ef1dfe3f608"
integrity sha512-VrZ7eOd3T1Fk4XWNXMgiGBK/z0MG48BWG2uQNU4I72fkQuKUTZpl+u9k+CxEG0twMVzSmXEEz12z5Fnw1jIQFA==
parse5@^3.0.1:
version "3.0.3"
resolved "https://registry.yarnpkg.com/parse5/-/parse5-3.0.3.tgz#042f792ffdd36851551cf4e9e066b3874ab45b5c"
integrity sha512-rgO9Zg5LLLkfJF9E6CCmXlSE4UVceloys8JrFqCcHloC3usd/kJCyPDwH2SOlzix2j3xaP9sUX3e8+kvkuleAA==
dependencies:
"@types/node" "*"
parseqs@0.0.5:
version "0.0.5"
resolved "https://registry.yarnpkg.com/parseqs/-/parseqs-0.0.5.tgz#d5208a3738e46766e291ba2ea173684921a8b89d"
@ -10695,6 +10752,11 @@ utils-merge@1.0.1:
resolved "https://registry.yarnpkg.com/utils-merge/-/utils-merge-1.0.1.tgz#9f95710f50a267947b2ccc124741c1028427e713"
integrity sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM=
uuid@^2.0.2:
version "2.0.3"
resolved "https://registry.yarnpkg.com/uuid/-/uuid-2.0.3.tgz#67e2e863797215530dff318e5bf9dcebfd47b21a"
integrity sha1-Z+LoY3lyFVMN/zGOW/nc6/1Hsho=
uuid@^3.0.1, uuid@^3.1.0, uuid@^3.3.2:
version "3.3.2"
resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.3.2.tgz#1b4af4955eb3077c501c23872fc6513811587131"
@ -11280,6 +11342,28 @@ void-elements@^2.0.0:
resolved "https://registry.yarnpkg.com/void-elements/-/void-elements-2.0.1.tgz#c066afb582bb1cb4128d60ea92392e94d5e9dbec"
integrity sha1-wGavtYK7HLQSjWDqkjkulNXp2+w=
vow-fs@^0.3.6:
version "0.3.6"
resolved "https://registry.yarnpkg.com/vow-fs/-/vow-fs-0.3.6.tgz#2d4c59be22e2bf2618ddf597ab4baa923be7200d"
integrity sha1-LUxZviLivyYY3fWXq0uqkjvnIA0=
dependencies:
glob "^7.0.5"
uuid "^2.0.2"
vow "^0.4.7"
vow-queue "^0.4.1"
vow-queue@^0.4.1:
version "0.4.3"
resolved "https://registry.yarnpkg.com/vow-queue/-/vow-queue-0.4.3.tgz#4ba8f64b56e9212c0dbe57f1405aeebd54cce78d"
integrity sha512-/poAKDTFL3zYbeQg7cl4BGcfP4sGgXKrHnRFSKj97dteUFu8oyXMwIcdwu8NSx/RmPGIuYx1Bik/y5vU4H/VKw==
dependencies:
vow "^0.4.17"
vow@^0.4.17, vow@^0.4.19, vow@^0.4.7:
version "0.4.19"
resolved "https://registry.yarnpkg.com/vow/-/vow-0.4.19.tgz#cc5ef4d6bb6972d830830a7c9ecf8ad834a7c525"
integrity sha512-S+0+CiQlbUhTNWMlJdqo/ARuXOttXdvw5ACGyh1W97NFHUdwt3Fzyaus03Kvdmo733dwnYS9AGJSDg0Zu8mNfA==
vscode-windows-registry@1.0.1:
version "1.0.1"
resolved "https://registry.yarnpkg.com/vscode-windows-registry/-/vscode-windows-registry-1.0.1.tgz#bc9f765563eb6dc1c9ad9a41f9eaacc84dfadc7c"