fix: update list item lexer and parser (#803)

* fix: CommonMark 264

* fix: muya list behavior

src/muya/lib/contentState/enterCtrl.js

@@ -112,9 +112,7 @@ const enterCtrl = ContentState => {
       } else {
         newBlock = this.createBlockLi()
         newBlock.listItemType = parent.listItemType
-        if (parent.listItemType === 'bullet') {
-          newBlock.bulletListItemMarker = parent.bulletListItemMarker
-        }
+        newBlock.bulletMarkerOrDelimiter = parent.bulletMarkerOrDelimiter
       }
       newBlock.isLooseListItem = parent.isLooseListItem
       this.insertAfter(newBlock, parent)
@@ -334,9 +332,7 @@ const enterCtrl = ContentState => {
             newBlock = this.chopBlockByCursor(block.children[0], start.key, start.offset)
             newBlock = this.createBlockLi(newBlock)
             newBlock.listItemType = block.listItemType
-            if (block.listItemType === 'bullet') {
-              newBlock.bulletListItemMarker = block.bulletListItemMarker
-            }
+            newBlock.bulletMarkerOrDelimiter = block.bulletMarkerOrDelimiter
           }
           newBlock.isLooseListItem = block.isLooseListItem
         }
@@ -357,9 +353,7 @@ const enterCtrl = ContentState => {
           } else {
             newBlock = this.createBlockLi()
             newBlock.listItemType = block.listItemType
-            if (block.listItemType === 'bullet') {
-              newBlock.bulletListItemMarker = block.bulletListItemMarker
-            }
+            newBlock.bulletMarkerOrDelimiter = block.bulletMarkerOrDelimiter
           }
           newBlock.isLooseListItem = block.isLooseListItem
         } else {

src/muya/lib/contentState/updateCtrl.js

@@ -5,7 +5,7 @@ import { CLASS_OR_ID } from '../config'
 const INLINE_UPDATE_FRAGMENTS = [
   '^([*+-]\\s)', // Bullet list
   '^(\\[[x\\s]{1}\\]\\s)', // Task list
-  '^(\\d+\\.\\s)', // Order list
+  '^(\\d{1,9}(?:\\.|\\))\\s)', // Order list
   '^\\s{0,3}(#{1,6})(?=\\s{1,}|$)', // ATX headings
   '^\\s{0,3}(\\={3,}|\\-{3,})(?=\\s{1,}|$)', // Setext headings
   '^(>).+', // Block quote
@@ -124,6 +124,8 @@ const updateCtrl = ContentState => {
     if (block.type === 'span') {
       block = this.getParent(block)
     }
+
+    const cleanMarker = marker ? marker.trim() : null
     const { preferLooseListItem } = this
     const parent = this.getParent(block)
     const wrapperTag = type === 'order' ? 'ol' : 'ul' // `bullet` => `ul` and `order` => `ol`
@@ -131,6 +133,7 @@ const updateCtrl = ContentState => {
     const startOffset = start.offset
     const endOffset = end.offset
     const newBlock = this.createBlock('li')
+
     if (/^h\d$/.test(block.type)) {
       delete block.marker
       delete block.headingStyle
@@ -156,18 +159,47 @@ const updateCtrl = ContentState => {
         this.insertBefore(paragraphBefore, block)
       }
     }
+
     const preSibling = this.getPreSibling(block)
     const nextSibling = this.getNextSibling(block)
     newBlock.listItemType = type
     newBlock.isLooseListItem = preferLooseListItem
 
-    if (type === 'task' || type === 'bullet') {
+    let bulletMarkerOrDelimiter
+    if (type === 'order') {
+      bulletMarkerOrDelimiter = (cleanMarker && cleanMarker.length >= 2) ? cleanMarker.slice(-1) : '.'
+    } else {
       const { bulletListMarker } = this
-      const bulletListItemMarker = marker ? marker.charAt(0) : bulletListMarker
-      newBlock.bulletListItemMarker = bulletListItemMarker
+      bulletMarkerOrDelimiter = marker ? marker.charAt(0) : bulletListMarker
+    }
+    newBlock.bulletMarkerOrDelimiter = bulletMarkerOrDelimiter
+
+    // Special cases for CommonMark 264 and 265: Changing the bullet or ordered list delimiter starts a new list.
+    let startNewList = false
+    if (preSibling && /^(ol|ul)$/.test(preSibling.type)) {
+      const bullet = preSibling.children[0].bulletMarkerOrDelimiter
+      startNewList = bulletMarkerOrDelimiter !== bullet
+    }
+    if (nextSibling && /^(ol|ul)$/.test(nextSibling.type)) {
+      const bullet = nextSibling.children[0].bulletMarkerOrDelimiter
+      startNewList = startNewList || bulletMarkerOrDelimiter !== bullet
     }
 
-    if (
+    if (startNewList) {
+      // Create a new list when changing list type, bullet or list delimiter
+      const listBlock = this.createBlock(wrapperTag)
+      listBlock.listType = type
+      if (wrapperTag === 'ol') {
+        const start = cleanMarker ? cleanMarker.slice(0, -1) : 1
+        listBlock.start = /^\d+$/.test(start) ? start : 1
+      }
+      this.appendChild(listBlock, newBlock)
+      this.insertBefore(listBlock, block)
+      this.removeBlock(block)
+
+    // --------------------------------
+    // Same list type or new list
+    } else if (
       preSibling &&
       preSibling.listType === type &&
       this.checkSameLooseType(preSibling, preferLooseListItem) &&
@@ -186,7 +218,6 @@ const updateCtrl = ContentState => {
       this.checkSameLooseType(preSibling, preferLooseListItem)
     ) {
       this.appendChild(preSibling, newBlock)
-
       this.removeBlock(block)
     } else if (
       nextSibling &&
@@ -201,13 +232,12 @@ const updateCtrl = ContentState => {
       this.checkSameLooseType(parent, preferLooseListItem)
     ) {
       this.insertBefore(newBlock, block)
-
       this.removeBlock(block)
     } else {
       const listBlock = this.createBlock(wrapperTag)
       listBlock.listType = type
       if (wrapperTag === 'ol') {
-        const start = marker.split('.')[0]
+        const start = cleanMarker ? cleanMarker.slice(0, -1) : 1
         listBlock.start = /^\d+$/.test(start) ? start : 1
       }
       this.appendChild(listBlock, newBlock)

src/muya/lib/parser/marked/README.md

@@ -0,0 +1,27 @@
+# Marked
+
+This folder contains a patched [Marked.js](https://github.com/markedjs/marked/) version based on `v0.6.1` commit [6eec528e5d6e08ea751251f9dc195d052caf4a79](https://github.com/markedjs/marked/commit/6eec528e5d6e08ea751251f9dc195d052caf4a79).
+
+## Changes
+
+### Features
+
+- Markdown Extra: frontmatter and inline and block math
+- GFM like: emojis
+
+### (Inline) Lexer
+
+- `disableInline` mode
+- Custom list and list item implementation based on an older marked.js version
+- Slightly modified definition due `disableInline`
+- More token information like list item bullet type
+
+### Renderer
+
+- Emoji renderer
+- Frontmatter renderer
+- Inline and block (`multiplemath`) math renderer
+
+## License
+
+[MIT](LICENSE)

src/muya/lib/parser/marked/blockRules.js

@@ -44,7 +44,7 @@ block.def = edit(block.def).
   getRegex()
 
 block.checkbox = /^\[([ xX])\] +/
-block.bullet = /(?:[*+-]|\d{1,9}\.)/
+block.bullet = /(?:[*+-]|\d{1,9}(?:\.|\)))/ // patched: support "(" as ordered list delimiter too
 block.item = /^( *)(bull) ?[^\n]*(?:\n(?!\1bull ?)[^\n]*)*/
 block.item = edit(block.item, 'gm').
   replace(/bull/g, block.bullet).

src/muya/lib/parser/marked/lexer.js

@@ -198,18 +198,20 @@ Lexer.prototype.token = function (src, top) {
       continue
     }
 
+    // NOTE: Complete list lexer part is a custom implementation based on an older marked.js version.
+
     // list
     cap = this.rules.list.exec(src)
     if (cap) {
       src = src.substring(cap[0].length)
       bull = cap[2]
-      const isordered = bull.length > 1 && /\d/.test(bull)
+      let isOrdered = bull.length > 1 && /\d{1,9}/.test(bull)
 
       this.tokens.push({
         type: 'list_start',
-        ordered: isordered,
-        listType: bull.length > 1 ? (/\d/.test(bull) ? 'order' : 'task') : 'bullet',
-        start: isordered ? +bull : ''
+        ordered: isOrdered,
+        listType: bull.length > 1 ? (/\d{1,9}/.test(bull) ? 'order' : 'task') : 'bullet',
+        start: isOrdered ? +(bull.slice(0, -1)) : ''
       })
 
       let next = false
@@ -228,9 +230,38 @@ Lexer.prototype.token = function (src, top) {
         // Remove the list item's bullet
         // so it is seen as the next token.
         space = item.length
-        item = item.replace(/^ *([*+-]|\d+\.) */, '')
+        let newBull
+        item = item.replace(/^ *([*+-]|\d+(?:\.|\))) */, function (m, p1) {
+          // Get and remove list item bullet
+          newBull = p1 || bull
+          return ''
+        })
 
-        if (this.options.gfm) {
+        // Changing the bullet or ordered list delimiter starts a new list (CommonMark 264 and 265)
+        //   - unordered, unordered --> bull !== newBull --> new list (e.g "-" --> "*")
+        //   - ordered, ordered --> lastChar !== lastChar --> new list (e.g "." --> ")")
+        //   - else --> new list (e.g. ordered --> unordered)
+        const newIsOrdered = bull.length > 1 && /\d{1,9}/.test(newBull)
+        if (i !== 0 &&
+          ((!isOrdered && !newIsOrdered && bull !== newBull) ||
+          (isOrdered && newIsOrdered && bull.slice(-1) !== newBull.slice(-1)) ||
+          ((isOrdered && !newIsOrdered) || (!isOrdered && newIsOrdered)))) {
+          this.tokens.push({
+            type: 'list_end'
+          })
+
+          // Start a new list
+          bull = newBull
+          isOrdered = newIsOrdered
+          this.tokens.push({
+            type: 'list_start',
+            ordered: isOrdered,
+            listType: bull.length > 1 ? (/\d{1,9}/.test(bull) ? 'order' : 'task') : 'bullet',
+            start: isOrdered ? +(bull.slice(0, -1)) : ''
+          })
+        }
+
+        if (!isOrdered && this.options.gfm) {
           checked = this.rules.checkbox.exec(item)
           if (checked) {
             checked = checked[1] === 'x' || checked[1] === 'X'
@@ -269,7 +300,7 @@ Lexer.prototype.token = function (src, top) {
 
         // Determine whether item is loose or not. If previous item is loose
         // this item is also loose.
-        loose = next = next || /^ *([*+-]|\d+\.)( +\S+\n\n(?!\s*$)|\n\n(?!\s*$))/.test(itemWithBullet)
+        loose = next = next || /^ *([*+-]|\d{1,9}(?:\.|\)))( +\S+\n\n(?!\s*$)|\n\n(?!\s*$))/.test(itemWithBullet)
 
         // Check if previous line ends with a new line.
         if (!loose && (i !== 0 || l > 1) && prevItem.length !== 0 && prevItem.charAt(prevItem.length - 1) === '\n') {
@@ -289,10 +320,11 @@ Lexer.prototype.token = function (src, top) {
           listItemIndices.push(this.tokens.length)
         }
 
+        const isOrderedListItem = /\d/.test(bull)
         this.tokens.push({
           checked: checked,
-          listItemType: bull.length > 1 ? (/\d/.test(bull) ? 'order' : 'task') : 'bullet',
-          bulletListItemMarker: /\d/.test(bull) ? '' : bull.charAt(0),
+          listItemType: bull.length > 1 ? (isOrderedListItem ? 'order' : 'task') : 'bullet',
+          bulletMarkerOrDelimiter: isOrderedListItem ? bull.slice(-1) : bull.charAt(0),
           type: loose ? 'loose_item_start' : 'list_item_start'
         })
 

src/muya/lib/parser/marked/parser.js

@@ -165,23 +165,23 @@ Parser.prototype.tok = function () {
     }
     case 'list_item_start': {
       let body = ''
-      const { checked, listItemType, bulletListItemMarker } = this.token
+      const { checked } = this.token
 
       while (this.next().type !== 'list_item_end') {
         body += this.token.type === 'text' ? this.parseText() : this.tok()
       }
 
-      return this.renderer.listitem(body, checked, listItemType, bulletListItemMarker, false)
+      return this.renderer.listitem(body, checked)
     }
     case 'loose_item_start': {
       let body = ''
-      const { checked, listItemType, bulletListItemMarker } = this.token
+      const { checked } = this.token
 
       while (this.next().type !== 'list_item_end') {
         body += this.tok()
       }
 
-      return this.renderer.listitem(body, checked, listItemType, bulletListItemMarker, true)
+      return this.renderer.listitem(body, checked)
     }
     case 'html': {
       // TODO parse inline content if parameter markdown=1

src/muya/lib/parser/marked/renderer.js

@@ -96,7 +96,7 @@ Renderer.prototype.list = function (body, ordered, start, taskList) {
   return '<' + type + startatt + '>\n' + body + '</' + type + '>\n'
 }
 
-Renderer.prototype.listitem = function (text, checked, listItemType, bulletListItemMarker, loose) {
+Renderer.prototype.listitem = function (text, checked) {
   // normal list
   if (checked === undefined) {
     return '<li>' + text + '</li>\n'

src/muya/lib/parser/render/renderBlock/renderContainerBlock.js

@@ -83,9 +83,7 @@ export default function renderContainerBlock (block, cursor, activeBlocks, match
       default:
         break
     }
-    if (block.bulletListItemMarker) {
-      Object.assign(data.dataset, { marker: block.bulletListItemMarker })
-    }
+    Object.assign(data.dataset, { marker: block.bulletMarkerOrDelimiter })
     selector += block.isLooseListItem ? `.${CLASS_OR_ID['AG_LOOSE_LIST_ITEM']}` : `.${CLASS_OR_ID['AG_TIGHT_LIST_ITEM']}`
   }
   if (block.type === 'ol') {

src/muya/lib/utils/exportMarkdown.js

@@ -23,8 +23,14 @@ class ExportMarkdown {
 
   translateBlocks2Markdown (blocks, indent = '') {
     const result = []
+    // helper for CommonMark 264
+    let lastListBullet = ''
 
     for (const block of blocks) {
+      if (block.type !== 'ul' && block.type !== 'ol') {
+        lastListBullet = ''
+      }
+
       switch (block.type) {
         case 'p': {
           this.insertLineBreak(result, indent, true)
@@ -88,9 +94,16 @@ class ExportMarkdown {
           break
         }
         case 'ul': {
-          const insertNewLine = this.isLooseParentList
+          let insertNewLine = this.isLooseParentList
           this.isLooseParentList = true
 
+          // Start a new list without separation due changing the bullet or ordered list delimiter starts a new list.
+          const { bulletMarkerOrDelimiter } = block.children[0]
+          if (lastListBullet && lastListBullet !== bulletMarkerOrDelimiter) {
+            insertNewLine = false
+          }
+          lastListBullet = bulletMarkerOrDelimiter
+
           this.insertLineBreak(result, indent, insertNewLine)
           this.listType.push({ type: 'ul' })
           result.push(this.normalizeList(block, indent))
@@ -98,9 +111,16 @@ class ExportMarkdown {
           break
         }
         case 'ol': {
-          const insertNewLine = this.isLooseParentList
+          let insertNewLine = this.isLooseParentList
           this.isLooseParentList = true
 
+          // Start a new list without separation due changing the bullet or ordered list delimiter starts a new list.
+          const { bulletMarkerOrDelimiter } = block.children[0]
+          if (lastListBullet && lastListBullet !== bulletMarkerOrDelimiter) {
+            insertNewLine = false
+          }
+          lastListBullet = bulletMarkerOrDelimiter
+
           this.insertLineBreak(result, indent, insertNewLine)
           const listCount = block.start !== undefined ? block.start : 1
           this.listType.push({ type: 'ol', listCount })
@@ -283,18 +303,19 @@ class ExportMarkdown {
   normalizeListItem (block, indent) {
     const result = []
     const listInfo = this.listType[this.listType.length - 1]
-    let { children, bulletListItemMarker } = block
+    let { children, bulletMarkerOrDelimiter } = block
     let itemMarker
 
     if (listInfo.type === 'ul') {
-      itemMarker = bulletListItemMarker ? `${bulletListItemMarker} ` : '- '
+      itemMarker = bulletMarkerOrDelimiter ? `${bulletMarkerOrDelimiter} ` : '- '
       if (block.listItemType === 'task') {
         const firstChild = children[0]
         itemMarker += firstChild.checked ? '[x] ' : '[ ] '
         children = children.slice(1)
       }
     } else {
-      itemMarker = `${listInfo.listCount++}. `
+      const delimiter = bulletMarkerOrDelimiter ? bulletMarkerOrDelimiter : '.'
+      itemMarker = `${listInfo.listCount++}${delimiter} `
     }
 
     const newIndent = indent + ' '.repeat(itemMarker.length)

src/muya/lib/utils/importMarkdown.js

@@ -200,10 +200,10 @@ const importRegister = ContentState => {
         }
         case 'loose_item_start':
         case 'list_item_start': {
-          const { listItemType, bulletListItemMarker, checked, type } = token
+          const { listItemType, bulletMarkerOrDelimiter, checked, type } = token
           block = this.createBlock('li')
           block.listItemType = checked !== undefined ? 'task' : listItemType
-          block.bulletListItemMarker = bulletListItemMarker
+          block.bulletMarkerOrDelimiter = bulletMarkerOrDelimiter
           block.isLooseListItem = type === 'loose_item_start'
           if (checked !== undefined) {
             const input = this.createBlock('input')

test/unit/data/common/Blockquotes.md

@@ -12,6 +12,10 @@ foo
 - > bar
 - baz
 
+> Use it if you're quoting a person, a song or whatever.
+
+> You can use *italic* or lists inside them also.
+
 ## Failing Tests
 
 ```
@@ -36,11 +40,5 @@ paragraph.
   > This is a blockquote
   > inside a list item.
 
-* bar`
-```
-
-```
-> Use it if you're quoting a person, a song or whatever.
-
-> You can use *italic* or lists inside them also.
+* bar
 ```

test/unit/data/common/Lists.md

@@ -18,6 +18,12 @@ To start an ordered list, write this:
 
 ---
 
+1) this starts a list *with* numbers
+2) this will show as number "2"
+3) this will show as number "3"
+
+---
+
 - foo
   - bar
     - baz
@@ -61,14 +67,75 @@ To start an ordered list, write this:
 
 ---
 
-## Failing Tests
+- foo
+- bar
++ baz
+* foobar
+* qux
 
-```
-* an asterisk starts an unordered list
-* and this is another item in the list
-+ or you can also use the + character
-- or the - character
-```
+---
+
+1. foo
+2. bar
+4) baz
+
+---
+
+1. foo
+2. bar
+1) baz
+
+---
+
+- foo
+- bar
++ foobar
++ baz
+
+---
+
+- foo
+- bar
+* foobar
+* baz
+
+---
+
+- foo
+- bar
+* foobar
+* baz
++ qux
++ quux
+
+---
+
+- foo
+- bar
+1. foobar
+2. baz
+
+---
+
+1. foo
+2. bar
+- foobar
+- baz
+
+---
+
+1. foo
+2. bar
+1) foobar
+2) baz
+
+---
+
+- foo
+- 
+- bar
+
+## Failing Tests
 
 ```
 1. this starts a list *with* numbers

test/unit/data/gfm/Lists.md

@@ -5,3 +5,15 @@ To start a check list, write this:
 - [ ] this is not checked
 - [ ] this is too
 - [x] but this is checked
+
+---
+
+* [x] this is checked
+* [ ] this is not checked
+* [x] but this is checked
+
+---
+
++ [ ] this is not checked
++ [ ] this is too
++ [x] but this is checked