Add file encoding support (#1438)

* Add file encoding support

* Update documentation

docs/PREFERENCES.md

@@ -19,21 +19,23 @@ Preferences can be controlled and modified in the settings window or via the `pr
 
 #### Editor
 
-| Key                      | Type    | Defaut             | Description                                                                                                                                                           |
+| Key                    | Type    | Defaut             | Description                                                                                                                                                           |
-| ------------------------ | ------- | ------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| ---------------------- | ------- | ------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| fontSize                 | Number  | 16                 | Font size in pixels. 12 ~ 32                                                                                                                                          |
+| fontSize               | Number  | 16                 | Font size in pixels. 12 ~ 32                                                                                                                                          |
-| editorFontFamily         | String  | Open Sans          | Font Family                                                                                                                                                           |
+| editorFontFamily       | String  | Open Sans          | Font Family                                                                                                                                                           |
-| lineHeight               | Number  | 1.6                | Line Height                                                                                                                                                           |
+| lineHeight             | Number  | 1.6                | Line Height                                                                                                                                                           |
-| autoPairBracket          | Boolean | true               | Automatically brackets when editing                                                                                                                                   |
+| autoPairBracket        | Boolean | true               | Automatically brackets when editing                                                                                                                                   |
-| autoPairMarkdownSyntax   | Boolean | true               | Autocomplete markdown syntax                                                                                                                                          |
+| autoPairMarkdownSyntax | Boolean | true               | Autocomplete markdown syntax                                                                                                                                          |
-| autoPairQuote            | Boolean | true               | Automatic completion of quotes                                                                                                                                        |
+| autoPairQuote          | Boolean | true               | Automatic completion of quotes                                                                                                                                        |
-| endOfLine                | String  | default            | The newline character used at the end of each line. The default value is default, which will be selected according to your system intelligence. `lf` `crlf` `default` |
+| endOfLine              | String  | default            | The newline character used at the end of each line. The default value is default, which will be selected according to your system intelligence. `lf` `crlf` `default` |
-| textDirection            | String  | ltr                | The writing text direction, optional value: `ltr` or `rtl`                                                                                                            |
+| textDirection          | String  | ltr                | The writing text direction, optional value: `ltr` or `rtl`                                                                                                            |
-| codeFontSize             | Number  | 14                 | Font size on code block, the range is 12 ~ 28                                                                                                                         |
+| codeFontSize           | Number  | 14                 | Font size on code block, the range is 12 ~ 28                                                                                                                         |
-| codeFontFamily           | String  | `DejaVu Sans Mono` | Code font family                                                                                                                                                      |
+| codeFontFamily         | String  | `DejaVu Sans Mono` | Code font family                                                                                                                                                      |
-| trimUnnecessaryCodeBlockEmptyLines | Boolean | true               | Whether to trim the beginning and end empty line in Code block                                                                                                         |
+| trimUnnecessaryCodeBlockEmptyLines | Boolean | true   | Whether to trim the beginning and end empty line in Code block                                                                                                        |
-| hideQuickInsertHint      | Boolean | false              | Hide hint for quickly creating paragraphs                                                                                                                             |
+| hideQuickInsertHint    | Boolean | false              | Hide hint for quickly creating paragraphs                                                                                                                             |
-| imageDropAction          | String  | folder             | The default behavior after paste or drag the image to Mark Text, upload it to the image cloud (if configured), move to the specified folder, insert the path          |
+| imageDropAction        | String  | folder             | The default behavior after paste or drag the image to Mark Text, upload it to the image cloud (if configured), move to the specified folder, insert the path          |
+| defaultEncoding        | String  | `utf8`             | The default file encoding                                                                                                                                             |
+| autoGuessEncoding      | Boolean | true               | Try to automatically guess the file encoding when opening files                                                                                                       |
 
 #### Markdown
 
@@ -46,9 +48,17 @@ Preferences can be controlled and modified in the settings window or via the `pr
 | tabSize             | Number  | 4       | The number of spaces a tab is equal to                                                                                               |
 | listIndentation     | String  | 1       | The list indentation of sub list items or paragraphs, optional value `dfm`, `tab` or number 1~4                                      |
 | frontmatterType     | String  | `-`     | The frontmatter type: `-` (YAML), `+` (TOML), `;` (JSON) or `{` (JSON)                                                               |
+#### Theme
 
+| Key   | Type   | Default | Description                                                           |
+| ----- | ------ | ------- | --------------------------------------------------------------------- |
+| theme | String | light   | `dark`, `graphite`, `material-dark`, `one-dark`, `light` or `ulysses` |
 
-#### View
+#### Editable via file
+
+These entires don't have a settings option and need to be changed manually.
+
+##### View
 
 | Key                           | Type    | Default | Description                                        |
 | ----------------------------- | ------- | ------- | -------------------------------------------------- |
@@ -58,8 +68,13 @@ Preferences can be controlled and modified in the settings window or via the `pr
 
 \*: These options are default/fallback values that are used if not session is loaded and are overwritten by the menu entries.
 
-#### Theme
+##### File system
 
-| Key   | Type   | Default | Description                                                           |
+| Key                  | Type             | Default | Description                                                                                                                                                      |
-| ----- | ------ | ------- | --------------------------------------------------------------------- |
+| -------------------- | ---------------- | ------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| theme | String | light   | `dark`, `graphite`, `material-dark`, `one-dark`, `light` or `ulysses` |
+| searchExclusions     | Array of Strings | `[]`    | The filename exclusions for the file searcher. Default: `'*.markdown', '*.mdown', '*.mkdn', '*.md', '*.mkd', '*.mdwn', '*.mdtxt', '*.mdtext', '*.text', '*.txt'` |
+| searchMaxFileSize    | String           | `""`    | The maximum file size to search in (e.g. 50K or 10MB). Default: unlimited                                                                                        |
+| searchIncludeHidden  | Boolean          | false   | Search hidden files and directories                                                                                                                              |
+| searchNoIgnore       | Boolean          | false   | Don't respect ignore files such as `.gitignore`.                                                                                                                 |
+| searchFollowSymlinks | Boolean          | true    | Whether to follow symbolic links.                                                                                                                                |
+| watcherUsePolling    | Boolean          | false   | Whether to use polling to receive file changes. Polling may leads to high CPU utilization.                                                                       |

package.json

@@ -37,6 +37,7 @@
     "@octokit/rest": "^16.30.1",
     "arg": "^4.1.1",
     "axios": "^0.19.0",
+    "ced": "^1.0.0",
     "chokidar": "^3.2.1",
     "codemirror": "^5.49.0",
     "command-exists": "^1.2.8",
@@ -58,6 +59,7 @@
     "fuzzaldrin": "^2.1.0",
     "github-markdown-css": "^3.0.1",
     "html-tags": "^3.0.0",
+    "iconv-lite": "^0.5.0",
     "joplin-turndown-plugin-gfm": "^1.0.9",
     "katex": "^0.11.1",
     "keyboard-layout": "^2.0.16",

src/common/encoding.js

@@ -0,0 +1,52 @@
+export const ENCODING_NAME_MAP = {
+  utf8: 'UTF-8',
+  utf16be: 'UTF-16 BE',
+  utf16le: 'UTF-16 LE',
+  utf32be: 'UTF-32 BE',
+  utf32le: 'UTF-32 LE',
+  ascii: 'Western (ISO 8859-1)',
+  latin3: 'Western (ISO 8859-3)',
+  iso885915: 'Western (ISO 8859-15)',
+  cp1252: 'Western (Windows 1252)',
+  arabic: 'Arabic (ISO 8859-6)',
+  cp1256: 'Arabic (Windows 1256)',
+  latin4: 'Baltic (ISO 8859-4)',
+  cp1257: 'Baltic (Windows 1257)',
+  iso88592: 'Central European (ISO 8859-2)',
+  windows1250: 'Central European (Windows 1250)',
+  cp866: 'Cyrillic (CP 866)',
+  iso88595: 'Cyrillic (ISO 8859-5)',
+  koi8r: 'Cyrillic (KOI8-R)',
+  koi8u: 'Cyrillic (KOI8-U)',
+  cp1251: 'Cyrillic (Windows 1251)',
+  iso885913: 'Estonian (ISO 8859-13)',
+  greek: 'Greek (ISO 8859-7)',
+  cp1253: 'Greek (Windows 1253)',
+  hebrew: 'Hebrew (ISO 8859-8)',
+  cp1255: 'Hebrew (Windows 1255)',
+  latin5: 'Turkish (ISO 8859-9)',
+  cp1254: 'Turkish (Windows 1254)',
+  gb2312: 'Simplified Chinese (GB2312)',
+  gb18030: 'Simplified Chinese (GB18030)',
+  gbk: 'Simplified Chinese (GBK)',
+  big5: 'Traditional Chinese (Big5)',
+  big5hkscs: 'Traditional Chinese (Big5-HKSCS)',
+  shiftjis: 'Japanese (Shift JIS)',
+  eucjp: 'Japanese (EUC-JP)',
+  euckr: 'Korean (EUC-KR)',
+  latin6: 'Nordic (ISO 8859-10)'
+}
+
+/**
+ * Try to translate the encoding.
+ *
+ * @param {Encoding} enc The encoding object.
+ */
+export const getEncodingName = enc => {
+  const { encoding, isBom } = enc
+  let str = ENCODING_NAME_MAP[encoding] || encoding
+  if (isBom) {
+    str += ' with BOM'
+  }
+  return str
+}

src/main/config.js

@@ -39,6 +39,7 @@ export const defaultPreferenceWinOptions = {
 export const PANDOC_EXTENSIONS = [
   'html',
   'docx',
+  'odt',
   'latex',
   'tex',
   'ltx',

src/main/filesystem/encoding.js

@@ -0,0 +1,74 @@
+import ced from 'ced'
+
+const CED_ICONV_ENCODINGS = {
+  'BIG5-CP950': 'big5',
+  KSC: 'euckr',
+  'ISO-2022-KR': 'euckr',
+  GB: 'gb2312',
+  ISO_2022_CN: 'gb2312',
+  JIS: 'shiftjis',
+  SJS: 'shiftjis',
+  Unicode: 'utf8',
+
+  // Map ASCII to UTF-8
+  'ASCII-7-bit': 'utf8',
+  ASCII: 'utf8',
+  MACINTOSH: 'utf8'
+}
+
+// Byte Order Mark's to detect endianness and encoding.
+const BOM_ENCODINGS = {
+  utf8: [0xEF, 0xBB, 0xBF],
+  utf16be: [0xFE, 0xFF],
+  utf16le: [0xFF, 0xFE]
+}
+
+const checkSequence = (buffer, sequence) => {
+  if (buffer.length < sequence.length) {
+    return false
+  }
+  return sequence.every((v, i) => v === buffer[i])
+}
+
+/**
+ * Guess the encoding from the buffer.
+ *
+ * @param {Buffer} buffer
+ * @param {boolean} autoGuessEncoding
+ * @returns {Encoding}
+ */
+export const guessEncoding = (buffer, autoGuessEncoding) => {
+  let isBom = false
+  let encoding = 'utf8'
+
+  // Detect UTF8- and UTF16-BOM encodings.
+  for (const [key, value] of Object.entries(BOM_ENCODINGS)) {
+    if (checkSequence(buffer, value)) {
+      return { encoding: key, isBom: true }
+    }
+  }
+
+  // // Try to detect binary files. Text files should not containt four 0x00 characters.
+  // let zeroSeenCounter = 0
+  // for (let i = 0; i < Math.min(buffer.byteLength, 256); ++i) {
+  //   if (buffer[i] === 0x00) {
+  //     if (zeroSeenCounter >= 3) {
+  //       return { encoding: 'binary', isBom: false }
+  //     }
+  //     zeroSeenCounter++
+  //   } else {
+  //     zeroSeenCounter = 0
+  //   }
+  // }
+
+  // Auto guess encoding, otherwise use UTF8.
+  if (autoGuessEncoding) {
+    encoding = ced(buffer)
+    if (CED_ICONV_ENCODINGS[encoding]) {
+      encoding = CED_ICONV_ENCODINGS[encoding]
+    } else {
+      encoding = encoding.toLowerCase().replace(/-_/g, '')
+    }
+  }
+  return { encoding, isBom }
+}

src/main/filesystem/index.js

@@ -22,11 +22,11 @@ export const normalizeAndResolvePath = pathname => {
   return path.resolve(pathname)
 }
 
-export const writeFile = (pathname, content, extension) => {
+export const writeFile = (pathname, content, extension, options = 'utf-8') => {
   if (!pathname) {
     return Promise.reject(new Error('[ERROR] Cannot save file without path.'))
   }
   pathname = !extension || pathname.endsWith(extension) ? pathname : `${pathname}${extension}`
 
-  return fs.outputFile(pathname, content, 'utf-8')
+  return fs.outputFile(pathname, content, options)
 }

src/main/filesystem/markdown.js

@@ -1,10 +1,12 @@
 import fs from 'fs-extra'
 import path from 'path'
 import log from 'electron-log'
+import iconv from 'iconv-lite'
 import { LINE_ENDING_REG, LF_LINE_ENDING_REG, CRLF_LINE_ENDING_REG } from '../config'
 import { isDirectory } from 'common/filesystem'
 import { isMarkdownFileOrLink } from 'common/filesystem/paths'
 import { normalizeAndResolvePath, writeFile } from '../filesystem'
+import { guessEncoding } from './encoding'
 
 const getLineEnding = lineEnding => {
   if (lineEnding === 'lf') {
@@ -51,46 +53,47 @@ export const normalizeMarkdownPath = pathname => {
  * @param {IMarkdownDocumentOptions} options The markdown document options
  */
 export const writeMarkdownFile = (pathname, content, options) => {
-  const { adjustLineEndingOnSave, encoding, lineEnding } = options
+  const { adjustLineEndingOnSave, lineEnding } = options
+  const { encoding, isBom } = options.encoding
   const extension = path.extname(pathname) || '.md'
 
-  if (encoding === 'utf8bom') {
-    content = '\uFEFF' + content
-  }
-
   if (adjustLineEndingOnSave) {
     content = convertLineEndings(content, lineEnding)
   }
 
+  const buffer = iconv.encode(content, encoding, { addBOM: isBom })
+
   // TODO(@fxha): "safeSaveDocuments" using temporary file and rename syscall.
-  return writeFile(pathname, content, extension)
+  return writeFile(pathname, buffer, extension, undefined)
 }
 
 /**
  * Reads the contents of a markdown file.
  *
  * @param {string} pathname The path to the markdown file.
- * @param {string} preferedEOL The prefered EOL.
+ * @param {string} preferedEol The prefered EOL.
  * @returns {IMarkdownDocumentRaw} Returns a raw markdown document.
  */
-export const loadMarkdownFile = async (pathname, preferedEOL) => {
+export const loadMarkdownFile = async (pathname, preferedEol, autoGuessEncoding = true) => {
-  let markdown = await fs.readFile(path.resolve(pathname), 'utf-8')
+  // TODO: Use streams to not buffer the file multiple times and only guess
+  //       encoding on the first 256/512 bytes.
 
-  // Check UTF-8 BOM (EF BB BF) encoding
+  let buffer = await fs.readFile(path.resolve(pathname))
-  const isUtf8BomEncoded = markdown.length >= 1 && markdown.charCodeAt(0) === 0xFEFF
+
-  if (isUtf8BomEncoded) {
+  const encoding = guessEncoding(buffer, autoGuessEncoding)
-    markdown.splice(0, 1)
+  const supported = iconv.encodingExists(encoding.encoding)
+  if (!supported) {
+    throw new Error(`"${encoding.encoding}" encoding is not supported.`)
   }
 
-  // TODO(@fxha): Check for more file encodings and whether the file is binary but for now expect UTF-8.
+  let markdown = iconv.decode(buffer, encoding.encoding)
-  const encoding = isUtf8BomEncoded ? 'utf8bom' : 'utf8'
 
   // Detect line ending
   const isLf = LF_LINE_ENDING_REG.test(markdown)
   const isCrlf = CRLF_LINE_ENDING_REG.test(markdown)
   const isMixedLineEndings = isLf && isCrlf
   const isUnknownEnding = !isLf && !isCrlf
-  let lineEnding = preferedEOL
+  let lineEnding = preferedEol
   if (isLf && !isCrlf) {
     lineEnding = 'lf'
   } else if (isCrlf && !isLf) {

src/main/filesystem/watcher.js

@@ -18,7 +18,7 @@ const EVENT_NAME = {
   file: 'AGANI::update-file'
 }
 
-const add = async (win, pathname, type, endOfLine) => {
+const add = async (win, pathname, type, endOfLine, autoGuessEncoding) => {
   const stats = await fs.stat(pathname)
   const birthTime = stats.birthtime
   const isMarkdown = hasMarkdownExtension(pathname)
@@ -33,7 +33,7 @@ const add = async (win, pathname, type, endOfLine) => {
   if (isMarkdown) {
     // HACK: But this should be removed completely in #1034/#1035.
     try {
-      const data = await loadMarkdownFile(pathname, endOfLine)
+      const data = await loadMarkdownFile(pathname, endOfLine, autoGuessEncoding)
       file.data = data
     } catch (err) {
       // Only notify user about opened files.
@@ -61,7 +61,7 @@ const unlink = (win, pathname, type) => {
   })
 }
 
-const change = async (win, pathname, type, endOfLine) => {
+const change = async (win, pathname, type, endOfLine, autoGuessEncoding) => {
   // No need to update the tree view if the file content has changed.
   if (type === 'dir') return
 
@@ -70,7 +70,7 @@ const change = async (win, pathname, type, endOfLine) => {
     // HACK: Markdown data should be removed completely in #1034/#1035 and
     // should be only loaded after user interaction.
     try {
-      const data = await loadMarkdownFile(pathname, endOfLine)
+      const data = await loadMarkdownFile(pathname, endOfLine, autoGuessEncoding)
       const file = {
         pathname,
         data
@@ -179,12 +179,16 @@ class Watcher {
     watcher
       .on('add', pathname => {
         if (!this._shouldIgnoreEvent(win.id, pathname, type)) {
-          add(win, pathname, type, this._preferences.getPreferedEOL())
+          const eol = this._preferences.getPreferedEol()
+          const autoGuessEncoding = this._preferences.getItem('autoGuessEncoding')
+          add(win, pathname, type, eol, autoGuessEncoding)
         }
       })
       .on('change', pathname => {
         if (!this._shouldIgnoreEvent(win.id, pathname, type)) {
-          change(win, pathname, type, this._preferences.getPreferedEOL())
+          const eol = this._preferences.getPreferedEol()
+          const autoGuessEncoding = this._preferences.getItem('autoGuessEncoding')
+          change(win, pathname, type, eol, autoGuessEncoding)
         }
       })
       .on('unlink', pathname => unlink(win, pathname, type))
@@ -204,7 +208,9 @@ class Watcher {
           }
           renameTimer = setTimeout(async () => {
             renameTimer = null
-            if (disposed) return
+            if (disposed) {
+              return
+            }
 
             const fileExists = await exists(watchPath)
             if (fileExists) {

src/main/preferences/index.js

@@ -122,7 +122,7 @@ class Preference extends EventEmitter {
     })
   }
 
-  getPreferedEOL () {
+  getPreferedEol () {
     const endOfLine = this.getItem('endOfLine')
     if (endOfLine === 'lf') {
       return 'lf'

src/main/preferences/schema.json

@@ -1,7 +1,8 @@
 {
   "autoSave": {
     "description": "General--Automatically save the content being edited.",
-    "type": "boolean"
+    "type": "boolean",
+    "default": false
   },
   "autoSaveDelay": {
     "description": "General--The time in ms after a change that the file is saved.",
@@ -56,6 +57,7 @@
     "description": "General--The language Mark Text use.",
     "type": "string"
   },
+
   "editorFontFamily": {
     "description": "Editor--editor font family",
     "type": "string",
@@ -114,8 +116,55 @@
       "default",
       "lf",
       "crlf"
+    ],
+    "default": "default"
+  },
+  "defaultEncoding": {
+    "description": "Editor--The default file encoding.",
+    "default": "utf8",
+    "enum": [
+      "utf8",
+      "utf16be",
+      "utf16le",
+      "utf32be",
+      "utf32le",
+      "latin3",
+      "iso885915",
+      "cp1252",
+      "arabic",
+      "cp1256",
+      "latin4",
+      "cp1257",
+      "iso88592",
+      "windows1250",
+      "cp866",
+      "iso88595",
+      "koi8r",
+      "koi8u",
+      "cp1251",
+      "iso885913",
+      "greek",
+      "cp1253",
+      "hebrew",
+      "cp1255",
+      "latin5",
+      "cp1254",
+      "gb2312",
+      "gb18030",
+      "gbk",
+      "big5",
+      "big5hkscs",
+      "shiftjis",
+      "eucjp",
+      "euckr",
+      "latin6"
     ]
   },
+  "autoGuessEncoding": {
+    "description": "Editor--Try to automatically guess the file encoding when opening files.",
+    "type": "boolean",
+    "default": true
+  },
   "textDirection": {
     "description": "Editor--The writing text direction",
     "enum": [
@@ -127,6 +176,7 @@
     "description": "Editor--Hide hint for quickly creating paragraphs",
     "type": "boolean"
   },
+
   "preferLooseListItem": {
     "description": "Markdown--The preferred list type",
     "type": "boolean"
@@ -177,10 +227,12 @@
       "{"
     ]
   },
+
   "theme": {
     "description": "Theme--Select the theme used in Mark Text",
     "type": "string"
   },
+
   "imageInsertAction": {
     "description": "Image--The default behavior after insert image from local folder",
     "enum": [
@@ -189,6 +241,7 @@
       "path"
     ]
   },
+
   "sideBarVisibility": {
     "description": "View--Whether the side bar is visible.",
     "type": "boolean"
@@ -201,6 +254,7 @@
     "description": "View--Whether the source-code mode is enabled by default.",
     "type": "boolean"
   },
+
   "searchExclusions": {
     "description": "Searcher--List of glob patterns to exclude from search.",
     "type": "array",

src/main/windows/base.js

@@ -102,6 +102,27 @@ class BaseWindow extends EventEmitter {
   _buildUrlString (windowId, env, userPreference) {
     return this._buildUrlWithSettings(windowId, env, userPreference).toString()
   }
+
+  _getPreferredBackgroundColor (theme) {
+    // Hardcode the theme background color and show the window direct for the fastet window ready time.
+    // Later with custom themes we need the background color (e.g. from meta information) and wait
+    // that the window is loaded and then pass theme data to the renderer.
+    switch (theme) {
+      case 'dark':
+        return '#282828'
+      case 'material-dark':
+        return '#34393f'
+      case 'ulysses':
+        return '#f3f3f3'
+      case 'graphite':
+        return '#f7f7f7'
+      case 'one-dark':
+        return '#282c34'
+      case 'light':
+      default:
+        return '#ffffff'
+    }
+  }
 }
 
 export default BaseWindow

src/main/windows/editor.js

@@ -80,7 +80,7 @@ class EditorWindow extends BaseWindow {
       // Restore and focus window
       this.bringToFront()
 
-      const lineEnding = preferences.getPreferedEOL()
+      const lineEnding = preferences.getPreferedEol()
       appMenu.updateLineEndingMenu(this.id, lineEnding)
 
       win.webContents.send('mt::bootstrap-editor', {
@@ -211,10 +211,11 @@ class EditorWindow extends BaseWindow {
 
     const { browserWindow } = this
     const { preferences } = this._accessor
-    const eol = preferences.getPreferedEOL()
+    const eol = preferences.getPreferedEol()
+    const autoGuessEncoding = preferences.getItem('autoGuessEncoding')
 
     for (const { filePath, options, selected } of fileList) {
-      loadMarkdownFile(filePath, eol).then(rawDocument => {
+      loadMarkdownFile(filePath, eol, autoGuessEncoding).then(rawDocument => {
         if (this.lifecycle === WindowLifecycle.READY) {
           this._doOpenTab(rawDocument, options, selected)
         } else {
@@ -363,7 +364,7 @@ class EditorWindow extends BaseWindow {
       this.lifecycle = WindowLifecycle.READY
       const { preferences } = this._accessor
       const { sideBarVisibility, tabBarVisibility, sourceCodeModeEnabled } = preferences.getAll()
-      const lineEnding = preferences.getPreferedEOL()
+      const lineEnding = preferences.getPreferedEol()
       browserWindow.webContents.send('mt::bootstrap-editor', {
         addBlankTab: true,
         markdownList: [],
@@ -396,27 +397,6 @@ class EditorWindow extends BaseWindow {
 
   // --- private ---------------------------------
 
-  _getPreferredBackgroundColor (theme) {
-    // Hardcode the theme background color and show the window direct for the fastet window ready time.
-    // Later with custom themes we need the background color (e.g. from meta information) and wait
-    // that the window is loaded and then pass theme data to the renderer.
-    switch (theme) {
-      case 'dark':
-        return '#282828'
-      case 'material-dark':
-        return '#34393f'
-      case 'ulysses':
-        return '#f3f3f3'
-      case 'graphite':
-        return '#f7f7f7'
-      case 'one-dark':
-        return '#282c34'
-      case 'light':
-      default:
-        return '#ffffff'
-    }
-  }
-
   /**
    * Open a new new tab from the markdown document.
    *

src/main/windows/setting.js

@@ -28,7 +28,7 @@ class SettingWindow extends BaseWindow {
     }
 
     // Enable native or custom/frameless window and titlebar
-    const { titleBarStyle } = preferences.getAll()
+    const { titleBarStyle, theme } = preferences.getAll()
     if (!isOsx) {
       winOptions.titleBarStyle = 'default'
       if (titleBarStyle === 'native') {
@@ -36,6 +36,8 @@ class SettingWindow extends BaseWindow {
       }
     }
 
+    winOptions.backgroundColor = this._getPreferredBackgroundColor(theme)
+
     let win = this.browserWindow = new BrowserWindow(winOptions)
     this.id = win.id
 

src/renderer/prefComponents/editor/config.js

@@ -1,3 +1,5 @@
+import { ENCODING_NAME_MAP } from 'common/encoding'
+
 export const endOfLineOptions = [{
   label: 'Default',
   value: 'default'
@@ -16,3 +18,16 @@ export const textDirectionOptions = [{
   label: 'Right to Left',
   value: 'rtl'
 }]
+
+let defaultEncodingOptions = null
+export const getDefaultEncodingOptions = () => {
+  if (defaultEncodingOptions) {
+    return defaultEncodingOptions
+  }
+
+  defaultEncodingOptions = []
+  for (const [value, label] of Object.entries(ENCODING_NAME_MAP)) {
+    defaultEncodingOptions.push({ label, value })
+  }
+  return defaultEncodingOptions
+}

src/renderer/prefComponents/editor/index.vue

@@ -62,29 +62,41 @@
     ></bool>
     <separator></separator>
     <cur-select
-      description="The default end of line character, if you select default, which will be selected according to your system intelligence."
+      description="The default end of line character. If you select default, the ending will be selected according to your system intelligence."
       :value="endOfLine"
       :options="endOfLineOptions"
       :onChange="value => onSelectChange('endOfLine', value)"
     ></cur-select>
+    <separator></separator>
+    <cur-select
+      description="The default file encoding."
+      :value="defaultEncoding"
+      :options="defaultEncodingOptions"
+      :onChange="value => onSelectChange('defaultEncoding', value)"
+    ></cur-select>
+    <bool
+      description="Try to automatically guess the file encoding when opening files."
+      :bool="autoGuessEncoding"
+      :onChange="value => onSelectChange('autoGuessEncoding', value)"
+    ></bool>
+    <separator></separator>
     <cur-select
       description="The writing text direction."
       :value="textDirection"
       :options="textDirectionOptions"
       :onChange="value => onSelectChange('textDirection', value)"
     ></cur-select>
-    <separator></separator>
     <bool
       description="Hide hint for quickly creating paragraphs."
-      :input="hideQuickInsertHint"
+      :bool="hideQuickInsertHint"
       :onChange="value => onSelectChange('hideQuickInsertHint', value)"
     ></bool>
     <separator></separator>
     <text-box
       description="Defines the maximum editor area width. An empty string or suffixes of ch (characters), px (pixels) or % (percentage) are allowed."
-      :input="editorLineWidth"
+      :bool="editorLineWidth"
       :regexValidator="/^(?:$|[0-9]+(?:ch|px|%)$)/"
-      defaultValue="The default value from the current theme"
+      defaultValue="Default value from current theme"
       :onChange="value => onSelectChange('editorLineWidth', value)"
     ></text-box>
   </div>
@@ -100,7 +112,8 @@ import Separator from '../common/separator'
 import TextBox from '../common/textBox'
 import {
   endOfLineOptions,
-  textDirectionOptions
+  textDirectionOptions,
+  getDefaultEncodingOptions
 } from './config'
 
 export default {
@@ -115,6 +128,7 @@ export default {
   data () {
     this.endOfLineOptions = endOfLineOptions
     this.textDirectionOptions = textDirectionOptions
+    this.defaultEncodingOptions = getDefaultEncodingOptions()
     return {}
   },
   computed: {
@@ -131,7 +145,9 @@ export default {
       codeFontFamily: state => state.preferences.codeFontFamily,
       trimUnnecessaryCodeBlockEmptyLines: state => state.preferences.trimUnnecessaryCodeBlockEmptyLines,
       hideQuickInsertHint: state => state.preferences.hideQuickInsertHint,
-      editorLineWidth: state => state.preferences.editorLineWidth
+      editorLineWidth: state => state.preferences.editorLineWidth,
+      defaultEncoding: state => state.preferences.defaultEncoding,
+      autoGuessEncoding: state => state.preferences.autoGuessEncoding
     })
   },
   methods: {

src/renderer/store/editor.js

@@ -11,7 +11,6 @@ import notice from '../services/notification'
 const autoSaveTimers = new Map()
 
 const state = {
-  lineEnding: 'lf',
   currentFile: {},
   tabs: [],
   listToc: [], // Just use for deep equal check. and replace with new toc if needed.
@@ -270,11 +269,6 @@ const mutations = {
     })
   },
 
-  // TODO: Remove "SET_GLOBAL_LINE_ENDING" because nowhere used.
-  SET_GLOBAL_LINE_ENDING (state, ending) {
-    state.lineEnding = ending
-  },
-
   // Push a tab specific notification on stack that never disappears.
   PUSH_TAB_NOTIFICATION (state, data) {
     const defaultAction = () => {}
@@ -606,8 +600,8 @@ const actions = {
         sourceCodeModeEnabled
       } = config
 
-      commit('SET_GLOBAL_LINE_ENDING', lineEnding)
       dispatch('SEND_INITIALIZED')
+      commit('SET_USER_PREFERENCE', { endOfLine: lineEnding })
       commit('SET_LAYOUT', {
         rightColumn: 'files',
         showSideBar: !!sideBarVisibility,
@@ -714,15 +708,17 @@ const actions = {
    * @param {{markdown?: string, selected?: boolean}} obj Optional markdown string
    * and whether the tab should become the selected tab (true if not set).
    */
-  NEW_UNTITLED_TAB ({ commit, state, dispatch }, { markdown: markdownString, selected }) {
+  NEW_UNTITLED_TAB ({ commit, state, dispatch, rootState }, { markdown: markdownString, selected }) {
     // If not set select the tab.
     if (selected == null) {
       selected = true
     }
 
     dispatch('SHOW_TAB_VIEW', false)
-    const { tabs, lineEnding } = state
+
-    const fileState = getBlankFileState(tabs, lineEnding, markdownString)
+    const { defaultEncoding, endOfLine } = rootState.preferences
+    const { tabs } = state
+    const fileState = getBlankFileState(tabs, defaultEncoding, endOfLine, markdownString)
 
     if (selected) {
       const { id, markdown } = fileState

src/renderer/store/help.js

@@ -12,7 +12,10 @@ export const defaultFileState = {
   pathname: '',
   filename: 'Untitled-1',
   markdown: '',
-  encoding: 'utf8', // Currently just "utf8" or "utf8bom"
+  encoding: {
+    encoding: 'utf8',
+    isBom: false
+  },
   lineEnding: 'lf', // lf or crlf
   adjustLineEndingOnSave: false, // convert editor buffer (LF) to CRLF when saving
   history: {
@@ -65,8 +68,8 @@ export const getFileStateFromData = data => {
   })
 }
 
-export const getBlankFileState = (tabs, lineEnding = 'lf', markdown = '') => {
+export const getBlankFileState = (tabs, defaultEncoding = 'utf8', lineEnding = 'lf', markdown = '') => {
-  const fileState = JSON.parse(JSON.stringify(defaultFileState))
+  const fileState = cloneObj(defaultFileState, true)
   let untitleId = Math.max(...tabs.map(f => {
     if (f.pathname === '') {
       return +f.filename.split('-')[1]
@@ -77,11 +80,12 @@ export const getBlankFileState = (tabs, lineEnding = 'lf', markdown = '') => {
 
   const id = getUniqueId()
 
-  // We may pass muarkdown=null as parameter.
+  // We may pass markdown=null as parameter.
   if (markdown == null) {
     markdown = ''
   }
 
+  fileState.encoding.encoding = defaultEncoding
   return Object.assign(fileState, {
     lineEnding,
     adjustLineEndingOnSave: lineEnding.toLowerCase() === 'crlf',
@@ -94,7 +98,7 @@ export const getBlankFileState = (tabs, lineEnding = 'lf', markdown = '') => {
 export const getSingleFileState = ({ id = getUniqueId(), markdown, filename, pathname, options }) => {
   // TODO(refactor:renderer/editor): Replace this function with `createDocumentState`.
 
-  const fileState = JSON.parse(JSON.stringify(defaultFileState))
+  const fileState = cloneObj(defaultFileState, true)
   const { encoding, lineEnding, adjustLineEndingOnSave = 'ltr' } = options
 
   assertLineEnding(adjustLineEndingOnSave, lineEnding)

src/renderer/store/preferences.js

@@ -26,6 +26,8 @@ const state = {
   autoPairMarkdownSyntax: true,
   autoPairQuote: true,
   endOfLine: 'default',
+  defaultEncoding: 'utf8',
+  autoGuessEncoding: true,
   textDirection: 'ltr',
   hideQuickInsertHint: false,
   imageInsertAction: 'folder',

static/preference.json

@@ -23,6 +23,8 @@
   "autoPairMarkdownSyntax": true,
   "autoPairQuote": true,
   "endOfLine": "default",
+  "defaultEncoding": "utf8",
+  "autoGuessEncoding": true,
   "textDirection": "ltr",
   "hideQuickInsertHint": false,
   "imageInsertAction": "path",

yarn.lock

@@ -1795,6 +1795,13 @@ binary-extensions@^2.0.0:
   resolved "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.0.0.tgz#23c0df14f6a88077f5f986c0d167ec03c3d5537c"
   integrity sha512-Phlt0plgpIIBOGTT/ehfFnbNlfsDEiqmzE2KRXoX1bLIlir4X/MR+zSyBEkL05ffWgnRSf/DXv+WrUAVr93/ow==
 
+bindings@^1.3.0:
+  version "1.5.0"
+  resolved "https://registry.yarnpkg.com/bindings/-/bindings-1.5.0.tgz#10353c9e945334bc0511a6d90b38fbc7c9c504df"
+  integrity sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==
+  dependencies:
+    file-uri-to-path "1.0.0"
+
 bl@^1.0.0:
   version "1.2.2"
   resolved "https://registry.npmjs.org/bl/-/bl-1.2.2.tgz#a160911717103c07410cef63ef51b397c025af9c"
@@ -2239,6 +2246,13 @@ caseless@~0.12.0:
   resolved "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz#1b681c21ff84033c826543090689420d187151dc"
   integrity sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw=
 
+ced@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/ced/-/ced-1.0.0.tgz#09899dbcda52083b80a65f165c65f9055d03ab03"
+  integrity sha512-Ud3ltdMCO3XMaclGtBGAuV+rNTx/lOwXukEf2pjtmk8CjGRhgACUtbHSQnty/wDn3ZPSz+gv1FS5jiiXH5g8Bw==
+  dependencies:
+    bindings "^1.3.0"
+
 center-align@^0.1.1:
   version "0.1.3"
   resolved "https://registry.npmjs.org/center-align/-/center-align-0.1.3.tgz#aa0d32629b6ee972200411cbd4461c907bc2b7ad"
@@ -5049,6 +5063,11 @@ file-loader@^4.1.0:
     loader-utils "^1.2.3"
     schema-utils "^2.0.0"
 
+file-uri-to-path@1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz#553a7b8446ff6f684359c445f1e37a05dacc33dd"
+  integrity sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==
+
 filesize@^3.6.1:
   version "3.6.1"
   resolved "https://registry.npmjs.org/filesize/-/filesize-3.6.1.tgz#090bb3ee01b6f801a8a8be99d31710b3422bb317"
@@ -5998,7 +6017,7 @@ iconv-lite@0.4, iconv-lite@0.4.24, iconv-lite@^0.4.17, iconv-lite@^0.4.24, iconv
 
 iconv-lite@^0.5.0:
   version "0.5.0"
-  resolved "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.5.0.tgz#59cdde0a2a297cc2aeb0c6445a195ee89f127550"
+  resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.5.0.tgz#59cdde0a2a297cc2aeb0c6445a195ee89f127550"
   integrity sha512-NnEhI9hIEKHOzJ4f697DMz9IQEXr/MMJ5w64vN2/4Ai+wRnvV7SBrL0KLoRlwaKVghOc7LQ5YkPLuX146b6Ydw==
   dependencies:
     safer-buffer ">= 2.1.2 < 3"