Refactor MFM

Co-authored-by: syuilo syuilotan@yahoo.co.jp
This commit is contained in:
Aya Morisawa 2018-12-20 19:41:04 +09:00
parent e0b107a3a0
commit e9f8897fe2
No known key found for this signature in database
GPG key ID: 3E64865D70D579F2
14 changed files with 588 additions and 518 deletions

View file

@ -80,8 +80,8 @@ export default (opts: Opts = {}) => ({
const ast = parse(this.appearNote.text); const ast = parse(this.appearNote.text);
// TODO: 再帰的にURL要素がないか調べる // TODO: 再帰的にURL要素がないか調べる
return unique(ast return unique(ast
.filter(t => ((t.name == 'url' || t.name == 'link') && t.props.url && !t.props.silent)) .filter(t => ((t.node.type == 'url' || t.node.type == 'link') && t.node.props.url && !t.node.props.silent))
.map(t => t.props.url)); .map(t => t.node.props.url));
} else { } else {
return null; return null;
} }

View file

@ -52,8 +52,8 @@ export default Vue.extend({
if (this.message.text) { if (this.message.text) {
const ast = parse(this.message.text); const ast = parse(this.message.text);
return unique(ast return unique(ast
.filter(t => ((t.name == 'url' || t.name == 'link') && t.props.url && !t.silent)) .filter(t => ((t.node.type == 'url' || t.node.type == 'link') && t.node.props.url && !t.node.props.silent))
.map(t => t.props.url)); .map(t => t.node.props.url));
} else { } else {
return null; return null;
} }

View file

@ -1,6 +1,6 @@
import Vue, { VNode } from 'vue'; import Vue, { VNode } from 'vue';
import { length } from 'stringz'; import { length } from 'stringz';
import { Node } from '../../../../../mfm/parser'; import { MfmForest } from '../../../../../mfm/parser';
import parse from '../../../../../mfm/parse'; import parse from '../../../../../mfm/parse';
import MkUrl from './url.vue'; import MkUrl from './url.vue';
import MkMention from './mention.vue'; import MkMention from './mention.vue';
@ -9,16 +9,11 @@ import MkFormula from './formula.vue';
import MkGoogle from './google.vue'; import MkGoogle from './google.vue';
import syntaxHighlight from '../../../../../mfm/syntax-highlight'; import syntaxHighlight from '../../../../../mfm/syntax-highlight';
import { host } from '../../../config'; import { host } from '../../../config';
import { preorderF, countNodesF } from '../../../../../prelude/tree';
function getTextCount(tokens: Node[]): number { function sumTextsLength(ts: MfmForest): number {
const rootCount = sum(tokens.filter(x => x.name === 'text').map(x => length(x.props.text))); const textNodes = preorderF(ts).filter(n => n.type === 'text');
const childrenCount = sum(tokens.filter(x => x.children).map(x => getTextCount(x.children))); return sum(textNodes.map(x => length(x.props.text)));
return rootCount + childrenCount;
}
function getChildrenCount(tokens: Node[]): number {
const countTree = tokens.filter(x => x.children).map(x => getChildrenCount(x.children));
return countTree.length + sum(countTree);
} }
export default Vue.component('misskey-flavored-markdown', { export default Vue.component('misskey-flavored-markdown', {
@ -27,10 +22,6 @@ export default Vue.component('misskey-flavored-markdown', {
type: String, type: String,
required: true required: true
}, },
ast: {
type: [],
required: false
},
shouldBreak: { shouldBreak: {
type: Boolean, type: Boolean,
default: true default: true
@ -55,17 +46,15 @@ export default Vue.component('misskey-flavored-markdown', {
render(createElement) { render(createElement) {
if (this.text == null || this.text == '') return; if (this.text == null || this.text == '') return;
const ast = this.ast == null ? const ast = parse(this.text, this.plainText);
parse(this.text, this.plainText) : // Parse text to ast
this.ast as Node[];
let bigCount = 0; let bigCount = 0;
let motionCount = 0; let motionCount = 0;
const genEl = (ast: Node[]) => concat(ast.map((token): VNode[] => { const genEl = (ast: MfmForest) => concat(ast.map((token): VNode[] => {
switch (token.name) { switch (token.node.type) {
case 'text': { case 'text': {
const text = token.props.text.replace(/(\r\n|\n|\r)/g, '\n'); const text = token.node.props.text.replace(/(\r\n|\n|\r)/g, '\n');
if (this.shouldBreak) { if (this.shouldBreak) {
const x = text.split('\n') const x = text.split('\n')
@ -95,7 +84,7 @@ export default Vue.component('misskey-flavored-markdown', {
case 'big': { case 'big': {
bigCount++; bigCount++;
const isLong = getTextCount(token.children) > 10 || getChildrenCount(token.children) > 5; const isLong = sumTextsLength(token.children) > 10 || countNodesF(token.children) > 5;
const isMany = bigCount > 3; const isMany = bigCount > 3;
return (createElement as any)('strong', { return (createElement as any)('strong', {
attrs: { attrs: {
@ -122,7 +111,7 @@ export default Vue.component('misskey-flavored-markdown', {
case 'motion': { case 'motion': {
motionCount++; motionCount++;
const isLong = getTextCount(token.children) > 10 || getChildrenCount(token.children) > 5; const isLong = sumTextsLength(token.children) > 10 || countNodesF(token.children) > 5;
const isMany = motionCount > 3; const isMany = motionCount > 3;
return (createElement as any)('span', { return (createElement as any)('span', {
attrs: { attrs: {
@ -139,7 +128,7 @@ export default Vue.component('misskey-flavored-markdown', {
return [createElement(MkUrl, { return [createElement(MkUrl, {
key: Math.random(), key: Math.random(),
props: { props: {
url: token.props.url, url: token.node.props.url,
target: '_blank', target: '_blank',
style: 'color:var(--mfmLink);' style: 'color:var(--mfmLink);'
} }
@ -150,9 +139,9 @@ export default Vue.component('misskey-flavored-markdown', {
return [createElement('a', { return [createElement('a', {
attrs: { attrs: {
class: 'link', class: 'link',
href: token.props.url, href: token.node.props.url,
target: '_blank', target: '_blank',
title: token.props.url, title: token.node.props.url,
style: 'color:var(--mfmLink);' style: 'color:var(--mfmLink);'
} }
}, genEl(token.children))]; }, genEl(token.children))];
@ -162,8 +151,8 @@ export default Vue.component('misskey-flavored-markdown', {
return [createElement(MkMention, { return [createElement(MkMention, {
key: Math.random(), key: Math.random(),
props: { props: {
host: (token.props.host == null && this.author && this.author.host != null ? this.author.host : token.props.host) || host, host: (token.node.props.host == null && this.author && this.author.host != null ? this.author.host : token.node.props.host) || host,
username: token.props.username username: token.node.props.username
} }
})]; })];
} }
@ -172,10 +161,10 @@ export default Vue.component('misskey-flavored-markdown', {
return [createElement('router-link', { return [createElement('router-link', {
key: Math.random(), key: Math.random(),
attrs: { attrs: {
to: `/tags/${encodeURIComponent(token.props.hashtag)}`, to: `/tags/${encodeURIComponent(token.node.props.hashtag)}`,
style: 'color:var(--mfmHashtag);' style: 'color:var(--mfmHashtag);'
} }
}, `#${token.props.hashtag}`)]; }, `#${token.node.props.hashtag}`)];
} }
case 'blockCode': { case 'blockCode': {
@ -184,7 +173,7 @@ export default Vue.component('misskey-flavored-markdown', {
}, [ }, [
createElement('code', { createElement('code', {
domProps: { domProps: {
innerHTML: syntaxHighlight(token.props.code) innerHTML: syntaxHighlight(token.node.props.code)
} }
}) })
])]; ])];
@ -193,7 +182,7 @@ export default Vue.component('misskey-flavored-markdown', {
case 'inlineCode': { case 'inlineCode': {
return [createElement('code', { return [createElement('code', {
domProps: { domProps: {
innerHTML: syntaxHighlight(token.props.code) innerHTML: syntaxHighlight(token.node.props.code)
} }
})]; })];
} }
@ -227,8 +216,8 @@ export default Vue.component('misskey-flavored-markdown', {
return [createElement('mk-emoji', { return [createElement('mk-emoji', {
key: Math.random(), key: Math.random(),
attrs: { attrs: {
emoji: token.props.emoji, emoji: token.node.props.emoji,
name: token.props.name name: token.node.props.name
}, },
props: { props: {
customEmojis: this.customEmojis || customEmojis, customEmojis: this.customEmojis || customEmojis,
@ -242,7 +231,7 @@ export default Vue.component('misskey-flavored-markdown', {
return [createElement(MkFormula, { return [createElement(MkFormula, {
key: Math.random(), key: Math.random(),
props: { props: {
formula: token.props.formula formula: token.node.props.formula
} }
})]; })];
} }
@ -252,13 +241,13 @@ export default Vue.component('misskey-flavored-markdown', {
return [createElement(MkGoogle, { return [createElement(MkGoogle, {
key: Math.random(), key: Math.random(),
props: { props: {
q: token.props.query q: token.node.props.query
} }
})]; })];
} }
default: { default: {
console.log('unknown ast type:', token.name); console.log('unknown ast type:', token.node.type);
return []; return [];
} }

View file

@ -2,10 +2,10 @@ const jsdom = require('jsdom');
const { JSDOM } = jsdom; const { JSDOM } = jsdom;
import config from '../config'; import config from '../config';
import { INote } from '../models/note'; import { INote } from '../models/note';
import { Node } from './parser';
import { intersperse } from '../prelude/array'; import { intersperse } from '../prelude/array';
import { MfmForest, MfmTree } from './parser';
export default (tokens: Node[], mentionedRemoteUsers: INote['mentionedRemoteUsers'] = []) => { export default (tokens: MfmForest, mentionedRemoteUsers: INote['mentionedRemoteUsers'] = []) => {
if (tokens == null) { if (tokens == null) {
return null; return null;
} }
@ -14,11 +14,11 @@ export default (tokens: Node[], mentionedRemoteUsers: INote['mentionedRemoteUser
const doc = window.document; const doc = window.document;
function appendChildren(children: Node[], targetElement: any): void { function appendChildren(children: MfmForest, targetElement: any): void {
for (const child of children.map(n => handlers[n.name](n))) targetElement.appendChild(child); for (const child of children.map(t => handlers[t.node.type](t))) targetElement.appendChild(child);
} }
const handlers: { [key: string]: (token: Node) => any } = { const handlers: { [key: string]: (token: MfmTree) => any } = {
bold(token) { bold(token) {
const el = doc.createElement('b'); const el = doc.createElement('b');
appendChildren(token.children, el); appendChildren(token.children, el);
@ -58,7 +58,7 @@ export default (tokens: Node[], mentionedRemoteUsers: INote['mentionedRemoteUser
blockCode(token) { blockCode(token) {
const pre = doc.createElement('pre'); const pre = doc.createElement('pre');
const inner = doc.createElement('code'); const inner = doc.createElement('code');
inner.innerHTML = token.props.code; inner.innerHTML = token.node.props.code;
pre.appendChild(inner); pre.appendChild(inner);
return pre; return pre;
}, },
@ -70,39 +70,39 @@ export default (tokens: Node[], mentionedRemoteUsers: INote['mentionedRemoteUser
}, },
emoji(token) { emoji(token) {
return doc.createTextNode(token.props.emoji ? token.props.emoji : `:${token.props.name}:`); return doc.createTextNode(token.node.props.emoji ? token.node.props.emoji : `:${token.node.props.name}:`);
}, },
hashtag(token) { hashtag(token) {
const a = doc.createElement('a'); const a = doc.createElement('a');
a.href = `${config.url}/tags/${token.props.hashtag}`; a.href = `${config.url}/tags/${token.node.props.hashtag}`;
a.textContent = `#${token.props.hashtag}`; a.textContent = `#${token.node.props.hashtag}`;
a.setAttribute('rel', 'tag'); a.setAttribute('rel', 'tag');
return a; return a;
}, },
inlineCode(token) { inlineCode(token) {
const el = doc.createElement('code'); const el = doc.createElement('code');
el.textContent = token.props.code; el.textContent = token.node.props.code;
return el; return el;
}, },
math(token) { math(token) {
const el = doc.createElement('code'); const el = doc.createElement('code');
el.textContent = token.props.formula; el.textContent = token.node.props.formula;
return el; return el;
}, },
link(token) { link(token) {
const a = doc.createElement('a'); const a = doc.createElement('a');
a.href = token.props.url; a.href = token.node.props.url;
appendChildren(token.children, a); appendChildren(token.children, a);
return a; return a;
}, },
mention(token) { mention(token) {
const a = doc.createElement('a'); const a = doc.createElement('a');
const { username, host, acct } = token.props; const { username, host, acct } = token.node.props;
switch (host) { switch (host) {
case 'github.com': case 'github.com':
a.href = `https://github.com/${username}`; a.href = `https://github.com/${username}`;
@ -133,7 +133,7 @@ export default (tokens: Node[], mentionedRemoteUsers: INote['mentionedRemoteUser
text(token) { text(token) {
const el = doc.createElement('span'); const el = doc.createElement('span');
const nodes = (token.props.text as string).split('\n').map(x => doc.createTextNode(x)); const nodes = (token.node.props.text as string).split('\n').map(x => doc.createTextNode(x));
for (const x of intersperse('br', nodes)) { for (const x of intersperse('br', nodes)) {
el.appendChild(x === 'br' ? doc.createElement('br') : x); el.appendChild(x === 'br' ? doc.createElement('br') : x);
@ -144,15 +144,15 @@ export default (tokens: Node[], mentionedRemoteUsers: INote['mentionedRemoteUser
url(token) { url(token) {
const a = doc.createElement('a'); const a = doc.createElement('a');
a.href = token.props.url; a.href = token.node.props.url;
a.textContent = token.props.url; a.textContent = token.node.props.url;
return a; return a;
}, },
search(token) { search(token) {
const a = doc.createElement('a'); const a = doc.createElement('a');
a.href = `https://www.google.com/?#q=${token.props.query}`; a.href = `https://www.google.com/?#q=${token.node.props.query}`;
a.textContent = token.props.content; a.textContent = token.node.props.content;
return a; return a;
} }
}; };

View file

@ -1,40 +1,36 @@
import parser, { Node, plainParser } from './parser'; import parser, { plainParser, MfmForest, MfmTree } from './parser';
import * as A from '../prelude/array'; import * as A from '../prelude/array';
import * as S from '../prelude/string'; import * as S from '../prelude/string';
import { createTree, createLeaf } from '../prelude/tree';
export default (source: string, plainText = false): Node[] => { function concatTextTrees(ts: MfmForest): MfmTree {
return createLeaf({ type: 'text', props: { text: S.concat(ts.map(x => x.node.props.text)) } });
}
function concatIfTextTrees(ts: MfmForest): MfmForest {
return ts[0].node.type === 'text' ? [concatTextTrees(ts)] : ts;
}
function concatConsecutiveTextTrees(ts: MfmForest): MfmForest {
const us = A.concat(A.groupOn(t => t.node.type, ts).map(concatIfTextTrees));
return us.map(t => createTree(t.node, concatConsecutiveTextTrees(t.children)));
}
function isEmptyTextTree(t: MfmTree): boolean {
return t.node.type == 'text' && t.node.props.text === '';
}
function removeEmptyTextNodes(ts: MfmForest): MfmForest {
return ts
.filter(t => !isEmptyTextTree(t))
.map(t => createTree(t.node, removeEmptyTextNodes(t.children)));
}
export default (source: string, plainText = false): MfmForest => {
if (source == null || source == '') { if (source == null || source == '') {
return null; return null;
} }
let nodes: Node[] = plainText ? plainParser.root.tryParse(source) : parser.root.tryParse(source); const raw = plainText ? plainParser.root.tryParse(source) : parser.root.tryParse(source) as MfmForest;
return removeEmptyTextNodes(concatConsecutiveTextTrees(raw));
const combineText = (es: Node[]): Node =>
({ name: 'text', props: { text: S.concat(es.map(e => e.props.text)) } });
const concatText = (nodes: Node[]): Node[] =>
A.concat(A.groupOn(x => x.name, nodes).map(es =>
es[0].name === 'text' ? [combineText(es)] : es
));
const concatTextRecursive = (es: Node[]): void => {
for (const x of es.filter(x => x.children)) {
x.children = concatText(x.children);
concatTextRecursive(x.children);
}
};
nodes = concatText(nodes);
concatTextRecursive(nodes);
const removeEmptyTextNodes = (nodes: Node[]) => {
for (const n of nodes.filter(n => n.children)) {
n.children = removeEmptyTextNodes(n.children);
}
return nodes.filter(n => !(n.name == 'text' && n.props.text == ''));
};
nodes = removeEmptyTextNodes(nodes);
return nodes;
}; };

File diff suppressed because one or more lines are too long

View file

@ -0,0 +1,9 @@
import { EmojiNode, MfmForest } from '../mfm/parser';
import { preorderF } from '../prelude/tree';
import { unique } from '../prelude/array';
export default function(mfmForest: MfmForest): string[] {
const emojiNodes = preorderF(mfmForest).filter(x => x.type === 'emoji') as EmojiNode[];
const emojis = emojiNodes.filter(x => x.props.name && x.props.name.length <= 100).map(x => x.props.name);
return unique(emojis);
}

View file

@ -0,0 +1,9 @@
import { HashtagNode, MfmForest } from '../mfm/parser';
import { preorderF } from '../prelude/tree';
import { unique } from '../prelude/array';
export default function(mfmForest: MfmForest): string[] {
const hashtagNodes = preorderF(mfmForest).filter(x => x.type === 'hashtag') as HashtagNode[];
const hashtags = hashtagNodes.map(x => x.props.hashtag);
return unique(hashtags);
}

View file

@ -1,19 +1,10 @@
import parse from '../mfm/parse'; // test is located in test/extract-mentions
import { Node, IMentionNode } from '../mfm/parser';
export default function(tokens: ReturnType<typeof parse>): IMentionNode['props'][] { import { MentionNode, MfmForest } from '../mfm/parser';
const mentions: IMentionNode['props'][] = []; import { preorderF } from '../prelude/tree';
const extract = (tokens: Node[]) => { export default function(mfmForest: MfmForest): MentionNode['props'][] {
for (const x of tokens.filter(x => x.name === 'mention')) { // TODO: 重複を削除
mentions.push(x.props); const mentionNodes = preorderF(mfmForest).filter(x => x.type === 'mention') as MentionNode[];
} return mentionNodes.map(x => x.props);
for (const x of tokens.filter(x => x.children)) {
extract(x.children);
}
};
extract(tokens);
return mentions;
} }

36
src/prelude/tree.ts Normal file
View file

@ -0,0 +1,36 @@
import { concat, sum } from './array';
export type Tree<T> = {
node: T,
children: Forest<T>;
};
export type Forest<T> = Tree<T>[];
export function createLeaf<T>(node: T): Tree<T> {
return { node, children: [] };
}
export function createTree<T>(node: T, children: Forest<T>): Tree<T> {
return { node, children };
}
export function hasChildren<T>(t: Tree<T>): boolean {
return t.children.length !== 0;
}
export function preorder<T>(t: Tree<T>): T[] {
return [t.node, ...preorderF(t.children)];
}
export function preorderF<T>(ts: Forest<T>): T[] {
return concat(ts.map(preorder));
}
export function countNodes<T>(t: Tree<T>): number {
return preorder(t).length;
}
export function countNodesF<T>(ts: Forest<T>): number {
return sum(ts.map(countNodes));
}

View file

@ -7,7 +7,7 @@ import { publishToFollowers } from '../../../../services/i/update';
import define from '../../define'; import define from '../../define';
import getDriveFileUrl from '../../../../misc/get-drive-file-url'; import getDriveFileUrl from '../../../../misc/get-drive-file-url';
import parse from '../../../../mfm/parse'; import parse from '../../../../mfm/parse';
import { extractEmojis } from '../../../../services/note/create'; import extractEmojis from '../../../../misc/extract-emojis';
const langmap = require('langmap'); const langmap = require('langmap');
export const meta = { export const meta = {

View file

@ -24,12 +24,13 @@ import isQuote from '../../misc/is-quote';
import notesChart from '../../chart/notes'; import notesChart from '../../chart/notes';
import perUserNotesChart from '../../chart/per-user-notes'; import perUserNotesChart from '../../chart/per-user-notes';
import { erase, unique } from '../../prelude/array'; import { erase } from '../../prelude/array';
import insertNoteUnread from './unread'; import insertNoteUnread from './unread';
import registerInstance from '../register-instance'; import registerInstance from '../register-instance';
import Instance from '../../models/instance'; import Instance from '../../models/instance';
import { Node } from '../../mfm/parser';
import extractMentions from '../../misc/extract-mentions'; import extractMentions from '../../misc/extract-mentions';
import extractEmojis from '../../misc/extract-emojis';
import extractHashtags from '../../misc/extract-hashtags';
type NotificationType = 'reply' | 'renote' | 'quote' | 'mention'; type NotificationType = 'reply' | 'renote' | 'quote' | 'mention';
@ -466,44 +467,6 @@ async function insertNote(user: IUser, data: Option, tags: string[], emojis: str
} }
} }
function extractHashtags(tokens: ReturnType<typeof parse>): string[] {
const hashtags: string[] = [];
const extract = (tokens: Node[]) => {
for (const x of tokens.filter(x => x.name === 'hashtag')) {
hashtags.push(x.props.hashtag);
}
for (const x of tokens.filter(x => x.children)) {
extract(x.children);
}
};
// Extract hashtags
extract(tokens);
return unique(hashtags);
}
export function extractEmojis(tokens: ReturnType<typeof parse>): string[] {
const emojis: string[] = [];
const extract = (tokens: Node[]) => {
for (const x of tokens.filter(x => x.name === 'emoji')) {
if (x.props.name && x.props.name.length <= 100) {
emojis.push(x.props.name);
}
}
for (const x of tokens.filter(x => x.children)) {
extract(x.children);
}
};
// Extract emojis
extract(tokens);
return unique(emojis);
}
function index(note: INote) { function index(note: INote) {
if (note.text == null || config.elasticsearch == null) return; if (note.text == null || config.elasticsearch == null) return;

48
test/extract-mentions.ts Normal file
View file

@ -0,0 +1,48 @@
import * as assert from 'assert';
import extractMentions from '../src/misc/extract-mentions';
import parse from '../src/mfm/parse';
describe('Extract mentions', () => {
it('simple', () => {
const ast = parse('@foo @bar @baz');
const mentions = extractMentions(ast);
assert.deepStrictEqual(mentions, [{
username: 'foo',
acct: '@foo',
canonical: '@foo',
host: null
}, {
username: 'bar',
acct: '@bar',
canonical: '@bar',
host: null
}, {
username: 'baz',
acct: '@baz',
canonical: '@baz',
host: null
}]);
});
it('nested', () => {
const ast = parse('@foo **@bar** @baz');
const mentions = extractMentions(ast);
assert.deepStrictEqual(mentions, [{
username: 'foo',
acct: '@foo',
canonical: '@foo',
host: null
}, {
username: 'bar',
acct: '@bar',
canonical: '@bar',
host: null
}, {
username: 'baz',
acct: '@baz',
canonical: '@baz',
host: null
}]);
});
});

File diff suppressed because it is too large Load diff