support custom emoji in editor and completions

AndrewRyanChama · AndrewRyanChama · commit f71b1e3c8fe6 · 2022-03-19T14:34:31.000-07:00
diff --git a/res/css/views/elements/_RichText.scss b/res/css/views/elements/_RichText.scss
@@ -14,6 +14,14 @@
     padding-left: 0;
 }
 
+.mx_CustomEmojiPill {
+    display: inline-flex;
+    align-items: center;
+    vertical-align: middle;
+    padding-left: 1px;
+    font-size: 0;
+}
+
 a.mx_Pill {
     text-overflow: ellipsis;
     white-space: nowrap;
diff --git a/res/css/views/rooms/_BasicMessageComposer.scss b/res/css/views/rooms/_BasicMessageComposer.scss
@@ -73,6 +73,24 @@ limitations under the License.
                 }
             }
 
+            span.mx_CustomEmojiPill {
+                position: relative;
+                user-select: all;
+
+                // avatar psuedo element
+                &::before {
+                    content: var(--avatar-letter);
+                    width: $font-18px;
+                    height: $font-18px;
+                    background: var(--avatar-background), $background;
+                    color: $avatar-initial-color;
+                    background-repeat: no-repeat;
+                    background-size: $font-18px;
+                    text-align: center;
+                    font-weight: normal;
+                }
+            }
+
             span.mx_UserPill {
                 cursor: pointer;
             }
diff --git a/src/Markdown.ts b/src/Markdown.ts
@@ -41,6 +41,9 @@ function isAllowedHtmlTag(node: commonmark.Node): boolean {
     if (node.literal != null &&
         node.literal.match('^<((div|span) data-mx-maths="[^"]*"|/(div|span))>$') != null) {
         return true;
+    } else if (node.literal != null &&
+        node.literal.match('^<img data-mx-emoticon') != null) {
+        return true;
     }
 
     // Regex won't work for tags with attrs, but we only
diff --git a/src/autocomplete/Autocompleter.ts b/src/autocomplete/Autocompleter.ts
@@ -37,11 +37,11 @@ export interface ISelectionRange {
 }
 
 export interface ICompletion {
-    type: "at-room" | "command" | "community" | "room" | "user";
+    type?: "at-room" | "command" | "community" | "room" | "user" | "customEmoji";
     completion: string;
     completionId?: string;
     component?: ReactElement;
-    range: ISelectionRange;
+    range?: ISelectionRange;
     command?: string;
     suffix?: string;
     // If provided, apply a LINK entity to the completion with the
diff --git a/src/autocomplete/EmojiProvider.tsx b/src/autocomplete/EmojiProvider.tsx
@@ -18,9 +18,10 @@ limitations under the License.
 */
 
 import React from 'react';
-import { uniq, sortBy } from 'lodash';
+import { uniq, sortBy, ListIteratee } from 'lodash';
 import EMOTICON_REGEX from 'emojibase-regex/emoticon';
 import { Room } from 'matrix-js-sdk/src/models/room';
+import { MatrixEvent } from 'matrix-js-sdk/src/matrix';
 
 import { _t } from '../languageHandler';
 import AutocompleteProvider from './AutocompleteProvider';
@@ -30,6 +31,7 @@ import { ICompletion, ISelectionRange } from './Autocompleter';
 import SettingsStore from "../settings/SettingsStore";
 import { EMOJI, IEmoji } from '../emoji';
 import { TimelineRenderingType } from '../contexts/RoomContext';
+import { mediaFromMxc } from '../customisations/Media';
 
 const LIMIT = 20;
 
@@ -38,10 +40,16 @@ const LIMIT = 20;
 const EMOJI_REGEX = new RegExp('(' + EMOTICON_REGEX.source + '|(?:^|\\s):[+-\\w]*:?)$', 'g');
 
 interface ISortedEmoji {
-    emoji: IEmoji;
+    emoji: IEmoji | ICustomEmoji;
     _orderBy: number;
 }
 
+export interface ICustomEmoji {
+    shortcodes: string[];
+    emoticon?: string;
+    url: string;
+}
+
 const SORTED_EMOJI: ISortedEmoji[] = EMOJI.sort((a, b) => {
     if (a.group === b.group) {
         return a.order - b.order;
@@ -65,6 +73,7 @@ function score(query, space) {
 export default class EmojiProvider extends AutocompleteProvider {
     matcher: QueryMatcher<ISortedEmoji>;
     nameMatcher: QueryMatcher<ISortedEmoji>;
+    customEmojiMatcher: QueryMatcher<ISortedEmoji>;
 
     constructor(room: Room, renderingType?: TimelineRenderingType) {
         super({ commandRegex: EMOJI_REGEX, renderingType });
@@ -74,11 +83,42 @@ export default class EmojiProvider extends AutocompleteProvider {
             // For matching against ascii equivalents
             shouldMatchWordsOnly: false,
         });
-        this.nameMatcher = new QueryMatcher(SORTED_EMOJI, {
+        this.nameMatcher = new QueryMatcher<ISortedEmoji>(SORTED_EMOJI, {
             keys: ['emoji.annotation'],
             // For removing punctuation
             shouldMatchWordsOnly: true,
         });
+
+        // Load this room's image sets.
+        const loadedImages: ICustomEmoji[] = [];
+        const imageSetEvents = room.currentState.getStateEvents('im.ponies.room_emotes');
+        imageSetEvents.forEach(imageSetEvent => {
+            this.loadImageSet(loadedImages, imageSetEvent);
+        });
+        const sortedCustomImages = loadedImages.map((emoji, index) => ({
+            emoji,
+            // Include the index so that we can preserve the original order
+            _orderBy: index,
+        }));
+        this.customEmojiMatcher = new QueryMatcher<ISortedEmoji>(sortedCustomImages, {
+            keys: [],
+            funcs: [o => o.emoji.shortcodes.map(s => `:${s}:`)],
+            shouldMatchWordsOnly: true,
+        });
+    }
+
+    private loadImageSet(loadedImages: ICustomEmoji[], imageSetEvent: MatrixEvent): void {
+        const images = imageSetEvent.getContent().images;
+        if (!images) {
+            return;
+        }
+        for (const imageKey in images) {
+            const imageData = images[imageKey];
+            loadedImages.push({
+                shortcodes: [imageKey],
+                url: imageData.url,
+            });
+        }
     }
 
     async getCompletions(
@@ -91,17 +131,23 @@ export default class EmojiProvider extends AutocompleteProvider {
             return []; // don't give any suggestions if the user doesn't want them
         }
 
-        let completions = [];
+        let completionResult: ICompletion[] = [];
         const { command, range } = this.getCurrentCommand(query, selection);
 
         if (command && command[0].length > 2) {
+            let completions: ISortedEmoji[] = [];
+
+            // find completions
             const matchedString = command[0];
             completions = this.matcher.match(matchedString, limit);
 
             // Do second match with shouldMatchWordsOnly in order to match against 'name'
-            completions = completions.concat(this.nameMatcher.match(matchedString));
+            completions = completions.concat(this.nameMatcher.match(matchedString, limit));
+
+            // do a match for the custom emoji
+            completions = completions.concat(this.customEmojiMatcher.match(matchedString, limit));
 
-            const sorters = [];
+            const sorters: ListIteratee<ISortedEmoji>[] = [];
             // make sure that emoticons come first
             sorters.push(c => score(matchedString, c.emoji.emoticon || ""));
 
@@ -121,17 +167,41 @@ export default class EmojiProvider extends AutocompleteProvider {
             sorters.push(c => c._orderBy);
             completions = sortBy(uniq(completions), sorters);
 
-            completions = completions.map(c => ({
-                completion: c.emoji.unicode,
-                component: (
-                    <PillCompletion title={`:${c.emoji.shortcodes[0]}:`} aria-label={c.emoji.unicode}>
-                        <span>{ c.emoji.unicode }</span>
-                    </PillCompletion>
-                ),
-                range,
-            })).slice(0, LIMIT);
+            completionResult = completions.map(c => {
+                if ('unicode' in c.emoji) {
+                    return {
+                        completion: c.emoji.unicode,
+                        component: (
+                            <PillCompletion title={`:${c.emoji.shortcodes[0]}:`} aria-label={c.emoji.unicode}>
+                                <span>{ c.emoji.unicode }</span>
+                            </PillCompletion>
+                        ),
+                        range,
+                    };
+                } else {
+                    const mediaUrl = mediaFromMxc(c.emoji.url).getThumbnailOfSourceHttp(24, 24, 'scale');
+                    return {
+                        completion: c.emoji.shortcodes[0],
+                        type: "customEmoji",
+                        completionId: c.emoji.url,
+                        component: (
+                            <PillCompletion title={`:${c.emoji.shortcodes[0]}:`}>
+                                <img
+                                    className="mx_BaseAvatar_image"
+                                    src={mediaUrl}
+                                    alt={c.emoji.shortcodes[0]}
+                                    style={{
+                                        width: '24px',
+                                        height: '24px',
+                                    }} />
+                            </PillCompletion>
+                        ),
+                        range,
+                    } as const;
+                }
+            }).slice(0, LIMIT);
         }
-        return completions;
+        return completionResult;
     }
 
     getName() {
diff --git a/src/autocomplete/UserProvider.tsx b/src/autocomplete/UserProvider.tsx
@@ -54,7 +54,7 @@ export default class UserProvider extends AutocompleteProvider {
             renderingType,
         });
         this.room = room;
-        this.matcher = new QueryMatcher([], {
+        this.matcher = new QueryMatcher<RoomMember>([], {
             keys: ['name'],
             funcs: [obj => obj.userId.slice(1)], // index by user id minus the leading '@'
             shouldMatchWordsOnly: false,
diff --git a/src/editor/autocomplete.ts b/src/editor/autocomplete.ts
@@ -109,6 +109,8 @@ export default class AutocompleteWrapperModel {
             case "command":
                 // command needs special handling for auto complete, but also renders as plain texts
                 return [(this.partCreator as CommandPartCreator).command(text)];
+            case "customEmoji":
+                return [this.partCreator.customEmoji(text, completionId)];
             default:
                 // used for emoji and other plain text completion replacement
                 return this.partCreator.plainWithEmoji(text);
diff --git a/src/editor/parts.ts b/src/editor/parts.ts
@@ -31,14 +31,15 @@ import * as Avatar from "../Avatar";
 import defaultDispatcher from "../dispatcher/dispatcher";
 import { Action } from "../dispatcher/actions";
 import SettingsStore from "../settings/SettingsStore";
+import { mediaFromMxc } from "../customisations/Media";
 
 interface ISerializedPart {
     type: Type.Plain | Type.Newline | Type.Emoji | Type.Command | Type.PillCandidate;
     text: string;
 }
 
 interface ISerializedPillPart {
-    type: Type.AtRoomPill | Type.RoomPill | Type.UserPill;
+    type: Type.AtRoomPill | Type.RoomPill | Type.UserPill | Type.CustomEmoji;
     text: string;
     resourceId?: string;
 }
@@ -49,6 +50,7 @@ export enum Type {
     Plain = "plain",
     Newline = "newline",
     Emoji = "emoji",
+    CustomEmoji = "custom-emoji",
     Command = "command",
     UserPill = "user-pill",
     RoomPill = "room-pill",
@@ -80,7 +82,7 @@ interface IPillCandidatePart extends Omit<IBasePart, "type" | "createAutoComplet
 }
 
 interface IPillPart extends Omit<IBasePart, "type" | "resourceId"> {
-    type: Type.AtRoomPill | Type.RoomPill | Type.UserPill;
+    type: Type.AtRoomPill | Type.RoomPill | Type.UserPill | Type.CustomEmoji;
     resourceId: string;
 }
 
@@ -403,6 +405,34 @@ class EmojiPart extends BasePart implements IBasePart {
     }
 }
 
+class CustomEmojiPart extends PillPart implements IPillPart {
+    protected get className(): string {
+        return "mx_CustomEmojiPill";
+    }
+    protected setAvatar(node: HTMLElement): void {
+        const url = mediaFromMxc(this.resourceId).getThumbnailOfSourceHttp(24, 24, "crop");
+        this.setAvatarVars(node, url, this.text[0]);
+    }
+    constructor(shortCode: string, url: string) {
+        super(url, shortCode);
+    }
+    protected acceptsInsertion(chr: string): boolean {
+        return false;
+    }
+
+    protected acceptsRemoval(position: number, chr: string): boolean {
+        return false;
+    }
+
+    public get type(): IPillPart["type"] {
+        return Type.CustomEmoji;
+    }
+
+    public get canEdit(): boolean {
+        return false;
+    }
+}
+
 class RoomPillPart extends PillPart {
     constructor(resourceId: string, label: string, private room: Room) {
         super(resourceId, label);
@@ -574,6 +604,8 @@ export class PartCreator {
                 return this.newline();
             case Type.Emoji:
                 return this.emoji(part.text);
+            case Type.CustomEmoji:
+                return this.customEmoji(part.text, part.resourceId);
             case Type.AtRoomPill:
                 return this.atRoomPill(part.text);
             case Type.PillCandidate:
@@ -645,6 +677,10 @@ export class PartCreator {
         return parts;
     }
 
+    public customEmoji(shortcode: string, url: string) {
+        return new CustomEmojiPart(shortcode, url);
+    }
+
     public createMentionParts(
         insertTrailingCharacter: boolean,
         displayName: string,
diff --git a/src/editor/serialize.ts b/src/editor/serialize.ts
@@ -17,6 +17,7 @@ limitations under the License.
 
 import { AllHtmlEntities } from 'html-entities';
 import cheerio from 'cheerio';
+import _ from 'lodash';
 
 import Markdown from '../Markdown';
 import { makeGenericPermalink } from "../utils/permalinks/Permalinks";
@@ -44,6 +45,10 @@ export function mdSerialize(model: EditorModel): string {
             case Type.UserPill:
                 return html +
                     `[${part.text.replace(/[[\\\]]/g, c => "\\" + c)}](${makeGenericPermalink(part.resourceId)})`;
+            case Type.CustomEmoji:
+                return html +
+                    `<img data-mx-emoticon height="18" src="${encodeURI(part.resourceId)}"`
+                    + ` title=":${_.escape(part.text)}:" alt=":${_.escape(part.text)}:">`;
         }
     }, "");
 }
@@ -176,6 +181,8 @@ export function textSerialize(model: EditorModel): string {
                 return text + `${part.resourceId}`;
             case Type.UserPill:
                 return text + `${part.text}`;
+            case Type.CustomEmoji:
+                return text + `:${part.text}:`;
         }
     }, "");
 }
diff --git a/test/editor/serialize-test.js b/test/editor/serialize-test.js
@@ -38,6 +38,13 @@ describe('editor/serialize', function() {
         const html = htmlSerializeIfNeeded(model, {});
         expect(html).toBeFalsy();
     });
+    it('custom emoji pill turns message into html', function() {
+        const pc = createPartCreator();
+        const model = new EditorModel([pc.customEmoji("poggers", "mxc://matrix.org/test")]);
+        const html = htmlSerializeIfNeeded(model, {});
+        expect(html).toBe("<img data-mx-emoticon height=\"18\" src=\"mxc://matrix.org/test\""
+        + " title=\":poggers:\" alt=\":poggers:\">");
+    });
     it('any markdown turns message into html', function() {
         const pc = createPartCreator();
         const model = new EditorModel([pc.plain("*hello* world")]);

Original file line number	Diff line number	Diff line change
`@@ -41,6 +41,9 @@ function isAllowedHtmlTag(node: commonmark.Node): boolean {`
`41`	`41`	`if (node.literal != null &&`
`42`	`42`	`node.literal.match('^<((div\|span) data-mx-maths="[^"]*"\|/(div\|span))>$') != null) {`
`43`	`43`	`return true;`
	`44`	`+ } else if (node.literal != null &&`
	`45`	`+ node.literal.match('^<img data-mx-emoticon') != null) {`
	`46`	`+ return true;`
`44`	`47`	`}`
`45`	`48`
`46`	`49`	`// Regex won't work for tags with attrs, but we only`
Original file line number	Diff line number	Diff line change
`@@ -17,6 +17,7 @@ limitations under the License.`
`17`	`17`
`18`	`18`	`import { AllHtmlEntities } from 'html-entities';`
`19`	`19`	`import cheerio from 'cheerio';`
	`20`	`+import _ from 'lodash';`
`20`	`21`
`21`	`22`	`import Markdown from '../Markdown';`
`22`	`23`	`import { makeGenericPermalink } from "../utils/permalinks/Permalinks";`
`@@ -44,6 +45,10 @@ export function mdSerialize(model: EditorModel): string {`
`44`	`45`	`case Type.UserPill:`
`45`	`46`	`return html +`
`46`	`47`	`[${part.text.replace(/[[\\\]]/g, c => "\\" + c)}](${makeGenericPermalink(part.resourceId)})`;
	`48`	`+ case Type.CustomEmoji:`
	`49`	`+ return html +`
	`50`	+ `<img data-mx-emoticon height="18" src="${encodeURI(part.resourceId)}"`
	`51`	+ + ` title=":${_.escape(part.text)}:" alt=":${_.escape(part.text)}:">`;
`47`	`52`	`}`
`48`	`53`	`}, "");`
`49`	`54`	`}`
`@@ -176,6 +181,8 @@ export function textSerialize(model: EditorModel): string {`
`176`	`181`	return text + `${part.resourceId}`;
`177`	`182`	`case Type.UserPill:`
`178`	`183`	return text + `${part.text}`;
	`184`	`+ case Type.CustomEmoji:`
	`185`	+ return text + `:${part.text}:`;
`179`	`186`	`}`
`180`	`187`	`}, "");`
`181`	`188`	`}`