Refactor formatter (#17828)

* Refactor formatter

* Move custom emoji pre-rendering logic to view helpers

* Move more methods out of Formatter

* Fix code style issues

* Remove Formatter

* Add inline poll options to RSS feeds

* Remove unused helper method

* Fix code style issues

* Various fixes and improvements

* Fix test

app/chewy/statuses_index.rb

@@ -57,7 +57,7 @@ class StatusesIndex < Chewy::Index
     field :id, type: 'long'
     field :account_id, type: 'long'
 
-    field :text, type: 'text', value: ->(status) { [status.spoiler_text, Formatter.instance.plaintext(status)].concat(status.ordered_media_attachments.map(&:description)).concat(status.preloadable_poll ? status.preloadable_poll.options : []).join("\n\n") } do
+    field :text, type: 'text', value: ->(status) { [status.spoiler_text, PlainTextFormatter.new(status.text, status.local?).to_s].concat(status.ordered_media_attachments.map(&:description)).concat(status.preloadable_poll ? status.preloadable_poll.options : []).join("\n\n") } do
       field :stemmed, type: 'text', analyzer: 'content'
     end
 

app/controllers/api/web/embeds_controller.rb

@@ -15,7 +15,7 @@ class Api::Web::EmbedsController < Api::Web::BaseController
     return not_found if oembed.nil?
 
     begin
-      oembed[:html] = Formatter.instance.sanitize(oembed[:html], Sanitize::Config::MASTODON_OEMBED)
+      oembed[:html] = Sanitize.fragment(oembed[:html], Sanitize::Config::MASTODON_OEMBED)
     rescue ArgumentError
       return not_found
     end

app/helpers/accounts_helper.rb

@@ -2,10 +2,12 @@
 
 module AccountsHelper
   def display_name(account, **options)
+    str = account.display_name.presence || account.username
+
     if options[:custom_emojify]
-      Formatter.instance.format_display_name(account, **options)
+      prerender_custom_emojis(h(str), account.emojis)
     else
-      account.display_name.presence || account.username
+      str
     end
   end
 

app/helpers/admin/trends/statuses_helper.rb

@@ -12,9 +12,6 @@ module Admin::Trends::StatusesHelper
 
     return '' if text.blank?
 
-    html = Formatter.instance.send(:encode, text)
-    html = Formatter.instance.send(:encode_custom_emojis, html, status.emojis, prefers_autoplay?)
-
-    html.html_safe # rubocop:disable Rails/OutputSafety
+    prerender_custom_emojis(h(text), status.emojis)
   end
 end

app/helpers/application_helper.rb

@@ -239,4 +239,8 @@ module ApplicationHelper
       end
     end.values
   end
+
+  def prerender_custom_emojis(html, custom_emojis)
+    EmojiFormatter.new(html, custom_emojis, animate: prefers_autoplay?).to_s
+  end
 end

app/helpers/formatting_helper.rb

+# frozen_string_literal: true
+
+module FormattingHelper
+  def html_aware_format(text, local, options = {})
+    HtmlAwareFormatter.new(text, local, options).to_s
+  end
+
+  def linkify(text, options = {})
+    TextFormatter.new(text, options).to_s
+  end
+
+  def extract_plain_text(text, local)
+    PlainTextFormatter.new(text, local).to_s
+  end
+
+  def status_content_format(status)
+    html_aware_format(status.text, status.local?, preloaded_accounts: [status.account] + (status.respond_to?(:active_mentions) ? status.active_mentions.map(&:account) : []))
+  end
+end

app/helpers/routing_helper.rb

@@ -2,6 +2,7 @@
 
 module RoutingHelper
   extend ActiveSupport::Concern
+
   include Rails.application.routes.url_helpers
   include ActionView::Helpers::AssetTagHelper
   include Webpacker::Helper
@@ -22,8 +23,6 @@ module RoutingHelper
     full_asset_url(asset_pack_path(source, **options))
   end
 
-  private
-
   def use_storage?
     Rails.configuration.x.use_s3 || Rails.configuration.x.use_swift
   end

app/helpers/statuses_helper.rb

@@ -113,20 +113,6 @@ module StatusesHelper
     end
   end
 
-  private
-
-  def simplified_text(text)
-    text.dup.tap do |new_text|
-      URI.extract(new_text).each do |url|
-        new_text.gsub!(url, '')
-      end
-
-      new_text.gsub!(Account::MENTION_RE, '')
-      new_text.gsub!(Tag::HASHTAG_RE, '')
-      new_text.gsub!(/\s+/, '')
-    end
-  end
-
   def embedded_view?
     params[:controller] == EMBEDDED_CONTROLLER && params[:action] == EMBEDDED_ACTION
   end

app/lib/activitypub/activity/create.rb

 # frozen_string_literal: true
 
 class ActivityPub::Activity::Create < ActivityPub::Activity
+  include FormattingHelper
+
   def perform
     dereference_object!
 
@@ -367,7 +369,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
   end
 
   def converted_text
-    Formatter.instance.linkify([@status_parser.title.presence, @status_parser.spoiler_text.presence, @status_parser.url || @status_parser.uri].compact.join("\n\n"))
+    linkify([@status_parser.title.presence, @status_parser.spoiler_text.presence, @status_parser.url || @status_parser.uri].compact.join("\n\n"))
   end
 
   def unsupported_media_type?(mime_type)

app/lib/emoji_formatter.rb

+# frozen_string_literal: true
+
+class EmojiFormatter
+  include RoutingHelper
+
+  DISALLOWED_BOUNDING_REGEX = /[[:alnum:]:]/.freeze
+
+  attr_reader :html, :custom_emojis, :options
+
+  # @param [ActiveSupport::SafeBuffer] html
+  # @param [Array<CustomEmoji>] custom_emojis
+  # @param [Hash] options
+  # @option options [Boolean] :animate
+  def initialize(html, custom_emojis, options = {})
+    raise ArgumentError unless html.html_safe?
+
+    @html = html
+    @custom_emojis = custom_emojis
+    @options = options
+  end
+
+  def to_s
+    return html if custom_emojis.empty? || html.blank?
+
+    i                     = -1
+    tag_open_index        = nil
+    inside_shortname      = false
+    shortname_start_index = -1
+    invisible_depth       = 0
+    last_index            = 0
+    result                = ''.dup
+
+    while i + 1 < html.size
+      i += 1
+
+      if invisible_depth.zero? && inside_shortname && html[i] == ':'
+        inside_shortname = false
+        shortcode = html[shortname_start_index + 1..i - 1]
+        char_after = html[i + 1]
+
+        next unless (char_after.nil? || !DISALLOWED_BOUNDING_REGEX.match?(char_after)) && (emoji = emoji_map[shortcode])
+
+        result << html[last_index..shortname_start_index - 1] if shortname_start_index.positive?
+        result << image_for_emoji(shortcode, emoji)
+        last_index = i + 1
+      elsif tag_open_index && html[i] == '>'
+        tag = html[tag_open_index..i]
+        tag_open_index = nil
+
+        if invisible_depth.positive?
+          invisible_depth += count_tag_nesting(tag)
+        elsif tag == '<span class="invisible">'
+          invisible_depth = 1
+        end
+      elsif html[i] == '<'
+        tag_open_index = i
+        inside_shortname = false
+      elsif !tag_open_index && html[i] == ':' && (i.zero? || !DISALLOWED_BOUNDING_REGEX.match?(html[i - 1]))
+        inside_shortname = true
+        shortname_start_index = i
+      end
+    end
+
+    result << html[last_index..-1]
+
+    result.html_safe # rubocop:disable Rails/OutputSafety
+  end
+
+  private
+
+  def emoji_map
+    @emoji_map ||= custom_emojis.each_with_object({}) { |e, h| h[e.shortcode] = [full_asset_url(e.image.url), full_asset_url(e.image.url(:static))] }
+  end
+
+  def count_tag_nesting(tag)
+    if tag[1] == '/'
+      -1
+    elsif tag[-2] == '/'
+      0
+    else
+      1
+    end
+  end
+
+  def image_for_emoji(shortcode, emoji)
+    original_url, static_url = emoji
+
+    if animate?
+      image_tag(original_url, draggable: false, class: 'emojione', alt: ":#{shortcode}:", title: ":#{shortcode}:")
+    else
+      image_tag(original_url, draggable: false, class: 'emojione custom-emoji', alt: ":#{shortcode}:", title: ":#{shortcode}:", data: { original: original_url, static: static_url })
+    end
+  end
+
+  def animate?
+    @options[:animate]
+  end
+end

app/lib/extractor.rb

@@ -5,18 +5,34 @@ module Extractor
 
   module_function
 
-  # :yields: username, list_slug, start, end
+  def extract_entities_with_indices(text, options = {}, &block)
+    entities = begin
+      extract_urls_with_indices(text, options) +
+        extract_hashtags_with_indices(text, check_url_overlap: false) +
+        extract_mentions_or_lists_with_indices(text) +
+        extract_extra_uris_with_indices(text)
+    end
+
+    return [] if entities.empty?
+
+    entities = remove_overlapping_entities(entities)
+    entities.each(&block) if block_given?
+    entities
+  end
+
   def extract_mentions_or_lists_with_indices(text)
-    return [] unless Twitter::TwitterText::Regex[:at_signs].match?(text)
+    return [] unless text && Twitter::TwitterText::Regex[:at_signs].match?(text)
 
     possible_entries = []
 
-    text.to_s.scan(Account::MENTION_RE) do |screen_name, _|
+    text.scan(Account::MENTION_RE) do |screen_name, _|
       match_data = $LAST_MATCH_INFO
-      after = $'
+      after      = $'
+
       unless Twitter::TwitterText::Regex[:end_mention_match].match?(after)
         start_position = match_data.char_begin(1) - 1
-        end_position = match_data.char_end(1)
+        end_position   = match_data.char_end(1)
+
         possible_entries << {
           screen_name: screen_name,
           indices: [start_position, end_position],
@@ -29,36 +45,70 @@ module Extractor
         yield mention[:screen_name], mention[:indices].first, mention[:indices].last
       end
     end
+
     possible_entries
   end
 
-  def extract_hashtags_with_indices(text, **)
-    return [] unless /#/.match?(text)
+  def extract_hashtags_with_indices(text, _options = {})
+    return [] unless text&.index('#')
+
+    possible_entries = []
 
-    tags = []
     text.scan(Tag::HASHTAG_RE) do |hash_text, _|
-      match_data = $LAST_MATCH_INFO
+      match_data     = $LAST_MATCH_INFO
       start_position = match_data.char_begin(1) - 1
-      end_position = match_data.char_end(1)
-      after = $'
+      end_position   = match_data.char_end(1)
+      after          = $'
+
       if %r{\A://}.match?(after)
         hash_text.match(/(.+)(https?\Z)/) do |matched|
-          hash_text = matched[1]
+          hash_text     = matched[1]
           end_position -= matched[2].codepoint_length
         end
       end
 
-      tags << {
+      possible_entries << {
         hashtag: hash_text,
         indices: [start_position, end_position],
       }
     end
 
-    tags.each { |tag| yield tag[:hashtag], tag[:indices].first, tag[:indices].last } if block_given?
-    tags
+    if block_given?
+      possible_entries.each do |tag|
+        yield tag[:hashtag], tag[:indices].first, tag[:indices].last
+      end
+    end
+
+    possible_entries
   end
 
   def extract_cashtags_with_indices(_text)
-    [] # always returns empty array
+    []
+  end
+
+  def extract_extra_uris_with_indices(text)
+    return [] unless text&.index(':')
+
+    possible_entries = []
+
+    text.scan(Twitter::TwitterText::Regex[:valid_extended_uri]) do
+      valid_uri_match_data = $LAST_MATCH_INFO
+
+      start_position = valid_uri_match_data.char_begin(3)
+      end_position   = valid_uri_match_data.char_end(3)
+
+      possible_entries << {
+        url: valid_uri_match_data[3],
+        indices: [start_position, end_position],
+      }
+    end
+
+    if block_given?
+      possible_entries.each do |url|
+        yield url[:url], url[:indices].first, url[:indices].last
+      end
+    end
+
+    possible_entries
   end
 end

app/lib/feed_manager.rb

@@ -5,6 +5,7 @@ require 'singleton'
 class FeedManager
   include Singleton
   include Redisable
+  include FormattingHelper
 
   # Maximum number of items stored in a single feed
   MAX_ITEMS = 400
@@ -445,7 +446,7 @@ class FeedManager
     status         = status.reblog if status.reblog?
 
     combined_text = [
-      Formatter.instance.plaintext(status),
+      extract_plain_text(status.text, status.local?),
       status.spoiler_text,
       status.preloadable_poll ? status.preloadable_poll.options.join("\n\n") : nil,
       status.ordered_media_attachments.map(&:description).join("\n\n"),

app/lib/formatter.rb

-# frozen_string_literal: true
-
-require 'singleton'
-
-class Formatter
-  include Singleton
-  include RoutingHelper
-
-  include ActionView::Helpers::TextHelper
-
-  def format(status, **options)
-    if status.respond_to?(:reblog?) && status.reblog?
-      prepend_reblog = status.reblog.account.acct
-      status         = status.proper
-    else
-      prepend_reblog = false
-    end
-
-    raw_content = status.text
-
-    if options[:inline_poll_options] && status.preloadable_poll
-      raw_content = raw_content + "\n\n" + status.preloadable_poll.options.map { |title| "[ ] #{title}" }.join("\n")
-    end
-
-    return '' if raw_content.blank?
-
-    unless status.local?
-      html = reformat(raw_content)
-      html = encode_custom_emojis(html, status.emojis, options[:autoplay]) if options[:custom_emojify]
-      return html.html_safe # rubocop:disable Rails/OutputSafety
-    end
-
-    linkable_accounts = status.respond_to?(:active_mentions) ? status.active_mentions.map(&:account) : []
-    linkable_accounts << status.account
-
-    html = raw_content
-    html = "RT @#{prepend_reblog} #{html}" if prepend_reblog
-    html = encode_and_link_urls(html, linkable_accounts)
-    html = encode_custom_emojis(html, status.emojis, options[:autoplay]) if options[:custom_emojify]
-    html = simple_format(html, {}, sanitize: false)
-    html = html.delete("\n")
-
-    html.html_safe # rubocop:disable Rails/OutputSafety
-  end
-
-  def reformat(html)
-    sanitize(html, Sanitize::Config::MASTODON_STRICT)
-  rescue ArgumentError
-    ''
-  end
-
-  def plaintext(status)
-    return status.text if status.local?
-
-    text = status.text.gsub(/(<br \/>|<br>|<\/p>)+/) { |match| "#{match}\n" }
-    strip_tags(text)
-  end
-
-  def simplified_format(account, **options)
-    return '' if account.note.blank?
-
-    html = account.local? ? linkify(account.note) : reformat(account.note)
-    html = encode_custom_emojis(html, account.emojis, options[:autoplay]) if options[:custom_emojify]
-    html.html_safe # rubocop:disable Rails/OutputSafety
-  end
-
-  def sanitize(html, config)
-    Sanitize.fragment(html, config)
-  end
-
-  def format_spoiler(status, **options)
-    html = encode(status.spoiler_text)
-    html = encode_custom_emojis(html, status.emojis, options[:autoplay])
-    html.html_safe # rubocop:disable Rails/OutputSafety
-  end
-
-  def format_poll_option(status, option, **options)
-    html = encode(option.title)
-    html = encode_custom_emojis(html, status.emojis, options[:autoplay])
-    html.html_safe # rubocop:disable Rails/OutputSafety
-  end
-
-  def format_display_name(account, **options)
-    html = encode(account.display_name.presence || account.username)
-    html = encode_custom_emojis(html, account.emojis, options[:autoplay]) if options[:custom_emojify]
-    html.html_safe # rubocop:disable Rails/OutputSafety
-  end
-
-  def format_field(account, str, **options)
-    html = account.local? ? encode_and_link_urls(str, me: true, with_domain: true) : reformat(str)
-    html = encode_custom_emojis(html, account.emojis, options[:autoplay]) if options[:custom_emojify]
-    html.html_safe # rubocop:disable Rails/OutputSafety
-  end
-
-  def linkify(text)
-    html = encode_and_link_urls(text)
-    html = simple_format(html, {}, sanitize: false)
-    html = html.delete("\n")
-
-    html.html_safe # rubocop:disable Rails/OutputSafety
-  end
-
-  private
-
-  def html_entities
-    @html_entities ||= HTMLEntities.new
-  end
-
-  def encode(html)
-    html_entities.encode(html)
-  end
-
-  def encode_and_link_urls(html, accounts = nil, options = {})
-    entities = utf8_friendly_extractor(html, extract_url_without_protocol: false)
-
-    if accounts.is_a?(Hash)
-      options  = accounts
-      accounts = nil
-    end
-
-    rewrite(html.dup, entities) do |entity|
-      if entity[:url]
-        link_to_url(entity, options)
-      elsif entity[:hashtag]
-        link_to_hashtag(entity)
-      elsif entity[:screen_name]
-        link_to_mention(entity, accounts, options)
-      end
-    end
-  end
-
-  def count_tag_nesting(tag)
-    if tag[1] == '/' then -1
-    elsif tag[-2] == '/' then 0
-    else 1
-    end
-  end
-
-  # rubocop:disable Metrics/BlockNesting
-  def encode_custom_emojis(html, emojis, animate = false)
-    return html if emojis.empty?
-
-    emoji_map = emojis.each_with_object({}) { |e, h| h[e.shortcode] = [full_asset_url(e.image.url), full_asset_url(e.image.url(:static))] }
-
-    i                     = -1
-    tag_open_index        = nil
-    inside_shortname      = false
-    shortname_start_index = -1
-    invisible_depth       = 0
-
-    while i + 1 < html.size
-      i += 1
-
-      if invisible_depth.zero? && inside_shortname && html[i] == ':'
-        shortcode = html[shortname_start_index + 1..i - 1]
-        emoji     = emoji_map[shortcode]
-
-        if emoji
-          original_url, static_url = emoji
-          replacement = begin
-            if animate
-              image_tag(original_url, draggable: false, class: 'emojione', alt: ":#{shortcode}:", title: ":#{shortcode}:")
-            else
-              image_tag(original_url, draggable: false, class: 'emojione custom-emoji', alt: ":#{shortcode}:", title: ":#{shortcode}:", data: { original: original_url, static: static_url })
-            end
-          end
-          before_html = shortname_start_index.positive? ? html[0..shortname_start_index - 1] : ''
-          html        = before_html + replacement + html[i + 1..-1]
-          i          += replacement.size - (shortcode.size + 2) - 1
-        else
-          i -= 1
-        end
-
-        inside_shortname = false
-      elsif tag_open_index && html[i] == '>'
-        tag = html[tag_open_index..i]
-        tag_open_index = nil
-        if invisible_depth.positive?
-          invisible_depth += count_tag_nesting(tag)
-        elsif tag == '<span class="invisible">'
-          invisible_depth = 1
-        end
-      elsif html[i] == '<'
-        tag_open_index   = i
-        inside_shortname = false
-      elsif !tag_open_index && html[i] == ':'
-        inside_shortname      = true
-        shortname_start_index = i
-      end
-    end
-
-    html
-  end
-  # rubocop:enable Metrics/BlockNesting
-
-  def rewrite(text, entities)
-    text = text.to_s
-
-    # Sort by start index
-    entities = entities.sort_by do |entity|
-      indices = entity.respond_to?(:indices) ? entity.indices : entity[:indices]
-      indices.first
-    end
-
-    result = []
-
-    last_index = entities.reduce(0) do |index, entity|
-      indices = entity.respond_to?(:indices) ? entity.indices : entity[:indices]
-      result << encode(text[index...indices.first])
-      result << yield(entity)
-      indices.last
-    end
-
-    result << encode(text[last_index..-1])
-
-    result.flatten.join
-  end
-
-  def utf8_friendly_extractor(text, options = {})
-    # Note: I couldn't obtain list_slug with @user/list-name format
-    # for mention so this requires additional check
-    special = Extractor.extract_urls_with_indices(text, options)
-    standard = Extractor.extract_entities_with_indices(text, options)
-    extra = Extractor.extract_extra_uris_with_indices(text, options)
-
-    Extractor.remove_overlapping_entities(special + standard + extra)
-  end
-
-  def link_to_url(entity, options = {})
-    url        = Addressable::URI.parse(entity[:url])
-    html_attrs = { target: '_blank', rel: 'nofollow noopener noreferrer' }
-
-    html_attrs[:rel] = "me #{html_attrs[:rel]}" if options[:me]
-
-    Twitter::TwitterText::Autolink.send(:link_to_text, entity, link_html(entity[:url]), url, html_attrs)
-  rescue Addressable::URI::InvalidURIError, IDN::Idna::IdnaError
-    encode(entity[:url])
-  end
-
-  def link_to_mention(entity, linkable_accounts, options = {})
-    acct = entity[:screen_name]
-
-    return link_to_account(acct, options) unless linkable_accounts
-
-    same_username_hits = 0
-    account = nil
-    username, domain = acct.split('@')
-    domain = nil if TagManager.instance.local_domain?(domain)
-
-    linkable_accounts.each do |item|
-      same_username = item.username.casecmp(username).zero?
-      same_domain   = item.domain.nil? ? domain.nil? : item.domain.casecmp(domain)&.zero?
-
-      if same_username && !same_domain
-        same_username_hits += 1
-      elsif same_username && same_domain
-        account = item
-      end
-    end
-
-    account ? mention_html(account, with_domain: same_username_hits.positive? || options[:with_domain]) : "@#{encode(acct)}"
-  end
-
-  def link_to_account(acct, options = {})
-    username, domain = acct.split('@')
-
-    domain  = nil if TagManager.instance.local_domain?(domain)
-    account = EntityCache.instance.mention(username, domain)
-
-    account ? mention_html(account, with_domain: options[:with_domain]) : "@#{encode(acct)}"
-  end
-
-  def link_to_hashtag(entity)
-    hashtag_html(entity[:hashtag])
-  end
-
-  def link_html(url)
-    url    = Addressable::URI.parse(url).to_s
-    prefix = url.match(/\A(https?:\/\/(www\.)?|xmpp:)/).to_s
-    text   = url[prefix.length, 30]
-    suffix = url[prefix.length + 30..-1]
-    cutoff = url[prefix.length..-1].length > 30
-
-    "<span class=\"invisible\">#{encode(prefix)}</span><span class=\"#{cutoff ? 'ellipsis' : ''}\">#{encode(text)}</span><span class=\"invisible\">#{encode(suffix)}</span>"
-  end
-
-  def hashtag_html(tag)
-    "<a href=\"#{encode(tag_url(tag))}\" class=\"mention hashtag\" rel=\"tag\">#<span>#{encode(tag)}</span></a>"
-  end
-
-  def mention_html(account, with_domain: false)
-    "<span class=\"h-card\"><a href=\"#{encode(ActivityPub::TagManager.instance.url_for(account))}\" class=\"u-url mention\">@<span>#{encode(with_domain ? account.pretty_acct : account.username)}</span></a></span>"
-  end
-end

app/lib/html_aware_formatter.rb

+# frozen_string_literal: true
+
+class HtmlAwareFormatter
+  attr_reader :text, :local, :options
+
+  alias local? local
+
+  # @param [String] text
+  # @param [Boolean] local
+  # @param [Hash] options
+  def initialize(text, local, options = {})
+    @text    = text
+    @local   = local
+    @options = options
+  end
+
+  def to_s
+    return ''.html_safe if text.blank?
+
+    if local?
+      linkify
+    else
+      reformat.html_safe # rubocop:disable Rails/OutputSafety
+    end
+  rescue ArgumentError
+    ''.html_safe
+  end
+
+  private
+
+  def reformat
+    Sanitize.fragment(text, Sanitize::Config::MASTODON_STRICT)
+  end
+
+  def linkify
+    TextFormatter.new(text, options).to_s
+  end
+end

app/lib/plain_text_formatter.rb

+# frozen_string_literal: true
+
+class PlainTextFormatter
+  include ActionView::Helpers::TextHelper
+
+  NEWLINE_TAGS_RE = /(<br \/>|<br>|<\/p>)+/.freeze
+
+  attr_reader :text, :local
+
+  alias local? local
+
+  def initialize(text, local)
+    @text  = text
+    @local = local
+  end
+
+  def to_s
+    if local?
+      text
+    else
+      strip_tags(insert_newlines).chomp
+    end
+  end
+
+  private
+
+  def insert_newlines
+    text.gsub(NEWLINE_TAGS_RE) { |match| "#{match}\n" }
+  end
+end

app/lib/rss/serializer.rb

 # frozen_string_literal: true
 
 class RSS::Serializer
+  include FormattingHelper
+
   private
 
   def render_statuses(builder, statuses)
@@ -9,7 +11,7 @@ class RSS::Serializer
         item.title(status_title(status))
             .link(ActivityPub::TagManager.instance.url_for(status))
             .pub_date(status.created_at)
-            .description(status.spoiler_text.presence || Formatter.instance.format(status, inline_poll_options: true).to_str)
+            .description(status_description(status))
 
         status.ordered_media_attachments.each do |media|
           item.enclosure(full_asset_url(media.file.url(:original, false)), media.file.content_type, media.file.size)
@@ -19,9 +21,8 @@ class RSS::Serializer
   end
 
   def status_title(status)
-    return "#{status.account.acct} deleted status" if status.destroyed?
-
     preview = status.proper.spoiler_text.presence || status.proper.text
+
     if preview.length > 30 || preview[0, 30].include?("\n")
       preview = preview[0, 30]
       preview = preview[0, preview.index("\n").presence || 30] + '…'
@@ -35,4 +36,20 @@ class RSS::Serializer
       "#{status.account.acct}: #{preview}"
     end
   end
+
+  def status_description(status)
+    if status.proper.spoiler_text?
+      status.proper.spoiler_text
+    else
+      html = status_content_format(status.proper).to_str
+      after_html = ''
+
+      if status.proper.preloadable_poll
+        poll_options_html = status.proper.preloadable_poll.options.map { |o| "[ ] #{o}" }.join('<br />')
+        after_html = "<p>#{poll_options_html}</p>"
+      end
+
+      "#{html}#{after_html}"
+    end
+  end
 end

app/lib/text_formatter.rb

+# frozen_string_literal: true
+
+class TextFormatter
+  include ActionView::Helpers::TextHelper
+  include ERB::Util
+  include RoutingHelper
+
+  URL_PREFIX_REGEX = /\A(https?:\/\/(www\.)?|xmpp:)/.freeze
+
+  DEFAULT_REL = %w(nofollow noopener noreferrer).freeze
+
+  DEFAULT_OPTIONS = {
+    multiline: true,
+  }.freeze
+
+  attr_reader :text, :options
+
+  # @param [String] text
+  # @param [Hash] options
+  # @option options [Boolean] :multiline
+  # @option options [Boolean] :with_domains
+  # @option options [Boolean] :with_rel_me
+  # @option options [Array<Account>] :preloaded_accounts
+  def initialize(text, options = {})
+    @text    = text
+    @options = DEFAULT_OPTIONS.merge(options)
+  end
+
+  def entities
+    @entities ||= Extractor.extract_entities_with_indices(text, extract_url_without_protocol: false)
+  end
+
+  def to_s
+    return ''.html_safe if text.blank?
+
+    html = rewrite do |entity|
+      if entity[:url]
+        link_to_url(entity)
+      elsif entity[:hashtag]
+        link_to_hashtag(entity)
+      elsif entity[:screen_name]
+        link_to_mention(entity)
+      end
+    end
+
+    html = simple_format(html, {}, sanitize: false).delete("\n") if multiline?
+
+    html.html_safe # rubocop:disable Rails/OutputSafety
+  end
+
+  private
+
+  def rewrite
+    entities.sort_by! do |entity|
+      entity[:indices].first
+    end
+
+    result = ''.dup
+
+    last_index = entities.reduce(0) do |index, entity|
+      indices = entity[:indices]
+      result << h(text[index...indices.first])
+      result << yield(entity)
+      indices.last
+    end
+
+    result << h(text[last_index..-1])
+
+    result
+  end
+
+  def link_to_url(entity)
+    url = Addressable::URI.parse(entity[:url]).to_s
+    rel = with_rel_me? ? (DEFAULT_REL + %w(me)) : DEFAULT_REL
+
+    prefix      = url.match(URL_PREFIX_REGEX).to_s
+    display_url = url[prefix.length, 30]
+    suffix      = url[prefix.length + 30..-1]
+    cutoff      = url[prefix.length..-1].length > 30
+
+    <<~HTML.squish
+      <a href="#{h(url)}" target="_blank" rel="#{rel.join(' ')}"><span class="invisible">#{h(prefix)}</span><span class="#{cutoff ? 'ellipsis' : ''}">#{h(display_url)}</span><span class="invisible">#{h(suffix)}</span></a>
+    HTML
+  rescue Addressable::URI::InvalidURIError, IDN::Idna::IdnaError
+    h(entity[:url])
+  end
+
+  def link_to_hashtag(entity)
+    hashtag = entity[:hashtag]
+    url     = tag_url(hashtag)
+
+    <<~HTML.squish
+      <a href="#{h(url)}" class="mention hashtag" rel="tag">#<span>#{h(hashtag)}</span></a>
+    HTML
+  end
+
+  def link_to_mention(entity)
+    username, domain = entity[:screen_name].split('@')
+    domain           = nil if local_domain?(domain)
+    account          = nil
+
+    if preloaded_accounts?
+      same_username_hits = 0
+
+      preloaded_accounts.each do |other_account|
+        same_username = other_account.username.casecmp(username).zero?
+        same_domain   = other_account.domain.nil? ? domain.nil? : other_account.domain.casecmp(domain)&.zero?
+
+        if same_username && !same_domain
+          same_username_hits += 1
+        elsif same_username && same_domain
+          account = other_account
+        end
+      end
+    else
+      account = entity_cache.mention(username, domain)
+    end
+
+    return "@#{h(entity[:screen_name])}" if account.nil?
+
+    url = ActivityPub::TagManager.instance.url_for(account)
+    display_username = same_username_hits&.positive? || with_domains? ? account.pretty_acct : account.username
+
+    <<~HTML.squish
+      <span class="h-card"><a href="#{h(url)}" class="u-url mention">@<span>#{h(display_username)}</span></a></span>
+    HTML
+  end
+
+  def entity_cache
+    @entity_cache ||= EntityCache.instance
+  end
+
+  def tag_manager
+    @tag_manager ||= TagManager.instance
+  end
+
+  delegate :local_domain?, to: :tag_manager
+
+  def multiline?
+    options[:multiline]
+  end
+
+  def with_domains?
+    options[:with_domains]
+  end
+
+  def with_rel_me?
+    options[:with_rel_me]
+  end
+
+  def preloaded_accounts
+    options[:preloaded_accounts]
+  end
+
+  def preloaded_accounts?
+    preloaded_accounts.present?
+  end
+end

app/mailers/application_mailer.rb

@@ -5,6 +5,7 @@ class ApplicationMailer < ActionMailer::Base
 
   helper :application
   helper :instance
+  helper :formatting
 
   protected
 

app/serializers/activitypub/actor_serializer.rb

@@ -2,6 +2,7 @@
 
 class ActivityPub::ActorSerializer < ActivityPub::Serializer
   include RoutingHelper
+  include FormattingHelper
 
   context :security
 
@@ -102,7 +103,7 @@ class ActivityPub::ActorSerializer < ActivityPub::Serializer
   end
 
   def summary
-    object.suspended? ? '' : Formatter.instance.simplified_format(object)
+    object.suspended? ? '' : html_aware_format(object.note, object.local?)
   end
 
   def icon
@@ -185,6 +186,8 @@ class ActivityPub::ActorSerializer < ActivityPub::Serializer
   end
 
   class Account::FieldSerializer < ActivityPub::Serializer
+    include FormattingHelper
+
     attributes :type, :name, :value
 
     def type
@@ -192,7 +195,7 @@ class ActivityPub::ActorSerializer < ActivityPub::Serializer
     end
 
     def value
-      Formatter.instance.format_field(object.account, object.value)
+      html_aware_format(object.value, object.account.value?, with_rel_me: true, with_domains: true, multiline: false)
     end
   end
 

app/serializers/activitypub/note_serializer.rb

 # frozen_string_literal: true
 
 class ActivityPub::NoteSerializer < ActivityPub::Serializer
+  include FormattingHelper
+
   context_extensions :atom_uri, :conversation, :sensitive, :voters_count
 
   attributes :id, :type, :summary,
@@ -39,11 +41,11 @@ class ActivityPub::NoteSerializer < ActivityPub::Serializer
   end
 
   def content
-    Formatter.instance.format(object)
+    status_content_format(object)
   end
 
   def content_map
-    { object.language => Formatter.instance.format(object) }
+    { object.language => content }
   end
 
   def replies