From d1ecc323e7d435d0ffb11056a53b52d3345868f0 Mon Sep 17 00:00:00 2001 From: Claire Date: Thu, 3 Feb 2022 14:07:29 +0100 Subject: [PATCH 001/419] Compact JSON-LD signed incoming activities (#17426) Co-authored-by: Puck Meerburg --- app/helpers/context_helper.rb | 54 +++++++++++++++++++ app/helpers/jsonld_helper.rb | 8 +++ app/lib/activitypub/adapter.rb | 51 +----------------- .../activitypub/process_collection_service.rb | 2 + 4 files changed, 66 insertions(+), 49 deletions(-) create mode 100644 app/helpers/context_helper.rb diff --git a/app/helpers/context_helper.rb b/app/helpers/context_helper.rb new file mode 100644 index 000000000..08cfa9c6d --- /dev/null +++ b/app/helpers/context_helper.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +module ContextHelper + NAMED_CONTEXT_MAP = { + activitystreams: 'https://www.w3.org/ns/activitystreams', + security: 'https://w3id.org/security/v1', + }.freeze + + CONTEXT_EXTENSION_MAP = { + manually_approves_followers: { 'manuallyApprovesFollowers' => 'as:manuallyApprovesFollowers' }, + sensitive: { 'sensitive' => 'as:sensitive' }, + hashtag: { 'Hashtag' => 'as:Hashtag' }, + moved_to: { 'movedTo' => { '@id' => 'as:movedTo', '@type' => '@id' } }, + also_known_as: { 'alsoKnownAs' => { '@id' => 'as:alsoKnownAs', '@type' => '@id' } }, + emoji: { 'toot' => 'http://joinmastodon.org/ns#', 'Emoji' => 'toot:Emoji' }, + featured: { 'toot' => 'http://joinmastodon.org/ns#', 'featured' => { '@id' => 'toot:featured', '@type' => '@id' }, 'featuredTags' => { '@id' => 'toot:featuredTags', '@type' => '@id' } }, + property_value: { 'schema' => 'http://schema.org#', 'PropertyValue' => 'schema:PropertyValue', 'value' => 'schema:value' }, + atom_uri: { 'ostatus' => 'http://ostatus.org#', 'atomUri' => 'ostatus:atomUri' }, + conversation: { 'ostatus' => 'http://ostatus.org#', 'inReplyToAtomUri' => 'ostatus:inReplyToAtomUri', 'conversation' => 'ostatus:conversation' }, + focal_point: { 'toot' => 'http://joinmastodon.org/ns#', 'focalPoint' => { '@container' => '@list', '@id' => 'toot:focalPoint' } }, + blurhash: { 'toot' => 'http://joinmastodon.org/ns#', 'blurhash' => 'toot:blurhash' }, + discoverable: { 'toot' => 'http://joinmastodon.org/ns#', 'discoverable' => 'toot:discoverable' }, + voters_count: { 'toot' => 'http://joinmastodon.org/ns#', 'votersCount' => 'toot:votersCount' }, + olm: { 'toot' => 'http://joinmastodon.org/ns#', 'Device' => 'toot:Device', 'Ed25519Signature' => 'toot:Ed25519Signature', 'Ed25519Key' => 'toot:Ed25519Key', 'Curve25519Key' => 'toot:Curve25519Key', 'EncryptedMessage' => 'toot:EncryptedMessage', 'publicKeyBase64' => 'toot:publicKeyBase64', 'deviceId' => 'toot:deviceId', 'claim' => { '@type' => '@id', '@id' => 'toot:claim' }, 'fingerprintKey' => { '@type' => '@id', '@id' => 'toot:fingerprintKey' }, 'identityKey' => { '@type' => '@id', '@id' => 'toot:identityKey' }, 'devices' => { '@type' => '@id', '@id' => 'toot:devices' }, 'messageFranking' => 'toot:messageFranking', 'messageType' => 'toot:messageType', 'cipherText' => 'toot:cipherText' }, + suspended: { 'toot' => 'http://joinmastodon.org/ns#', 'suspended' => 'toot:suspended' }, + }.freeze + + def full_context + serialized_context(NAMED_CONTEXT_MAP, CONTEXT_EXTENSION_MAP) + end + + def serialized_context(named_contexts_map, context_extensions_map) + context_array = [] + + named_contexts = named_contexts_map.keys + context_extensions = context_extensions_map.keys + + named_contexts.each do |key| + context_array << NAMED_CONTEXT_MAP[key] + end + + extensions = context_extensions.each_with_object({}) do |key, h| + h.merge!(CONTEXT_EXTENSION_MAP[key]) + end + + context_array << extensions unless extensions.empty? + + if context_array.size == 1 + context_array.first + else + context_array + end + end +end diff --git a/app/helpers/jsonld_helper.rb b/app/helpers/jsonld_helper.rb index c24d2ddf1..841f27746 100644 --- a/app/helpers/jsonld_helper.rb +++ b/app/helpers/jsonld_helper.rb @@ -1,6 +1,8 @@ # frozen_string_literal: true module JsonLdHelper + include ContextHelper + def equals_or_includes?(haystack, needle) haystack.is_a?(Array) ? haystack.include?(needle) : haystack == needle end @@ -69,6 +71,12 @@ module JsonLdHelper graph.dump(:normalize) end + def compact(json) + compacted = JSON::LD::API.compact(json.without('signature'), full_context, documentLoader: method(:load_jsonld_context)) + compacted['signature'] = json['signature'] + compacted + end + def fetch_resource(uri, id, on_behalf_of = nil) unless id json = fetch_resource_without_id_validation(uri, on_behalf_of) diff --git a/app/lib/activitypub/adapter.rb b/app/lib/activitypub/adapter.rb index 776e1d3da..098b6296f 100644 --- a/app/lib/activitypub/adapter.rb +++ b/app/lib/activitypub/adapter.rb @@ -1,29 +1,7 @@ # frozen_string_literal: true class ActivityPub::Adapter < ActiveModelSerializers::Adapter::Base - NAMED_CONTEXT_MAP = { - activitystreams: 'https://www.w3.org/ns/activitystreams', - security: 'https://w3id.org/security/v1', - }.freeze - - CONTEXT_EXTENSION_MAP = { - manually_approves_followers: { 'manuallyApprovesFollowers' => 'as:manuallyApprovesFollowers' }, - sensitive: { 'sensitive' => 'as:sensitive' }, - hashtag: { 'Hashtag' => 'as:Hashtag' }, - moved_to: { 'movedTo' => { '@id' => 'as:movedTo', '@type' => '@id' } }, - also_known_as: { 'alsoKnownAs' => { '@id' => 'as:alsoKnownAs', '@type' => '@id' } }, - emoji: { 'toot' => 'http://joinmastodon.org/ns#', 'Emoji' => 'toot:Emoji' }, - featured: { 'toot' => 'http://joinmastodon.org/ns#', 'featured' => { '@id' => 'toot:featured', '@type' => '@id' }, 'featuredTags' => { '@id' => 'toot:featuredTags', '@type' => '@id' } }, - property_value: { 'schema' => 'http://schema.org#', 'PropertyValue' => 'schema:PropertyValue', 'value' => 'schema:value' }, - atom_uri: { 'ostatus' => 'http://ostatus.org#', 'atomUri' => 'ostatus:atomUri' }, - conversation: { 'ostatus' => 'http://ostatus.org#', 'inReplyToAtomUri' => 'ostatus:inReplyToAtomUri', 'conversation' => 'ostatus:conversation' }, - focal_point: { 'toot' => 'http://joinmastodon.org/ns#', 'focalPoint' => { '@container' => '@list', '@id' => 'toot:focalPoint' } }, - blurhash: { 'toot' => 'http://joinmastodon.org/ns#', 'blurhash' => 'toot:blurhash' }, - discoverable: { 'toot' => 'http://joinmastodon.org/ns#', 'discoverable' => 'toot:discoverable' }, - voters_count: { 'toot' => 'http://joinmastodon.org/ns#', 'votersCount' => 'toot:votersCount' }, - olm: { 'toot' => 'http://joinmastodon.org/ns#', 'Device' => 'toot:Device', 'Ed25519Signature' => 'toot:Ed25519Signature', 'Ed25519Key' => 'toot:Ed25519Key', 'Curve25519Key' => 'toot:Curve25519Key', 'EncryptedMessage' => 'toot:EncryptedMessage', 'publicKeyBase64' => 'toot:publicKeyBase64', 'deviceId' => 'toot:deviceId', 'claim' => { '@type' => '@id', '@id' => 'toot:claim' }, 'fingerprintKey' => { '@type' => '@id', '@id' => 'toot:fingerprintKey' }, 'identityKey' => { '@type' => '@id', '@id' => 'toot:identityKey' }, 'devices' => { '@type' => '@id', '@id' => 'toot:devices' }, 'messageFranking' => 'toot:messageFranking', 'messageType' => 'toot:messageType', 'cipherText' => 'toot:cipherText' }, - suspended: { 'toot' => 'http://joinmastodon.org/ns#', 'suspended' => 'toot:suspended' }, - }.freeze + include ContextHelper def self.default_key_transform :camel_lower @@ -34,7 +12,7 @@ class ActivityPub::Adapter < ActiveModelSerializers::Adapter::Base end def serializable_hash(options = nil) - named_contexts = {} + named_contexts = { activitystreams: NAMED_CONTEXT_MAP['activitystreams'] } context_extensions = {} options = serialization_options(options) @@ -44,29 +22,4 @@ class ActivityPub::Adapter < ActiveModelSerializers::Adapter::Base { '@context' => serialized_context(named_contexts, context_extensions) }.merge(serialized_hash) end - - private - - def serialized_context(named_contexts_map, context_extensions_map) - context_array = [] - - named_contexts = [:activitystreams] + named_contexts_map.keys - context_extensions = context_extensions_map.keys - - named_contexts.each do |key| - context_array << NAMED_CONTEXT_MAP[key] - end - - extensions = context_extensions.each_with_object({}) do |key, h| - h.merge!(CONTEXT_EXTENSION_MAP[key]) - end - - context_array << extensions unless extensions.empty? - - if context_array.size == 1 - context_array.first - else - context_array - end - end end diff --git a/app/services/activitypub/process_collection_service.rb b/app/services/activitypub/process_collection_service.rb index 170e6709c..5f3d63bb3 100644 --- a/app/services/activitypub/process_collection_service.rb +++ b/app/services/activitypub/process_collection_service.rb @@ -8,6 +8,8 @@ class ActivityPub::ProcessCollectionService < BaseService @json = Oj.load(body, mode: :strict) @options = options + @json = compact(@json) if @json['signature'].is_a?(Hash) + return if !supported_context? || (different_actor? && verify_account!.nil?) || suspended_actor? || @account.local? case @json['type'] From 948235592aa31c63033f7dc2d20a82115ca50149 Mon Sep 17 00:00:00 2001 From: Claire Date: Thu, 3 Feb 2022 14:07:43 +0100 Subject: [PATCH 002/419] Fix response_to_recipient? CTE (#17427) --- app/services/notify_service.rb | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/app/services/notify_service.rb b/app/services/notify_service.rb index 09e28b76b..0f3516d28 100644 --- a/app/services/notify_service.rb +++ b/app/services/notify_service.rb @@ -73,9 +73,11 @@ class NotifyService < BaseService # Using an SQL CTE to avoid unneeded back-and-forth with SQL server in case of long threads !Status.count_by_sql([<<-SQL.squish, id: @notification.target_status.in_reply_to_id, recipient_id: @recipient.id, sender_id: @notification.from_account.id]).zero? - WITH RECURSIVE ancestors(id, in_reply_to_id, replying_to_sender) AS ( + WITH RECURSIVE ancestors(id, in_reply_to_id, replying_to_sender, path) AS ( SELECT - s.id, s.in_reply_to_id, (CASE + s.id, + s.in_reply_to_id, + (CASE WHEN s.account_id = :recipient_id THEN EXISTS ( SELECT * @@ -84,7 +86,8 @@ class NotifyService < BaseService ) ELSE FALSE - END) + END), + ARRAY[s.id] FROM statuses s WHERE s.id = :id UNION ALL @@ -100,10 +103,11 @@ class NotifyService < BaseService ) ELSE FALSE - END) + END), + st.path || s.id FROM ancestors st JOIN statuses s ON s.id = st.in_reply_to_id - WHERE st.replying_to_sender IS FALSE + WHERE st.replying_to_sender IS FALSE AND NOT s.id = ANY(path) ) SELECT COUNT(*) FROM ancestors st From c8b1e72a4febd0922e22c3bdbba9165507de23bb Mon Sep 17 00:00:00 2001 From: Claire Date: Thu, 3 Feb 2022 14:09:04 +0100 Subject: [PATCH 003/419] Fix compacted JSON-LD possibly causing compatibility issues on forwarding (#17428) --- app/helpers/jsonld_helper.rb | 72 ++++++++++++++++ .../activitypub/process_collection_service.rb | 18 +++- spec/helpers/jsonld_helper_spec.rb | 82 +++++++++++++++++++ 3 files changed, 170 insertions(+), 2 deletions(-) diff --git a/app/helpers/jsonld_helper.rb b/app/helpers/jsonld_helper.rb index 841f27746..c6557817d 100644 --- a/app/helpers/jsonld_helper.rb +++ b/app/helpers/jsonld_helper.rb @@ -77,6 +77,78 @@ module JsonLdHelper compacted end + # Patches a JSON-LD document to avoid compatibility issues on redistribution + # + # Since compacting a JSON-LD document against Mastodon's built-in vocabulary + # means other extension namespaces will be expanded, malformed JSON-LD + # attributes lost, and some values “unexpectedly” compacted this method + # patches the following likely sources of incompatibility: + # - 'https://www.w3.org/ns/activitystreams#Public' being compacted to + # 'as:Public' (for instance, pre-3.4.0 Mastodon does not understand + # 'as:Public') + # - single-item arrays being compacted to the item itself (`[foo]` being + # compacted to `foo`) + # + # It is not always possible for `patch_for_forwarding!` to produce a document + # deemed safe for forwarding. Use `safe_for_forwarding?` to check the status + # of the output document. + # + # @param original [Hash] The original JSON-LD document used as reference + # @param compacted [Hash] The compacted JSON-LD document to be patched + # @return [void] + def patch_for_forwarding!(original, compacted) + original.without('@context', 'signature').each do |key, value| + next if value.nil? || !compacted.key?(key) + + compacted_value = compacted[key] + if value.is_a?(Hash) && compacted_value.is_a?(Hash) + patch_for_forwarding!(value, compacted_value) + elsif value.is_a?(Array) + compacted_value = [compacted_value] unless compacted_value.is_a?(Array) + return if value.size != compacted_value.size + + compacted[key] = value.zip(compacted_value).map do |v, vc| + if v.is_a?(Hash) && vc.is_a?(Hash) + patch_for_forwarding!(v, vc) + vc + elsif v == 'https://www.w3.org/ns/activitystreams#Public' && vc == 'as:Public' + v + else + vc + end + end + elsif value == 'https://www.w3.org/ns/activitystreams#Public' && compacted_value == 'as:Public' + compacted[key] = value + end + end + end + + # Tests whether a JSON-LD compaction is deemed safe for redistribution, + # that is, if it doesn't change its meaning to consumers that do not actually + # handle JSON-LD, but rely on values being serialized in a certain way. + # + # See `patch_for_forwarding!` for details. + # + # @param original [Hash] The original JSON-LD document used as reference + # @param compacted [Hash] The compacted JSON-LD document to be patched + # @return [Boolean] Whether the patched document is deemed safe + def safe_for_forwarding?(original, compacted) + original.without('@context', 'signature').all? do |key, value| + compacted_value = compacted[key] + return false unless value.class == compacted_value.class + + if value.is_a?(Hash) + safe_for_forwarding?(value, compacted_value) + elsif value.is_a?(Array) + value.zip(compacted_value).all? do |v, vc| + v.is_a?(Hash) ? (vc.is_a?(Hash) && safe_for_forwarding?(v, vc)) : v == vc + end + else + value == compacted_value + end + end + end + def fetch_resource(uri, id, on_behalf_of = nil) unless id json = fetch_resource_without_id_validation(uri, on_behalf_of) diff --git a/app/services/activitypub/process_collection_service.rb b/app/services/activitypub/process_collection_service.rb index 5f3d63bb3..eb008c40a 100644 --- a/app/services/activitypub/process_collection_service.rb +++ b/app/services/activitypub/process_collection_service.rb @@ -5,13 +5,27 @@ class ActivityPub::ProcessCollectionService < BaseService def call(body, account, **options) @account = account - @json = Oj.load(body, mode: :strict) + @json = original_json = Oj.load(body, mode: :strict) @options = options - @json = compact(@json) if @json['signature'].is_a?(Hash) + begin + @json = compact(@json) if @json['signature'].is_a?(Hash) + rescue JSON::LD::JsonLdError => e + Rails.logger.debug "Error when compacting JSON-LD document for #{value_or_id(@json['actor'])}: #{e.message}" + @json = original_json.without('signature') + end return if !supported_context? || (different_actor? && verify_account!.nil?) || suspended_actor? || @account.local? + if @json['signature'].present? + # We have verified the signature, but in the compaction step above, might + # have introduced incompatibilities with other servers that do not + # normalize the JSON-LD documents (for instance, previous Mastodon + # versions), so skip redistribution if we can't get a safe document. + patch_for_forwarding!(original_json, @json) + @json.delete('signature') unless safe_for_forwarding?(original_json, @json) + end + case @json['type'] when 'Collection', 'CollectionPage' process_items @json['items'] diff --git a/spec/helpers/jsonld_helper_spec.rb b/spec/helpers/jsonld_helper_spec.rb index 883a88b14..744a14f26 100644 --- a/spec/helpers/jsonld_helper_spec.rb +++ b/spec/helpers/jsonld_helper_spec.rb @@ -89,4 +89,86 @@ describe JsonLdHelper do expect(fetch_resource_without_id_validation('https://host.test/')).to eq({}) end end + + context 'compaction and forwarding' do + let(:json) do + { + '@context' => [ + 'https://www.w3.org/ns/activitystreams', + 'https://w3id.org/security/v1', + { + 'obsolete' => 'http://ostatus.org#', + 'convo' => 'obsolete:conversation', + 'new' => 'https://obscure-unreleased-test.joinmastodon.org/#', + }, + ], + 'type' => 'Create', + 'to' => ['https://www.w3.org/ns/activitystreams#Public'], + 'object' => { + 'id' => 'https://example.com/status', + 'type' => 'Note', + 'inReplyTo' => nil, + 'convo' => 'https://example.com/conversation', + 'tag' => [ + { + 'type' => 'Mention', + 'href' => ['foo'], + } + ], + }, + 'signature' => { + 'type' => 'RsaSignature2017', + 'created' => '2022-02-02T12:00:00Z', + 'creator' => 'https://example.com/actor#main-key', + 'signatureValue' => 'some-sig', + }, + } + end + + describe '#compact' do + it 'properly compacts JSON-LD with alternative context definitions' do + expect(compact(json).dig('object', 'conversation')).to eq 'https://example.com/conversation' + end + + it 'compacts single-item arrays' do + expect(compact(json).dig('object', 'tag', 'href')).to eq 'foo' + end + + it 'compacts the activistreams Public collection' do + expect(compact(json)['to']).to eq 'as:Public' + end + + it 'properly copies signature' do + expect(compact(json)['signature']).to eq json['signature'] + end + end + + describe 'patch_for_forwarding!' do + it 'properly patches incompatibilities' do + json['object'].delete('convo') + compacted = compact(json) + patch_for_forwarding!(json, compacted) + expect(compacted['to']).to eq ['https://www.w3.org/ns/activitystreams#Public'] + expect(compacted.dig('object', 'tag', 0, 'href')).to eq ['foo'] + expect(safe_for_forwarding?(json, compacted)).to eq true + end + end + + describe 'safe_for_forwarding?' do + it 'deems a safe compacting as such' do + json['object'].delete('convo') + compacted = compact(json) + deemed_compatible = patch_for_forwarding!(json, compacted) + expect(compacted['to']).to eq ['https://www.w3.org/ns/activitystreams#Public'] + expect(safe_for_forwarding?(json, compacted)).to eq true + end + + it 'deems an unsafe compacting as such' do + compacted = compact(json) + deemed_compatible = patch_for_forwarding!(json, compacted) + expect(compacted['to']).to eq ['https://www.w3.org/ns/activitystreams#Public'] + expect(safe_for_forwarding?(json, compacted)).to eq false + end + end + end end From 3413f1c44bd95980292f7efae5bb940c5e477cfc Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Thu, 3 Feb 2022 14:21:38 +0100 Subject: [PATCH 004/419] Forward-port version bump to 3.4.6 (#17434) --- CHANGELOG.md | 16 ++++++++++++++++ lib/mastodon/version.rb | 2 +- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9deff5a0d..8e9d6ea1d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,12 +3,28 @@ Changelog All notable changes to this project will be documented in this file. +## [3.4.6] - 2022-02-03 +### Fixed + +- Fix `mastodon:webpush:generate_vapid_key` task requiring a functional environment ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/17338)) +- Fix spurious errors when receiving an Add activity for a private post ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/17425)) + +### Security + +- Fix error-prone SQL queries ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/15828)) +- Fix not compacting incoming signed JSON-LD activities ([puckipedia](https://github.com/mastodon/mastodon/pull/17426), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/17428)) (CVE-2022-24307) +- Fix insufficient sanitization of report comments ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/17430)) +- Fix stop condition of a Common Table Expression ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/17427)) +- Disable legacy XSS filtering ([Wonderfall](https://github.com/mastodon/mastodon/pull/17289)) + ## [3.4.5] - 2022-01-31 ### Added + - Add more advanced migration tests ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/17393)) - Add github workflow to build Docker images ([unasuke](https://github.com/mastodon/mastodon/pull/16973), [Gargron](https://github.com/mastodon/mastodon/pull/16980), [Gargron](https://github.com/mastodon/mastodon/pull/17000)) ### Fixed + - Fix some old migrations failing when skipping releases ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/17394)) - Fix migrations script failing in certain edge cases ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/17398)) - Fix Docker build ([tribela](https://github.com/mastodon/mastodon/pull/17188)) diff --git a/lib/mastodon/version.rb b/lib/mastodon/version.rb index 1ba45aeca..d71b5b4ac 100644 --- a/lib/mastodon/version.rb +++ b/lib/mastodon/version.rb @@ -13,7 +13,7 @@ module Mastodon end def patch - 5 + 6 end def flags From 50ab3f3dcb6b00109fa1462a5ca0228563abb99b Mon Sep 17 00:00:00 2001 From: Alexandra Catalina Date: Thu, 3 Feb 2022 12:29:20 -0800 Subject: [PATCH 005/419] Update tootsuite/mastodon Docker tag to v3.4.6 (#17436) Co-authored-by: Renovate Bot --- chart/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chart/values.yaml b/chart/values.yaml index caac3eba0..dc476b1c5 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -8,7 +8,7 @@ image: # built from the most recent commit # # tag: latest - tag: v3.4.5 + tag: v3.4.6 # use `Always` when using `latest` tag pullPolicy: IfNotPresent From e0263c73694a75a881d2fb18dd825135e4a5ca4c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 5 Feb 2022 13:02:42 +0900 Subject: [PATCH 006/419] Bump http-link-header from 1.0.3 to 1.0.4 (#17414) Bumps [http-link-header](https://github.com/jhermsmeier/node-http-link-header) from 1.0.3 to 1.0.4. - [Release notes](https://github.com/jhermsmeier/node-http-link-header/releases) - [Changelog](https://github.com/jhermsmeier/node-http-link-header/blob/master/CHANGELOG.md) - [Commits](https://github.com/jhermsmeier/node-http-link-header/compare/v1.0.3...v1.0.4) --- updated-dependencies: - dependency-name: http-link-header dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index e6e9e7a64..74a0ddd63 100644 --- a/package.json +++ b/package.json @@ -98,7 +98,7 @@ "font-awesome": "^4.7.0", "glob": "^7.2.0", "history": "^4.10.1", - "http-link-header": "^1.0.3", + "http-link-header": "^1.0.4", "immutable": "^4.0.0", "imports-loader": "^1.2.0", "intersection-observer": "^0.12.0", diff --git a/yarn.lock b/yarn.lock index 9d161c006..2f78fb1f7 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5471,10 +5471,10 @@ http-errors@~1.6.2: setprototypeof "1.1.0" statuses ">= 1.4.0 < 2" -http-link-header@^1.0.3: - version "1.0.3" - resolved "https://registry.yarnpkg.com/http-link-header/-/http-link-header-1.0.3.tgz#abbc2cdc5e06dd7e196a4983adac08a2d085ec90" - integrity sha512-nARK1wSKoBBrtcoESlHBx36c1Ln/gnbNQi1eB6MeTUefJIT3NvUOsV15bClga0k38f0q/kN5xxrGSDS3EFnm9w== +http-link-header@^1.0.4: + version "1.0.4" + resolved "https://registry.yarnpkg.com/http-link-header/-/http-link-header-1.0.4.tgz#f4efc76c6151ed0ba0d1a2d679798a18854a4a99" + integrity sha512-Cnv3Q+FF+35avekdnH/ML8dls++tdnSgrvUIWw0YEszrWeLSuw5Iq1vyCVTb5v0rEUgFTy0x4shxXyrO0MDUzw== "http-parser-js@>=0.4.0 <0.4.11": version "0.4.10" From e001e116da86733d6179ffc3a2ead1e7a3cf8410 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 5 Feb 2022 13:02:57 +0900 Subject: [PATCH 007/419] Bump sidekiq-scheduler from 3.1.0 to 3.1.1 (#17407) Bumps [sidekiq-scheduler](https://github.com/moove-it/sidekiq-scheduler) from 3.1.0 to 3.1.1. - [Release notes](https://github.com/moove-it/sidekiq-scheduler/releases) - [Commits](https://github.com/moove-it/sidekiq-scheduler/compare/v3.1.0...v3.1.1) --- updated-dependencies: - dependency-name: sidekiq-scheduler dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Gemfile.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index dc5b33964..f2e053765 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -208,7 +208,7 @@ GEM multi_json encryptor (3.0.0) erubi (1.10.0) - et-orbi (1.2.4) + et-orbi (1.2.6) tzinfo excon (0.76.0) fabrication (2.24.0) @@ -252,7 +252,7 @@ GEM fog-json (>= 1.0) ipaddress (>= 0.8) formatador (0.2.5) - fugit (1.4.5) + fugit (1.5.2) et-orbi (~> 1.1, >= 1.1.8) raabro (~> 1.4) fuubar (2.5.1) @@ -551,7 +551,7 @@ GEM nokogiri (>= 1.10.5) rexml ruby2_keywords (0.0.5) - rufus-scheduler (3.7.0) + rufus-scheduler (3.8.1) fugit (~> 1.1, >= 1.1.6) safety_net_attestation (0.4.0) jwt (~> 2.0) @@ -569,7 +569,7 @@ GEM redis (>= 4.2.0) sidekiq-bulk (0.2.0) sidekiq - sidekiq-scheduler (3.1.0) + sidekiq-scheduler (3.1.1) e2mmap redis (>= 3, < 5) rufus-scheduler (~> 3.2) From bfe5ad5fee3e0ccc8ab480b311af1cf3c8fdcb2c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 5 Feb 2022 13:03:06 +0900 Subject: [PATCH 008/419] Bump redis from 4.0.2 to 4.0.3 (#17412) Bumps [redis](https://github.com/redis/node-redis) from 4.0.2 to 4.0.3. - [Release notes](https://github.com/redis/node-redis/releases) - [Changelog](https://github.com/redis/node-redis/blob/master/CHANGELOG.md) - [Commits](https://github.com/redis/node-redis/compare/redis@4.0.2...redis@4.0.3) --- updated-dependencies: - dependency-name: redis dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- package.json | 2 +- yarn.lock | 40 +++++++++++++++++++++++----------------- 2 files changed, 24 insertions(+), 18 deletions(-) diff --git a/package.json b/package.json index 74a0ddd63..ebbec6afc 100644 --- a/package.json +++ b/package.json @@ -143,7 +143,7 @@ "react-swipeable-views": "^0.14.0", "react-textarea-autosize": "^8.3.3", "react-toggle": "^4.1.2", - "redis": "^4.0.2", + "redis": "^4.0.3", "redux": "^4.1.2", "redux-immutable": "^4.0.0", "redux-thunk": "^2.4.1", diff --git a/yarn.lock b/yarn.lock index 2f78fb1f7..158d3a1b8 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1384,32 +1384,37 @@ "@types/yargs" "^16.0.0" chalk "^4.0.0" -"@node-redis/bloom@^1.0.0": +"@node-redis/bloom@1.0.1": version "1.0.1" resolved "https://registry.yarnpkg.com/@node-redis/bloom/-/bloom-1.0.1.tgz#144474a0b7dc4a4b91badea2cfa9538ce0a1854e" integrity sha512-mXEBvEIgF4tUzdIN89LiYsbi6//EdpFA7L8M+DHCvePXg+bfHWi+ct5VI6nHUFQE5+ohm/9wmgihCH3HSkeKsw== -"@node-redis/client@^1.0.2": - version "1.0.2" - resolved "https://registry.yarnpkg.com/@node-redis/client/-/client-1.0.2.tgz#7f09fb739675728fbc6e73536f7cd1be99bf7b8f" - integrity sha512-C+gkx68pmTnxfV+y4pzasvCH3s4UGHNOAUNhdJxGI27aMdnXNDZct7ffDHBL7bAZSGv9FSwCP5PeYvEIEKGbiA== +"@node-redis/client@1.0.3": + version "1.0.3" + resolved "https://registry.yarnpkg.com/@node-redis/client/-/client-1.0.3.tgz#ece282b7ee07283d744e6ab1fa72f2d47641402c" + integrity sha512-IXNgOG99PHGL3NxN3/e8J8MuX+H08I+OMNmheGmZBXngE0IntaCQwwrd7NzmiHA+zH3SKHiJ+6k3P7t7XYknMw== dependencies: cluster-key-slot "1.1.0" generic-pool "3.8.2" redis-parser "3.0.0" yallist "4.0.0" -"@node-redis/json@^1.0.2": +"@node-redis/graph@1.0.0": + version "1.0.0" + resolved "https://registry.yarnpkg.com/@node-redis/graph/-/graph-1.0.0.tgz#baf8eaac4a400f86ea04d65ec3d65715fd7951ab" + integrity sha512-mRSo8jEGC0cf+Rm7q8mWMKKKqkn6EAnA9IA2S3JvUv/gaWW/73vil7GLNwion2ihTptAm05I9LkepzfIXUKX5g== + +"@node-redis/json@1.0.2": version "1.0.2" resolved "https://registry.yarnpkg.com/@node-redis/json/-/json-1.0.2.tgz#8ad2d0f026698dc1a4238cc3d1eb099a3bee5ab8" integrity sha512-qVRgn8WfG46QQ08CghSbY4VhHFgaTY71WjpwRBGEuqGPfWwfRcIf3OqSpR7Q/45X+v3xd8mvYjywqh0wqJ8T+g== -"@node-redis/search@^1.0.2": +"@node-redis/search@1.0.2": version "1.0.2" resolved "https://registry.yarnpkg.com/@node-redis/search/-/search-1.0.2.tgz#8cfc91006ea787df801d41410283e1f59027f818" integrity sha512-gWhEeji+kTAvzZeguUNJdMSZNH2c5dv3Bci8Nn2f7VGuf6IvvwuZDSBOuOlirLVgayVuWzAG7EhwaZWK1VDnWQ== -"@node-redis/time-series@^1.0.1": +"@node-redis/time-series@1.0.1": version "1.0.1" resolved "https://registry.yarnpkg.com/@node-redis/time-series/-/time-series-1.0.1.tgz#703149f8fa4f6fff377c61a0873911e7c1ba5cc3" integrity sha512-+nTn6EewVj3GlUXPuD3dgheWqo219jTxlo6R+pg24OeVvFHx9aFGGiyOgj3vBPhWUdRZ0xMcujXV5ki4fbLyMw== @@ -9147,16 +9152,17 @@ redis-parser@3.0.0: dependencies: redis-errors "^1.0.0" -redis@^4.0.2: - version "4.0.2" - resolved "https://registry.yarnpkg.com/redis/-/redis-4.0.2.tgz#096cf716842731a24f34c7c3a996c143e2b133bb" - integrity sha512-Ip1DJ/lwuvtJz9AZ6pl1Bv33fWzk5d3iQpGzsXpi04ErkT4fq0pfGOm4k/p9DHmPGieEIOWvJ9xmIeQMooLybg== +redis@^4.0.3: + version "4.0.3" + resolved "https://registry.yarnpkg.com/redis/-/redis-4.0.3.tgz#f60931175de6f5b5727240a08e58a9ed5cf0f9de" + integrity sha512-SJMRXvgiQUYN0HaWwWv002J5ZgkhYXOlbLomzcrL3kP42yRNZ8Jx5nvLYhVpgmf10xcDpanFOxxJkphu2eyIFQ== dependencies: - "@node-redis/bloom" "^1.0.0" - "@node-redis/client" "^1.0.2" - "@node-redis/json" "^1.0.2" - "@node-redis/search" "^1.0.2" - "@node-redis/time-series" "^1.0.1" + "@node-redis/bloom" "1.0.1" + "@node-redis/client" "1.0.3" + "@node-redis/graph" "1.0.0" + "@node-redis/json" "1.0.2" + "@node-redis/search" "1.0.2" + "@node-redis/time-series" "1.0.1" redux-immutable@^4.0.0: version "4.0.0" From 6a649e91311b69d7a0c7c71dbb2e9662e6b06662 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 5 Feb 2022 13:03:12 +0900 Subject: [PATCH 009/419] Bump brakeman from 5.2.0 to 5.2.1 (#17410) Bumps [brakeman](https://github.com/presidentbeef/brakeman) from 5.2.0 to 5.2.1. - [Release notes](https://github.com/presidentbeef/brakeman/releases) - [Changelog](https://github.com/presidentbeef/brakeman/blob/main/CHANGES.md) - [Commits](https://github.com/presidentbeef/brakeman/compare/v5.2.0...v5.2.1) --- updated-dependencies: - dependency-name: brakeman dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index f2e053765..5ecddec12 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -106,7 +106,7 @@ GEM ffi (~> 1.14) bootsnap (1.10.2) msgpack (~> 1.2) - brakeman (5.2.0) + brakeman (5.2.1) browser (4.2.0) brpoplpush-redis_script (0.1.2) concurrent-ruby (~> 1.0, >= 1.0.5) From e03e7ac290f1458e105acc1bcfd7c0a3b04826ff Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Sat, 5 Feb 2022 05:06:34 +0100 Subject: [PATCH 010/419] Fix error on account relationships page in admin UI (#17444) --- .../admin/relationships_controller.rb | 3 ++- app/views/admin/relationships/index.html.haml | 23 ++++++++++--------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/app/controllers/admin/relationships_controller.rb b/app/controllers/admin/relationships_controller.rb index f8a95cfc8..085ded21c 100644 --- a/app/controllers/admin/relationships_controller.rb +++ b/app/controllers/admin/relationships_controller.rb @@ -9,7 +9,8 @@ module Admin def index authorize :account, :index? - @accounts = RelationshipFilter.new(@account, filter_params).results.page(params[:page]).per(PER_PAGE) + @accounts = RelationshipFilter.new(@account, filter_params).results.includes(:account_stat, user: [:ips, :invite_request]).page(params[:page]).per(PER_PAGE) + @form = Form::AccountBatch.new end private diff --git a/app/views/admin/relationships/index.html.haml b/app/views/admin/relationships/index.html.haml index 907477f24..60b9b5b25 100644 --- a/app/views/admin/relationships/index.html.haml +++ b/app/views/admin/relationships/index.html.haml @@ -24,16 +24,17 @@ %hr.spacer/ -.table-wrapper - %table.table - %thead - %tr - %th= t('admin.accounts.username') - %th= t('admin.accounts.role') - %th= t('admin.accounts.most_recent_ip') - %th= t('admin.accounts.most_recent_activity') - %th - %tbody - = render partial: 'admin/accounts/account', collection: @accounts += form_for(@form, url: batch_admin_accounts_path) do |f| + .batch-table + .batch-table__toolbar + %label.batch-table__toolbar__select.batch-checkbox-all + = check_box_tag :batch_checkbox_all, nil, false + .batch-table__toolbar__actions + = f.button safe_join([fa_icon('lock'), t('admin.accounts.perform_full_suspension')]), name: :suspend, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') } + .batch-table__body + - if @accounts.empty? + = nothing_here 'nothing-here--under-tabs' + - else + = render partial: 'admin/accounts/account', collection: @accounts, locals: { f: f } = paginate @accounts From 5f48ec9e4264065072d3ad67a1224df76cb892da Mon Sep 17 00:00:00 2001 From: Claire Date: Sat, 5 Feb 2022 10:27:24 +0100 Subject: [PATCH 011/419] Make theme-selection fall back to default ones if configured is not found --- app/controllers/application_controller.rb | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index 08cca0734..42f3081f1 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -137,13 +137,12 @@ class ApplicationController < ActionController::Base end def current_flavour - return Setting.flavour unless Themes.instance.flavours.include? current_user&.setting_flavour - current_user.setting_flavour + [current_user&.setting_flavour, Setting.flavour, 'glitch', 'vanilla'].find { |flavour| Themes.instance.flavours.include?(flavour) } end def current_skin - return Setting.skin unless Themes.instance.skins_for(current_flavour).include? current_user&.setting_skin - current_user.setting_skin + skins = Themes.instance.skins_for(current_flavour) + [current_user&.setting_skin, Setting.skin, 'default'].find { |skin| skins.include?(skin) } end def respond_with_error(code) From 08f44d1953d465825fd250452efbca6fc8d82dc3 Mon Sep 17 00:00:00 2001 From: Claire Date: Sat, 5 Feb 2022 10:58:51 +0100 Subject: [PATCH 012/419] Move glitch-soc-specific theming methods to ThemingConcern --- app/controllers/application_controller.rb | 9 --------- app/controllers/concerns/theming_concern.rb | 9 +++++++++ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index 42f3081f1..0f948ff5f 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -136,15 +136,6 @@ class ApplicationController < ActionController::Base @current_session = SessionActivation.find_by(session_id: cookies.signed['_session_id']) if cookies.signed['_session_id'].present? end - def current_flavour - [current_user&.setting_flavour, Setting.flavour, 'glitch', 'vanilla'].find { |flavour| Themes.instance.flavours.include?(flavour) } - end - - def current_skin - skins = Themes.instance.skins_for(current_flavour) - [current_user&.setting_skin, Setting.skin, 'default'].find { |skin| skins.include?(skin) } - end - def respond_with_error(code) respond_to do |format| format.any do diff --git a/app/controllers/concerns/theming_concern.rb b/app/controllers/concerns/theming_concern.rb index 1ee3256c0..425554072 100644 --- a/app/controllers/concerns/theming_concern.rb +++ b/app/controllers/concerns/theming_concern.rb @@ -10,6 +10,15 @@ module ThemingConcern private + def current_flavour + [current_user&.setting_flavour, Setting.flavour, 'glitch', 'vanilla'].find { |flavour| Themes.instance.flavours.include?(flavour) } + end + + def current_skin + skins = Themes.instance.skins_for(current_flavour) + [current_user&.setting_skin, Setting.skin, 'default'].find { |skin| skins.include?(skin) } + end + def valid_pack_data?(data, pack_name) data['pack'].is_a?(Hash) && [String, Hash].any? { |c| data['pack'][pack_name].is_a?(c) } end From 097c4903f1755f6657631759cbcf237f2966ede3 Mon Sep 17 00:00:00 2001 From: potpro Date: Sun, 6 Feb 2022 01:29:54 +0900 Subject: [PATCH 013/419] Update build-image.yml (#17454) --- .github/workflows/build-image.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml index 58f2813d3..0aaea6b1f 100644 --- a/.github/workflows/build-image.yml +++ b/.github/workflows/build-image.yml @@ -11,6 +11,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 + - uses: docker/setup-qemu-action@v1 - uses: docker/setup-buildx-action@v1 - uses: docker/login-action@v1 with: @@ -28,6 +29,7 @@ jobs: - uses: docker/build-push-action@v2 with: context: . + platforms: linux/amd64,linux/arm64 push: true tags: ${{ steps.meta.outputs.tags }} cache-from: type=registry,ref=tootsuite/mastodon:latest From 92658f0fb0cf6cb582126f41f7132bde80f77657 Mon Sep 17 00:00:00 2001 From: Claire Date: Sun, 6 Feb 2022 15:31:03 +0100 Subject: [PATCH 014/419] Fix instance actor not being dereferenceable (#17457) * Add tests * Fix instance actor not being dereferenceable * Fix tests * Fix tests for real --- app/controllers/instance_actors_controller.rb | 1 + .../instance_actors_controller_spec.rb | 55 +++++++++++++++++++ 2 files changed, 56 insertions(+) create mode 100644 spec/controllers/instance_actors_controller_spec.rb diff --git a/app/controllers/instance_actors_controller.rb b/app/controllers/instance_actors_controller.rb index b3b5476e2..0853897f2 100644 --- a/app/controllers/instance_actors_controller.rb +++ b/app/controllers/instance_actors_controller.rb @@ -3,6 +3,7 @@ class InstanceActorsController < ApplicationController include AccountControllerConcern + skip_before_action :check_account_confirmation skip_around_action :set_locale def show diff --git a/spec/controllers/instance_actors_controller_spec.rb b/spec/controllers/instance_actors_controller_spec.rb new file mode 100644 index 000000000..f64a7d2ca --- /dev/null +++ b/spec/controllers/instance_actors_controller_spec.rb @@ -0,0 +1,55 @@ +require 'rails_helper' + +RSpec.describe InstanceActorsController, type: :controller do + describe 'GET #show' do + context 'as JSON' do + let(:format) { 'json' } + + shared_examples 'shared behavior' do + before do + get :show, params: { format: format } + end + + it 'returns http success' do + expect(response).to have_http_status(200) + end + + it 'returns application/activity+json' do + expect(response.media_type).to eq 'application/activity+json' + end + + it 'does not set cookies' do + expect(response.cookies).to be_empty + expect(response.headers['Set-Cookies']).to be nil + end + + it 'does not set sessions' do + expect(session).to be_empty + end + + it 'returns public Cache-Control header' do + expect(response.headers['Cache-Control']).to include 'public' + end + + it 'renders account' do + json = body_as_json + expect(json).to include(:id, :type, :preferredUsername, :inbox, :publicKey, :inbox, :outbox, :url) + end + end + + before do + allow(controller).to receive(:authorized_fetch_mode?).and_return(authorized_fetch_mode) + end + + context 'without authorized fetch mode' do + let(:authorized_fetch_mode) { false } + it_behaves_like 'shared behavior' + end + + context 'with authorized fetch mode' do + let(:authorized_fetch_mode) { true } + it_behaves_like 'shared behavior' + end + end + end +end From 0d2cf3cd4a73ffcf0dfba24ea38be2e36528a4b7 Mon Sep 17 00:00:00 2001 From: Claire Date: Mon, 7 Feb 2022 13:14:48 +0100 Subject: [PATCH 015/419] Fix errors when multiple Delete are received for a given actor (#17460) --- app/workers/activitypub/processing_worker.rb | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/app/workers/activitypub/processing_worker.rb b/app/workers/activitypub/processing_worker.rb index cef595319..37e316354 100644 --- a/app/workers/activitypub/processing_worker.rb +++ b/app/workers/activitypub/processing_worker.rb @@ -6,7 +6,10 @@ class ActivityPub::ProcessingWorker sidekiq_options backtrace: true, retry: 8 def perform(account_id, body, delivered_to_account_id = nil) - ActivityPub::ProcessCollectionService.new.call(body, Account.find(account_id), override_timestamps: true, delivered_to_account_id: delivered_to_account_id, delivery: true) + account = Account.find_by(id: account_id) + return if account.nil? + + ActivityPub::ProcessCollectionService.new.call(body, account, override_timestamps: true, delivered_to_account_id: delivered_to_account_id, delivery: true) rescue ActiveRecord::RecordInvalid => e Rails.logger.debug "Error processing incoming ActivityPub object: #{e}" end From 73a782391ca3bc5cbb24fb98065f6a5f4d64f22c Mon Sep 17 00:00:00 2001 From: Claire Date: Mon, 7 Feb 2022 17:06:43 +0100 Subject: [PATCH 016/419] Fix replies collection incorrectly looping (#17462) * Refactor tests * Add tests * Fix replies collection incorrectly looping --- .../activitypub/replies_controller.rb | 30 +- .../activitypub/replies_controller_spec.rb | 351 ++++++++---------- 2 files changed, 180 insertions(+), 201 deletions(-) diff --git a/app/controllers/activitypub/replies_controller.rb b/app/controllers/activitypub/replies_controller.rb index fde6c861f..4ff7cfa08 100644 --- a/app/controllers/activitypub/replies_controller.rb +++ b/app/controllers/activitypub/replies_controller.rb @@ -63,15 +63,29 @@ class ActivityPub::RepliesController < ActivityPub::BaseController end def next_page - only_other_accounts = !(@replies&.last&.account_id == @account.id && @replies.size == DESCENDANTS_LIMIT) + if only_other_accounts? + # Only consider remote accounts + return nil if @replies.size < DESCENDANTS_LIMIT - account_status_replies_url( - @account, - @status, - page: true, - min_id: only_other_accounts && !only_other_accounts? ? nil : @replies&.last&.id, - only_other_accounts: only_other_accounts - ) + account_status_replies_url( + @account, + @status, + page: true, + min_id: @replies&.last&.id, + only_other_accounts: true + ) + else + # For now, we're serving only self-replies, but next page might be other accounts + next_only_other_accounts = @replies&.last&.account_id != @account.id || @replies.size < DESCENDANTS_LIMIT + + account_status_replies_url( + @account, + @status, + page: true, + min_id: next_only_other_accounts ? nil : @replies&.last&.id, + only_other_accounts: next_only_other_accounts + ) + end end def page_params diff --git a/spec/controllers/activitypub/replies_controller_spec.rb b/spec/controllers/activitypub/replies_controller_spec.rb index bf82fd020..a2c7f336f 100644 --- a/spec/controllers/activitypub/replies_controller_spec.rb +++ b/spec/controllers/activitypub/replies_controller_spec.rb @@ -4,8 +4,9 @@ require 'rails_helper' RSpec.describe ActivityPub::RepliesController, type: :controller do let(:status) { Fabricate(:status, visibility: parent_visibility) } - let(:remote_reply_id) { nil } - let(:remote_account) { nil } + let(:remote_account) { Fabricate(:account, domain: 'foobar.com') } + let(:remote_reply_id) { 'https://foobar.com/statuses/1234' } + let(:remote_querier) { nil } shared_examples 'cachable response' do it 'does not set cookies' do @@ -23,8 +24,151 @@ RSpec.describe ActivityPub::RepliesController, type: :controller do end end + shared_examples 'common behavior' do + context 'when status is private' do + let(:parent_visibility) { :private } + + it 'returns http not found' do + expect(response).to have_http_status(404) + end + end + + context 'when status is direct' do + let(:parent_visibility) { :direct } + + it 'returns http not found' do + expect(response).to have_http_status(404) + end + end + end + + shared_examples 'disallowed access' do + context 'when status is public' do + let(:parent_visibility) { :public } + + it 'returns http not found' do + expect(response).to have_http_status(404) + end + end + + it_behaves_like 'common behavior' + end + + shared_examples 'allowed access' do + context 'when account is permanently suspended' do + let(:parent_visibility) { :public } + + before do + status.account.suspend! + status.account.deletion_request.destroy + end + + it 'returns http gone' do + expect(response).to have_http_status(410) + end + end + + context 'when account is temporarily suspended' do + let(:parent_visibility) { :public } + + before do + status.account.suspend! + end + + it 'returns http forbidden' do + expect(response).to have_http_status(403) + end + end + + context 'when status is public' do + let(:parent_visibility) { :public } + let(:json) { body_as_json } + let(:page_json) { json[:first] } + + it 'returns http success' do + expect(response).to have_http_status(200) + end + + it 'returns application/activity+json' do + expect(response.media_type).to eq 'application/activity+json' + end + + it_behaves_like 'cachable response' + + context 'without only_other_accounts' do + it "returns items with thread author's replies" do + expect(page_json).to be_a Hash + expect(page_json[:items]).to be_an Array + expect(page_json[:items].size).to eq 1 + expect(page_json[:items].all? { |item| item[:to].include?(ActivityPub::TagManager::COLLECTIONS[:public]) || item[:cc].include?(ActivityPub::TagManager::COLLECTIONS[:public]) }).to be true + end + + context 'when there are few self-replies' do + it 'points next to replies from other people' do + expect(page_json).to be_a Hash + expect(Addressable::URI.parse(page_json[:next]).query.split('&')).to include('only_other_accounts=true', 'page=true') + end + end + + context 'when there are many self-replies' do + before do + 10.times { Fabricate(:status, account: status.account, thread: status, visibility: :public) } + end + + it 'points next to other self-replies' do + expect(page_json).to be_a Hash + expect(Addressable::URI.parse(page_json[:next]).query.split('&')).to include('only_other_accounts=false', 'page=true') + end + end + end + + context 'with only_other_accounts' do + let(:only_other_accounts) { 'true' } + + it 'returns items with other public or unlisted replies' do + expect(page_json).to be_a Hash + expect(page_json[:items]).to be_an Array + expect(page_json[:items].size).to eq 3 + end + + it 'only inlines items that are local and public or unlisted replies' do + inlined_replies = page_json[:items].select { |x| x.is_a?(Hash) } + public_collection = ActivityPub::TagManager::COLLECTIONS[:public] + expect(inlined_replies.all? { |item| item[:to].include?(public_collection) || item[:cc].include?(public_collection) }).to be true + expect(inlined_replies.all? { |item| ActivityPub::TagManager.instance.local_uri?(item[:id]) }).to be true + end + + it 'uses ids for remote toots' do + remote_replies = page_json[:items].select { |x| !x.is_a?(Hash) } + expect(remote_replies.all? { |item| item.is_a?(String) && !ActivityPub::TagManager.instance.local_uri?(item) }).to be true + end + + context 'when there are few replies' do + it 'does not have a next page' do + expect(page_json).to be_a Hash + expect(page_json[:next]).to be_nil + end + end + + context 'when there are many replies' do + before do + 10.times { Fabricate(:status, thread: status, visibility: :public) } + end + + it 'points next to other replies' do + expect(page_json).to be_a Hash + expect(Addressable::URI.parse(page_json[:next]).query.split('&')).to include('only_other_accounts=true', 'page=true') + end + end + end + end + + it_behaves_like 'common behavior' + end + before do - allow(controller).to receive(:signed_request_account).and_return(remote_account) + stub_const 'ActivityPub::RepliesController::DESCENDANTS_LIMIT', 5 + allow(controller).to receive(:signed_request_account).and_return(remote_querier) Fabricate(:status, thread: status, visibility: :public) Fabricate(:status, thread: status, visibility: :public) @@ -32,215 +176,36 @@ RSpec.describe ActivityPub::RepliesController, type: :controller do Fabricate(:status, account: status.account, thread: status, visibility: :public) Fabricate(:status, account: status.account, thread: status, visibility: :private) - Fabricate(:status, account: remote_account, thread: status, visibility: :public, uri: remote_reply_id) if remote_reply_id + Fabricate(:status, account: remote_account, thread: status, visibility: :public, uri: remote_reply_id) end describe 'GET #index' do + subject(:response) { get :index, params: { account_username: status.account.username, status_id: status.id, only_other_accounts: only_other_accounts } } + let(:only_other_accounts) { nil } + context 'with no signature' do - subject(:response) { get :index, params: { account_username: status.account.username, status_id: status.id } } - subject(:body) { body_as_json } - - context 'when account is permanently suspended' do - let(:parent_visibility) { :public } - - before do - status.account.suspend! - status.account.deletion_request.destroy - end - - it 'returns http gone' do - expect(response).to have_http_status(410) - end - end - - context 'when account is temporarily suspended' do - let(:parent_visibility) { :public } - - before do - status.account.suspend! - end - - it 'returns http forbidden' do - expect(response).to have_http_status(403) - end - end - - context 'when status is public' do - let(:parent_visibility) { :public } - - it 'returns http success' do - expect(response).to have_http_status(200) - end - - it 'returns application/activity+json' do - expect(response.media_type).to eq 'application/activity+json' - end - - it_behaves_like 'cachable response' - - it 'returns items with account\'s own replies' do - expect(body[:first]).to be_a Hash - expect(body[:first][:items]).to be_an Array - expect(body[:first][:items].size).to eq 1 - expect(body[:first][:items].all? { |item| item[:to].include?(ActivityPub::TagManager::COLLECTIONS[:public]) || item[:cc].include?(ActivityPub::TagManager::COLLECTIONS[:public]) }).to be true - end - end - - context 'when status is private' do - let(:parent_visibility) { :private } - - it 'returns http not found' do - expect(response).to have_http_status(404) - end - end - - context 'when status is direct' do - let(:parent_visibility) { :direct } - - it 'returns http not found' do - expect(response).to have_http_status(404) - end - end + it_behaves_like 'allowed access' end context 'with signature' do - let(:remote_account) { Fabricate(:account, domain: 'example.com') } - let(:only_other_accounts) { nil } + let(:remote_querier) { Fabricate(:account, domain: 'example.com') } - context do - before do - get :index, params: { account_username: status.account.username, status_id: status.id, only_other_accounts: only_other_accounts } - end - - context 'when status is public' do - let(:parent_visibility) { :public } - - it 'returns http success' do - expect(response).to have_http_status(200) - end - - it 'returns application/activity+json' do - expect(response.media_type).to eq 'application/activity+json' - end - - it_behaves_like 'cachable response' - - context 'without only_other_accounts' do - it 'returns items with account\'s own replies' do - json = body_as_json - - expect(json[:first]).to be_a Hash - expect(json[:first][:items]).to be_an Array - expect(json[:first][:items].size).to eq 1 - expect(json[:first][:items].all? { |item| item[:to].include?(ActivityPub::TagManager::COLLECTIONS[:public]) || item[:cc].include?(ActivityPub::TagManager::COLLECTIONS[:public]) }).to be true - end - end - - context 'with only_other_accounts' do - let(:only_other_accounts) { 'true' } - - it 'returns items with other public or unlisted replies' do - json = body_as_json - - expect(json[:first]).to be_a Hash - expect(json[:first][:items]).to be_an Array - expect(json[:first][:items].size).to eq 2 - expect(json[:first][:items].all? { |item| item[:to].include?(ActivityPub::TagManager::COLLECTIONS[:public]) || item[:cc].include?(ActivityPub::TagManager::COLLECTIONS[:public]) }).to be true - end - - context 'with remote responses' do - let(:remote_reply_id) { 'foo' } - - it 'returned items are all inlined local toots or are ids' do - json = body_as_json - - expect(json[:first]).to be_a Hash - expect(json[:first][:items]).to be_an Array - expect(json[:first][:items].size).to eq 3 - expect(json[:first][:items].all? { |item| item.is_a?(Hash) ? ActivityPub::TagManager.instance.local_uri?(item[:id]) : item.is_a?(String) }).to be true - expect(json[:first][:items]).to include remote_reply_id - end - end - end - end - - context 'when status is private' do - let(:parent_visibility) { :private } - - it 'returns http not found' do - expect(response).to have_http_status(404) - end - end - - context 'when status is direct' do - let(:parent_visibility) { :direct } - - it 'returns http not found' do - expect(response).to have_http_status(404) - end - end - end + it_behaves_like 'allowed access' context 'when signed request account is blocked' do before do - status.account.block!(remote_account) - get :index, params: { account_username: status.account.username, status_id: status.id } + status.account.block!(remote_querier) end - context 'when status is public' do - let(:parent_visibility) { :public } - - it 'returns http not found' do - expect(response).to have_http_status(404) - end - end - - context 'when status is private' do - let(:parent_visibility) { :private } - - it 'returns http not found' do - expect(response).to have_http_status(404) - end - end - - context 'when status is direct' do - let(:parent_visibility) { :direct } - - it 'returns http not found' do - expect(response).to have_http_status(404) - end - end + it_behaves_like 'disallowed access' end context 'when signed request account is domain blocked' do before do - status.account.block_domain!(remote_account.domain) - get :index, params: { account_username: status.account.username, status_id: status.id } + status.account.block_domain!(remote_querier.domain) end - context 'when status is public' do - let(:parent_visibility) { :public } - - it 'returns http not found' do - expect(response).to have_http_status(404) - end - end - - context 'when status is private' do - let(:parent_visibility) { :private } - - it 'returns http not found' do - expect(response).to have_http_status(404) - end - end - - context 'when status is direct' do - let(:parent_visibility) { :direct } - - it 'returns http not found' do - expect(response).to have_http_status(404) - end - end + it_behaves_like 'disallowed access' end end end From f1f6ddd5362f40e287857750f5e102206bd0e169 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Mon, 7 Feb 2022 18:16:31 +0100 Subject: [PATCH 017/419] Fix structured data parsing from links choking on bad data (#17403) * Fix structured data parsing from links choking on bad data - Fix og:url meta tag being prioritized over canonical link tag - Fix structured data parsing choking on commented-out CDATA declarations - Fix HTML entities in title, description, provider_name, author_name - Change structured data parsing to attempt every JSON-LD script tag * Remove unnecessary slash escapes from CDATA regex pattern --- app/lib/link_details_extractor.rb | 53 ++++++++-- spec/lib/link_details_extractor_spec.rb | 122 ++++++++++++++++++++++++ 2 files changed, 166 insertions(+), 9 deletions(-) diff --git a/app/lib/link_details_extractor.rb b/app/lib/link_details_extractor.rb index 56ad0717b..d2bcf0c25 100644 --- a/app/lib/link_details_extractor.rb +++ b/app/lib/link_details_extractor.rb @@ -3,6 +3,19 @@ class LinkDetailsExtractor include ActionView::Helpers::TagHelper + # Some publications wrap their JSON-LD data in their + + + HTML + + describe '#title' do + it 'returns the title from structured data' do + expect(subject.title).to eq 'Foo' + end + end + + describe '#description' do + it 'returns the description from structured data' do + expect(subject.description).to eq 'Bar' + end + end + + describe '#provider_name' do + it 'returns the provider name from structured data' do + expect(subject.provider_name).to eq 'Baz' + end + end + + describe '#author_name' do + it 'returns the author name from structured data' do + expect(subject.author_name).to eq 'Hoge' + end + end + end + + context 'but the first tag is invalid JSON' do + let(:html) { <<-HTML } + + + + + + + + HTML + + describe '#title' do + it 'returns the title from structured data' do + expect(subject.title).to eq 'Foo' + end + end + + describe '#description' do + it 'returns the description from structured data' do + expect(subject.description).to eq 'Bar' + end + end + + describe '#provider_name' do + it 'returns the provider name from structured data' do + expect(subject.provider_name).to eq 'Baz' + end + end + + describe '#author_name' do + it 'returns the author name from structured data' do + expect(subject.author_name).to eq 'Hoge' + end + end + end + end end From 52c1b86964caddb99e01ff36e928a524bf66ec0e Mon Sep 17 00:00:00 2001 From: Claire Date: Mon, 7 Feb 2022 19:57:06 +0100 Subject: [PATCH 018/419] Fix Ruby 2.5 incompatibility (#17465) --- app/workers/scheduler/accounts_statuses_cleanup_scheduler.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/workers/scheduler/accounts_statuses_cleanup_scheduler.rb b/app/workers/scheduler/accounts_statuses_cleanup_scheduler.rb index f42d4bca6..7195f0ff9 100644 --- a/app/workers/scheduler/accounts_statuses_cleanup_scheduler.rb +++ b/app/workers/scheduler/accounts_statuses_cleanup_scheduler.rb @@ -66,7 +66,7 @@ class Scheduler::AccountsStatusesCleanupScheduler end def compute_budget - threads = Sidekiq::ProcessSet.new.filter { |x| x['queues'].include?('push') }.map { |x| x['concurrency'] }.sum + threads = Sidekiq::ProcessSet.new.select { |x| x['queues'].include?('push') }.map { |x| x['concurrency'] }.sum [PER_THREAD_BUDGET * threads, MAX_BUDGET].min end From 35850f8195b633c60215461ebde48d2e8725fbd2 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Tue, 8 Feb 2022 01:53:49 +0100 Subject: [PATCH 019/419] Fix localization of cold-start follow recommendations (#17479) --- .../account_suggestions/global_source.rb | 2 +- .../follow_recommendations/show.html.haml | 2 +- .../follow_recommendations_scheduler.rb | 29 +++++++++++-------- 3 files changed, 19 insertions(+), 14 deletions(-) diff --git a/app/models/account_suggestions/global_source.rb b/app/models/account_suggestions/global_source.rb index ac764de50..03ed1b6c2 100644 --- a/app/models/account_suggestions/global_source.rb +++ b/app/models/account_suggestions/global_source.rb @@ -6,7 +6,7 @@ class AccountSuggestions::GlobalSource < AccountSuggestions::Source end def get(account, skip_account_ids: [], limit: 40) - account_ids = account_ids_for_locale(account.user_locale) - [account.id] - skip_account_ids + account_ids = account_ids_for_locale(I18n.locale.to_str.split(/[_-]/).first) - [account.id] - skip_account_ids as_ordered_suggestions( scope(account).where(id: account_ids), diff --git a/app/views/admin/follow_recommendations/show.html.haml b/app/views/admin/follow_recommendations/show.html.haml index 5b949a165..48c1ad1f2 100644 --- a/app/views/admin/follow_recommendations/show.html.haml +++ b/app/views/admin/follow_recommendations/show.html.haml @@ -13,7 +13,7 @@ .filter-subset.filter-subset--with-select %strong= t('admin.follow_recommendations.language') .input.select.optional - = select_tag :language, options_for_select(I18n.available_locales.map { |key| [human_locale(key), key]}, @language) + = select_tag :language, options_for_select(I18n.available_locales.map { |key| key.to_s.split(/[_-]/).first.to_sym }.uniq.map { |key| [human_locale(key), key]}, @language) .filter-subset %strong= t('admin.follow_recommendations.status') diff --git a/app/workers/scheduler/follow_recommendations_scheduler.rb b/app/workers/scheduler/follow_recommendations_scheduler.rb index effc63e59..084619cbd 100644 --- a/app/workers/scheduler/follow_recommendations_scheduler.rb +++ b/app/workers/scheduler/follow_recommendations_scheduler.rb @@ -16,28 +16,33 @@ class Scheduler::FollowRecommendationsScheduler AccountSummary.refresh FollowRecommendation.refresh - fallback_recommendations = FollowRecommendation.order(rank: :desc).limit(SET_SIZE).index_by(&:account_id) + fallback_recommendations = FollowRecommendation.order(rank: :desc).limit(SET_SIZE) - I18n.available_locales.each do |locale| + I18n.available_locales.map { |locale| locale.to_s.split(/[_-]/).first }.uniq.each do |locale| recommendations = begin if AccountSummary.safe.filtered.localized(locale).exists? # We can skip the work if no accounts with that language exist - FollowRecommendation.localized(locale).order(rank: :desc).limit(SET_SIZE).index_by(&:account_id) + FollowRecommendation.localized(locale).order(rank: :desc).limit(SET_SIZE).map { |recommendation| [recommendation.account_id, recommendation.rank] } else - {} + [] end end # Use language-agnostic results if there are not enough language-specific ones - missing = SET_SIZE - recommendations.keys.size + missing = SET_SIZE - recommendations.size + + if missing.positive? && fallback_recommendations.size.positive? + max_fallback_rank = fallback_recommendations.first.rank || 0 + + # Language-specific results should be above language-agnostic ones, + # otherwise language-agnostic ones will always overshadow them + recommendations.map! { |(account_id, rank)| [account_id, rank + max_fallback_rank] } - if missing.positive? added = 0 - # Avoid duplicate results - fallback_recommendations.each_value do |recommendation| - next if recommendations.key?(recommendation.account_id) + fallback_recommendations.each do |recommendation| + next if recommendations.any? { |(account_id, _)| account_id == recommendation.account_id } - recommendations[recommendation.account_id] = recommendation + recommendations << [recommendation.account_id, recommendation.rank] added += 1 break if added >= missing @@ -47,8 +52,8 @@ class Scheduler::FollowRecommendationsScheduler redis.pipelined do redis.del(key(locale)) - recommendations.each_value do |recommendation| - redis.zadd(key(locale), recommendation.rank, recommendation.account_id) + recommendations.each do |(account_id, rank)| + redis.zadd(key(locale), rank, account_id) end end end From 85b86fe28c62b8c3b34de20a292b158526355ddd Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Tue, 8 Feb 2022 02:34:56 +0100 Subject: [PATCH 020/419] Add global `locale` param (#17464) - Remove the session-based locale stickyness --- app/controllers/concerns/localized.rb | 29 ++++++++++++--------------- config/application.rb | 11 ++++++---- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/app/controllers/concerns/localized.rb b/app/controllers/concerns/localized.rb index fe1142f34..f7b62f09c 100644 --- a/app/controllers/concerns/localized.rb +++ b/app/controllers/concerns/localized.rb @@ -7,27 +7,24 @@ module Localized around_action :set_locale end - def set_locale - locale = current_user.locale if respond_to?(:user_signed_in?) && user_signed_in? - locale ||= session[:locale] ||= default_locale - locale = default_locale unless I18n.available_locales.include?(locale.to_sym) - - I18n.with_locale(locale) do - yield - end + def set_locale(&block) + I18n.with_locale(requested_locale || I18n.default_locale, &block) end private - def default_locale - if ENV['DEFAULT_LOCALE'].present? - I18n.default_locale - else - request_locale || I18n.default_locale - end + def requested_locale + requested_locale_name = available_locale_or_nil(params[:locale]) + requested_locale_name ||= available_locale_or_nil(current_user.locale) if respond_to?(:user_signed_in?) && user_signed_in? + requested_locale_name ||= http_accept_language if ENV['DEFAULT_LOCALE'].blank? + requested_locale_name end - def request_locale - http_accept_language.language_region_compatible_from(I18n.available_locales) + def http_accept_language + HttpAcceptLanguage::Parser.new(request.headers.fetch('Accept-Language')).language_region_compatible_from(I18n.available_locales) if request.headers.key?('Accept-Language') + end + + def available_locale_or_nil(locale_name) + locale_name.to_sym if locale_name.present? && I18n.available_locales.include?(locale_name.to_sym) end end diff --git a/config/application.rb b/config/application.rb index 561722884..c6f775162 100644 --- a/config/application.rb +++ b/config/application.rb @@ -149,10 +149,14 @@ module Mastodon :'zh-TW', ] - config.i18n.default_locale = ENV['DEFAULT_LOCALE']&.to_sym + config.i18n.default_locale = begin + custom_default_locale = ENV['DEFAULT_LOCALE']&.to_sym - unless config.i18n.available_locales.include?(config.i18n.default_locale) - config.i18n.default_locale = :en + if config.i18n.available_locales.include?(custom_default_locale) + custom_default_locale + else + :en + end end # config.paths.add File.join('app', 'api'), glob: File.join('**', '*.rb') @@ -169,7 +173,6 @@ module Mastodon Doorkeeper::Application.send :include, ApplicationExtension Doorkeeper::AccessToken.send :include, AccessTokenExtension Devise::FailureApp.send :include, AbstractController::Callbacks - Devise::FailureApp.send :include, HttpAcceptLanguage::EasyAccess Devise::FailureApp.send :include, Localized end end From b6d7726ecbc833abd00f6a9d36b24d9776cfe623 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Tue, 8 Feb 2022 02:41:17 +0100 Subject: [PATCH 021/419] Remove language detection through cld3 (#17478) * Remove language detection through cld3 * Update app/helpers/languages_helper.rb Co-authored-by: Yamagishi Kazutoshi Co-authored-by: Yamagishi Kazutoshi --- Gemfile | 2 - Gemfile.lock | 5 - app/helpers/languages_helper.rb | 291 +++++++++++++----- app/helpers/settings_helper.rb | 2 +- app/lib/activitypub/activity/create.rb | 6 +- app/lib/language_detector.rb | 101 ------ app/lib/link_details_extractor.rb | 9 +- app/models/user.rb | 4 + .../process_status_update_service.rb | 6 +- app/services/post_status_service.rb | 7 +- app/validators/import_validator.rb | 2 + .../settings/preferences/other/show.html.haml | 4 +- config/locales/en.yml | 2 +- lib/tasks/repo.rake | 5 +- spec/helpers/languages_helper_spec.rb | 6 +- spec/lib/language_detector_spec.rb | 134 -------- 16 files changed, 238 insertions(+), 348 deletions(-) delete mode 100644 app/lib/language_detector.rb delete mode 100644 spec/lib/language_detector_spec.rb diff --git a/Gemfile b/Gemfile index afed1ac94..e869e5f7a 100644 --- a/Gemfile +++ b/Gemfile @@ -29,9 +29,7 @@ gem 'addressable', '~> 2.8' gem 'bootsnap', '~> 1.10.2', require: false gem 'browser' gem 'charlock_holmes', '~> 0.7.7' -gem 'iso-639' gem 'chewy', '~> 7.2' -gem 'cld3', '~> 3.4.4' gem 'devise', '~> 4.8' gem 'devise-two-factor', '~> 4.0' diff --git a/Gemfile.lock b/Gemfile.lock index 5ecddec12..f7dd292dd 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -152,8 +152,6 @@ GEM elasticsearch (>= 7.12.0, < 7.14.0) elasticsearch-dsl chunky_png (1.4.0) - cld3 (3.4.4) - ffi (>= 1.1.0, < 1.16.0) climate_control (0.2.0) coderay (1.1.3) color_diff (0.1) @@ -301,7 +299,6 @@ GEM terminal-table (>= 1.5.1) idn-ruby (0.1.4) ipaddress (0.8.3) - iso-639 (0.3.5) jmespath (1.5.0) json (2.5.1) json-canonicalization (0.3.0) @@ -698,7 +695,6 @@ DEPENDENCIES capybara (~> 3.36) charlock_holmes (~> 0.7.7) chewy (~> 7.2) - cld3 (~> 3.4.4) climate_control (~> 0.2) color_diff (~> 0.1) concurrent-ruby @@ -725,7 +721,6 @@ DEPENDENCIES httplog (~> 1.5.0) i18n-tasks (~> 0.9) idn-ruby - iso-639 json-ld json-ld-preloaded (~> 3.2) kaminari (~> 1.2) diff --git a/app/helpers/languages_helper.rb b/app/helpers/languages_helper.rb index 730724208..f3ed7b314 100644 --- a/app/helpers/languages_helper.rb +++ b/app/helpers/languages_helper.rb @@ -1,94 +1,237 @@ # frozen_string_literal: true module LanguagesHelper - HUMAN_LOCALES = { - af: 'Afrikaans', - ar: 'العربية', - ast: 'Asturianu', - bg: 'Български', - bn: 'বাংলা', - br: 'Breton', - ca: 'Català', - co: 'Corsu', - cs: 'Čeština', - cy: 'Cymraeg', - da: 'Dansk', - de: 'Deutsch', - el: 'Ελληνικά', - en: 'English', - eo: 'Esperanto', + ISO_639_1 = { + aa: ['Afar', 'Afaraf'].freeze, + ab: ['Abkhaz', 'аҧсуа бызшәа'].freeze, + ae: ['Avestan', 'avesta'].freeze, + af: ['Afrikaans', 'Afrikaans'].freeze, + ak: ['Akan', 'Akan'].freeze, + am: ['Amharic', 'አማርኛ'].freeze, + an: ['Aragonese', 'aragonés'].freeze, + ar: ['Arabic', 'اللغة العربية'].freeze, + as: ['Assamese', 'অসমীয়া'].freeze, + av: ['Avaric', 'авар мацӀ'].freeze, + ay: ['Aymara', 'aymar aru'].freeze, + az: ['Azerbaijani', 'azərbaycan dili'].freeze, + ba: ['Bashkir', 'башҡорт теле'].freeze, + be: ['Belarusian', 'беларуская мова'].freeze, + bg: ['Bulgarian', 'български език'].freeze, + bh: ['Bihari', 'भोजपुरी'].freeze, + bi: ['Bislama', 'Bislama'].freeze, + bm: ['Bambara', 'bamanankan'].freeze, + bn: ['Bengali', 'বাংলা'].freeze, + bo: ['Tibetan', 'བོད་ཡིག'].freeze, + br: ['Breton', 'brezhoneg'].freeze, + bs: ['Bosnian', 'bosanski jezik'].freeze, + ca: ['Catalan', 'Català'].freeze, + ce: ['Chechen', 'нохчийн мотт'].freeze, + ch: ['Chamorro', 'Chamoru'].freeze, + co: ['Corsican', 'corsu'].freeze, + cr: ['Cree', 'ᓀᐦᐃᔭᐍᐏᐣ'].freeze, + cs: ['Czech', 'čeština'].freeze, + cu: ['Old Church Slavonic', 'ѩзыкъ словѣньскъ'].freeze, + cv: ['Chuvash', 'чӑваш чӗлхи'].freeze, + cy: ['Welsh', 'Cymraeg'].freeze, + da: ['Danish', 'dansk'].freeze, + de: ['German', 'Deutsch'].freeze, + dv: ['Divehi', 'Dhivehi'].freeze, + dz: ['Dzongkha', 'རྫོང་ཁ'].freeze, + ee: ['Ewe', 'Eʋegbe'].freeze, + el: ['Greek', 'Ελληνικά'].freeze, + en: ['English', 'English'].freeze, + eo: ['Esperanto', 'Esperanto'].freeze, + es: ['Spanish', 'Español'].freeze, + et: ['Estonian', 'eesti'].freeze, + eu: ['Basque', 'euskara'].freeze, + fa: ['Persian', 'فارسی'].freeze, + ff: ['Fula', 'Fulfulde'].freeze, + fi: ['Finnish', 'suomi'].freeze, + fj: ['Fijian', 'Vakaviti'].freeze, + fo: ['Faroese', 'føroyskt'].freeze, + fr: ['French', 'Français'].freeze, + fy: ['Western Frisian', 'Frysk'].freeze, + ga: ['Irish', 'Gaeilge'].freeze, + gd: ['Scottish Gaelic', 'Gàidhlig'].freeze, + gl: ['Galician', 'galego'].freeze, + gu: ['Gujarati', 'ગુજરાતી'].freeze, + gv: ['Manx', 'Gaelg'].freeze, + ha: ['Hausa', 'هَوُسَ'].freeze, + he: ['Hebrew', 'עברית'].freeze, + hi: ['Hindi', 'हिन्दी'].freeze, + ho: ['Hiri Motu', 'Hiri Motu'].freeze, + hr: ['Croatian', 'Hrvatski'].freeze, + ht: ['Haitian', 'Kreyòl ayisyen'].freeze, + hu: ['Hungarian', 'magyar'].freeze, + hy: ['Armenian', 'Հայերեն'].freeze, + hz: ['Herero', 'Otjiherero'].freeze, + ia: ['Interlingua', 'Interlingua'].freeze, + id: ['Indonesian', 'Bahasa Indonesia'].freeze, + ie: ['Interlingue', 'Interlingue'].freeze, + ig: ['Igbo', 'Asụsụ Igbo'].freeze, + ii: ['Nuosu', 'ꆈꌠ꒿ Nuosuhxop'].freeze, + ik: ['Inupiaq', 'Iñupiaq'].freeze, + io: ['Ido', 'Ido'].freeze, + is: ['Icelandic', 'Íslenska'].freeze, + it: ['Italian', 'Italiano'].freeze, + iu: ['Inuktitut', 'ᐃᓄᒃᑎᑐᑦ'].freeze, + ja: ['Japanese', '日本語'].freeze, + jv: ['Javanese', 'basa Jawa'].freeze, + ka: ['Georgian', 'ქართული'].freeze, + kg: ['Kongo', 'Kikongo'].freeze, + ki: ['Kikuyu', 'Gĩkũyũ'].freeze, + kj: ['Kwanyama', 'Kuanyama'].freeze, + kk: ['Kazakh', 'қазақ тілі'].freeze, + kl: ['Kalaallisut', 'kalaallisut'].freeze, + km: ['Khmer', 'ខេមរភាសា'].freeze, + kn: ['Kannada', 'ಕನ್ನಡ'].freeze, + ko: ['Korean', '한국어'].freeze, + kr: ['Kanuri', 'Kanuri'].freeze, + ks: ['Kashmiri', 'कश्मीरी'].freeze, + ku: ['Kurdish', 'Kurdî'].freeze, + kv: ['Komi', 'коми кыв'].freeze, + kw: ['Cornish', 'Kernewek'].freeze, + ky: ['Kyrgyz', 'Кыргызча'].freeze, + la: ['Latin', 'latine'].freeze, + lb: ['Luxembourgish', 'Lëtzebuergesch'].freeze, + lg: ['Ganda', 'Luganda'].freeze, + li: ['Limburgish', 'Limburgs'].freeze, + ln: ['Lingala', 'Lingála'].freeze, + lo: ['Lao', 'ພາສາ'].freeze, + lt: ['Lithuanian', 'lietuvių kalba'].freeze, + lu: ['Luba-Katanga', 'Tshiluba'].freeze, + lv: ['Latvian', 'latviešu valoda'].freeze, + mg: ['Malagasy', 'fiteny malagasy'].freeze, + mh: ['Marshallese', 'Kajin M̧ajeļ'].freeze, + mi: ['Māori', 'te reo Māori'].freeze, + mk: ['Macedonian', 'македонски јазик'].freeze, + ml: ['Malayalam', 'മലയാളം'].freeze, + mn: ['Mongolian', 'Монгол хэл'].freeze, + mr: ['Marathi', 'मराठी'].freeze, + ms: ['Malay', 'Bahasa Malaysia'].freeze, + mt: ['Maltese', 'Malti'].freeze, + my: ['Burmese', 'ဗမာစာ'].freeze, + na: ['Nauru', 'Ekakairũ Naoero'].freeze, + nb: ['Norwegian Bokmål', 'Norsk bokmål'].freeze, + nd: ['Northern Ndebele', 'isiNdebele'].freeze, + ne: ['Nepali', 'नेपाली'].freeze, + ng: ['Ndonga', 'Owambo'].freeze, + nl: ['Dutch', 'Nederlands'].freeze, + nn: ['Norwegian Nynorsk', 'Norsk nynorsk'].freeze, + no: ['Norwegian', 'Norsk'].freeze, + nr: ['Southern Ndebele', 'isiNdebele'].freeze, + nv: ['Navajo', 'Diné bizaad'].freeze, + ny: ['Chichewa', 'chiCheŵa'].freeze, + oc: ['Occitan', 'occitan'].freeze, + oj: ['Ojibwe', 'ᐊᓂᔑᓈᐯᒧᐎᓐ'].freeze, + om: ['Oromo', 'Afaan Oromoo'].freeze, + or: ['Oriya', 'ଓଡ଼ିଆ'].freeze, + os: ['Ossetian', 'ирон æвзаг'].freeze, + pa: ['Panjabi', 'ਪੰਜਾਬੀ'].freeze, + pi: ['Pāli', 'पाऴि'].freeze, + pl: ['Polish', 'Polski'].freeze, + ps: ['Pashto', 'پښتو'].freeze, + pt: ['Portuguese', 'Português'].freeze, + qu: ['Quechua', 'Runa Simi'].freeze, + rm: ['Romansh', 'rumantsch grischun'].freeze, + rn: ['Kirundi', 'Ikirundi'].freeze, + ro: ['Romanian', 'Română'].freeze, + ru: ['Russian', 'Русский'].freeze, + rw: ['Kinyarwanda', 'Ikinyarwanda'].freeze, + sa: ['Sanskrit', 'संस्कृतम्'].freeze, + sc: ['Sardinian', 'sardu'].freeze, + sd: ['Sindhi', 'सिन्धी'].freeze, + se: ['Northern Sami', 'Davvisámegiella'].freeze, + sg: ['Sango', 'yângâ tî sängö'].freeze, + si: ['Sinhala', 'සිංහල'].freeze, + sk: ['Slovak', 'slovenčina'].freeze, + sl: ['Slovenian', 'slovenščina'].freeze, + sn: ['Shona', 'chiShona'].freeze, + so: ['Somali', 'Soomaaliga'].freeze, + sq: ['Albanian', 'Shqip'].freeze, + sr: ['Serbian', 'српски језик'].freeze, + ss: ['Swati', 'SiSwati'].freeze, + st: ['Southern Sotho', 'Sesotho'].freeze, + su: ['Sundanese', 'Basa Sunda'].freeze, + sv: ['Swedish', 'Svenska'].freeze, + sw: ['Swahili', 'Kiswahili'].freeze, + ta: ['Tamil', 'தமிழ்'].freeze, + te: ['Telugu', 'తెలుగు'].freeze, + tg: ['Tajik', 'тоҷикӣ'].freeze, + th: ['Thai', 'ไทย'].freeze, + ti: ['Tigrinya', 'ትግርኛ'].freeze, + tk: ['Turkmen', 'Türkmen'].freeze, + tl: ['Tagalog', 'Wikang Tagalog'].freeze, + tn: ['Tswana', 'Setswana'].freeze, + to: ['Tonga', 'faka Tonga'].freeze, + tr: ['Turkish', 'Türkçe'].freeze, + ts: ['Tsonga', 'Xitsonga'].freeze, + tt: ['Tatar', 'татар теле'].freeze, + tw: ['Twi', 'Twi'].freeze, + ty: ['Tahitian', 'Reo Tahiti'].freeze, + ug: ['Uyghur', 'ئۇيغۇرچە‎'].freeze, + uk: ['Ukrainian', 'Українська'].freeze, + ur: ['Urdu', 'اردو'].freeze, + uz: ['Uzbek', 'Ўзбек'].freeze, + ve: ['Venda', 'Tshivenḓa'].freeze, + vi: ['Vietnamese', 'Tiếng Việt'].freeze, + vo: ['Volapük', 'Volapük'].freeze, + wa: ['Walloon', 'walon'].freeze, + wo: ['Wolof', 'Wollof'].freeze, + xh: ['Xhosa', 'isiXhosa'].freeze, + yi: ['Yiddish', 'ייִדיש'].freeze, + yo: ['Yoruba', 'Yorùbá'].freeze, + za: ['Zhuang', 'Saɯ cueŋƅ'].freeze, + zh: ['Chinese', '中文'].freeze, + zu: ['Zulu', 'isiZulu'].freeze, + }.freeze + + ISO_639_3 = { + ast: ['Asturian', 'Asturianu'].freeze, + kab: ['Kabyle', 'Taqbaylit'].freeze, + kmr: ['Northern Kurdish', 'Kurmancî'].freeze, + zgh: ['Standard Moroccan Tamazight', 'ⵜⴰⵎⴰⵣⵉⵖⵜ'].freeze, + }.freeze + + SUPPORTED_LOCALES = {}.merge(ISO_639_1).merge(ISO_639_3).freeze + + # For ISO-639-1 and ISO-639-3 language codes, we have their official + # names, but for some translations, we need the names of the + # regional variants specifically + REGIONAL_LOCALE_NAMES = { 'es-AR': 'Español (Argentina)', 'es-MX': 'Español (México)', - es: 'Español', - et: 'Eesti', - eu: 'Euskara', - fa: 'فارسی', - fi: 'Suomi', - fr: 'Français', - ga: 'Gaeilge', - gd: 'Gàidhlig', - gl: 'Galego', - he: 'עברית', - hi: 'हिन्दी', - hr: 'Hrvatski', - hu: 'Magyar', - hy: 'Հայերեն', - id: 'Bahasa Indonesia', - io: 'Ido', - is: 'Íslenska', - it: 'Italiano', - ja: '日本語', - ka: 'ქართული', - kab: 'Taqbaylit', - kk: 'Қазақша', - kmr: 'Kurmancî', - kn: 'ಕನ್ನಡ', - ko: '한국어', - ku: 'سۆرانی', - lt: 'Lietuvių', - lv: 'Latviešu', - mk: 'Македонски', - ml: 'മലയാളം', - mr: 'मराठी', - ms: 'Bahasa Melayu', - nl: 'Nederlands', - nn: 'Nynorsk', - no: 'Norsk', - oc: 'Occitan', - pl: 'Polski', 'pt-BR': 'Português (Brasil)', 'pt-PT': 'Português (Portugal)', - pt: 'Português', - ro: 'Română', - ru: 'Русский', - sa: 'संस्कृतम्', - sc: 'Sardu', - si: 'සිංහල', - sk: 'Slovenčina', - sl: 'Slovenščina', - sq: 'Shqip', 'sr-Latn': 'Srpski (latinica)', - sr: 'Српски', - sv: 'Svenska', - ta: 'தமிழ்', - te: 'తెలుగు', - th: 'ไทย', - tr: 'Türkçe', - uk: 'Українська', - ur: 'اُردُو', - vi: 'Tiếng Việt', - zgh: 'ⵜⴰⵎⴰⵣⵉⵖⵜ', 'zh-CN': '简体中文', 'zh-HK': '繁體中文（香港）', 'zh-TW': '繁體中文（臺灣）', - zh: '中文', }.freeze def human_locale(locale) if locale == 'und' I18n.t('generic.none') + elsif (supported_locale = SUPPORTED_LOCALES[locale.to_sym]) + supported_locale[1] + elsif (regional_locale = REGIONAL_LOCALE_NAMES[locale.to_sym]) + regional_locale else - HUMAN_LOCALES[locale.to_sym] || locale + locale end end + + def valid_locale_or_nil(str) + return if str.blank? + + code, = str.to_s.split(/[_-]/) # Strip out the region from e.g. en_US or ja-JP + + return unless valid_locale?(code) + + code + end + + def valid_locale?(locale) + SUPPORTED_LOCALES.key?(locale.to_sym) + end end diff --git a/app/helpers/settings_helper.rb b/app/helpers/settings_helper.rb index 23739d1cd..3d5592867 100644 --- a/app/helpers/settings_helper.rb +++ b/app/helpers/settings_helper.rb @@ -2,7 +2,7 @@ module SettingsHelper def filterable_languages - LanguageDetector.instance.language_names.select(&LanguagesHelper::HUMAN_LOCALES.method(:key?)) + LanguagesHelper::SUPPORTED_LOCALES.keys end def hash_to_object(hash) diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb index 33998c477..ea8d146d4 100644 --- a/app/lib/activitypub/activity/create.rb +++ b/app/lib/activitypub/activity/create.rb @@ -112,7 +112,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity url: @status_parser.url || @status_parser.uri, account: @account, text: converted_object_type? ? converted_text : (@status_parser.text || ''), - language: @status_parser.language || detected_language, + language: @status_parser.language, spoiler_text: converted_object_type? ? '' : (@status_parser.spoiler_text || ''), created_at: @status_parser.created_at, edited_at: @status_parser.edited_at, @@ -370,10 +370,6 @@ class ActivityPub::Activity::Create < ActivityPub::Activity Formatter.instance.linkify([@status_parser.title.presence, @status_parser.spoiler_text.presence, @status_parser.url || @status_parser.uri].compact.join("\n\n")) end - def detected_language - LanguageDetector.instance.detect(@status_parser.text, @account) if supported_object_type? - end - def unsupported_media_type?(mime_type) mime_type.present? && !MediaAttachment.supported_mime_types.include?(mime_type) end diff --git a/app/lib/language_detector.rb b/app/lib/language_detector.rb deleted file mode 100644 index 40452eddc..000000000 --- a/app/lib/language_detector.rb +++ /dev/null @@ -1,101 +0,0 @@ -# frozen_string_literal: true - -class LanguageDetector - include Singleton - - WORDS_THRESHOLD = 4 - RELIABLE_CHARACTERS_RE = /[\p{Hebrew}\p{Arabic}\p{Syriac}\p{Thaana}\p{Nko}\p{Han}\p{Katakana}\p{Hiragana}\p{Hangul}\p{Thai}]+/m - - def initialize - @identifier = CLD3::NNetLanguageIdentifier.new(1, 2048) - end - - def detect(text, account) - input_text = prepare_text(text) - - return if input_text.blank? - - detect_language_code(input_text) || default_locale(account) - end - - def language_names - @language_names = CLD3::TaskContextParams::LANGUAGE_NAMES.map { |name| iso6391(name.to_s).to_sym }.uniq - end - - private - - def prepare_text(text) - simplify_text(text).strip - end - - def unreliable_input?(text) - !reliable_input?(text) - end - - def reliable_input?(text) - sufficient_text_length?(text) || language_specific_character_set?(text) - end - - def sufficient_text_length?(text) - text.split(/\s+/).size >= WORDS_THRESHOLD - end - - def language_specific_character_set?(text) - words = text.scan(RELIABLE_CHARACTERS_RE) - - if words.present? - words.reduce(0) { |acc, elem| acc + elem.size }.to_f / text.size > 0.3 - else - false - end - end - - def detect_language_code(text) - return if unreliable_input?(text) - - result = @identifier.find_language(text) - - iso6391(result.language.to_s).to_sym if result&.reliable? - end - - def iso6391(bcp47) - iso639 = bcp47.split('-').first - - # CLD3 returns grandfathered language code for Hebrew - return 'he' if iso639 == 'iw' - - ISO_639.find(iso639).alpha2 - end - - def simplify_text(text) - new_text = remove_html(text) - new_text.gsub!(FetchLinkCardService::URL_PATTERN, '\1') - new_text.gsub!(Account::MENTION_RE, '') - new_text.gsub!(Tag::HASHTAG_RE) { |string| string.gsub(/[#_]/, '#' => '', '_' => ' ').gsub(/[a-z][A-Z]|[a-zA-Z][\d]/) { |s| s.insert(1, ' ') }.downcase } - new_text.gsub!(/:#{CustomEmoji::SHORTCODE_RE_FRAGMENT}:/, '') - new_text.gsub!(/\s+/, ' ') - new_text - end - - def new_scrubber - scrubber = Rails::Html::PermitScrubber.new - scrubber.tags = %w(br p) - scrubber - end - - def scrubber - @scrubber ||= new_scrubber - end - - def remove_html(text) - text = Loofah.fragment(text).scrub!(scrubber).to_s - text.gsub!('
', "\n") - text.gsub!('

', "\n\n") - text.gsub!(/(^

|<\/p>$)/, '') - text - end - - def default_locale(account) - account.user_locale&.to_sym || I18n.default_locale if account.local? - end -end diff --git a/app/lib/link_details_extractor.rb b/app/lib/link_details_extractor.rb index d2bcf0c25..fabbd244d 100644 --- a/app/lib/link_details_extractor.rb +++ b/app/lib/link_details_extractor.rb @@ -2,6 +2,7 @@ class LinkDetailsExtractor include ActionView::Helpers::TagHelper + include LanguagesHelper # Some publications wrap their JSON-LD data in their } } - - it 'does not include the HTML in the URL' do - is_expected.to include '"http://example.com/blahblahblahblah/a"' - end - - it 'escapes the HTML' do - is_expected.to include '<script>alert("Hello")</script>' - end - end - - context 'given text containing HTML code (script tag)' do - let(:text) { '' } - - it 'escapes the HTML' do - is_expected.to include '

<script>alert("Hello")</script>

' - end - end - - context 'given text containing HTML (XSS attack)' do - let(:text) { %q{} } - - it 'escapes the HTML' do - is_expected.to include '

<img src="javascript:alert('XSS');">

' - end - end - - context 'given an invalid URL' do - let(:text) { 'http://www\.google\.com' } - - it 'outputs the raw URL' do - is_expected.to eq '

http://www\.google\.com

' - end - end - - context 'given text containing a hashtag' do - let(:text) { '#hashtag' } - - it 'creates a hashtag link' do - is_expected.to include '/tags/hashtag" class="mention hashtag" rel="tag">#hashtag' - end - end - - context 'given text containing a hashtag with Unicode chars' do - let(:text) { '#hashtagタグ' } - - it 'creates a hashtag link' do - is_expected.to include '/tags/hashtag%E3%82%BF%E3%82%B0" class="mention hashtag" rel="tag">#hashtagタグ' - end - end - - context 'given a stand-alone xmpp: URI' do - let(:text) { 'xmpp:user@instance.com' } - - it 'matches the full URI' do - is_expected.to include 'href="xmpp:user@instance.com"' - end - end - - context 'given a an xmpp: URI with a query-string' do - let(:text) { 'please join xmpp:muc@instance.com?join right now' } - - it 'matches the full URI' do - is_expected.to include 'href="xmpp:muc@instance.com?join"' - end - end - - context 'given text containing a magnet: URI' do - let(:text) { 'wikipedia gives this example of a magnet uri: magnet:?xt=urn:btih:c12fe1c06bba254a9dc9f519b335aa7c1367a88a' } - - it 'matches the full URI' do - is_expected.to include 'href="magnet:?xt=urn:btih:c12fe1c06bba254a9dc9f519b335aa7c1367a88a"' - end - end - end - - describe '#format_spoiler' do - subject { Formatter.instance.format_spoiler(status) } - - context 'given a post containing plain text' do - let(:status) { Fabricate(:status, text: 'text', spoiler_text: 'Secret!', uri: nil) } - - it 'Returns the spoiler text' do - is_expected.to eq 'Secret!' - end - end - - context 'given a post with an emoji shortcode at the start' do - let!(:emoji) { Fabricate(:custom_emoji) } - let(:status) { Fabricate(:status, text: 'text', spoiler_text: ':coolcat: Secret!', uri: nil) } - let(:text) { ':coolcat: Beep boop' } - - it 'converts the shortcode to an image tag' do - is_expected.to match(/:coolcat:@alice Hello world' - end - end - - context 'given a post containing plain text' do - let(:status) { Fabricate(:status, text: 'text', uri: nil) } - - it 'paragraphizes the text' do - is_expected.to eq '

text

' - end - end - - context 'given a post containing line feeds' do - let(:status) { Fabricate(:status, text: "line\nfeed", uri: nil) } - - it 'removes line feeds' do - is_expected.not_to include "\n" - end - end - - context 'given a post containing linkable mentions' do - let(:status) { Fabricate(:status, mentions: [ Fabricate(:mention, account: local_account) ], text: '@alice') } - - it 'creates a mention link' do - is_expected.to include '@alice' - end - end - - context 'given a post containing unlinkable mentions' do - let(:status) { Fabricate(:status, text: '@alice', uri: nil) } - - it 'does not create a mention link' do - is_expected.to include '@alice' - end - end - - context do - subject do - status = Fabricate(:status, text: text, uri: nil) - Formatter.instance.format(status) - end - - include_examples 'encode and link URLs' - end - - context 'given a post with custom_emojify option' do - let!(:emoji) { Fabricate(:custom_emoji) } - let(:status) { Fabricate(:status, account: local_account, text: text) } - - subject { Formatter.instance.format(status, custom_emojify: true) } - - context 'given a post with an emoji shortcode at the start' do - let(:text) { ':coolcat: Beep boop' } - - it 'converts the shortcode to an image tag' do - is_expected.to match(/

:coolcat::coolcat: Beep boop
' } - - it 'converts the shortcode to an image tag' do - is_expected.to match(/

:coolcat:Beep :coolcat: boop

' } - - it 'converts the shortcode to an image tag' do - is_expected.to match(/Beep :coolcat::coolcat::coolcat:

' } - - it 'does not touch the shortcodes' do - is_expected.to match(/

:coolcat::coolcat:<\/p>/) - end - end - - context 'given a post with an emoji shortcode at the end' do - let(:text) { '

Beep boop
:coolcat:

' } - - it 'converts the shortcode to an image tag' do - is_expected.to match(/
:coolcat:alert("Hello")' } - - it 'strips the scripts' do - is_expected.to_not include '' - end - end - - context 'given a post containing malicious classes' do - let(:text) { 'Show more' } - - it 'strips the malicious classes' do - is_expected.to_not include 'status__content__spoiler-link' - end - end - end - - describe '#plaintext' do - subject { Formatter.instance.plaintext(status) } - - context 'given a post with local status' do - let(:status) { Fabricate(:status, text: '

a text by a nerd who uses an HTML tag in text

', uri: nil) } - - it 'returns the raw text' do - is_expected.to eq '

a text by a nerd who uses an HTML tag in text

' - end - end - - context 'given a post with remote status' do - let(:status) { Fabricate(:status, account: remote_account, text: '') } - - it 'returns tag-stripped text' do - is_expected.to eq '' - end - end - end - - describe '#simplified_format' do - subject { Formatter.instance.simplified_format(account) } - - context 'given a post with local status' do - let(:account) { Fabricate(:account, domain: nil, note: text) } - - context 'given a post containing linkable mentions for local accounts' do - let(:text) { '@alice' } - - before { local_account } - - it 'creates a mention link' do - is_expected.to eq '

@alice

' - end - end - - context 'given a post containing linkable mentions for remote accounts' do - let(:text) { '@bob@remote.test' } - - before { remote_account } - - it 'creates a mention link' do - is_expected.to eq '

@bob

' - end - end - - context 'given a post containing unlinkable mentions' do - let(:text) { '@alice' } - - it 'does not create a mention link' do - is_expected.to eq '

@alice

' - end - end - - context 'given a post with custom_emojify option' do - let!(:emoji) { Fabricate(:custom_emoji) } - - before { account.note = text } - subject { Formatter.instance.simplified_format(account, custom_emojify: true) } - - context 'given a post with an emoji shortcode at the start' do - let(:text) { ':coolcat: Beep boop' } - - it 'converts the shortcode to an image tag' do - is_expected.to match(/

:coolcat:alert("Hello")' } - let(:account) { Fabricate(:account, domain: 'remote', note: text) } - - it 'reformats' do - is_expected.to_not include '' - end - - context 'with custom_emojify option' do - let!(:emoji) { Fabricate(:custom_emoji, domain: remote_account.domain) } - - before { remote_account.note = text } - - subject { Formatter.instance.simplified_format(remote_account, custom_emojify: true) } - - context 'given a post with an emoji shortcode at the start' do - let(:text) { '

:coolcat: Beep boop
' } - - it 'converts shortcode to image tag' do - is_expected.to match(/

:coolcat:Beep :coolcat: boop

' } - - it 'converts shortcode to image tag' do - is_expected.to match(/Beep :coolcat::coolcat::coolcat:

' } - - it 'does not touch the shortcodes' do - is_expected.to match(/

:coolcat::coolcat:<\/p>/) - end - end - - context 'given a post with an emoji shortcode at the end' do - let(:text) { '

Beep boop
:coolcat:

' } - - it 'converts shortcode to image tag' do - is_expected.to match(/
:coolcat:alert("Hello")' } - - subject { Formatter.instance.sanitize(html, Sanitize::Config::MASTODON_STRICT) } - - it 'sanitizes' do - is_expected.to eq '' - end - end -end diff --git a/spec/lib/html_aware_formatter.rb b/spec/lib/html_aware_formatter.rb new file mode 100644 index 000000000..18d23abf5 --- /dev/null +++ b/spec/lib/html_aware_formatter.rb @@ -0,0 +1,44 @@ +require 'rails_helper' + +RSpec.describe HtmlAwareFormatter do + describe '#to_s' do + subject { described_class.new(text, local).to_s } + + context 'when local' do + let(:local) { true } + let(:text) { 'Foo bar' } + + it 'returns formatted text' do + is_expected.to eq '

Foo bar

' + end + end + + context 'when remote' do + let(:local) { false } + + context 'given plain text' do + let(:text) { 'Beep boop' } + + it 'keeps the plain text' do + is_expected.to include 'Beep boop' + end + end + + context 'given text containing script tags' do + let(:text) { '' } + + it 'strips the scripts' do + is_expected.to_not include '' + end + end + + context 'given text containing malicious classes' do + let(:text) { 'Show more' } + + it 'strips the malicious classes' do + is_expected.to_not include 'status__content__spoiler-link' + end + end + end + end +end diff --git a/spec/lib/plain_text_formatter_spec.rb b/spec/lib/plain_text_formatter_spec.rb new file mode 100644 index 000000000..c3d0ee630 --- /dev/null +++ b/spec/lib/plain_text_formatter_spec.rb @@ -0,0 +1,24 @@ +require 'rails_helper' + +RSpec.describe PlainTextFormatter do + describe '#to_s' do + subject { described_class.new(status.text, status.local?).to_s } + + context 'given a post with local status' do + let(:status) { Fabricate(:status, text: '

a text by a nerd who uses an HTML tag in text

', uri: nil) } + + it 'returns the raw text' do + is_expected.to eq '

a text by a nerd who uses an HTML tag in text

' + end + end + + context 'given a post with remote status' do + let(:remote_account) { Fabricate(:account, domain: 'remote.test', username: 'bob', url: 'https://remote.test/') } + let(:status) { Fabricate(:status, account: remote_account, text: '

Hello

') } + + it 'returns tag-stripped text' do + is_expected.to eq 'Hello' + end + end + end +end diff --git a/spec/lib/text_formatter_spec.rb b/spec/lib/text_formatter_spec.rb new file mode 100644 index 000000000..52a9d2498 --- /dev/null +++ b/spec/lib/text_formatter_spec.rb @@ -0,0 +1,313 @@ +require 'rails_helper' + +RSpec.describe TextFormatter do + describe '#to_s' do + let(:preloaded_accounts) { nil } + + subject { described_class.new(text, preloaded_accounts: preloaded_accounts).to_s } + + context 'given text containing plain text' do + let(:text) { 'text' } + + it 'paragraphizes the text' do + is_expected.to eq '

text

' + end + end + + context 'given text containing line feeds' do + let(:text) { "line\nfeed" } + + it 'removes line feeds' do + is_expected.not_to include "\n" + end + end + + context 'given text containing linkable mentions' do + let(:preloaded_accounts) { [Fabricate(:account, username: 'alice')] } + let(:text) { '@alice' } + + it 'creates a mention link' do + is_expected.to include '@alice' + end + end + + context 'given text containing unlinkable mentions' do + let(:preloaded_accounts) { [] } + let(:text) { '@alice' } + + it 'does not create a mention link' do + is_expected.to include '@alice' + end + end + + context 'given a stand-alone medium URL' do + let(:text) { 'https://hackernoon.com/the-power-to-build-communities-a-response-to-mark-zuckerberg-3f2cac9148a4' } + + it 'matches the full URL' do + is_expected.to include 'href="https://hackernoon.com/the-power-to-build-communities-a-response-to-mark-zuckerberg-3f2cac9148a4"' + end + end + + context 'given a stand-alone google URL' do + let(:text) { 'http://google.com' } + + it 'matches the full URL' do + is_expected.to include 'href="http://google.com"' + end + end + + context 'given a stand-alone URL with a newer TLD' do + let(:text) { 'http://example.gay' } + + it 'matches the full URL' do + is_expected.to include 'href="http://example.gay"' + end + end + + context 'given a stand-alone IDN URL' do + let(:text) { 'https://nic.みんな/' } + + it 'matches the full URL' do + is_expected.to include 'href="https://nic.みんな/"' + end + + it 'has display URL' do + is_expected.to include 'nic.みんな/' + end + end + + context 'given a URL with a trailing period' do + let(:text) { 'http://www.mcmansionhell.com/post/156408871451/50-states-of-mcmansion-hell-scottsdale-arizona. ' } + + it 'matches the full URL but not the period' do + is_expected.to include 'href="http://www.mcmansionhell.com/post/156408871451/50-states-of-mcmansion-hell-scottsdale-arizona"' + end + end + + context 'given a URL enclosed with parentheses' do + let(:text) { '(http://google.com/)' } + + it 'matches the full URL but not the parentheses' do + is_expected.to include 'href="http://google.com/"' + end + end + + context 'given a URL with a trailing exclamation point' do + let(:text) { 'http://www.google.com!' } + + it 'matches the full URL but not the exclamation point' do + is_expected.to include 'href="http://www.google.com"' + end + end + + context 'given a URL with a trailing single quote' do + let(:text) { "http://www.google.com'" } + + it 'matches the full URL but not the single quote' do + is_expected.to include 'href="http://www.google.com"' + end + end + + context 'given a URL with a trailing angle bracket' do + let(:text) { 'http://www.google.com>' } + + it 'matches the full URL but not the angle bracket' do + is_expected.to include 'href="http://www.google.com"' + end + end + + context 'given a URL with a query string' do + context 'with escaped unicode character' do + let(:text) { 'https://www.ruby-toolbox.com/search?utf8=%E2%9C%93&q=autolink' } + + it 'matches the full URL' do + is_expected.to include 'href="https://www.ruby-toolbox.com/search?utf8=%E2%9C%93&q=autolink"' + end + end + + context 'with unicode character' do + let(:text) { 'https://www.ruby-toolbox.com/search?utf8=✓&q=autolink' } + + it 'matches the full URL' do + is_expected.to include 'href="https://www.ruby-toolbox.com/search?utf8=✓&q=autolink"' + end + end + + context 'with unicode character at the end' do + let(:text) { 'https://www.ruby-toolbox.com/search?utf8=✓' } + + it 'matches the full URL' do + is_expected.to include 'href="https://www.ruby-toolbox.com/search?utf8=✓"' + end + end + + context 'with escaped and not escaped unicode characters' do + let(:text) { 'https://www.ruby-toolbox.com/search?utf8=%E2%9C%93&utf81=✓&q=autolink' } + + it 'preserves escaped unicode characters' do + is_expected.to include 'href="https://www.ruby-toolbox.com/search?utf8=%E2%9C%93&utf81=✓&q=autolink"' + end + end + end + + context 'given a URL with parentheses in it' do + let(:text) { 'https://en.wikipedia.org/wiki/Diaspora_(software)' } + + it 'matches the full URL' do + is_expected.to include 'href="https://en.wikipedia.org/wiki/Diaspora_(software)"' + end + end + + context 'given a URL in quotation marks' do + let(:text) { '"https://example.com/"' } + + it 'does not match the quotation marks' do + is_expected.to include 'href="https://example.com/"' + end + end + + context 'given a URL in angle brackets' do + let(:text) { '' } + + it 'does not match the angle brackets' do + is_expected.to include 'href="https://example.com/"' + end + end + + context 'given a URL with Japanese path string' do + let(:text) { 'https://ja.wikipedia.org/wiki/日本' } + + it 'matches the full URL' do + is_expected.to include 'href="https://ja.wikipedia.org/wiki/日本"' + end + end + + context 'given a URL with Korean path string' do + let(:text) { 'https://ko.wikipedia.org/wiki/대한민국' } + + it 'matches the full URL' do + is_expected.to include 'href="https://ko.wikipedia.org/wiki/대한민국"' + end + end + + context 'given a URL with a full-width space' do + let(:text) { 'https://example.com/　abc123' } + + it 'does not match the full-width space' do + is_expected.to include 'href="https://example.com/"' + end + end + + context 'given a URL in Japanese quotation marks' do + let(:text) { '「[https://example.org/」' } + + it 'does not match the quotation marks' do + is_expected.to include 'href="https://example.org/"' + end + end + + context 'given a URL with Simplified Chinese path string' do + let(:text) { 'https://baike.baidu.com/item/中华人民共和国' } + + it 'matches the full URL' do + is_expected.to include 'href="https://baike.baidu.com/item/中华人民共和国"' + end + end + + context 'given a URL with Traditional Chinese path string' do + let(:text) { 'https://zh.wikipedia.org/wiki/臺灣' } + + it 'matches the full URL' do + is_expected.to include 'href="https://zh.wikipedia.org/wiki/臺灣"' + end + end + + context 'given a URL containing unsafe code (XSS attack, visible part)' do + let(:text) { %q{http://example.com/bb} } + + it 'does not include the HTML in the URL' do + is_expected.to include '"http://example.com/b"' + end + + it 'escapes the HTML' do + is_expected.to include '<del>b</del>' + end + end + + context 'given a URL containing unsafe code (XSS attack, invisible part)' do + let(:text) { %q{http://example.com/blahblahblahblah/a} } + + it 'does not include the HTML in the URL' do + is_expected.to include '"http://example.com/blahblahblahblah/a"' + end + + it 'escapes the HTML' do + is_expected.to include '<script>alert("Hello")</script>' + end + end + + context 'given text containing HTML code (script tag)' do + let(:text) { '' } + + it 'escapes the HTML' do + is_expected.to include '

<script>alert("Hello")</script>

' + end + end + + context 'given text containing HTML (XSS attack)' do + let(:text) { %q{} } + + it 'escapes the HTML' do + is_expected.to include '

<img src="javascript:alert('XSS');">

' + end + end + + context 'given an invalid URL' do + let(:text) { 'http://www\.google\.com' } + + it 'outputs the raw URL' do + is_expected.to eq '

http://www\.google\.com

' + end + end + + context 'given text containing a hashtag' do + let(:text) { '#hashtag' } + + it 'creates a hashtag link' do + is_expected.to include '/tags/hashtag" class="mention hashtag" rel="tag">#hashtag' + end + end + + context 'given text containing a hashtag with Unicode chars' do + let(:text) { '#hashtagタグ' } + + it 'creates a hashtag link' do + is_expected.to include '/tags/hashtag%E3%82%BF%E3%82%B0" class="mention hashtag" rel="tag">#hashtagタグ' + end + end + + context 'given text with a stand-alone xmpp: URI' do + let(:text) { 'xmpp:user@instance.com' } + + it 'matches the full URI' do + is_expected.to include 'href="xmpp:user@instance.com"' + end + end + + context 'given text with an xmpp: URI with a query-string' do + let(:text) { 'please join xmpp:muc@instance.com?join right now' } + + it 'matches the full URI' do + is_expected.to include 'href="xmpp:muc@instance.com?join"' + end + end + + context 'given text containing a magnet: URI' do + let(:text) { 'wikipedia gives this example of a magnet uri: magnet:?xt=urn:btih:c12fe1c06bba254a9dc9f519b335aa7c1367a88a' } + + it 'matches the full URI' do + is_expected.to include 'href="magnet:?xt=urn:btih:c12fe1c06bba254a9dc9f519b335aa7c1367a88a"' + end + end + end +end From 07f8b4d1b19f734d04e69daeb4c3421ef9767aac Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Sat, 26 Mar 2022 02:54:11 +0100 Subject: [PATCH 339/419] Bump version to 3.5.0rc2 (#17855) --- CHANGELOG.md | 6 ++++++ lib/mastodon/version.rb | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 52a62a213..f0305d148 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -81,6 +81,7 @@ All notable changes to this project will be documented in this file. - Add lazy loading for emoji picker in web UI ([mashirozx](https://github.com/mastodon/mastodon/pull/16907), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/17011)) - Add single option votes tooltip in polls in web UI ([Brawaru](https://github.com/mastodon/mastodon/pull/16849)) - Add confirmation modal when closing media edit modal with unsaved changes in web UI ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/16518)) +- Add hint about missing media attachment description in web UI ([Gargron](https://github.com/mastodon/mastodon/pull/17845)) - Add support for fetching Create and Announce activities by URI in ActivityPub ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/16383)) - Add `S3_FORCE_SINGLE_REQUEST` environment variable ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/16866)) - Add `OMNIAUTH_ONLY` environment variable ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/17288), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/17345)) @@ -130,6 +131,11 @@ All notable changes to this project will be documented in this file. ### Fixed +- Fix IDN domains not being rendered correctly in a few left-over places ([Gargron](https://github.com/mastodon/mastodon/pull/17848)) +- Fix Sanskrit translation not being used in web UI ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/17820)) +- Fix Kurdish languages having the wrong language codes ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/17812)) +- Fix pghero making database schema suggestions ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/17807)) +- Fix encoding glitch in the OpenGraph description of a profile page ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/17821)) - Fix web manifest not permitting PWA usage from alternate domains ([HolgerHuo](https://github.com/mastodon/mastodon/pull/16714)) - Fix not being able to edit media attachments for scheduled posts ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/17690)) - Fix subscribed relay activities being recorded as boosts ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/17571)) diff --git a/lib/mastodon/version.rb b/lib/mastodon/version.rb index f6e437d3a..acaa978bb 100644 --- a/lib/mastodon/version.rb +++ b/lib/mastodon/version.rb @@ -17,7 +17,7 @@ module Mastodon end def flags - 'rc1' + 'rc2' end def suffix From 24e78969ae97501aad18595eb3af8c7338a1cb7c Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Sat, 26 Mar 2022 04:02:19 +0100 Subject: [PATCH 340/419] Fix typo (#17875) --- app/serializers/activitypub/actor_serializer.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/serializers/activitypub/actor_serializer.rb b/app/serializers/activitypub/actor_serializer.rb index bd1648348..30f86aae3 100644 --- a/app/serializers/activitypub/actor_serializer.rb +++ b/app/serializers/activitypub/actor_serializer.rb @@ -195,7 +195,7 @@ class ActivityPub::ActorSerializer < ActivityPub::Serializer end def value - html_aware_format(object.value, object.account.value?, with_rel_me: true, with_domains: true, multiline: false) + html_aware_format(object.value, object.account.local?, with_rel_me: true, with_domains: true, multiline: false) end end From d7d049aab7578028492e73671769f0a350e34203 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Sat, 26 Mar 2022 04:29:36 +0100 Subject: [PATCH 341/419] Bump version to 3.5.0rc3 (#17876) --- lib/mastodon/version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/mastodon/version.rb b/lib/mastodon/version.rb index acaa978bb..b1bd692a5 100644 --- a/lib/mastodon/version.rb +++ b/lib/mastodon/version.rb @@ -17,7 +17,7 @@ module Mastodon end def flags - 'rc2' + 'rc3' end def suffix From a4e1830b5f15118bf2532401005376a0c6e896e6 Mon Sep 17 00:00:00 2001 From: mayaeh Date: Sat, 26 Mar 2022 10:52:51 +0900 Subject: [PATCH 342/419] [Glitch] Add a hashtag public link to the trending hashtag page Port 52813830bee5607332b49bee2916956286ec5dc1 to glitch-soc Co-authored-by: Claire Co-authored-by: Eugen Rochko Co-authored-by: Claire Signed-off-by: Claire --- app/javascript/flavours/glitch/components/admin/Counter.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/app/javascript/flavours/glitch/components/admin/Counter.js b/app/javascript/flavours/glitch/components/admin/Counter.js index ecb242950..a4d6cef41 100644 --- a/app/javascript/flavours/glitch/components/admin/Counter.js +++ b/app/javascript/flavours/glitch/components/admin/Counter.js @@ -33,6 +33,7 @@ export default class Counter extends React.PureComponent { label: PropTypes.string.isRequired, href: PropTypes.string, params: PropTypes.object, + target: PropTypes.string, }; state = { @@ -54,7 +55,7 @@ export default class Counter extends React.PureComponent { } render () { - const { label, href } = this.props; + const { label, href, target } = this.props; const { loading, data } = this.state; let content; @@ -100,7 +101,7 @@ export default class Counter extends React.PureComponent { if (href) { return ( - + {inner} ); From e6a159a64869927cca5535943cdf3a280aeb5394 Mon Sep 17 00:00:00 2001 From: Claire Date: Mon, 28 Mar 2022 01:16:02 +0200 Subject: [PATCH 343/419] =?UTF-8?q?Fix=20extra=20=E2=80=9Czero=E2=80=9D=20?= =?UTF-8?q?key=20in=20some=20plural=20translation=20strings=20(#17883)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config/locales/en.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/config/locales/en.yml b/config/locales/en.yml index 5fa3c012e..829cd61d0 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -168,7 +168,6 @@ en: previous_strikes_description_html: one: This account has one strike. other: This account has %{count} strikes. - zero: This account is in good standing. promote: Promote protocol: Protocol public: Public @@ -530,7 +529,6 @@ en: known_accounts: one: "%{count} known account" other: "%{count} known accounts" - zero: No known account moderation: all: All limited: Limited @@ -802,7 +800,6 @@ en: shared_by_over_week: one: Shared by one person over the last week other: Shared by %{count} people over the last week - zero: Shared by noone over the last week title: Trending links usage_comparison: Shared %{today} times today, compared to %{yesterday} yesterday pending_review: Pending review @@ -845,7 +842,6 @@ en: used_by_over_week: one: Used by one person over the last week other: Used by %{count} people over the last week - zero: Used by noone over the last week title: Trends warning_presets: add_new: Add new From 2c45859ca9076c0b9916922e0be21ff83fc3b143 Mon Sep 17 00:00:00 2001 From: Claire Date: Mon, 28 Mar 2022 01:17:17 +0200 Subject: [PATCH 344/419] Refactor account and status formatting (#17877) * Refactor status formatting * Add account formatting helpers * Remove StatusFormatter * Fixup * Fix copied typo --- app/chewy/statuses_index.rb | 4 +++- app/helpers/formatting_helper.rb | 12 ++++++++++-- app/lib/feed_manager.rb | 2 +- app/serializers/activitypub/actor_serializer.rb | 4 ++-- app/serializers/rest/account_serializer.rb | 4 ++-- app/views/accounts/_bio.html.haml | 4 ++-- app/views/admin/accounts/show.html.haml | 4 ++-- app/views/admin/reports/show.html.haml | 2 +- app/views/directories/index.html.haml | 2 +- app/views/notification_mailer/_status.text.erb | 2 +- app/views/notification_mailer/digest.text.erb | 2 +- 11 files changed, 26 insertions(+), 16 deletions(-) diff --git a/app/chewy/statuses_index.rb b/app/chewy/statuses_index.rb index d119f7cac..bfd61a048 100644 --- a/app/chewy/statuses_index.rb +++ b/app/chewy/statuses_index.rb @@ -1,6 +1,8 @@ # frozen_string_literal: true class StatusesIndex < Chewy::Index + include FormattingHelper + settings index: { refresh_interval: '15m' }, analysis: { filter: { english_stop: { @@ -57,7 +59,7 @@ class StatusesIndex < Chewy::Index field :id, type: 'long' field :account_id, type: 'long' - field :text, type: 'text', value: ->(status) { [status.spoiler_text, PlainTextFormatter.new(status.text, status.local?).to_s].concat(status.ordered_media_attachments.map(&:description)).concat(status.preloadable_poll ? status.preloadable_poll.options : []).join("\n\n") } do + field :text, type: 'text', value: ->(status) { [status.spoiler_text, extract_status_plain_text(status)].concat(status.ordered_media_attachments.map(&:description)).concat(status.preloadable_poll ? status.preloadable_poll.options : []).join("\n\n") } do field :stemmed, type: 'text', analyzer: 'content' end diff --git a/app/helpers/formatting_helper.rb b/app/helpers/formatting_helper.rb index 66e9e1e91..e11156999 100644 --- a/app/helpers/formatting_helper.rb +++ b/app/helpers/formatting_helper.rb @@ -9,11 +9,19 @@ module FormattingHelper TextFormatter.new(text, options).to_s end - def extract_plain_text(text, local) - PlainTextFormatter.new(text, local).to_s + def extract_status_plain_text(status) + PlainTextFormatter.new(status.text, status.local?).to_s end def status_content_format(status) html_aware_format(status.text, status.local?, preloaded_accounts: [status.account] + (status.respond_to?(:active_mentions) ? status.active_mentions.map(&:account) : [])) end + + def account_bio_format(account) + html_aware_format(account.note, account.local?) + end + + def account_field_value_format(field, with_rel_me: true) + html_aware_format(field.value, field.account.local?, with_rel_me: with_rel_me, with_domains: true, multiline: false) + end end diff --git a/app/lib/feed_manager.rb b/app/lib/feed_manager.rb index 53d1390d4..709450080 100644 --- a/app/lib/feed_manager.rb +++ b/app/lib/feed_manager.rb @@ -446,7 +446,7 @@ class FeedManager status = status.reblog if status.reblog? combined_text = [ - extract_plain_text(status.text, status.local?), + extract_status_plain_text(status), status.spoiler_text, status.preloadable_poll ? status.preloadable_poll.options.join("\n\n") : nil, status.ordered_media_attachments.map(&:description).join("\n\n"), diff --git a/app/serializers/activitypub/actor_serializer.rb b/app/serializers/activitypub/actor_serializer.rb index 30f86aae3..e6dd8040e 100644 --- a/app/serializers/activitypub/actor_serializer.rb +++ b/app/serializers/activitypub/actor_serializer.rb @@ -103,7 +103,7 @@ class ActivityPub::ActorSerializer < ActivityPub::Serializer end def summary - object.suspended? ? '' : html_aware_format(object.note, object.local?) + object.suspended? ? '' : account_bio_format(object) end def icon @@ -195,7 +195,7 @@ class ActivityPub::ActorSerializer < ActivityPub::Serializer end def value - html_aware_format(object.value, object.account.local?, with_rel_me: true, with_domains: true, multiline: false) + account_field_value_format(object) end end diff --git a/app/serializers/rest/account_serializer.rb b/app/serializers/rest/account_serializer.rb index 2f67e06b2..4cf7b253f 100644 --- a/app/serializers/rest/account_serializer.rb +++ b/app/serializers/rest/account_serializer.rb @@ -20,7 +20,7 @@ class REST::AccountSerializer < ActiveModel::Serializer attributes :name, :value, :verified_at def value - html_aware_format(object.value, object.account.local?, with_rel_me: true, with_domains: true, multiline: false) + account_field_value_format(object) end end @@ -35,7 +35,7 @@ class REST::AccountSerializer < ActiveModel::Serializer end def note - object.suspended? ? '' : html_aware_format(object.note, object.local?) + object.suspended? ? '' : account_bio_format(object) end def url diff --git a/app/views/accounts/_bio.html.haml b/app/views/accounts/_bio.html.haml index df4f9bdb8..e2539b1d4 100644 --- a/app/views/accounts/_bio.html.haml +++ b/app/views/accounts/_bio.html.haml @@ -10,12 +10,12 @@ - if field.verified? %span.verified__mark{ title: t('accounts.link_verified_on', date: l(field.verified_at)) } = fa_icon 'check' - = prerender_custom_emojis(html_aware_format(field.value, account.local?, with_rel_me: true, with_domains: true, multiline: false), account.emojis) + = prerender_custom_emojis(account_field_value_format(field), account.emojis) = account_badge(account) - if account.note.present? - .account__header__content.emojify= prerender_custom_emojis(html_aware_format(account.note, account.local?), account.emojis) + .account__header__content.emojify= prerender_custom_emojis(account_bio_format(account), account.emojis) .public-account-bio__extra = t 'accounts.joined', date: l(account.created_at, format: :month) diff --git a/app/views/admin/accounts/show.html.haml b/app/views/admin/accounts/show.html.haml index b252f3eac..1230294fe 100644 --- a/app/views/admin/accounts/show.html.haml +++ b/app/views/admin/accounts/show.html.haml @@ -21,11 +21,11 @@ - if field.verified? %span.verified__mark{ title: t('accounts.link_verified_on', date: l(field.verified_at)) } = fa_icon 'check' - = prerender_custom_emojis(html_aware_format(field.value, account.local?, with_rel_me: true, with_domains: true, multiline: false), account.emojis) + = prerender_custom_emojis(account_field_value_format(field, with_rel_me: false), account.emojis) - if account.note.present? %div - .account__header__content.emojify= prerender_custom_emojis(html_aware_format(account.note, account.local?), account.emojis) + .account__header__content.emojify= prerender_custom_emojis(account_bio_format(account), account.emojis) .dashboard__counters.admin-account-counters %div diff --git a/app/views/admin/reports/show.html.haml b/app/views/admin/reports/show.html.haml index 41fed2efb..cf960565f 100644 --- a/app/views/admin/reports/show.html.haml +++ b/app/views/admin/reports/show.html.haml @@ -27,7 +27,7 @@ = fa_icon('lock') if @report.target_account.locked? - if @report.target_account.note.present? .account-card__bio.emojify - = prerender_custom_emojis(html_aware_format(@report.target_account.note, @report.target_account.local?), @report.target_account.emojis) + = prerender_custom_emojis(account_bio_format(@report.target_account), @report.target_account.emojis) .account-card__actions .account-card__counters .account-card__counters__item diff --git a/app/views/directories/index.html.haml b/app/views/directories/index.html.haml index a032ddb8d..48f8c4bc2 100644 --- a/app/views/directories/index.html.haml +++ b/app/views/directories/index.html.haml @@ -34,7 +34,7 @@ = fa_icon('lock') if account.locked? - if account.note.present? .account-card__bio.emojify - = prerender_custom_emojis(html_aware_format(account.note, account.local?), account.emojis) + = prerender_custom_emojis(account_bio_format(account), account.emojis) - else .flex-spacer .account-card__actions diff --git a/app/views/notification_mailer/_status.text.erb b/app/views/notification_mailer/_status.text.erb index bf6d2b620..1dc8de739 100644 --- a/app/views/notification_mailer/_status.text.erb +++ b/app/views/notification_mailer/_status.text.erb @@ -3,6 +3,6 @@ > ---- > <% end %> -> <%= raw word_wrap(extract_plain_text(status.text, status.local?), break_sequence: "\n> ") %> +> <%= raw word_wrap(extract_status_plain_text(status), break_sequence: "\n> ") %> <%= raw t('application_mailer.view')%> <%= web_url("statuses/#{status.id}") %> diff --git a/app/views/notification_mailer/digest.text.erb b/app/views/notification_mailer/digest.text.erb index b767eb9c4..0f84a4ef0 100644 --- a/app/views/notification_mailer/digest.text.erb +++ b/app/views/notification_mailer/digest.text.erb @@ -5,7 +5,7 @@ * <%= raw t('notification_mailer.digest.mention', name: notification.from_account.pretty_acct) %> - <%= raw extract_plain_text(notification.target_status.text, notification.target_status.local?) %> + <%= raw extract_status_plain_text(notification.target_status) %> <%= raw t('application_mailer.view')%> <%= web_url("statuses/#{notification.target_status.id}") %> <% end %> From 56edc6552f71a1f58fd8ca5ea2f0603015be0c2c Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Mon, 28 Mar 2022 09:39:31 +0200 Subject: [PATCH 345/419] Add `SMTP_RETURN_PATH` environment variable to set bounce domain (#17886) --- config/environments/production.rb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/config/environments/production.rb b/config/environments/production.rb index 7fe381040..b003cce9e 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -91,11 +91,13 @@ Rails.application.configure do # E-mails outgoing_email_address = ENV.fetch('SMTP_FROM_ADDRESS', 'notifications@localhost') - outgoing_mail_domain = Mail::Address.new(outgoing_email_address).domain + outgoing_email_domain = Mail::Address.new(outgoing_email_address).domain + config.action_mailer.default_options = { from: outgoing_email_address, reply_to: ENV['SMTP_REPLY_TO'], - 'Message-ID': -> { "<#{Mail.random_tag}@#{outgoing_mail_domain}>" }, + return_path: ENV['SMTP_RETURN_PATH'], + message_id: -> { "<#{Mail.random_tag}@#{outgoing_email_domain}>" }, } config.action_mailer.smtp_settings = { From 30658924a80434e6a2bceb61267b911ea8d37898 Mon Sep 17 00:00:00 2001 From: Claire Date: Mon, 28 Mar 2022 12:43:58 +0200 Subject: [PATCH 346/419] Fix test-related issues (#17888) * Remove obsolete RSS::Serializer test Since #17828, RSS::Serializer no longer has specific code for deleted statuses, but it is never called on deleted statuses anyway. * Rename erroneously-named test files * Fix failing test * Fix test deprecation warnings * Update CircleCI Ruby orb 1.4.0 has a bug that does not match all the test files due to incorrect globbing --- .circleci/config.yml | 2 +- .../admin/accounts_controller_spec.rb | 10 ++-- ..._specs.rb => bookmarks_controller_spec.rb} | 13 ++++-- ...matter.rb => html_aware_formatter_spec.rb} | 0 spec/lib/rss/serializer_spec.rb | 7 --- spec/services/after_block_service_spec.rb | 8 ++-- spec/services/delete_account_service_spec.rb | 14 +++--- spec/services/mute_service_spec.rb | 22 ++++----- spec/services/notify_service_spec.rb | 46 +++++++++---------- spec/services/suspend_account_service_spec.rb | 12 ++--- .../unsuspend_account_service_spec.rb | 26 +++++------ 11 files changed, 70 insertions(+), 90 deletions(-) rename spec/controllers/settings/exports/{bookmarks_controller_specs.rb => bookmarks_controller_spec.rb} (54%) rename spec/lib/{html_aware_formatter.rb => html_aware_formatter_spec.rb} (100%) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4fcc8c618..b9228f996 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,7 +1,7 @@ version: 2.1 orbs: - ruby: circleci/ruby@1.4.0 + ruby: circleci/ruby@1.4.1 node: circleci/node@5.0.1 executors: diff --git a/spec/controllers/admin/accounts_controller_spec.rb b/spec/controllers/admin/accounts_controller_spec.rb index 0f71d697c..1779fb7c0 100644 --- a/spec/controllers/admin/accounts_controller_spec.rb +++ b/spec/controllers/admin/accounts_controller_spec.rb @@ -194,9 +194,7 @@ RSpec.describe Admin::AccountsController, type: :controller do end describe 'POST #unblock_email' do - subject do - -> { post :unblock_email, params: { id: account.id } } - end + subject { post :unblock_email, params: { id: account.id } } let(:current_user) { Fabricate(:user, admin: admin) } let(:account) { Fabricate(:account, suspended: true) } @@ -206,11 +204,11 @@ RSpec.describe Admin::AccountsController, type: :controller do let(:admin) { true } it 'succeeds in removing email blocks' do - is_expected.to change { CanonicalEmailBlock.where(reference_account: account).count }.from(1).to(0) + expect { subject }.to change { CanonicalEmailBlock.where(reference_account: account).count }.from(1).to(0) end it 'redirects to admin account path' do - subject.call + subject expect(response).to redirect_to admin_account_path(account.id) end end @@ -219,7 +217,7 @@ RSpec.describe Admin::AccountsController, type: :controller do let(:admin) { false } it 'fails to remove avatar' do - subject.call + subject expect(response).to have_http_status :forbidden end end diff --git a/spec/controllers/settings/exports/bookmarks_controller_specs.rb b/spec/controllers/settings/exports/bookmarks_controller_spec.rb similarity index 54% rename from spec/controllers/settings/exports/bookmarks_controller_specs.rb rename to spec/controllers/settings/exports/bookmarks_controller_spec.rb index 85761577b..a06c02e0c 100644 --- a/spec/controllers/settings/exports/bookmarks_controller_specs.rb +++ b/spec/controllers/settings/exports/bookmarks_controller_spec.rb @@ -3,11 +3,16 @@ require 'rails_helper' describe Settings::Exports::BookmarksController do render_views - describe 'GET #index' do - it 'returns a csv of the bookmarked toots' do - user = Fabricate(:user) - user.account.bookmarks.create!(status: Fabricate(:status, uri: 'https://foo.bar/statuses/1312')) + let(:user) { Fabricate(:user) } + let(:account) { Fabricate(:account, domain: 'foo.bar') } + let(:status) { Fabricate(:status, account: account, uri: 'https://foo.bar/statuses/1312') } + describe 'GET #index' do + before do + user.account.bookmarks.create!(status: status) + end + + it 'returns a csv of the bookmarked toots' do sign_in user, scope: :user get :index, format: :csv diff --git a/spec/lib/html_aware_formatter.rb b/spec/lib/html_aware_formatter_spec.rb similarity index 100% rename from spec/lib/html_aware_formatter.rb rename to spec/lib/html_aware_formatter_spec.rb diff --git a/spec/lib/rss/serializer_spec.rb b/spec/lib/rss/serializer_spec.rb index 0364d13de..1da45d302 100644 --- a/spec/lib/rss/serializer_spec.rb +++ b/spec/lib/rss/serializer_spec.rb @@ -13,13 +13,6 @@ describe RSS::Serializer do subject { RSS::Serializer.new.send(:status_title, status) } - context 'if destroyed?' do - it 'returns "#{account.acct} deleted status"' do - status.destroy! - expect(subject).to eq "#{account.acct} deleted status" - end - end - context 'on a toot with long text' do let(:text) { "This toot's text is longer than the allowed number of characters" } diff --git a/spec/services/after_block_service_spec.rb b/spec/services/after_block_service_spec.rb index fe5b26b2b..c09425d7c 100644 --- a/spec/services/after_block_service_spec.rb +++ b/spec/services/after_block_service_spec.rb @@ -1,9 +1,7 @@ require 'rails_helper' RSpec.describe AfterBlockService, type: :service do - subject do - -> { described_class.new.call(account, target_account) } - end + subject { described_class.new.call(account, target_account) } let(:account) { Fabricate(:account) } let(:target_account) { Fabricate(:account) } @@ -24,7 +22,7 @@ RSpec.describe AfterBlockService, type: :service do FeedManager.instance.push_to_home(account, other_account_status) FeedManager.instance.push_to_home(account, other_account_reblog) - is_expected.to change { + expect { subject }.to change { Redis.current.zrange(home_timeline_key, 0, -1) }.from([status.id.to_s, other_account_status.id.to_s, other_account_reblog.id.to_s]).to([other_account_status.id.to_s]) end @@ -43,7 +41,7 @@ RSpec.describe AfterBlockService, type: :service do FeedManager.instance.push_to_list(list, other_account_status) FeedManager.instance.push_to_list(list, other_account_reblog) - is_expected.to change { + expect { subject }.to change { Redis.current.zrange(list_timeline_key, 0, -1) }.from([status.id.to_s, other_account_status.id.to_s, other_account_reblog.id.to_s]).to([other_account_status.id.to_s]) end diff --git a/spec/services/delete_account_service_spec.rb b/spec/services/delete_account_service_spec.rb index 9c785fc17..1fbe4d07c 100644 --- a/spec/services/delete_account_service_spec.rb +++ b/spec/services/delete_account_service_spec.rb @@ -23,12 +23,10 @@ RSpec.describe DeleteAccountService, type: :service do let!(:account_note) { Fabricate(:account_note, account: account) } - subject do - -> { described_class.new.call(account) } - end + subject { described_class.new.call(account) } it 'deletes associated owned records' do - is_expected.to change { + expect { subject }.to change { [ account.statuses, account.media_attachments, @@ -43,7 +41,7 @@ RSpec.describe DeleteAccountService, type: :service do end it 'deletes associated target records' do - is_expected.to change { + expect { subject }.to change { [ AccountPin.where(target_account: account), ].map(&:count) @@ -51,7 +49,7 @@ RSpec.describe DeleteAccountService, type: :service do end it 'deletes associated target notifications' do - is_expected.to change { + expect { subject }.to change { [ 'poll', 'favourite', 'status', 'mention', 'follow' ].map { |type| Notification.where(type: type).count } @@ -73,7 +71,7 @@ RSpec.describe DeleteAccountService, type: :service do let!(:local_follower) { Fabricate(:account) } it 'sends a delete actor activity to all known inboxes' do - subject.call + subject expect(a_request(:post, "https://alice.com/inbox")).to have_been_made.once expect(a_request(:post, "https://bob.com/inbox")).to have_been_made.once end @@ -91,7 +89,7 @@ RSpec.describe DeleteAccountService, type: :service do let!(:local_follower) { Fabricate(:account) } it 'sends a reject follow to follower inboxes' do - subject.call + subject expect(a_request(:post, account.inbox_url)).to have_been_made.once end end diff --git a/spec/services/mute_service_spec.rb b/spec/services/mute_service_spec.rb index 4bb839b8d..bdec1c67b 100644 --- a/spec/services/mute_service_spec.rb +++ b/spec/services/mute_service_spec.rb @@ -1,9 +1,7 @@ require 'rails_helper' RSpec.describe MuteService, type: :service do - subject do - -> { described_class.new.call(account, target_account) } - end + subject { described_class.new.call(account, target_account) } let(:account) { Fabricate(:account) } let(:target_account) { Fabricate(:account) } @@ -21,45 +19,41 @@ RSpec.describe MuteService, type: :service do FeedManager.instance.push_to_home(account, status) FeedManager.instance.push_to_home(account, other_account_status) - is_expected.to change { + expect { subject }.to change { Redis.current.zrange(home_timeline_key, 0, -1) }.from([status.id.to_s, other_account_status.id.to_s]).to([other_account_status.id.to_s]) end end it 'mutes account' do - is_expected.to change { + expect { subject }.to change { account.muting?(target_account) }.from(false).to(true) end context 'without specifying a notifications parameter' do it 'mutes notifications from the account' do - is_expected.to change { + expect { subject }.to change { account.muting_notifications?(target_account) }.from(false).to(true) end end context 'with a true notifications parameter' do - subject do - -> { described_class.new.call(account, target_account, notifications: true) } - end + subject { described_class.new.call(account, target_account, notifications: true) } it 'mutes notifications from the account' do - is_expected.to change { + expect { subject }.to change { account.muting_notifications?(target_account) }.from(false).to(true) end end context 'with a false notifications parameter' do - subject do - -> { described_class.new.call(account, target_account, notifications: false) } - end + subject { described_class.new.call(account, target_account, notifications: false) } it 'does not mute notifications from the account' do - is_expected.to_not change { + expect { subject }.to_not change { account.muting_notifications?(target_account) }.from(false) end diff --git a/spec/services/notify_service_spec.rb b/spec/services/notify_service_spec.rb index 7433866b7..294c31b04 100644 --- a/spec/services/notify_service_spec.rb +++ b/spec/services/notify_service_spec.rb @@ -1,9 +1,7 @@ require 'rails_helper' RSpec.describe NotifyService, type: :service do - subject do - -> { described_class.new.call(recipient, type, activity) } - end + subject { described_class.new.call(recipient, type, activity) } let(:user) { Fabricate(:user) } let(:recipient) { user.account } @@ -11,42 +9,42 @@ RSpec.describe NotifyService, type: :service do let(:activity) { Fabricate(:follow, account: sender, target_account: recipient) } let(:type) { :follow } - it { is_expected.to change(Notification, :count).by(1) } + it { expect { subject }.to change(Notification, :count).by(1) } it 'does not notify when sender is blocked' do recipient.block!(sender) - is_expected.to_not change(Notification, :count) + expect { subject }.to_not change(Notification, :count) end it 'does not notify when sender is muted with hide_notifications' do recipient.mute!(sender, notifications: true) - is_expected.to_not change(Notification, :count) + expect { subject }.to_not change(Notification, :count) end it 'does notify when sender is muted without hide_notifications' do recipient.mute!(sender, notifications: false) - is_expected.to change(Notification, :count) + expect { subject }.to change(Notification, :count) end it 'does not notify when sender\'s domain is blocked' do recipient.block_domain!(sender.domain) - is_expected.to_not change(Notification, :count) + expect { subject }.to_not change(Notification, :count) end it 'does still notify when sender\'s domain is blocked but sender is followed' do recipient.block_domain!(sender.domain) recipient.follow!(sender) - is_expected.to change(Notification, :count) + expect { subject }.to change(Notification, :count) end it 'does not notify when sender is silenced and not followed' do sender.silence! - is_expected.to_not change(Notification, :count) + expect { subject }.to_not change(Notification, :count) end it 'does not notify when recipient is suspended' do recipient.suspend! - is_expected.to_not change(Notification, :count) + expect { subject }.to_not change(Notification, :count) end context 'for direct messages' do @@ -61,7 +59,7 @@ RSpec.describe NotifyService, type: :service do let(:enabled) { true } it 'does not notify' do - is_expected.to_not change(Notification, :count) + expect { subject }.to_not change(Notification, :count) end context 'if the message chain is initiated by recipient, but is not direct message' do @@ -70,7 +68,7 @@ RSpec.describe NotifyService, type: :service do let(:activity) { Fabricate(:mention, account: recipient, status: Fabricate(:status, account: sender, visibility: :direct, thread: reply_to)) } it 'does not notify' do - is_expected.to_not change(Notification, :count) + expect { subject }.to_not change(Notification, :count) end end @@ -81,7 +79,7 @@ RSpec.describe NotifyService, type: :service do let(:activity) { Fabricate(:mention, account: recipient, status: Fabricate(:status, account: sender, visibility: :direct, thread: dummy_reply)) } it 'does not notify' do - is_expected.to_not change(Notification, :count) + expect { subject }.to_not change(Notification, :count) end end @@ -91,7 +89,7 @@ RSpec.describe NotifyService, type: :service do let(:activity) { Fabricate(:mention, account: recipient, status: Fabricate(:status, account: sender, visibility: :direct, thread: reply_to)) } it 'does notify' do - is_expected.to change(Notification, :count) + expect { subject }.to change(Notification, :count) end end end @@ -100,7 +98,7 @@ RSpec.describe NotifyService, type: :service do let(:enabled) { false } it 'does notify' do - is_expected.to change(Notification, :count) + expect { subject }.to change(Notification, :count) end end end @@ -112,17 +110,17 @@ RSpec.describe NotifyService, type: :service do it 'shows reblogs by default' do recipient.follow!(sender) - is_expected.to change(Notification, :count) + expect { subject }.to change(Notification, :count) end it 'shows reblogs when explicitly enabled' do recipient.follow!(sender, reblogs: true) - is_expected.to change(Notification, :count) + expect { subject }.to change(Notification, :count) end it 'shows reblogs when disabled' do recipient.follow!(sender, reblogs: false) - is_expected.to change(Notification, :count) + expect { subject }.to change(Notification, :count) end end @@ -134,12 +132,12 @@ RSpec.describe NotifyService, type: :service do it 'does not notify when conversation is muted' do recipient.mute_conversation!(activity.status.conversation) - is_expected.to_not change(Notification, :count) + expect { subject }.to_not change(Notification, :count) end it 'does not notify when it is a reply to a blocked user' do recipient.block!(asshole) - is_expected.to_not change(Notification, :count) + expect { subject }.to_not change(Notification, :count) end end @@ -147,7 +145,7 @@ RSpec.describe NotifyService, type: :service do let(:sender) { recipient } it 'does not notify when recipient is the sender' do - is_expected.to_not change(Notification, :count) + expect { subject }.to_not change(Notification, :count) end end @@ -163,7 +161,7 @@ RSpec.describe NotifyService, type: :service do let(:enabled) { true } it 'sends email' do - is_expected.to change(ActionMailer::Base.deliveries, :count).by(1) + expect { subject }.to change(ActionMailer::Base.deliveries, :count).by(1) end end @@ -171,7 +169,7 @@ RSpec.describe NotifyService, type: :service do let(:enabled) { false } it "doesn't send email" do - is_expected.to_not change(ActionMailer::Base.deliveries, :count).from(0) + expect { subject }.to_not change(ActionMailer::Base.deliveries, :count).from(0) end end end diff --git a/spec/services/suspend_account_service_spec.rb b/spec/services/suspend_account_service_spec.rb index cf7eb257a..5d45e4ffd 100644 --- a/spec/services/suspend_account_service_spec.rb +++ b/spec/services/suspend_account_service_spec.rb @@ -5,9 +5,7 @@ RSpec.describe SuspendAccountService, type: :service do let!(:local_follower) { Fabricate(:user, current_sign_in_at: 1.hour.ago).account } let!(:list) { Fabricate(:list, account: local_follower) } - subject do - -> { described_class.new.call(account) } - end + subject { described_class.new.call(account) } before do allow(FeedManager.instance).to receive(:unmerge_from_home).and_return(nil) @@ -18,13 +16,13 @@ RSpec.describe SuspendAccountService, type: :service do end it "unmerges from local followers' feeds" do - subject.call + subject expect(FeedManager.instance).to have_received(:unmerge_from_home).with(account, local_follower) expect(FeedManager.instance).to have_received(:unmerge_from_list).with(account, list) end it 'marks account as suspended' do - is_expected.to change { account.suspended? }.from(false).to(true) + expect { subject }.to change { account.suspended? }.from(false).to(true) end end @@ -51,7 +49,7 @@ RSpec.describe SuspendAccountService, type: :service do end it 'sends an update actor to followers and reporters' do - subject.call + subject expect(a_request(:post, remote_follower.inbox_url).with { |req| match_update_actor_request(req, account) }).to have_been_made.once expect(a_request(:post, remote_reporter.inbox_url).with { |req| match_update_actor_request(req, account) }).to have_been_made.once end @@ -77,7 +75,7 @@ RSpec.describe SuspendAccountService, type: :service do end it 'sends a reject follow' do - subject.call + subject expect(a_request(:post, account.inbox_url).with { |req| match_reject_follow_request(req, account, local_followee) }).to have_been_made.once end end diff --git a/spec/services/unsuspend_account_service_spec.rb b/spec/services/unsuspend_account_service_spec.rb index 0593beb6f..3ac4cc085 100644 --- a/spec/services/unsuspend_account_service_spec.rb +++ b/spec/services/unsuspend_account_service_spec.rb @@ -5,9 +5,7 @@ RSpec.describe UnsuspendAccountService, type: :service do let!(:local_follower) { Fabricate(:user, current_sign_in_at: 1.hour.ago).account } let!(:list) { Fabricate(:list, account: local_follower) } - subject do - -> { described_class.new.call(account) } - end + subject { described_class.new.call(account) } before do allow(FeedManager.instance).to receive(:merge_into_home).and_return(nil) @@ -33,7 +31,7 @@ RSpec.describe UnsuspendAccountService, type: :service do end it 'marks account as unsuspended' do - is_expected.to change { account.suspended? }.from(true).to(false) + expect { subject }.to change { account.suspended? }.from(true).to(false) end include_examples 'common behavior' do @@ -47,13 +45,13 @@ RSpec.describe UnsuspendAccountService, type: :service do end it "merges back into local followers' feeds" do - subject.call + subject expect(FeedManager.instance).to have_received(:merge_into_home).with(account, local_follower) expect(FeedManager.instance).to have_received(:merge_into_list).with(account, list) end it 'sends an update actor to followers and reporters' do - subject.call + subject expect(a_request(:post, remote_follower.inbox_url).with { |req| match_update_actor_request(req, account) }).to have_been_made.once expect(a_request(:post, remote_reporter.inbox_url).with { |req| match_update_actor_request(req, account) }).to have_been_made.once end @@ -75,18 +73,18 @@ RSpec.describe UnsuspendAccountService, type: :service do end it 're-fetches the account' do - subject.call + subject expect(resolve_account_service).to have_received(:call).with(account) end it "merges back into local followers' feeds" do - subject.call + subject expect(FeedManager.instance).to have_received(:merge_into_home).with(account, local_follower) expect(FeedManager.instance).to have_received(:merge_into_list).with(account, list) end it 'marks account as unsuspended' do - is_expected.to change { account.suspended? }.from(true).to(false) + expect { subject }.to change { account.suspended? }.from(true).to(false) end end @@ -99,18 +97,18 @@ RSpec.describe UnsuspendAccountService, type: :service do end it 're-fetches the account' do - subject.call + subject expect(resolve_account_service).to have_received(:call).with(account) end it "does not merge back into local followers' feeds" do - subject.call + subject expect(FeedManager.instance).to_not have_received(:merge_into_home).with(account, local_follower) expect(FeedManager.instance).to_not have_received(:merge_into_list).with(account, list) end it 'does not mark the account as unsuspended' do - is_expected.not_to change { account.suspended? } + expect { subject }.not_to change { account.suspended? } end end @@ -120,12 +118,12 @@ RSpec.describe UnsuspendAccountService, type: :service do end it 're-fetches the account' do - subject.call + subject expect(resolve_account_service).to have_received(:call).with(account) end it "does not merge back into local followers' feeds" do - subject.call + subject expect(FeedManager.instance).to_not have_received(:merge_into_home).with(account, local_follower) expect(FeedManager.instance).to_not have_received(:merge_into_list).with(account, list) end From 61cefbebf717326bd6ec3923e67e3702a24a0b24 Mon Sep 17 00:00:00 2001 From: Claire Date: Mon, 28 Mar 2022 20:51:51 +0200 Subject: [PATCH 347/419] Add advanced text formatting back into glitch-soc --- app/helpers/formatting_helper.rb | 2 +- app/lib/advanced_text_formatter.rb | 131 +++++++++++ app/lib/html_aware_formatter.rb | 6 +- lib/sanitize_ext/sanitize_config.rb | 57 +++-- spec/lib/advanced_text_formatter_spec.rb | 274 +++++++++++++++++++++++ spec/lib/sanitize_config_spec.rb | 18 +- 6 files changed, 459 insertions(+), 29 deletions(-) create mode 100644 app/lib/advanced_text_formatter.rb create mode 100644 spec/lib/advanced_text_formatter_spec.rb diff --git a/app/helpers/formatting_helper.rb b/app/helpers/formatting_helper.rb index e11156999..2a622ae0b 100644 --- a/app/helpers/formatting_helper.rb +++ b/app/helpers/formatting_helper.rb @@ -14,7 +14,7 @@ module FormattingHelper end def status_content_format(status) - html_aware_format(status.text, status.local?, preloaded_accounts: [status.account] + (status.respond_to?(:active_mentions) ? status.active_mentions.map(&:account) : [])) + html_aware_format(status.text, status.local?, preloaded_accounts: [status.account] + (status.respond_to?(:active_mentions) ? status.active_mentions.map(&:account) : []), content_type: status.content_type) end def account_bio_format(account) diff --git a/app/lib/advanced_text_formatter.rb b/app/lib/advanced_text_formatter.rb new file mode 100644 index 000000000..5ce87d306 --- /dev/null +++ b/app/lib/advanced_text_formatter.rb @@ -0,0 +1,131 @@ +# frozen_string_literal: true + +class AdvancedTextFormatter < TextFormatter + class HTMLRenderer < Redcarpet::Render::HTML + def initialize(options, &block) + super(options) + @format_link = block + end + + def block_code(code, _language) + <<~HTML.squish + 
#{h(code).gsub("\n", '
')}
+ HTML + end + + def autolink(link, link_type) + return link if link_type == :email + @format_link.call(link) + end + end + + # @param [String] text + # @param [Hash] options + # @option options [Boolean] :multiline + # @option options [Boolean] :with_domains + # @option options [Boolean] :with_rel_me + # @option options [Array] :preloaded_accounts + # @option options [String] :content_type + def initialize(text, options = {}) + content_type = options.delete(:content_type) + super(text, options) + + @text = format_markdown(text) if content_type == 'text/markdown' + end + + # Differs from TextFormatter by not messing with newline after parsing + def to_s + return ''.html_safe if text.blank? + + html = rewrite do |entity| + if entity[:url] + link_to_url(entity) + elsif entity[:hashtag] + link_to_hashtag(entity) + elsif entity[:screen_name] + link_to_mention(entity) + end + end + + html.html_safe # rubocop:disable Rails/OutputSafety + end + + # Differs from `TextFormatter` by skipping HTML tags and entities + def entities + @entities ||= begin + gaps = [] + total_offset = 0 + + escaped = text.gsub(/<[^>]*>|&#[0-9]+;/) do |match| + total_offset += match.length - 1 + end_offset = Regexp.last_match.end(0) + gaps << [end_offset - total_offset, total_offset] + ' ' + end + + Extractor.extract_entities_with_indices(escaped, extract_url_without_protocol: false).map do |entity| + start_pos, end_pos = entity[:indices] + offset_idx = gaps.rindex { |gap| gap.first <= start_pos } + offset = offset_idx.nil? ? 0 : gaps[offset_idx].last + entity.merge(indices: [start_pos + offset, end_pos + offset]) + end + end + end + + private + + # Differs from `TextFormatter` in that it keeps HTML; but it sanitizes at the end to remain safe + def rewrite + entities.sort_by! do |entity| + entity[:indices].first + end + + result = ''.dup + + last_index = entities.reduce(0) do |index, entity| + indices = entity[:indices] + result << text[index...indices.first] + result << yield(entity) + indices.last + end + + result << text[last_index..-1] + + Sanitize.fragment(result, Sanitize::Config::MASTODON_OUTGOING) + end + + def format_markdown(html) + html = markdown_formatter.render(html) + html.delete("\r").delete("\n") + end + + def markdown_formatter + extensions = { + autolink: true, + no_intra_emphasis: true, + fenced_code_blocks: true, + disable_indented_code_blocks: true, + strikethrough: true, + lax_spacing: true, + space_after_headers: true, + superscript: true, + underline: true, + highlight: true, + footnotes: false, + } + + renderer = HTMLRenderer.new({ + filter_html: false, + escape_html: false, + no_images: true, + no_styles: true, + safe_links_only: true, + hard_wrap: true, + link_attributes: { target: '_blank', rel: 'nofollow noopener' }, + }) do |url| + link_to_url({ url: url }) + end + + Redcarpet::Markdown.new(renderer, extensions) + end +end diff --git a/app/lib/html_aware_formatter.rb b/app/lib/html_aware_formatter.rb index 64edba09b..7a1cd0340 100644 --- a/app/lib/html_aware_formatter.rb +++ b/app/lib/html_aware_formatter.rb @@ -33,6 +33,10 @@ class HtmlAwareFormatter end def linkify - TextFormatter.new(text, options).to_s + if %w(text/markdown text/html).include?(@options[:content_type]) + AdvancedTextFormatter.new(text, options).to_s + else + TextFormatter.new(text, options).to_s + end end end diff --git a/lib/sanitize_ext/sanitize_config.rb b/lib/sanitize_ext/sanitize_config.rb index ecaec2f84..935e1f4f6 100644 --- a/lib/sanitize_ext/sanitize_config.rb +++ b/lib/sanitize_ext/sanitize_config.rb @@ -55,18 +55,6 @@ class Sanitize end end - LINK_REL_TRANSFORMER = lambda do |env| - return unless env[:node_name] == 'a' and env[:node]['href'] - - node = env[:node] - - rel = (node['rel'] || '').split(' ') & ['tag'] - unless env[:config][:outgoing] && TagManager.instance.local_url?(node['href']) - rel += ['nofollow', 'noopener', 'noreferrer'] - end - node['rel'] = rel.join(' ') - end - UNSUPPORTED_HREF_TRANSFORMER = lambda do |env| return unless env[:node_name] == 'a' @@ -97,6 +85,7 @@ class Sanitize add_attributes: { 'a' => { + 'rel' => 'nofollow noopener noreferrer', 'target' => '_blank', }, }, @@ -110,7 +99,6 @@ class Sanitize CLASS_WHITELIST_TRANSFORMER, IMG_TAG_TRANSFORMER, UNSUPPORTED_HREF_TRANSFORMER, - LINK_REL_TRANSFORMER, ] ) @@ -135,5 +123,48 @@ class Sanitize 'source' => { 'src' => HTTP_PROTOCOLS } ) ) + + LINK_REL_TRANSFORMER = lambda do |env| + return unless env[:node_name] == 'a' && env[:node]['href'] + + node = env[:node] + + rel = (node['rel'] || '').split(' ') & ['tag'] + rel += ['nofollow', 'noopener', 'noreferrer'] unless TagManager.instance.local_url?(node['href']) + + if rel.empty? + node['rel']&.delete + else + node['rel'] = rel.join(' ') + end + end + + LINK_TARGET_TRANSFORMER = lambda do |env| + return unless env[:node_name] == 'a' && env[:node]['href'] + + node = env[:node] + if node['target'] != '_blank' && TagManager.instance.local_url?(node['href']) + node['target']&.delete + else + node['target'] = '_blank' + end + end + + MASTODON_OUTGOING ||= freeze_config MASTODON_STRICT.merge( + attributes: merge( + MASTODON_STRICT[:attributes], + 'a' => %w(href rel class title target) + ), + + add_attributes: {}, + + transformers: [ + CLASS_WHITELIST_TRANSFORMER, + IMG_TAG_TRANSFORMER, + UNSUPPORTED_HREF_TRANSFORMER, + LINK_REL_TRANSFORMER, + LINK_TARGET_TRANSFORMER, + ] + ) end end diff --git a/spec/lib/advanced_text_formatter_spec.rb b/spec/lib/advanced_text_formatter_spec.rb new file mode 100644 index 000000000..c097b86e1 --- /dev/null +++ b/spec/lib/advanced_text_formatter_spec.rb @@ -0,0 +1,274 @@ +require 'rails_helper' + +RSpec.describe AdvancedTextFormatter do + describe '#to_s' do + let(:preloaded_accounts) { nil } + let(:content_type) { 'text/markdown' } + + subject { described_class.new(text, preloaded_accounts: preloaded_accounts, content_type: content_type).to_s } + + context 'given a markdown source' do + let(:content_type) { 'text/markdown' } + + context 'given text containing plain text' do + let(:text) { 'text' } + + it 'paragraphizes the text' do + is_expected.to eq '

text

' + end + end + + context 'given text containing line feeds' do + let(:text) { "line\nfeed" } + + it 'removes line feeds' do + is_expected.not_to include "\n" + end + end + + context 'given some inline code using backticks' do + let(:text) { 'test `foo` bar' } + + it 'formats code using ' do + is_expected.to include 'test foo bar' + end + end + + context 'given some quote' do + let(:text) { "> foo\n\nbar" } + + it 'formats code using ' do + is_expected.to include '

foo

' + end + end + + context 'given text containing linkable mentions' do + let(:preloaded_accounts) { [Fabricate(:account, username: 'alice')] } + let(:text) { '@alice' } + + it 'creates a mention link' do + is_expected.to include '@alice' + end + end + + context 'given text containing unlinkable mentions' do + let(:preloaded_accounts) { [] } + let(:text) { '@alice' } + + it 'does not create a mention link' do + is_expected.to include '@alice' + end + end + + context 'given a stand-alone medium URL' do + let(:text) { 'https://hackernoon.com/the-power-to-build-communities-a-response-to-mark-zuckerberg-3f2cac9148a4' } + + it 'matches the full URL' do + is_expected.to include 'href="https://hackernoon.com/the-power-to-build-communities-a-response-to-mark-zuckerberg-3f2cac9148a4"' + end + end + + context 'given a stand-alone google URL' do + let(:text) { 'http://google.com' } + + it 'matches the full URL' do + is_expected.to include 'href="http://google.com"' + end + end + + context 'given a stand-alone URL with a newer TLD' do + let(:text) { 'http://example.gay' } + + it 'matches the full URL' do + is_expected.to include 'href="http://example.gay"' + end + end + + context 'given a stand-alone IDN URL' do + let(:text) { 'https://nic.みんな/' } + + it 'matches the full URL' do + is_expected.to include 'href="https://nic.みんな/"' + end + + it 'has display URL' do + is_expected.to include 'nic.みんな/' + end + end + + context 'given a URL with a trailing period' do + let(:text) { 'http://www.mcmansionhell.com/post/156408871451/50-states-of-mcmansion-hell-scottsdale-arizona. ' } + + it 'matches the full URL but not the period' do + is_expected.to include 'href="http://www.mcmansionhell.com/post/156408871451/50-states-of-mcmansion-hell-scottsdale-arizona"' + end + end + + context 'given a URL enclosed with parentheses' do + let(:text) { '(http://google.com/)' } + + it 'matches the full URL but not the parentheses' do + is_expected.to include 'href="http://google.com/"' + end + end + + context 'given a URL with a trailing exclamation point' do + let(:text) { 'http://www.google.com!' } + + it 'matches the full URL but not the exclamation point' do + is_expected.to include 'href="http://www.google.com"' + end + end + + context 'given a URL with a trailing single quote' do + let(:text) { "http://www.google.com'" } + + it 'matches the full URL but not the single quote' do + is_expected.to include 'href="http://www.google.com"' + end + end + end + + context 'given a URL with a trailing angle bracket' do + let(:text) { 'http://www.google.com>' } + + it 'matches the full URL but not the angle bracket' do + is_expected.to include 'href="http://www.google.com"' + end + end + + context 'given a URL with a query string' do + context 'with escaped unicode character' do + let(:text) { 'https://www.ruby-toolbox.com/search?utf8=%E2%9C%93&q=autolink' } + + it 'matches the full URL' do + is_expected.to include 'href="https://www.ruby-toolbox.com/search?utf8=%E2%9C%93&q=autolink"' + end + end + + context 'with unicode character' do + let(:text) { 'https://www.ruby-toolbox.com/search?utf8=✓&q=autolink' } + + it 'matches the full URL' do + is_expected.to include 'href="https://www.ruby-toolbox.com/search?utf8=✓&q=autolink"' + end + end + + context 'with unicode character at the end' do + let(:text) { 'https://www.ruby-toolbox.com/search?utf8=✓' } + + it 'matches the full URL' do + is_expected.to include 'href="https://www.ruby-toolbox.com/search?utf8=✓"' + end + end + + context 'with escaped and not escaped unicode characters' do + let(:text) { 'https://www.ruby-toolbox.com/search?utf8=%E2%9C%93&utf81=✓&q=autolink' } + + it 'preserves escaped unicode characters' do + is_expected.to include 'href="https://www.ruby-toolbox.com/search?utf8=%E2%9C%93&utf81=✓&q=autolink"' + end + end + + context 'given a URL with parentheses in it' do + let(:text) { 'https://en.wikipedia.org/wiki/Diaspora_(software)' } + + it 'matches the full URL' do + is_expected.to include 'href="https://en.wikipedia.org/wiki/Diaspora_(software)"' + end + end + + context 'given a URL in quotation marks' do + let(:text) { '"https://example.com/"' } + + it 'does not match the quotation marks' do + is_expected.to include 'href="https://example.com/"' + end + end + + context 'given a URL in angle brackets' do + let(:text) { '' } + + it 'does not match the angle brackets' do + is_expected.to include 'href="https://example.com/"' + end + end + + context 'given a URL containing unsafe code (XSS attack, invisible part)' do + let(:text) { %q{http://example.com/blahblahblahblah/a} } + + it 'does not include the HTML in the URL' do + is_expected.to include '"http://example.com/blahblahblahblah/a"' + end + + it 'does not include a script tag' do + is_expected.to_not include '' } + + it 'does not include a script tag' do + is_expected.to_not include '