Add import for custom filters (#39256)
This commit is contained in:
parent
2e3b81cc1e
commit
35f3748482
@ -13,6 +13,7 @@ class Settings::ImportsController < Settings::BaseController
|
|||||||
domain_blocking: 'blocked_domains_failures.csv',
|
domain_blocking: 'blocked_domains_failures.csv',
|
||||||
bookmarks: 'bookmarks_failures.csv',
|
bookmarks: 'bookmarks_failures.csv',
|
||||||
lists: 'lists_failures.csv',
|
lists: 'lists_failures.csv',
|
||||||
|
custom_filters: 'custom_filters_failures.json',
|
||||||
}.freeze
|
}.freeze
|
||||||
|
|
||||||
TYPE_TO_HEADERS_MAP = {
|
TYPE_TO_HEADERS_MAP = {
|
||||||
@ -61,6 +62,21 @@ class Settings::ImportsController < Settings::BaseController
|
|||||||
|
|
||||||
send_data export_data, filename: filename
|
send_data export_data, filename: filename
|
||||||
end
|
end
|
||||||
|
|
||||||
|
format.json do
|
||||||
|
filename = TYPE_TO_FILENAME_MAP[@bulk_import.type.to_sym]
|
||||||
|
|
||||||
|
data_collection = { custom_filters: [] }
|
||||||
|
@bulk_import.rows.find_each do |row|
|
||||||
|
case @bulk_import.type.to_sym
|
||||||
|
when :custom_filters
|
||||||
|
data_collection[:custom_filters] << row.data
|
||||||
|
end
|
||||||
|
end
|
||||||
|
export_data = JSON.generate(data_collection)
|
||||||
|
|
||||||
|
send_data export_data, filename: filename
|
||||||
|
end
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
|||||||
@ -8,6 +8,7 @@
|
|||||||
# finished_at :datetime
|
# finished_at :datetime
|
||||||
# imported_items :integer default(0), not null
|
# imported_items :integer default(0), not null
|
||||||
# likely_mismatched :boolean default(FALSE), not null
|
# likely_mismatched :boolean default(FALSE), not null
|
||||||
|
# missing_status :boolean default(FALSE), not null
|
||||||
# original_filename :string default(""), not null
|
# original_filename :string default(""), not null
|
||||||
# overwrite :boolean default(FALSE), not null
|
# overwrite :boolean default(FALSE), not null
|
||||||
# processed_items :integer default(0), not null
|
# processed_items :integer default(0), not null
|
||||||
@ -34,6 +35,7 @@ class BulkImport < ApplicationRecord
|
|||||||
domain_blocking: 3,
|
domain_blocking: 3,
|
||||||
bookmarks: 4,
|
bookmarks: 4,
|
||||||
lists: 5,
|
lists: 5,
|
||||||
|
custom_filters: 6,
|
||||||
}
|
}
|
||||||
|
|
||||||
enum :state, {
|
enum :state, {
|
||||||
|
|||||||
@ -58,18 +58,31 @@ class Form::Import
|
|||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
def guessed_type_json
|
||||||
|
:custom_filters if parse_json.keys.any?('custom_filters')
|
||||||
|
end
|
||||||
|
|
||||||
# Whether the uploaded CSV file seems to correspond to a different import type than the one selected
|
# Whether the uploaded CSV file seems to correspond to a different import type than the one selected
|
||||||
def likely_mismatched?
|
def likely_mismatched?
|
||||||
guessed_type.present? && guessed_type != type.to_sym
|
guessed_type.present? && guessed_type != type.to_sym
|
||||||
end
|
end
|
||||||
|
|
||||||
|
def likely_mismatched_json?
|
||||||
|
guessed_type_json.present? && guessed_type_json != type.to_sym
|
||||||
|
end
|
||||||
|
|
||||||
def save
|
def save
|
||||||
return false unless valid?
|
return false unless valid?
|
||||||
|
|
||||||
ApplicationRecord.transaction do
|
ApplicationRecord.transaction do
|
||||||
now = Time.now.utc
|
now = Time.now.utc
|
||||||
@bulk_import = current_account.bulk_imports.create(type: type, overwrite: overwrite || false, state: :unconfirmed, original_filename: data.original_filename, likely_mismatched: likely_mismatched?)
|
if content_type_is_json?
|
||||||
nb_items = BulkImportRow.insert_all(parsed_rows.map { |row| { bulk_import_id: bulk_import.id, data: row, created_at: now, updated_at: now } }).length
|
@bulk_import = current_account.bulk_imports.create(type: type, overwrite: overwrite || false, state: :unconfirmed, original_filename: data.original_filename, likely_mismatched: likely_mismatched_json?, missing_status: missing_status?)
|
||||||
|
nb_items = BulkImportRow.insert_all(json_data.map { |row| { bulk_import_id: bulk_import.id, data: row, created_at: now, updated_at: now } }).length
|
||||||
|
else
|
||||||
|
@bulk_import = current_account.bulk_imports.create(type: type, overwrite: overwrite || false, state: :unconfirmed, original_filename: data.original_filename, likely_mismatched: likely_mismatched?)
|
||||||
|
nb_items = BulkImportRow.insert_all(parsed_rows.map { |row| { bulk_import_id: bulk_import.id, data: row, created_at: now, updated_at: now } }).length
|
||||||
|
end
|
||||||
@bulk_import.update(total_items: nb_items)
|
@bulk_import.update(total_items: nb_items)
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
@ -82,6 +95,14 @@ class Form::Import
|
|||||||
self.overwrite = str.to_sym == :overwrite
|
self.overwrite = str.to_sym == :overwrite
|
||||||
end
|
end
|
||||||
|
|
||||||
|
def missing_status?
|
||||||
|
return false unless content_type_is_json?
|
||||||
|
|
||||||
|
import_statuses = json_data.pluck(:statuses).flatten.uniq
|
||||||
|
db_statuses = Status.where(uri: import_statuses)
|
||||||
|
import_statuses.count != db_statuses.count
|
||||||
|
end
|
||||||
|
|
||||||
private
|
private
|
||||||
|
|
||||||
def file_name_matches?(string)
|
def file_name_matches?(string)
|
||||||
@ -153,6 +174,21 @@ class Form::Import
|
|||||||
def validate_data
|
def validate_data
|
||||||
return if data.nil?
|
return if data.nil?
|
||||||
return errors.add(:data, I18n.t('imports.errors.too_large')) if data.size > FILE_SIZE_LIMIT
|
return errors.add(:data, I18n.t('imports.errors.too_large')) if data.size > FILE_SIZE_LIMIT
|
||||||
|
|
||||||
|
if content_type_is_json?
|
||||||
|
validate_json_data
|
||||||
|
else
|
||||||
|
validate_csv_data
|
||||||
|
end
|
||||||
|
rescue CSV::MalformedCSVError => e
|
||||||
|
errors.add(:data, I18n.t('imports.errors.invalid_csv_file', error: e.message))
|
||||||
|
rescue JSON::ParserError => e
|
||||||
|
errors.add(:data, I18n.t('imports.errors.invalid_json_file', error: e.message))
|
||||||
|
rescue EmptyFileError
|
||||||
|
errors.add(:data, I18n.t('imports.errors.empty'))
|
||||||
|
end
|
||||||
|
|
||||||
|
def validate_csv_data
|
||||||
return errors.add(:data, I18n.t('imports.errors.incompatible_type')) unless default_csv_headers.all? { |header| csv_data.headers.include?(header) }
|
return errors.add(:data, I18n.t('imports.errors.incompatible_type')) unless default_csv_headers.all? { |header| csv_data.headers.include?(header) }
|
||||||
|
|
||||||
errors.add(:data, I18n.t('imports.errors.over_rows_processing_limit', count: ROWS_PROCESSING_LIMIT)) if csv_row_count > ROWS_PROCESSING_LIMIT
|
errors.add(:data, I18n.t('imports.errors.over_rows_processing_limit', count: ROWS_PROCESSING_LIMIT)) if csv_row_count > ROWS_PROCESSING_LIMIT
|
||||||
@ -163,9 +199,26 @@ class Form::Import
|
|||||||
limit -= current_account.following_count unless overwrite
|
limit -= current_account.following_count unless overwrite
|
||||||
errors.add(:data, I18n.t('users.follow_limit_reached', limit: base_limit)) if csv_row_count > limit
|
errors.add(:data, I18n.t('users.follow_limit_reached', limit: base_limit)) if csv_row_count > limit
|
||||||
end
|
end
|
||||||
rescue CSV::MalformedCSVError => e
|
end
|
||||||
errors.add(:data, I18n.t('imports.errors.invalid_csv_file', error: e.message))
|
|
||||||
rescue EmptyFileError
|
def validate_json_data
|
||||||
errors.add(:data, I18n.t('imports.errors.empty'))
|
errors.add(:data, I18n.t('imports.errors.over_rows_processing_limit', count: ROWS_PROCESSING_LIMIT)) if json_data.count > ROWS_PROCESSING_LIMIT
|
||||||
|
errors.add(:data, I18n.t('imports.errors.incompatible_type')) unless allowed_type_for_json?
|
||||||
|
end
|
||||||
|
|
||||||
|
def content_type_is_json?
|
||||||
|
data.content_type == 'application/json'
|
||||||
|
end
|
||||||
|
|
||||||
|
def json_data
|
||||||
|
parse_json['custom_filters'].map(&:deep_symbolize_keys)
|
||||||
|
end
|
||||||
|
|
||||||
|
def parse_json
|
||||||
|
@parse_json ||= JSON.parse(data.read)
|
||||||
|
end
|
||||||
|
|
||||||
|
def allowed_type_for_json?
|
||||||
|
type.to_sym.in?(%i(custom_filters))
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|||||||
@ -42,6 +42,14 @@ class BulkImportRowService
|
|||||||
FollowService.new.call(@account, @target_account) unless @account.id == @target_account.id
|
FollowService.new.call(@account, @target_account) unless @account.id == @target_account.id
|
||||||
|
|
||||||
list.accounts << @target_account
|
list.accounts << @target_account
|
||||||
|
when :custom_filters
|
||||||
|
filter = @account.custom_filters.create!(title: @data['title'], context: @data['context'])
|
||||||
|
filter.keywords = @data['keywords_attributes'].map { |keyword| CustomFilterKeyword.new(keyword: keyword['keyword'], whole_word: keyword['whole_word']) }
|
||||||
|
filter.action = @data['action'].to_sym
|
||||||
|
filter.expires_at = @data['expires_at']
|
||||||
|
status_ids = Status.where(uri: @data['statuses']).ids
|
||||||
|
filter.statuses = status_ids.map { |status| CustomFilterStatus.new(status_id: status) } if status_ids.any?
|
||||||
|
filter.save!
|
||||||
end
|
end
|
||||||
|
|
||||||
true
|
true
|
||||||
|
|||||||
@ -18,6 +18,8 @@ class BulkImportService < BaseService
|
|||||||
import_bookmarks!
|
import_bookmarks!
|
||||||
when :lists
|
when :lists
|
||||||
import_lists!
|
import_lists!
|
||||||
|
when :custom_filters
|
||||||
|
import_custom_filters!
|
||||||
end
|
end
|
||||||
|
|
||||||
@import.update!(state: :finished, finished_at: Time.now.utc) if @import.processing_complete?
|
@import.update!(state: :finished, finished_at: Time.now.utc) if @import.processing_complete?
|
||||||
@ -182,4 +184,14 @@ class BulkImportService < BaseService
|
|||||||
[row.id]
|
[row.id]
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
def import_custom_filters!
|
||||||
|
rows = @import.rows.to_a
|
||||||
|
|
||||||
|
@account.custom_filters.destroy_all if @import.overwrite?
|
||||||
|
|
||||||
|
Import::RowWorker.push_bulk(rows) do |row|
|
||||||
|
[row.id]
|
||||||
|
end
|
||||||
|
end
|
||||||
end
|
end
|
||||||
|
|||||||
@ -5,7 +5,7 @@
|
|||||||
.field-group
|
.field-group
|
||||||
= f.input :type,
|
= f.input :type,
|
||||||
as: :grouped_select,
|
as: :grouped_select,
|
||||||
collection: { constructive: %i(following bookmarks lists), destructive: %i(muting blocking domain_blocking) },
|
collection: { constructive: %i(following bookmarks lists), destructive: %i(muting blocking domain_blocking custom_filters) },
|
||||||
group_label_method: ->(group) { I18n.t("imports.type_groups.#{group.first}") },
|
group_label_method: ->(group) { I18n.t("imports.type_groups.#{group.first}") },
|
||||||
group_method: :last,
|
group_method: :last,
|
||||||
hint: t('imports.preface'),
|
hint: t('imports.preface'),
|
||||||
@ -61,5 +61,9 @@
|
|||||||
|
|
||||||
%td
|
%td
|
||||||
- if import.failure_count.positive?
|
- if import.failure_count.positive?
|
||||||
= link_to_if import.state_finished?, import.failure_count, failures_settings_import_path(import, format: :csv) do
|
- if import.type == 'custom_filters'
|
||||||
= import.failure_count
|
= link_to_if import.state_finished?, import.failure_count, failures_settings_import_path(import, format: :json) do
|
||||||
|
= import.failure_count
|
||||||
|
- else
|
||||||
|
= link_to_if import.state_finished?, import.failure_count, failures_settings_import_path(import, format: :csv) do
|
||||||
|
= import.failure_count
|
||||||
|
|||||||
@ -4,6 +4,9 @@
|
|||||||
- if @bulk_import.likely_mismatched?
|
- if @bulk_import.likely_mismatched?
|
||||||
.flash-message.warning= t('imports.mismatched_types_warning')
|
.flash-message.warning= t('imports.mismatched_types_warning')
|
||||||
|
|
||||||
|
- if @bulk_import&.missing_status == true
|
||||||
|
.flash-message.warning= I18n.t('imports.errors.status_not_found_warning')
|
||||||
|
|
||||||
- if @bulk_import.overwrite?
|
- if @bulk_import.overwrite?
|
||||||
%p.hint= t("imports.overwrite_preambles.#{@bulk_import.type}_html", filename: @bulk_import.original_filename, count: @bulk_import.total_items)
|
%p.hint= t("imports.overwrite_preambles.#{@bulk_import.type}_html", filename: @bulk_import.original_filename, count: @bulk_import.total_items)
|
||||||
- else
|
- else
|
||||||
|
|||||||
@ -67,8 +67,8 @@ ignore_unused:
|
|||||||
- 'user_mailer.*.subject'
|
- 'user_mailer.*.subject'
|
||||||
- 'email_subscription_mailer.*'
|
- 'email_subscription_mailer.*'
|
||||||
- 'notification_mailer.*'
|
- 'notification_mailer.*'
|
||||||
- 'imports.overwrite_preambles.{following,blocking,muting,domain_blocking,bookmarks,lists}_html.*'
|
- 'imports.overwrite_preambles.{following,blocking,muting,domain_blocking,bookmarks,lists,custom_filters}_html.*'
|
||||||
- 'imports.preambles.{following,blocking,muting,domain_blocking,bookmarks,lists}_html.*'
|
- 'imports.preambles.{following,blocking,muting,domain_blocking,bookmarks,lists,custom_filters}_html.*'
|
||||||
- 'mail_subscriptions.unsubscribe.emails.*'
|
- 'mail_subscriptions.unsubscribe.emails.*'
|
||||||
- 'preferences.other' # some locales are missing other keys, therefore leading i18n-tasks to detect `preferences` as plural and not finding use
|
- 'preferences.other' # some locales are missing other keys, therefore leading i18n-tasks to detect `preferences` as plural and not finding use
|
||||||
- 'edit_profile.other' # some locales are missing other keys, therefore leading i18n-tasks to detect `preferences` as plural and not finding use
|
- 'edit_profile.other' # some locales are missing other keys, therefore leading i18n-tasks to detect `preferences` as plural and not finding use
|
||||||
|
|||||||
@ -1558,6 +1558,7 @@ en-GB:
|
|||||||
incompatible_type: Incompatible with the selected import type
|
incompatible_type: Incompatible with the selected import type
|
||||||
invalid_csv_file: 'Invalid CSV file. Error: %{error}'
|
invalid_csv_file: 'Invalid CSV file. Error: %{error}'
|
||||||
over_rows_processing_limit: contains more than %{count} rows
|
over_rows_processing_limit: contains more than %{count} rows
|
||||||
|
status_not_found_warning: Some of your filters hide specific posts that are not known by this server. These posts will not be automatically filtered if they are discovered later by the server.
|
||||||
too_large: File is too large
|
too_large: File is too large
|
||||||
failures: Failures
|
failures: Failures
|
||||||
imported: Imported
|
imported: Imported
|
||||||
@ -1574,6 +1575,9 @@ en-GB:
|
|||||||
bookmarks_html:
|
bookmarks_html:
|
||||||
one: You are about to <strong>replace your bookmarks</strong> with up to <strong>%{count} post</strong> from <strong>%{filename}</strong>.
|
one: You are about to <strong>replace your bookmarks</strong> with up to <strong>%{count} post</strong> from <strong>%{filename}</strong>.
|
||||||
other: You are about to <strong>replace your bookmarks</strong> with up to <strong>%{count} posts</strong> from <strong>%{filename}</strong>.
|
other: You are about to <strong>replace your bookmarks</strong> with up to <strong>%{count} posts</strong> from <strong>%{filename}</strong>.
|
||||||
|
custom_filters_html:
|
||||||
|
one: You are about to <strong>replace your filters</strong> with contents of <strong>%{filename}</strong>. Up to <strong>%{count} filter</strong> will be added to new filters.
|
||||||
|
other: You are about to <strong>replace your filters</strong> with contents of <strong>%{filename}</strong>. Up to <strong>%{count} filters</strong> will be added to new filters.
|
||||||
domain_blocking_html:
|
domain_blocking_html:
|
||||||
one: You are about to <strong>replace your domain block list</strong> with up to <strong>%{count} domain</strong> from <strong>%{filename}</strong>.
|
one: You are about to <strong>replace your domain block list</strong> with up to <strong>%{count} domain</strong> from <strong>%{filename}</strong>.
|
||||||
other: You are about to <strong>replace your domain block list</strong> with up to <strong>%{count} domains</strong> from <strong>%{filename}</strong>.
|
other: You are about to <strong>replace your domain block list</strong> with up to <strong>%{count} domains</strong> from <strong>%{filename}</strong>.
|
||||||
@ -1593,6 +1597,9 @@ en-GB:
|
|||||||
bookmarks_html:
|
bookmarks_html:
|
||||||
one: You are about to add up to <strong>%{count} post</strong> from <strong>%{filename}</strong> to your <strong>bookmarks</strong>.
|
one: You are about to add up to <strong>%{count} post</strong> from <strong>%{filename}</strong> to your <strong>bookmarks</strong>.
|
||||||
other: You are about to add up to <strong>%{count} posts</strong> from <strong>%{filename}</strong> to your <strong>bookmarks</strong>.
|
other: You are about to add up to <strong>%{count} posts</strong> from <strong>%{filename}</strong> to your <strong>bookmarks</strong>.
|
||||||
|
custom_filters_html:
|
||||||
|
one: You are about to add up to <strong>%{count} filter</strong> from <strong>%{filename}</strong> to your <strong>filters</strong>.
|
||||||
|
other: You are about to add up to <strong>%{count} filters</strong> from <strong>%{filename}</strong> to your <strong>filters</strong>.
|
||||||
domain_blocking_html:
|
domain_blocking_html:
|
||||||
one: You are about to <strong>block</strong> up to <strong>%{count} domain</strong> from <strong>%{filename}</strong>.
|
one: You are about to <strong>block</strong> up to <strong>%{count} domain</strong> from <strong>%{filename}</strong>.
|
||||||
other: You are about to <strong>block</strong> up to <strong>%{count} domains</strong> from <strong>%{filename}</strong>.
|
other: You are about to <strong>block</strong> up to <strong>%{count} domains</strong> from <strong>%{filename}</strong>.
|
||||||
@ -1629,6 +1636,7 @@ en-GB:
|
|||||||
types:
|
types:
|
||||||
blocking: Blocking list
|
blocking: Blocking list
|
||||||
bookmarks: Bookmarks
|
bookmarks: Bookmarks
|
||||||
|
custom_filters: Custom filters
|
||||||
domain_blocking: Domain blocking list
|
domain_blocking: Domain blocking list
|
||||||
following: Following list
|
following: Following list
|
||||||
lists: Lists
|
lists: Lists
|
||||||
|
|||||||
@ -1665,7 +1665,9 @@ en:
|
|||||||
empty: Empty CSV file
|
empty: Empty CSV file
|
||||||
incompatible_type: Incompatible with the selected import type
|
incompatible_type: Incompatible with the selected import type
|
||||||
invalid_csv_file: 'Invalid CSV file. Error: %{error}'
|
invalid_csv_file: 'Invalid CSV file. Error: %{error}'
|
||||||
|
invalid_json_file: 'Invalid JSON file. Error: %{error}'
|
||||||
over_rows_processing_limit: contains more than %{count} rows
|
over_rows_processing_limit: contains more than %{count} rows
|
||||||
|
status_not_found_warning: Some of your filters hide specific posts that are not known by this server. These posts will not be automatically filtered if they are discovered later by the server.
|
||||||
too_large: File is too large
|
too_large: File is too large
|
||||||
failures: Failures
|
failures: Failures
|
||||||
imported: Imported
|
imported: Imported
|
||||||
@ -1682,6 +1684,9 @@ en:
|
|||||||
bookmarks_html:
|
bookmarks_html:
|
||||||
one: You are about to <strong>replace your bookmarks</strong> with up to <strong>%{count} post</strong> from <strong>%{filename}</strong>.
|
one: You are about to <strong>replace your bookmarks</strong> with up to <strong>%{count} post</strong> from <strong>%{filename}</strong>.
|
||||||
other: You are about to <strong>replace your bookmarks</strong> with up to <strong>%{count} posts</strong> from <strong>%{filename}</strong>.
|
other: You are about to <strong>replace your bookmarks</strong> with up to <strong>%{count} posts</strong> from <strong>%{filename}</strong>.
|
||||||
|
custom_filters_html:
|
||||||
|
one: You are about to <strong>replace your filters</strong> with contents of <strong>%{filename}</strong>. Up to <strong>%{count} filter</strong> will be added to new filters.
|
||||||
|
other: You are about to <strong>replace your filters</strong> with contents of <strong>%{filename}</strong>. Up to <strong>%{count} filters</strong> will be added to new filters.
|
||||||
domain_blocking_html:
|
domain_blocking_html:
|
||||||
one: You are about to <strong>replace your domain block list</strong> with up to <strong>%{count} domain</strong> from <strong>%{filename}</strong>.
|
one: You are about to <strong>replace your domain block list</strong> with up to <strong>%{count} domain</strong> from <strong>%{filename}</strong>.
|
||||||
other: You are about to <strong>replace your domain block list</strong> with up to <strong>%{count} domains</strong> from <strong>%{filename}</strong>.
|
other: You are about to <strong>replace your domain block list</strong> with up to <strong>%{count} domains</strong> from <strong>%{filename}</strong>.
|
||||||
@ -1701,6 +1706,9 @@ en:
|
|||||||
bookmarks_html:
|
bookmarks_html:
|
||||||
one: You are about to add up to <strong>%{count} post</strong> from <strong>%{filename}</strong> to your <strong>bookmarks</strong>.
|
one: You are about to add up to <strong>%{count} post</strong> from <strong>%{filename}</strong> to your <strong>bookmarks</strong>.
|
||||||
other: You are about to add up to <strong>%{count} posts</strong> from <strong>%{filename}</strong> to your <strong>bookmarks</strong>.
|
other: You are about to add up to <strong>%{count} posts</strong> from <strong>%{filename}</strong> to your <strong>bookmarks</strong>.
|
||||||
|
custom_filters_html:
|
||||||
|
one: You are about to add up to <strong>%{count} filter</strong> from <strong>%{filename}</strong> to your <strong>filters</strong>.
|
||||||
|
other: You are about to add up to <strong>%{count} filters</strong> from <strong>%{filename}</strong> to your <strong>filters</strong>.
|
||||||
domain_blocking_html:
|
domain_blocking_html:
|
||||||
one: You are about to <strong>block</strong> up to <strong>%{count} domain</strong> from <strong>%{filename}</strong>.
|
one: You are about to <strong>block</strong> up to <strong>%{count} domain</strong> from <strong>%{filename}</strong>.
|
||||||
other: You are about to <strong>block</strong> up to <strong>%{count} domains</strong> from <strong>%{filename}</strong>.
|
other: You are about to <strong>block</strong> up to <strong>%{count} domains</strong> from <strong>%{filename}</strong>.
|
||||||
|
|||||||
@ -0,0 +1,7 @@
|
|||||||
|
# frozen_string_literal: true
|
||||||
|
|
||||||
|
class AddMissingStatusToBulkImport < ActiveRecord::Migration[8.1]
|
||||||
|
def change
|
||||||
|
add_column :bulk_imports, :missing_status, :boolean, null: false, default: false
|
||||||
|
end
|
||||||
|
end
|
||||||
@ -10,7 +10,7 @@
|
|||||||
#
|
#
|
||||||
# It's strongly recommended that you check this file into your version control system.
|
# It's strongly recommended that you check this file into your version control system.
|
||||||
|
|
||||||
ActiveRecord::Schema[8.1].define(version: 2026_05_05_155103) do
|
ActiveRecord::Schema[8.1].define(version: 2026_06_11_150940) do
|
||||||
# These are extensions that must be enabled in order to support this database
|
# These are extensions that must be enabled in order to support this database
|
||||||
enable_extension "pg_catalog.plpgsql"
|
enable_extension "pg_catalog.plpgsql"
|
||||||
|
|
||||||
@ -338,6 +338,7 @@ ActiveRecord::Schema[8.1].define(version: 2026_05_05_155103) do
|
|||||||
t.datetime "finished_at", precision: nil
|
t.datetime "finished_at", precision: nil
|
||||||
t.integer "imported_items", default: 0, null: false
|
t.integer "imported_items", default: 0, null: false
|
||||||
t.boolean "likely_mismatched", default: false, null: false
|
t.boolean "likely_mismatched", default: false, null: false
|
||||||
|
t.boolean "missing_status", default: false, null: false
|
||||||
t.string "original_filename", default: "", null: false
|
t.string "original_filename", default: "", null: false
|
||||||
t.boolean "overwrite", default: false, null: false
|
t.boolean "overwrite", default: false, null: false
|
||||||
t.integer "processed_items", default: 0, null: false
|
t.integer "processed_items", default: 0, null: false
|
||||||
|
|||||||
@ -169,6 +169,39 @@ RSpec.describe Settings::ImportsController do
|
|||||||
it_behaves_like 'export failed rows', 'following_accounts_failures.csv', "Account address,Show boosts,Notify on new posts,Languages\nfoo@bar,true,false,\nuser@bar,false,true,\"fr, de\"\n"
|
it_behaves_like 'export failed rows', 'following_accounts_failures.csv', "Account address,Show boosts,Notify on new posts,Languages\nfoo@bar,true,false,\nuser@bar,false,true,\"fr, de\"\n"
|
||||||
end
|
end
|
||||||
|
|
||||||
|
context 'with custom filters' do
|
||||||
|
subject { get :failures, params: { id: bulk_import.id }, format: :json }
|
||||||
|
|
||||||
|
let(:import_type) { 'custom_filters' }
|
||||||
|
let(:rows) do
|
||||||
|
[
|
||||||
|
{
|
||||||
|
'title' => 'random title',
|
||||||
|
'expires_at' => nil,
|
||||||
|
'context' => ['public', 'account'],
|
||||||
|
'action' => 'warn',
|
||||||
|
'keywords_attributes' => [{
|
||||||
|
'keyword' => 'all them keywords',
|
||||||
|
'whole_word' => true,
|
||||||
|
}, {
|
||||||
|
'keyword' => 'more keywords even',
|
||||||
|
'whole_word' => true,
|
||||||
|
}],
|
||||||
|
'statuses' => ['status'],
|
||||||
|
},
|
||||||
|
]
|
||||||
|
end
|
||||||
|
let(:bulk_import) { Fabricate(:bulk_import, account: user.account, type: import_type, state: :finished) }
|
||||||
|
|
||||||
|
before do
|
||||||
|
rows.each { |data| Fabricate(:bulk_import_row, bulk_import: bulk_import, data: data) }
|
||||||
|
bulk_import.update(total_items: bulk_import.rows.count, processed_items: bulk_import.rows.count, imported_items: 0)
|
||||||
|
end
|
||||||
|
|
||||||
|
it_behaves_like 'export failed rows', 'custom_filters_failures.json',
|
||||||
|
'{"custom_filters":[{"title":"random title","action":"warn","context":["public","account"],"statuses":["status"],"expires_at":null,"keywords_attributes":[{"keyword":"all them keywords","whole_word":true},{"keyword":"more keywords even","whole_word":true}]},{"title":"random title","action":"warn","context":["public","account"],"statuses":["status"],"expires_at":null,"keywords_attributes":[{"keyword":"all them keywords","whole_word":true},{"keyword":"more keywords even","whole_word":true}]}]}' # rubocop:disable Layout/LineLength
|
||||||
|
end
|
||||||
|
|
||||||
context 'with blocks' do
|
context 'with blocks' do
|
||||||
let(:import_type) { 'blocking' }
|
let(:import_type) { 'blocking' }
|
||||||
|
|
||||||
@ -291,5 +324,29 @@ RSpec.describe Settings::ImportsController do
|
|||||||
|
|
||||||
it_behaves_like 'unsuccessful import', 'following', 'empty.csv', 'merge'
|
it_behaves_like 'unsuccessful import', 'following', 'empty.csv', 'merge'
|
||||||
it_behaves_like 'unsuccessful import', 'following', 'empty.csv', 'overwrite'
|
it_behaves_like 'unsuccessful import', 'following', 'empty.csv', 'overwrite'
|
||||||
|
|
||||||
|
context 'with custom filter' do
|
||||||
|
subject { post :create, params: { form_import: { type: 'custom_filters', mode: mode, data: data } } }
|
||||||
|
|
||||||
|
describe 'successful import' do
|
||||||
|
let(:data) { fixture_file_upload('custom_filters.json', 'application/json') }
|
||||||
|
let(:mode) { 'merge' }
|
||||||
|
|
||||||
|
it 'creates an unconfirmed bulk_import with expected type and redirects', :aggregate_failures do
|
||||||
|
expect { subject }.to change { user.account.bulk_imports.pluck(:state, :type) }.from([]).to([['unconfirmed', 'custom_filters']])
|
||||||
|
expect(response).to redirect_to(settings_import_path(user.account.bulk_imports.first))
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
describe 'failing import' do
|
||||||
|
let(:mode) { 'merge' }
|
||||||
|
let(:data) { fixture_file_upload('empty.json', 'application/json') }
|
||||||
|
|
||||||
|
it 'does not creates an unconfirmed bulk_import', :aggregate_failures do
|
||||||
|
expect { subject }.to_not(change { user.account.bulk_imports.count })
|
||||||
|
expect(response.body).to include('field_with_errors')
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|||||||
32
spec/fixtures/files/custom_filters.json
vendored
Normal file
32
spec/fixtures/files/custom_filters.json
vendored
Normal file
@ -0,0 +1,32 @@
|
|||||||
|
{
|
||||||
|
"custom_filters": [
|
||||||
|
{
|
||||||
|
"title": "dfjswa",
|
||||||
|
"expires_at": null,
|
||||||
|
"context": ["home"],
|
||||||
|
"action": "warn",
|
||||||
|
"keywords_attributes": [{ "keyword": "dvshja", "whole_word": true }],
|
||||||
|
"statuses": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"title": "filter with a phrase as title",
|
||||||
|
"expires_at": null,
|
||||||
|
"context": ["home", "notifications", "public"],
|
||||||
|
"action": "warn",
|
||||||
|
"keywords_attributes": [
|
||||||
|
{ "keyword": "more words let's see ", "whole_word": true }
|
||||||
|
],
|
||||||
|
"statuses": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"title": "how do I add a status to a filter?",
|
||||||
|
"expires_at": null,
|
||||||
|
"context": ["public", "account"],
|
||||||
|
"action": "warn",
|
||||||
|
"keywords_attributes": [
|
||||||
|
{ "keyword": "something something", "whole_word": true }
|
||||||
|
],
|
||||||
|
"statuses": []
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
0
spec/fixtures/files/empty.json
vendored
Normal file
0
spec/fixtures/files/empty.json
vendored
Normal file
@ -43,6 +43,17 @@ RSpec.describe Form::Import do
|
|||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
describe 'when the import type is custom_filters' do
|
||||||
|
let(:data) { fixture_file_upload(import_file, content_type) }
|
||||||
|
let(:import_file) { File.open('spec/fixtures/files/custom_filters.json') }
|
||||||
|
let(:import_type) { 'custom_filters' }
|
||||||
|
let(:content_type) { 'application/json' }
|
||||||
|
|
||||||
|
it 'passes validation' do
|
||||||
|
expect(subject).to be_valid
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
context 'when the file too large' do
|
context 'when the file too large' do
|
||||||
let(:import_type) { 'following' }
|
let(:import_type) { 'following' }
|
||||||
let(:import_file) { 'imports.txt' }
|
let(:import_file) { 'imports.txt' }
|
||||||
@ -258,6 +269,69 @@ RSpec.describe Form::Import do
|
|||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
describe 'when importing json' do
|
||||||
|
let(:import_type) { 'custom_filters' }
|
||||||
|
let(:data) { fixture_file_upload('custom_filters.json', 'application/json') }
|
||||||
|
let(:import_mode) { 'merge' }
|
||||||
|
let(:expected_rows) do
|
||||||
|
[
|
||||||
|
{
|
||||||
|
'title' => 'dfjswa',
|
||||||
|
'expires_at' => nil,
|
||||||
|
'context' => ['home'],
|
||||||
|
'action' => 'warn',
|
||||||
|
'keywords_attributes' => [{ 'keyword' => 'dvshja', 'whole_word' => true }],
|
||||||
|
'statuses' => [],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
'title' => 'filter with a phrase as title',
|
||||||
|
'expires_at' => nil,
|
||||||
|
'context' => %w(home notifications public),
|
||||||
|
'action' => 'warn',
|
||||||
|
'keywords_attributes' => [
|
||||||
|
{ 'keyword' => "more words let's see ", 'whole_word' => true },
|
||||||
|
],
|
||||||
|
'statuses' => [],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
'title' => 'how do I add a status to a filter?',
|
||||||
|
'expires_at' => nil,
|
||||||
|
'context' => ['public', 'account'],
|
||||||
|
'action' => 'warn',
|
||||||
|
'keywords_attributes' => [
|
||||||
|
{ 'keyword' => 'something something', 'whole_word' => true },
|
||||||
|
],
|
||||||
|
'statuses' => [],
|
||||||
|
},
|
||||||
|
]
|
||||||
|
end
|
||||||
|
|
||||||
|
before do
|
||||||
|
subject.save
|
||||||
|
end
|
||||||
|
|
||||||
|
context 'with a BulkImport' do
|
||||||
|
let(:bulk_import) { account.bulk_imports.first }
|
||||||
|
|
||||||
|
it 'creates a bulk import with correct values' do
|
||||||
|
expect(bulk_import)
|
||||||
|
.to be_present
|
||||||
|
.and have_attributes(
|
||||||
|
type: eq(subject.type),
|
||||||
|
original_filename: eq(subject.data.original_filename),
|
||||||
|
likely_mismatched?: eq(subject.likely_mismatched_json?),
|
||||||
|
overwrite?: eq(!!subject.overwrite), # rubocop:disable Style/DoubleNegation
|
||||||
|
processed_items: eq(0),
|
||||||
|
imported_items: eq(0),
|
||||||
|
total_items: eq(bulk_import.rows.count),
|
||||||
|
state_unconfirmed?: be(true)
|
||||||
|
)
|
||||||
|
expect(bulk_import.rows.pluck(:data))
|
||||||
|
.to match_array(expected_rows)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
it_behaves_like('on successful import', 'following', 'merge', 'imports.txt', %w(user@example.com user@test.com).map { |acct| { 'acct' => acct } })
|
it_behaves_like('on successful import', 'following', 'merge', 'imports.txt', %w(user@example.com user@test.com).map { |acct| { 'acct' => acct } })
|
||||||
it_behaves_like('on successful import', 'following', 'overwrite', 'imports.txt', %w(user@example.com user@test.com).map { |acct| { 'acct' => acct } })
|
it_behaves_like('on successful import', 'following', 'overwrite', 'imports.txt', %w(user@example.com user@test.com).map { |acct| { 'acct' => acct } })
|
||||||
it_behaves_like('on successful import', 'blocking', 'merge', 'imports.txt', %w(user@example.com user@test.com).map { |acct| { 'acct' => acct } })
|
it_behaves_like('on successful import', 'blocking', 'merge', 'imports.txt', %w(user@example.com user@test.com).map { |acct| { 'acct' => acct } })
|
||||||
|
|||||||
@ -4,7 +4,10 @@ require 'rails_helper'
|
|||||||
|
|
||||||
RSpec.describe 'Settings Imports' do
|
RSpec.describe 'Settings Imports' do
|
||||||
describe 'POST /settings/imports' do
|
describe 'POST /settings/imports' do
|
||||||
before { sign_in Fabricate(:user) }
|
let(:user) { Fabricate(:user) }
|
||||||
|
let(:account) { user.account }
|
||||||
|
|
||||||
|
before { sign_in user }
|
||||||
|
|
||||||
it 'gracefully handles invalid nested params' do
|
it 'gracefully handles invalid nested params' do
|
||||||
post settings_imports_path(form_import: 'invalid')
|
post settings_imports_path(form_import: 'invalid')
|
||||||
@ -12,5 +15,22 @@ RSpec.describe 'Settings Imports' do
|
|||||||
expect(response)
|
expect(response)
|
||||||
.to have_http_status(400)
|
.to have_http_status(400)
|
||||||
end
|
end
|
||||||
|
|
||||||
|
describe 'with JSON' do
|
||||||
|
subject { post settings_imports_path, params: { form_import: { type: 'custom_filters', mode: 'merge', data: data } } }
|
||||||
|
|
||||||
|
let!(:data) { fixture_file_upload('custom_filters.json', 'application/json') }
|
||||||
|
let(:confirm) { post confirm_settings_import_path(id: user.account.bulk_imports.last.id) }
|
||||||
|
|
||||||
|
it 'redirects to confirm_settings_import_path' do
|
||||||
|
subject
|
||||||
|
expect(response).to have_http_status(302)
|
||||||
|
.and redirect_to(settings_import_path(id: user.account.bulk_imports.last.id))
|
||||||
|
expect(user.account.bulk_imports.last.state).to eq('unconfirmed')
|
||||||
|
confirm
|
||||||
|
expect(response).to have_http_status(302)
|
||||||
|
expect(user.account.bulk_imports.last.state).to eq('scheduled')
|
||||||
|
end
|
||||||
|
end
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|||||||
@ -329,6 +329,106 @@ RSpec.describe BulkImportService do
|
|||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
context 'when importing custom_filters' do
|
||||||
|
let(:import_type) { 'custom_filters' }
|
||||||
|
let!(:rows) do
|
||||||
|
[{
|
||||||
|
'title' => 'baz',
|
||||||
|
'expires_at' => nil,
|
||||||
|
'context' => ['home', 'notifications'],
|
||||||
|
'action' => 'warn',
|
||||||
|
'keywords_attributes' => [{
|
||||||
|
'keyword' => 'discourse',
|
||||||
|
'whole_word' => true,
|
||||||
|
}, {
|
||||||
|
'keyword' => 'something',
|
||||||
|
'whole_word' => false,
|
||||||
|
}],
|
||||||
|
'statuses' => ['http://localhost:3000/ap/users/116646814515254858/statuses/116681350935935708'],
|
||||||
|
}, {
|
||||||
|
'title' => 'buzz',
|
||||||
|
'expires_at' => nil,
|
||||||
|
'context' => ['notifications'],
|
||||||
|
'action' => 'warn',
|
||||||
|
'keywords_attributes' => [{
|
||||||
|
'keyword' => 'discourse',
|
||||||
|
'whole_word' => true,
|
||||||
|
}, {
|
||||||
|
'keyword' => 'something',
|
||||||
|
'whole_word' => false,
|
||||||
|
}],
|
||||||
|
'statuses' => [ActivityPub::TagManager.instance.uri_for(status)],
|
||||||
|
}].map { |data| import.rows.create!(data: data) }
|
||||||
|
end
|
||||||
|
let(:overwrite) { false }
|
||||||
|
let(:status) { Fabricate(:status, account: account, text: 'something something') }
|
||||||
|
let(:filter) { Fabricate(:custom_filter, account: account, title: 'a mazing title') }
|
||||||
|
let(:status_filter) { Fabricate(:custom_filter_status, custom_filter: filter, status: status) }
|
||||||
|
|
||||||
|
before do
|
||||||
|
status_filter
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'enqueues workers for the expected rows and updates filters, keywords and statuses after worker run' do
|
||||||
|
subject.call(import)
|
||||||
|
expect(row_worker_job_args).to match_array(rows.map(&:id))
|
||||||
|
stub_fetch_remote_and_drain_workers
|
||||||
|
expect(account.custom_filters.count).to eq(3)
|
||||||
|
expect(account.custom_filters.order(:phrase).last.statuses.count).to eq(1)
|
||||||
|
expect(account.custom_filters.last.keywords.count).to eq(2)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
context 'when importing custom_filters with overwrite' do
|
||||||
|
let(:import_type) { 'custom_filters' }
|
||||||
|
let!(:rows) do
|
||||||
|
[{
|
||||||
|
'title' => 'baz',
|
||||||
|
'expires_at' => nil,
|
||||||
|
'context' => ['home', 'notifications'],
|
||||||
|
'action' => 'warn',
|
||||||
|
'keywords_attributes' => [{
|
||||||
|
'keyword' => 'discourse',
|
||||||
|
'whole_word' => true,
|
||||||
|
}, {
|
||||||
|
'keyword' => 'something',
|
||||||
|
'whole_word' => false,
|
||||||
|
}],
|
||||||
|
'statuses' => ['http://localhost:3000/ap/users/116646814515254858/statuses/116681350935935708'],
|
||||||
|
}, {
|
||||||
|
'title' => 'buzz',
|
||||||
|
'expires_at' => nil,
|
||||||
|
'context' => ['notifications'],
|
||||||
|
'action' => 'warn',
|
||||||
|
'keywords_attributes' => [{
|
||||||
|
'keyword' => 'discourse',
|
||||||
|
'whole_word' => true,
|
||||||
|
}, {
|
||||||
|
'keyword' => 'something',
|
||||||
|
'whole_word' => false,
|
||||||
|
}],
|
||||||
|
'statuses' => [ActivityPub::TagManager.instance.uri_for(status)],
|
||||||
|
}].map { |data| import.rows.create!(data: data) }
|
||||||
|
end
|
||||||
|
let(:overwrite) { true }
|
||||||
|
let(:status) { Fabricate(:status, text: 'something something') }
|
||||||
|
let(:filter) { Fabricate(:custom_filter, account: account, title: 'a mazing title') }
|
||||||
|
let(:status_filter) { Fabricate(:custom_filter_status, custom_filter: filter, status: status) }
|
||||||
|
|
||||||
|
before do
|
||||||
|
status_filter
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'enqueues workers for the expected rows and updates filters, keywords and statuses after worker run' do
|
||||||
|
subject.call(import)
|
||||||
|
expect(row_worker_job_args).to match_array(rows.map(&:id))
|
||||||
|
stub_fetch_remote_and_drain_workers
|
||||||
|
expect(account.custom_filters.count).to eq(2)
|
||||||
|
expect(account.custom_filters.order(:phrase).last.statuses.count).to eq(1)
|
||||||
|
expect(account.custom_filters.order(:phrase).last.keywords.count).to eq(2)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
def row_worker_job_args
|
def row_worker_job_args
|
||||||
Import::RowWorker
|
Import::RowWorker
|
||||||
.jobs
|
.jobs
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user