Add import for custom filters (#39256)

app/controllers/settings/imports_controller.rb

@@ -13,6 +13,7 @@ class Settings::ImportsController < Settings::BaseController
     domain_blocking: 'blocked_domains_failures.csv',
     bookmarks: 'bookmarks_failures.csv',
     lists: 'lists_failures.csv',
+    custom_filters: 'custom_filters_failures.json',
   }.freeze
 
   TYPE_TO_HEADERS_MAP = {
@@ -61,6 +62,21 @@ class Settings::ImportsController < Settings::BaseController
 
         send_data export_data, filename: filename
       end
+
+      format.json do
+        filename = TYPE_TO_FILENAME_MAP[@bulk_import.type.to_sym]
+
+        data_collection = { custom_filters: [] }
+        @bulk_import.rows.find_each do |row|
+          case @bulk_import.type.to_sym
+          when :custom_filters
+            data_collection[:custom_filters] << row.data
+          end
+        end
+        export_data = JSON.generate(data_collection)
+
+        send_data export_data, filename: filename
+      end
     end
   end
 

app/models/bulk_import.rb

@@ -8,6 +8,7 @@
 #  finished_at       :datetime
 #  imported_items    :integer          default(0), not null
 #  likely_mismatched :boolean          default(FALSE), not null
+#  missing_status    :boolean          default(FALSE), not null
 #  original_filename :string           default(""), not null
 #  overwrite         :boolean          default(FALSE), not null
 #  processed_items   :integer          default(0), not null
@@ -34,6 +35,7 @@ class BulkImport < ApplicationRecord
     domain_blocking: 3,
     bookmarks: 4,
     lists: 5,
+    custom_filters: 6,
   }
 
   enum :state, {

app/models/form/import.rb

@@ -58,18 +58,31 @@ class Form::Import
     end
   end
 
+  def guessed_type_json
+    :custom_filters if parse_json.keys.any?('custom_filters')
+  end
+
   # Whether the uploaded CSV file seems to correspond to a different import type than the one selected
   def likely_mismatched?
     guessed_type.present? && guessed_type != type.to_sym
   end
 
+  def likely_mismatched_json?
+    guessed_type_json.present? && guessed_type_json != type.to_sym
+  end
+
   def save
     return false unless valid?
 
     ApplicationRecord.transaction do
       now = Time.now.utc
-      @bulk_import = current_account.bulk_imports.create(type: type, overwrite: overwrite || false, state: :unconfirmed, original_filename: data.original_filename, likely_mismatched: likely_mismatched?)
-      nb_items = BulkImportRow.insert_all(parsed_rows.map { |row| { bulk_import_id: bulk_import.id, data: row, created_at: now, updated_at: now } }).length
+      if content_type_is_json?
+        @bulk_import = current_account.bulk_imports.create(type: type, overwrite: overwrite || false, state: :unconfirmed, original_filename: data.original_filename, likely_mismatched: likely_mismatched_json?, missing_status: missing_status?)
+        nb_items = BulkImportRow.insert_all(json_data.map { |row| { bulk_import_id: bulk_import.id, data: row, created_at: now, updated_at: now } }).length
+      else
+        @bulk_import = current_account.bulk_imports.create(type: type, overwrite: overwrite || false, state: :unconfirmed, original_filename: data.original_filename, likely_mismatched: likely_mismatched?)
+        nb_items = BulkImportRow.insert_all(parsed_rows.map { |row| { bulk_import_id: bulk_import.id, data: row, created_at: now, updated_at: now } }).length
+      end
       @bulk_import.update(total_items: nb_items)
     end
   end
@@ -82,6 +95,14 @@ class Form::Import
     self.overwrite = str.to_sym == :overwrite
   end
 
+  def missing_status?
+    return false unless content_type_is_json?
+
+    import_statuses = json_data.pluck(:statuses).flatten.uniq
+    db_statuses = Status.where(uri: import_statuses)
+    import_statuses.count != db_statuses.count
+  end
+
   private
 
   def file_name_matches?(string)
@@ -153,6 +174,21 @@ class Form::Import
   def validate_data
     return if data.nil?
     return errors.add(:data, I18n.t('imports.errors.too_large')) if data.size > FILE_SIZE_LIMIT
+
+    if content_type_is_json?
+      validate_json_data
+    else
+      validate_csv_data
+    end
+  rescue CSV::MalformedCSVError => e
+    errors.add(:data, I18n.t('imports.errors.invalid_csv_file', error: e.message))
+  rescue JSON::ParserError => e
+    errors.add(:data, I18n.t('imports.errors.invalid_json_file', error: e.message))
+  rescue EmptyFileError
+    errors.add(:data, I18n.t('imports.errors.empty'))
+  end
+
+  def validate_csv_data
     return errors.add(:data, I18n.t('imports.errors.incompatible_type')) unless default_csv_headers.all? { |header| csv_data.headers.include?(header) }
 
     errors.add(:data, I18n.t('imports.errors.over_rows_processing_limit', count: ROWS_PROCESSING_LIMIT)) if csv_row_count > ROWS_PROCESSING_LIMIT
@@ -163,9 +199,26 @@ class Form::Import
       limit -= current_account.following_count unless overwrite
       errors.add(:data, I18n.t('users.follow_limit_reached', limit: base_limit)) if csv_row_count > limit
     end
-  rescue CSV::MalformedCSVError => e
-    errors.add(:data, I18n.t('imports.errors.invalid_csv_file', error: e.message))
-  rescue EmptyFileError
-    errors.add(:data, I18n.t('imports.errors.empty'))
+  end
+
+  def validate_json_data
+    errors.add(:data, I18n.t('imports.errors.over_rows_processing_limit', count: ROWS_PROCESSING_LIMIT)) if json_data.count > ROWS_PROCESSING_LIMIT
+    errors.add(:data, I18n.t('imports.errors.incompatible_type')) unless allowed_type_for_json?
+  end
+
+  def content_type_is_json?
+    data.content_type == 'application/json'
+  end
+
+  def json_data
+    parse_json['custom_filters'].map(&:deep_symbolize_keys)
+  end
+
+  def parse_json
+    @parse_json ||= JSON.parse(data.read)
+  end
+
+  def allowed_type_for_json?
+    type.to_sym.in?(%i(custom_filters))
   end
 end

app/services/bulk_import_row_service.rb

@@ -42,6 +42,14 @@ class BulkImportRowService
       FollowService.new.call(@account, @target_account) unless @account.id == @target_account.id
 
       list.accounts << @target_account
+    when :custom_filters
+      filter = @account.custom_filters.create!(title: @data['title'], context: @data['context'])
+      filter.keywords = @data['keywords_attributes'].map { |keyword| CustomFilterKeyword.new(keyword: keyword['keyword'], whole_word: keyword['whole_word']) }
+      filter.action = @data['action'].to_sym
+      filter.expires_at = @data['expires_at']
+      status_ids = Status.where(uri: @data['statuses']).ids
+      filter.statuses = status_ids.map { |status| CustomFilterStatus.new(status_id: status) } if status_ids.any?
+      filter.save!
     end
 
     true

app/services/bulk_import_service.rb

@@ -18,6 +18,8 @@ class BulkImportService < BaseService
       import_bookmarks!
     when :lists
       import_lists!
+    when :custom_filters
+      import_custom_filters!
     end
 
     @import.update!(state: :finished, finished_at: Time.now.utc) if @import.processing_complete?
@@ -182,4 +184,14 @@ class BulkImportService < BaseService
       [row.id]
     end
   end
+
+  def import_custom_filters!
+    rows = @import.rows.to_a
+
+    @account.custom_filters.destroy_all if @import.overwrite?
+
+    Import::RowWorker.push_bulk(rows) do |row|
+      [row.id]
+    end
+  end
 end

app/views/settings/imports/index.html.haml

@@ -5,7 +5,7 @@
   .field-group
     = f.input :type,
               as: :grouped_select,
-              collection: { constructive: %i(following bookmarks lists), destructive: %i(muting blocking domain_blocking) },
+              collection: { constructive: %i(following bookmarks lists), destructive: %i(muting blocking domain_blocking custom_filters) },
               group_label_method: ->(group) { I18n.t("imports.type_groups.#{group.first}") },
               group_method: :last,
               hint: t('imports.preface'),
@@ -61,5 +61,9 @@
 
             %td
               - if import.failure_count.positive?
-                = link_to_if import.state_finished?, import.failure_count, failures_settings_import_path(import, format: :csv) do
-                  = import.failure_count
+                - if import.type == 'custom_filters'
+                  = link_to_if import.state_finished?, import.failure_count, failures_settings_import_path(import, format: :json) do
+                    = import.failure_count
+                - else
+                  = link_to_if import.state_finished?, import.failure_count, failures_settings_import_path(import, format: :csv) do
+                    = import.failure_count

app/views/settings/imports/show.html.haml

@@ -4,6 +4,9 @@
 - if @bulk_import.likely_mismatched?
   .flash-message.warning= t('imports.mismatched_types_warning')
 
+- if @bulk_import&.missing_status == true
+  .flash-message.warning= I18n.t('imports.errors.status_not_found_warning')
+
 - if @bulk_import.overwrite?
   %p.hint= t("imports.overwrite_preambles.#{@bulk_import.type}_html", filename: @bulk_import.original_filename, count: @bulk_import.total_items)
 - else