diff --git a/app/workers/activitypub/delivery_worker.rb b/app/workers/activitypub/delivery_worker.rb
index ade7175c9d..8cd39f700c 100644
--- a/app/workers/activitypub/delivery_worker.rb
+++ b/app/workers/activitypub/delivery_worker.rb
@@ -38,7 +38,7 @@ class ActivityPub::DeliveryWorker
     if @inbox_url.present?
       if @performed
         failure_tracker.track_success!
-      else
+      elsif !@unsalvageable
         failure_tracker.track_failure!
       end
     end
@@ -62,9 +62,13 @@ class ActivityPub::DeliveryWorker
     stoplight_wrapper.run do
       request_pool.with(@host) do |http_client|
         build_request(http_client).perform do |response|
-          raise Mastodon::UnexpectedResponseError, response unless response_successful?(response) || response_error_unsalvageable?(response) || unsalvageable_authorization_failure?(response)
-
-          @performed = true
+          if response_successful?(response)
+            @performed = true
+          elsif response_error_unsalvageable?(response) || unsalvageable_authorization_failure?(response)
+            @unsalvageable = true
+          else
+            raise Mastodon::UnexpectedResponseError, response
+          end
         end
       end
     end