Merge branch 'feature/rerun_failed_importer_entries' into 'develop'

Add rake task to rerun failed and pending entries

Closes FDM/rdm-system/antleaf-projectmanagement#399

See merge request !348

hyrax/app/controllers/bulkrax/rerun_importers_controller.rb

@@ -3,10 +3,17 @@
 module Bulkrax
   class RerunImportersController < ApplicationController
     include BulkraxHelper
-    before_action :set_importer, only: [:rerun_failed_entries, :marked_as_complete]
+    before_action :set_importer, only: [:rerun_failed_entries, :rerun_panding_entries, :marked_as_complete]
 
     def rerun_failed_entries
-      rerun_failed_entries_in_last_run
+      total_rerun_entries = rerun_failed_or_pending_entries_in_last_run("Failed")
+      flash[:notice] = "#{total_rerun_entries} failed jobs queued"
+      redirect_to importers_path
+    end
+
+    def rerun_panding_entries
+      total_rerun_entries = rerun_failed_or_pending_entries_in_last_run("Pending")
+      flash[:notice] = "#{total_rerun_entries} pending jobs queued"
       redirect_to importers_path
     end
 

hyrax/app/helpers/bulkrax_helper.rb

 module BulkraxHelper
-  def rerun_failed_entries_in_last_run
+  def rerun_failed_or_pending_entries_in_last_run(status="Failed")
+    total_rerun_entries = 0
     current_run = @importer.last_run
 
-    if current_run&.failed_records&.positive?
-      ::Bulkrax::ScheduleRelationshipsJob.set(wait: 5.minutes).perform_later(importer_id: @importer.id)
-      @importer.status_info('Pending')
+    if current_run.failed_records.positive? || current_run.enqueued_records.positive?
+      total_rerun_entries = rerun_failed_or_pending_entries(@importer, status)
+    elsif current_run && @importer.parser.total.positive? && current_run.processed_records.zero?
+      ::Bulkrax::ImporterJob.send(
+        @importer.parser.perform_method,
+        @importer.id
+      )
+    end
+
+    total_rerun_entries
+  end
 
-      @importer.entries.each do |entry|
-        next if entry.status == 'Complete'
+  def rerun_failed_or_pending_entries(importer, status="Failed")
+    current_run = importer.last_run
+    total_rerun_entries = 0
+    ::Bulkrax::ScheduleRelationshipsJob.set(wait: 5.minutes).perform_later(importer_id: importer.id)
+    importer.status_info('Pending')
 
-        type = if entry.raw_metadata['model'] == 'CrcDataset'
-                 'Work'
-               else
-                 entry.raw_metadata['model']
-               end
+    importer.entries.each do |entry|
+      next unless entry.status == status
 
-        entry.status_info('Pending')
+      type = if entry.raw_metadata['model'] == 'CrcDataset'
+               'Work'
+             else
+               entry.raw_metadata['model']
+             end
+
+      entry.status_info('Pending')
+      if current_run.failed_records.positive?
         current_run.decrement!(:failed_records)
-        current_run.increment!(:enqueued_records)
-
-        if type == 'Collection'
-          current_run.decrement!(:failed_collections)
-        elsif type == 'Work'
-          current_run.decrement!(:failed_works)
-        elsif type == 'FileSet'
-          current_run.decrement!(:failed_file_sets)
-        elsif type == 'ComplexSubject'
-          current_run.decrement!(:failed_complex_subjects)
-        elsif type == 'ComplexSession'
-          current_run.decrement!(:failed_complex_sessions)
-        elsif type == 'ComplexModality'
-          current_run.decrement!(:failed_complex_modalities)
-        end
-
-        "Bulkrax::CrcDataset::Import#{type}Job".constantize.send(
-          entry.parser.perform_method,
-          entry.id,
-          @importer.importer_runs.last.id
-        )
       end
 
-      current_run.save
+      current_run.increment!(:enqueued_records)
 
-    elsif current_run && @importer.parser.total.positive? && current_run.processed_records.zero?
-      ::Bulkrax::ImporterJob.send(
-        @importer.parser.perform_method,
-        @importer.id
+      if type == 'Collection' && current_run.failed_collections.positive?
+        current_run.decrement!(:failed_collections)
+      elsif type == 'Work' && current_run.failed_works.positive?
+        current_run.decrement!(:failed_works)
+      elsif type == 'FileSet' && current_run.failed_file_sets.positive?
+        current_run.decrement!(:failed_file_sets)
+      elsif type == 'ComplexSubject' && current_run.failed_complex_subjects.positive?
+        current_run.decrement!(:failed_complex_subjects)
+      elsif type == 'ComplexSession' && current_run.failed_complex_sessions.positive?
+        current_run.decrement!(:failed_complex_sessions)
+      elsif type == 'ComplexModality' && current_run.failed_complex_modalities.positive?
+        current_run.decrement!(:failed_complex_modalities)
+      end
+
+      "Bulkrax::CrcDataset::Import#{type}Job".constantize.send(
+        entry.parser.perform_method,
+        entry.id,
+        importer.importer_runs.last.id
       )
+
+      current_run.save
+      total_rerun_entries +=1
     end
+
+    total_rerun_entries
   end
 end
\ No newline at end of file

hyrax/app/views/bulkrax/importers/index.html.erb

@@ -35,6 +35,7 @@
               <th scope="col"></th>
               <th scope="col"></th>
               <th scope="col"></th>
+              <th scope="col"></th>
             </tr>
           </thead>
           <tbody>
@@ -58,15 +59,25 @@
                 <td><%= link_to raw('<span class="glyphicon glyphicon-pencil"></span>'), edit_importer_path(importer) %></td>
                 <td><%= link_to raw('<span class="glyphicon glyphicon-remove"></span>'), importer, method: :delete, data: { confirm: 'Are you sure?' } %></td>
                 <td>
-                  <% if show_rerun_action?(importer) %>
-                    <%= link_to raw('<span class="glyphicon glyphicon-refresh"></span>'), rerun_failed_entries_path(importer_id: importer.id), method: :post, data: { confirm: 'Are you sure to re-run failed jobs?' } %>
-                  <% end %>
-                </td>
-
-                <td>
-                  <% if show_marked_as_complete_action?(importer) %>
-                    <%= link_to t('bulkrax.importer.actions.marked_as_complete'), marked_as_complete_path(importer_id: importer.id), method: :post, class:'btn btn-primary' %>
-                  <% end %>
+                  <div class="btn-group">
+                    <button class="btn btn-default dropdown-toggle" type="button" data-toggle="dropdown" aria-expanded="false"> 
+                      Actions
+                      <span class="caret" aria-hidden="true"></span>
+                    </button>
+                    <ul class="dropdown-menu">
+                      <li>
+                        <%= link_to "Re-run failed jobs", rerun_failed_entries_path(importer_id: importer.id), method: :post, class: "dropdown-item" %>
+                      </li>
+                      <li>
+                        <%= link_to "Re-run pending jobs", rerun_pending_entries_path(importer_id: importer.id), method: :post, class: "dropdown-item", data: { confirm: 'Are you sure to re-run pending jobs? It may create the same resource twice.' } %>
+                      </li>
+                      <li>
+                        <% if show_marked_as_complete_action?(importer) %>
+                          <%= link_to t('bulkrax.importer.actions.marked_as_complete'), marked_as_complete_path(importer_id: importer.id), method: :post, class:'dropdown-item' %>
+                        <% end %>
+                      </li>
+                    </ul>
+                  </div>
                 </td>
               </tr>
             <% end %>
@@ -85,7 +96,7 @@
       destroy: true, /* Reinitialize DataTable with config below */
       'columnDefs': [
           { 'orderable': true, 'targets': [...Array(10).keys()] },
-          { 'orderable': false, 'targets': [10, 11, 12] }
+          { 'orderable': false, 'targets': [14, 15, 16, 17] }
       ],
       'language': {
         'info': 'Showing _START_ to _END_ of _TOTAL_ importers',

hyrax/app/views/bulkrax/importers/show.html.erb

@@ -37,10 +37,8 @@
     <%= render partial: 'bulkrax/shared/bulkrax_errors', locals: {item: @importer} %>
 
     <p class="bulkrax-p-align">
-      <% if show_rerun_action?(@importer) %>
-        <%= link_to raw('<span class="glyphicon glyphicon-refresh"></span> Rerun'), rerun_failed_entries_path(importer_id: @importer.id), method: :post, data: { confirm: 'Are you sure to re-run failed jobs?' },class:'btn btn-primary' %>
-      <% end %>
-
+      <%= link_to "Re-run failed jobs", rerun_failed_entries_path(importer_id: @importer.id), method: :post, class:'btn btn-primary' %>
+      <%= link_to "Re-run pending jobs", rerun_pending_entries_path(importer_id: @importer.id), method: :post, class:'btn btn-primary', data: { confirm: 'Are you sure to re-run pending jobs? It may create the same resource twice.' } %>
       <% if show_marked_as_complete_action?(@importer) %>
         <%= link_to t('bulkrax.importer.actions.marked_as_complete'), marked_as_complete_path(importer_id: @importer.id), method: :post, class:'btn btn-primary' %>
       <% end %>

hyrax/config/routes.rb

@@ -68,6 +68,7 @@ end
 Bulkrax::Engine.routes.draw do
   concern :range_searchable, BlacklightRangeLimit::Routes::RangeSearchable.new
   post 'rerun_failed_entries', to: "rerun_importers#rerun_failed_entries"
+  post 'rerun_pending_entries', to: "rerun_importers#rerun_panding_entries"
   post 'marked_as_complete', to: "rerun_importers#marked_as_complete"
 end
 

hyrax/lib/tasks/fix_importer_failed_jobs.rake

+namespace :rdms do
+  desc 'Fix importer failed jobs for importer id'
+  task :"fix_importer_failed_jobs", [:importer_id] => :environment do |task, args|
+    puts "Starting fix"
+    importer_id = args.importer_id
+    importer = ::Bulkrax::Importer.find(importer_id)
+    ::BulkraxHelper.rerun_failed_or_pending_entries(importer)
+  end
+end