From 1ef6d378bf2ffad7c9df63e017c2588a78021329 Mon Sep 17 00:00:00 2001 From: Gyan Gupta <gyan@cottagelabs.com> Date: Tue, 10 Dec 2024 10:42:08 +0530 Subject: [PATCH 1/6] implements UI for import verification --- .../controllers/bulkrax/importers_controller.rb | 15 +++++++++++++-- hyrax/app/helpers/hyrax_helper.rb | 12 ++++++++++++ hyrax/app/services/verify_crc_dataset_import.rb | 9 +++++---- .../bulkrax/shared/_work_entries_tab.html.erb | 8 +++++++- hyrax/config/routes.rb | 3 +++ 5 files changed, 40 insertions(+), 7 deletions(-) diff --git a/hyrax/app/controllers/bulkrax/importers_controller.rb b/hyrax/app/controllers/bulkrax/importers_controller.rb index d16c9d75..31f7a1dd 100644 --- a/hyrax/app/controllers/bulkrax/importers_controller.rb +++ b/hyrax/app/controllers/bulkrax/importers_controller.rb @@ -14,7 +14,7 @@ module Bulkrax protect_from_forgery unless: -> { api_request? } before_action :token_authenticate!, if: -> { api_request? }, only: [:create, :update, :delete] before_action :authenticate_user!, unless: -> { api_request? } - before_action :set_importer, only: [:show, :edit, :update, :destroy] + before_action :set_importer, only: [:show, :edit, :update, :destroy, :verify_and_genrate_work] with_themed_layout 'dashboard' # GET /importers @@ -212,6 +212,17 @@ module Bulkrax send_content end + def verify_and_genrate_work + experiment_status, report_path = VerifyCRCDatasetImport.verify_experiment_and_report(@importer.id, params[:entry_id]) + + return head :not_found unless File.exists?(report_path) + + respond_to do |format| + format.csv { send_file report_path, type: 'application/csv', disposition: 'attachment' } + format.any { head :unsupported_media_type } + end + end + private def files_for_import(file, cloud_files) @@ -229,7 +240,7 @@ module Bulkrax # Use callbacks to share common setup or constraints between actions. def set_importer - @importer = Importer.find(params[:id]) + @importer = Importer.find(params[:id] || params[:importer_id]) end def importable_params diff --git a/hyrax/app/helpers/hyrax_helper.rb b/hyrax/app/helpers/hyrax_helper.rb index 125a8854..10900d67 100644 --- a/hyrax/app/helpers/hyrax_helper.rb +++ b/hyrax/app/helpers/hyrax_helper.rb @@ -187,4 +187,16 @@ module HyraxHelper "fa fa-file-o" end end + + def work_entry_varification_status(entry) + report_name_pattern = "analysis_report_#{entry.importerexporter_id}_#{entry.id}_*.csv" + matching_files = Dir.glob(File.join(ENV.fetch('DOWNLOAD_PATH', '/shared/downloads'), report_name_pattern)) + + return nil if matching_files.empty? + + newest_file = matching_files.max_by { |file| File.mtime(file) } + base_name, extension = File.basename(newest_file, ".*"), File.extname(newest_file) + + base_name.split('_')[-1].titleize + end end diff --git a/hyrax/app/services/verify_crc_dataset_import.rb b/hyrax/app/services/verify_crc_dataset_import.rb index 5c15153d..a3347ce0 100644 --- a/hyrax/app/services/verify_crc_dataset_import.rb +++ b/hyrax/app/services/verify_crc_dataset_import.rb @@ -2,6 +2,7 @@ require 'json' require 'csv' class VerifyCRCDatasetImport + extend ComplexHelper def self.verify_import(import_id) importer = ::Bulkrax::Importer.find(import_id) work_entries = importer.entries.select {|e| e.class.name == "Bulkrax::CrcFolderEntry"} @@ -19,9 +20,9 @@ class VerifyCRCDatasetImport verification_status, message, paths_compared, analysis = self.verify_experiment(entry_id, import_id) # prepare csv report - report_name = "analysis_report_#{import_id}_#{entry_id}_#{Time.now.strftime("%m-%d_%H-%M-%S")}.csv" + report_name = "analysis_report_#{import_id}_#{entry_id}_#{Time.now.strftime("%Y-%m-%d-%H-%M-%S")}_#{ verification_status ? "succes" : "failure"}.csv" report_path = File.join(ENV.fetch('DOWNLOAD_PATH', '/shared/downloads'), report_name) - headers = %w(id, status, message) + headers = %w(id status message) CSV.open(report_path, "w") do |csv| # Write headers @@ -135,7 +136,7 @@ class VerifyCRCDatasetImport all_parts = fp.split('/') if all_parts.size > 1 parts = all_parts[0...-1] - parts.map {|p| p.downcase!} + parts = parts.map {|p| p.length > 3 ? sanitize_title(p) :sanitize_title(p.rjust(3, '0'))} new_path = File.join(parts.join("/"), all_parts[-1]) sanitised_input_list[fp] = new_path end @@ -150,7 +151,7 @@ class VerifyCRCDatasetImport sanitised_input_list.each do |fp, new_path| parts = new_path.split('/') if parts.size == 3 - sanitised_input_list[fp] = File.join(parts[0], 'ses 01', parts[1], parts[2]) + sanitised_input_list[fp] = File.join(sanitize_title(parts[0]), 'ses 01', sanitize_title(parts[1]), parts[2]) end end end diff --git a/hyrax/app/views/bulkrax/shared/_work_entries_tab.html.erb b/hyrax/app/views/bulkrax/shared/_work_entries_tab.html.erb index 1fac7b9a..1c7f4f6a 100644 --- a/hyrax/app/views/bulkrax/shared/_work_entries_tab.html.erb +++ b/hyrax/app/views/bulkrax/shared/_work_entries_tab.html.erb @@ -7,6 +7,7 @@ <th>Status</th> <th>Errors</th> <th>Status Set At</th> + <th>Verification Status</th> <th>Actions</th> </tr> </thead> @@ -29,7 +30,12 @@ <td></td> <% end %> <td><%= e.status_at %></td> - <td><%= link_to raw('<span class="glyphicon glyphicon-info-sign"></span>'), entry_path %></td> + <td><%= work_entry_varification_status(e) %></td> + <td> + <% if work_entry_varification_status(e) != "Succes"%> + <%= link_to "Verify and Genrate report", url_for(controller: params[:controller], action: "verify_and_genrate_work", importer_id: 2, entry_id: 2, format: :csv), class:'btn btn-primary' %> + <% end %> + </td> </tr> <% end %> </tbody> diff --git a/hyrax/config/routes.rb b/hyrax/config/routes.rb index 5e64ee30..25ee606a 100644 --- a/hyrax/config/routes.rb +++ b/hyrax/config/routes.rb @@ -69,6 +69,9 @@ Bulkrax::Engine.routes.draw do concern :range_searchable, BlacklightRangeLimit::Routes::RangeSearchable.new post 'rerun_failed_entries', to: "rerun_importers#rerun_failed_entries" post 'marked_as_complete', to: "rerun_importers#marked_as_complete" + resources :importers do + get 'verify_and_genrate_work/:entry_id', to: "importers#verify_and_genrate_work" + end end Hyrax::Engine.routes.draw do -- GitLab From 4c6811f0fcfca144e39eca8be36bafc976eae83d Mon Sep 17 00:00:00 2001 From: Gyan Gupta <gyan@cottagelabs.com> Date: Tue, 10 Dec 2024 16:25:24 +0530 Subject: [PATCH 2/6] refactor some code --- .../bulkrax/importers_controller.rb | 12 +++++-- hyrax/app/helpers/hyrax_helper.rb | 31 +++++++++++++------ .../app/services/verify_crc_dataset_import.rb | 2 +- .../bulkrax/shared/_work_entries_tab.html.erb | 17 +++++++--- hyrax/config/routes.rb | 2 +- 5 files changed, 46 insertions(+), 18 deletions(-) diff --git a/hyrax/app/controllers/bulkrax/importers_controller.rb b/hyrax/app/controllers/bulkrax/importers_controller.rb index 31f7a1dd..fb2754e3 100644 --- a/hyrax/app/controllers/bulkrax/importers_controller.rb +++ b/hyrax/app/controllers/bulkrax/importers_controller.rb @@ -10,11 +10,12 @@ module Bulkrax include Bulkrax::DownloadBehavior include Bulkrax::API include Bulkrax::ValidationHelper + include HyraxHelper protect_from_forgery unless: -> { api_request? } before_action :token_authenticate!, if: -> { api_request? }, only: [:create, :update, :delete] before_action :authenticate_user!, unless: -> { api_request? } - before_action :set_importer, only: [:show, :edit, :update, :destroy, :verify_and_genrate_work] + before_action :set_importer, only: [:show, :edit, :update, :destroy, :verify_and_generate_work] with_themed_layout 'dashboard' # GET /importers @@ -212,8 +213,13 @@ module Bulkrax send_content end - def verify_and_genrate_work - experiment_status, report_path = VerifyCRCDatasetImport.verify_experiment_and_report(@importer.id, params[:entry_id]) + def verify_and_generate_work + entry = Bulkrax::Entry.find_by(id: params[:entry_id]) + varification_status, date, report_path = latest_varification_report_details(entry) + + if varification_status != 'Success' + experiment_status, report_path = VerifyCRCDatasetImport.verify_experiment_and_report(@importer.id, params[:entry_id]) + end return head :not_found unless File.exists?(report_path) diff --git a/hyrax/app/helpers/hyrax_helper.rb b/hyrax/app/helpers/hyrax_helper.rb index 10900d67..90ffd215 100644 --- a/hyrax/app/helpers/hyrax_helper.rb +++ b/hyrax/app/helpers/hyrax_helper.rb @@ -188,15 +188,28 @@ module HyraxHelper end end - def work_entry_varification_status(entry) - report_name_pattern = "analysis_report_#{entry.importerexporter_id}_#{entry.id}_*.csv" - matching_files = Dir.glob(File.join(ENV.fetch('DOWNLOAD_PATH', '/shared/downloads'), report_name_pattern)) - - return nil if matching_files.empty? - + def latest_varification_report_details(entry) + # Define the file pattern and search path + download_path = ENV.fetch('DOWNLOAD_PATH', '/shared/downloads') + report_pattern = "analysis_report_#{entry.importerexporter_id}_#{entry.id}_*.csv" + matching_files = Dir.glob(File.join(download_path, report_pattern)) + + # Return nil if no matching files are found + return [nil, nil, nil] if matching_files.empty? + + # Find the newest file based on modification time newest_file = matching_files.max_by { |file| File.mtime(file) } - base_name, extension = File.basename(newest_file, ".*"), File.extname(newest_file) - - base_name.split('_')[-1].titleize + + # Extract file details + base_name = File.basename(newest_file, ".*") + extension = File.extname(newest_file) + date_time_str, status = base_name.split('_')[-2..] + + # Parse the date and time + date, raw_time = date_time_str.split('T') + formatted_time = raw_time.gsub('-', ':') + parsed_datetime = Time.parse("#{date} #{formatted_time}") + + [status.titleize, parsed_datetime, newest_file] end end diff --git a/hyrax/app/services/verify_crc_dataset_import.rb b/hyrax/app/services/verify_crc_dataset_import.rb index a3347ce0..539141f6 100644 --- a/hyrax/app/services/verify_crc_dataset_import.rb +++ b/hyrax/app/services/verify_crc_dataset_import.rb @@ -20,7 +20,7 @@ class VerifyCRCDatasetImport verification_status, message, paths_compared, analysis = self.verify_experiment(entry_id, import_id) # prepare csv report - report_name = "analysis_report_#{import_id}_#{entry_id}_#{Time.now.strftime("%Y-%m-%d-%H-%M-%S")}_#{ verification_status ? "succes" : "failure"}.csv" + report_name = "analysis_report_#{import_id}_#{entry_id}_#{Time.now.strftime("%Y-%m-%dT%H-%M-%S")}_#{ verification_status ? "success" : "failure"}.csv" report_path = File.join(ENV.fetch('DOWNLOAD_PATH', '/shared/downloads'), report_name) headers = %w(id status message) diff --git a/hyrax/app/views/bulkrax/shared/_work_entries_tab.html.erb b/hyrax/app/views/bulkrax/shared/_work_entries_tab.html.erb index 1c7f4f6a..1a03aa8c 100644 --- a/hyrax/app/views/bulkrax/shared/_work_entries_tab.html.erb +++ b/hyrax/app/views/bulkrax/shared/_work_entries_tab.html.erb @@ -30,12 +30,21 @@ <td></td> <% end %> <td><%= e.status_at %></td> - <td><%= work_entry_varification_status(e) %></td> - <td> - <% if work_entry_varification_status(e) != "Succes"%> - <%= link_to "Verify and Genrate report", url_for(controller: params[:controller], action: "verify_and_genrate_work", importer_id: 2, entry_id: 2, format: :csv), class:'btn btn-primary' %> + <td class="text-center"> + <% varification_status, date, file_path = latest_varification_report_details(e) %> + <% if varification_status.present? %> + <% if varification_status == "Success" %> + <span class="glyphicon glyphicon-ok text-success", title="Verified successfully on (<%= date %>)"></span> <%= varification_status %> + <% else %> + <span class="glyphicon glyphicon-remove text-danger", title="Verification failed on (<%= date %>)"></span> <%= varification_status %> + <% end %> <% end %> </td> + <td> + <%= link_to raw('<span class="glyphicon glyphicon-info-sign"></span>'), entry_path %> + <% button_text = varification_status == "Success" ? "Download Report" : "Verify and Generate report" %> + <%= link_to button_text, url_for(controller: params[:controller], action: "verify_and_generate_work", importer_id: 2, entry_id: 2, format: :csv), class:'btn btn-primary' %> + </td> </tr> <% end %> </tbody> diff --git a/hyrax/config/routes.rb b/hyrax/config/routes.rb index 25ee606a..440c7c22 100644 --- a/hyrax/config/routes.rb +++ b/hyrax/config/routes.rb @@ -70,7 +70,7 @@ Bulkrax::Engine.routes.draw do post 'rerun_failed_entries', to: "rerun_importers#rerun_failed_entries" post 'marked_as_complete', to: "rerun_importers#marked_as_complete" resources :importers do - get 'verify_and_genrate_work/:entry_id', to: "importers#verify_and_genrate_work" + get 'verify_and_generate_work/:entry_id', to: "importers#verify_and_generate_work" end end -- GitLab From 60beca6409f43a4d91ec69a7271bc75803e24274 Mon Sep 17 00:00:00 2001 From: Gyan Gupta <gyan@cottagelabs.com> Date: Wed, 18 Dec 2024 22:06:43 +0530 Subject: [PATCH 3/6] fix som mistakes --- hyrax/app/views/bulkrax/shared/_work_entries_tab.html.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hyrax/app/views/bulkrax/shared/_work_entries_tab.html.erb b/hyrax/app/views/bulkrax/shared/_work_entries_tab.html.erb index 1a03aa8c..6eabde00 100644 --- a/hyrax/app/views/bulkrax/shared/_work_entries_tab.html.erb +++ b/hyrax/app/views/bulkrax/shared/_work_entries_tab.html.erb @@ -43,7 +43,7 @@ <td> <%= link_to raw('<span class="glyphicon glyphicon-info-sign"></span>'), entry_path %> <% button_text = varification_status == "Success" ? "Download Report" : "Verify and Generate report" %> - <%= link_to button_text, url_for(controller: params[:controller], action: "verify_and_generate_work", importer_id: 2, entry_id: 2, format: :csv), class:'btn btn-primary' %> + <%= link_to button_text, url_for(controller: params[:controller], action: "verify_and_generate_work", importer_id: e.importerexporter_id, entry_id: e.id, format: :csv), class:'btn btn-primary' %> </td> </tr> <% end %> -- GitLab From f640903aadd7f6956d98933f456f6886569ce1a8 Mon Sep 17 00:00:00 2001 From: Gyan Gupta <gyan@cottagelabs.com> Date: Thu, 19 Dec 2024 11:45:13 +0530 Subject: [PATCH 4/6] refactor code --- hyrax/app/assets/stylesheets/rdms.scss | 7 ++++ .../bulkrax/importers_controller.rb | 3 +- .../bulkrax/shared/_work_entries_tab.html.erb | 33 +++++++++++-------- 3 files changed, 28 insertions(+), 15 deletions(-) diff --git a/hyrax/app/assets/stylesheets/rdms.scss b/hyrax/app/assets/stylesheets/rdms.scss index d2d648a7..cb871418 100644 --- a/hyrax/app/assets/stylesheets/rdms.scss +++ b/hyrax/app/assets/stylesheets/rdms.scss @@ -370,3 +370,10 @@ form .field-wrapper label[required="required"]::after { margin: 4em auto 0; padding: 20px; } + +#work-entries { + .work-tab-actions { + display: flex; + gap: 10px; + } +} \ No newline at end of file diff --git a/hyrax/app/controllers/bulkrax/importers_controller.rb b/hyrax/app/controllers/bulkrax/importers_controller.rb index fb2754e3..bf175a0f 100644 --- a/hyrax/app/controllers/bulkrax/importers_controller.rb +++ b/hyrax/app/controllers/bulkrax/importers_controller.rb @@ -217,13 +217,14 @@ module Bulkrax entry = Bulkrax::Entry.find_by(id: params[:entry_id]) varification_status, date, report_path = latest_varification_report_details(entry) - if varification_status != 'Success' + if params[:varify] experiment_status, report_path = VerifyCRCDatasetImport.verify_experiment_and_report(@importer.id, params[:entry_id]) end return head :not_found unless File.exists?(report_path) respond_to do |format| + format.html {redirect_to bulkrax.importer_path(@importer.id) } format.csv { send_file report_path, type: 'application/csv', disposition: 'attachment' } format.any { head :unsupported_media_type } end diff --git a/hyrax/app/views/bulkrax/shared/_work_entries_tab.html.erb b/hyrax/app/views/bulkrax/shared/_work_entries_tab.html.erb index 6eabde00..c6236539 100644 --- a/hyrax/app/views/bulkrax/shared/_work_entries_tab.html.erb +++ b/hyrax/app/views/bulkrax/shared/_work_entries_tab.html.erb @@ -7,7 +7,6 @@ <th>Status</th> <th>Errors</th> <th>Status Set At</th> - <th>Verification Status</th> <th>Actions</th> </tr> </thead> @@ -30,20 +29,26 @@ <td></td> <% end %> <td><%= e.status_at %></td> - <td class="text-center"> - <% varification_status, date, file_path = latest_varification_report_details(e) %> - <% if varification_status.present? %> - <% if varification_status == "Success" %> - <span class="glyphicon glyphicon-ok text-success", title="Verified successfully on (<%= date %>)"></span> <%= varification_status %> - <% else %> - <span class="glyphicon glyphicon-remove text-danger", title="Verification failed on (<%= date %>)"></span> <%= varification_status %> - <% end %> - <% end %> - </td> <td> - <%= link_to raw('<span class="glyphicon glyphicon-info-sign"></span>'), entry_path %> - <% button_text = varification_status == "Success" ? "Download Report" : "Verify and Generate report" %> - <%= link_to button_text, url_for(controller: params[:controller], action: "verify_and_generate_work", importer_id: e.importerexporter_id, entry_id: e.id, format: :csv), class:'btn btn-primary' %> + <div class="work-tab-actions"> + <%= link_to raw('<span class="glyphicon glyphicon-info-sign"></span>'), entry_path %> + <% varification_status, date, file_path = latest_varification_report_details(e) %> + <% if varification_status.present? %> + <% if varification_status == "Success" %> + <span class="glyphicon glyphicon-ok text-success", title="Verified successfully on (<%= date %>)"></span> + <% else %> + <span class="glyphicon glyphicon-remove text-danger", title="Verification failed on (<%= date %>)"></span> + <% end %> + <%= link_to url_for(controller: params[:controller], action: "verify_and_generate_work", importer_id: e.importerexporter_id, entry_id: e.id, format: :csv) do %> + <span class="glyphicon glyphicon-download-alt"></span> + <% end %> + <% end %> + <% if !varification_status || varification_status == "Failure" %> + <%= link_to url_for(controller: params[:controller], action: "verify_and_generate_work", importer_id: e.importerexporter_id, entry_id: e.id, varify: true) do %> + <span class="glyphicon glyphicon-refresh"></span> + <% end %> + <% end%> + </div> </td> </tr> <% end %> -- GitLab From 1bc2c415cfd1b1c802934153cfd0a02bcef72a78 Mon Sep 17 00:00:00 2001 From: Anusha Ranganathan <anusha@cottagelabs.com> Date: Thu, 19 Dec 2024 12:56:55 +0530 Subject: [PATCH 5/6] Fixed spelling of verify --- hyrax/app/controllers/bulkrax/importers_controller.rb | 2 +- hyrax/app/views/bulkrax/shared/_work_entries_tab.html.erb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/hyrax/app/controllers/bulkrax/importers_controller.rb b/hyrax/app/controllers/bulkrax/importers_controller.rb index bf175a0f..284dac27 100644 --- a/hyrax/app/controllers/bulkrax/importers_controller.rb +++ b/hyrax/app/controllers/bulkrax/importers_controller.rb @@ -217,7 +217,7 @@ module Bulkrax entry = Bulkrax::Entry.find_by(id: params[:entry_id]) varification_status, date, report_path = latest_varification_report_details(entry) - if params[:varify] + if params[:verify] experiment_status, report_path = VerifyCRCDatasetImport.verify_experiment_and_report(@importer.id, params[:entry_id]) end diff --git a/hyrax/app/views/bulkrax/shared/_work_entries_tab.html.erb b/hyrax/app/views/bulkrax/shared/_work_entries_tab.html.erb index c6236539..2e7b0953 100644 --- a/hyrax/app/views/bulkrax/shared/_work_entries_tab.html.erb +++ b/hyrax/app/views/bulkrax/shared/_work_entries_tab.html.erb @@ -44,7 +44,7 @@ <% end %> <% end %> <% if !varification_status || varification_status == "Failure" %> - <%= link_to url_for(controller: params[:controller], action: "verify_and_generate_work", importer_id: e.importerexporter_id, entry_id: e.id, varify: true) do %> + <%= link_to url_for(controller: params[:controller], action: "verify_and_generate_work", importer_id: e.importerexporter_id, entry_id: e.id, verify: true) do %> <span class="glyphicon glyphicon-refresh"></span> <% end %> <% end%> -- GitLab From 61e1bfed5ee059b2e430a8e10a2caeb7d41e7ca0 Mon Sep 17 00:00:00 2001 From: Gyan Gupta <gyan@cottagelabs.com> Date: Thu, 19 Dec 2024 13:23:58 +0530 Subject: [PATCH 6/6] Minor fixes --- hyrax/app/views/bulkrax/shared/_work_entries_tab.html.erb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hyrax/app/views/bulkrax/shared/_work_entries_tab.html.erb b/hyrax/app/views/bulkrax/shared/_work_entries_tab.html.erb index 2e7b0953..6fcb1d2a 100644 --- a/hyrax/app/views/bulkrax/shared/_work_entries_tab.html.erb +++ b/hyrax/app/views/bulkrax/shared/_work_entries_tab.html.erb @@ -35,12 +35,12 @@ <% varification_status, date, file_path = latest_varification_report_details(e) %> <% if varification_status.present? %> <% if varification_status == "Success" %> - <span class="glyphicon glyphicon-ok text-success", title="Verified successfully on (<%= date %>)"></span> + <span class="glyphicon glyphicon-ok text-success" title="Verified successfully on (<%= date %>)"></span> <% else %> - <span class="glyphicon glyphicon-remove text-danger", title="Verification failed on (<%= date %>)"></span> + <span class="glyphicon glyphicon-remove text-danger" title="Verification failed on (<%= date %>)"></span> <% end %> <%= link_to url_for(controller: params[:controller], action: "verify_and_generate_work", importer_id: e.importerexporter_id, entry_id: e.id, format: :csv) do %> - <span class="glyphicon glyphicon-download-alt"></span> + <span class="glyphicon glyphicon-download-alt" title="Download csv report of last verification"></span> <% end %> <% end %> <% if !varification_status || varification_status == "Failure" %> -- GitLab