diff --git a/hyrax/app/services/verify_crc_dataset_import.rb b/hyrax/app/services/verify_crc_dataset_import.rb index 539141f67e83aff62ade8ba27a5a8cfed1e3e483..aea5668abe2cfd2d64e54c91bda3e743e810ec61 100644 --- a/hyrax/app/services/verify_crc_dataset_import.rb +++ b/hyrax/app/services/verify_crc_dataset_import.rb @@ -1,5 +1,6 @@ require 'json' require 'csv' +require 'find' class VerifyCRCDatasetImport extend ComplexHelper @@ -66,11 +67,11 @@ class VerifyCRCDatasetImport msg = "Error: Import entry #{entry_id} not found" return false, msg, paths_compared, nil end - paths_compared[source_identifier: entry.identifier] + paths_compared[:source_identifier] = entry.identifier # Check import id if import_id != entry.importerexporter_id - msg = "Error: Importer id #{import_id} does not match id associated with entry #{entry_id}" + msg = "Error: Importer id #{import_id} does not match id associated with entry #{entry.importerexporter_id}" return false, msg, paths_compared, nil end @@ -124,11 +125,16 @@ class VerifyCRCDatasetImport # Get files to compare - # -- Get list of files from import directory - input_list = Dir.glob(File.join(import_path, '**', '*')). - reject {|fn| File.directory?(fn) }. - reject{|fn| self.restricted_file_names?(fn.split('/')[-1])}. - map{ |fn| fn.sub(import_path, '')}.sort + # -- Get list of files from import directory including hidden files + input_list = [] + ::Find.find(import_path) do |fp| + unless FileTest.directory?(fp) + unless self.restricted_file_names?(fp.split('/')[-1]) + input_list << fp.sub(import_path, '') + end + end + end + input_list.sort # ---- downcase folder name sanitised_input_list = {} input_list.each do |fp| diff --git a/hyrax/lib/tasks/verify_crc_1280_import.rake b/hyrax/lib/tasks/verify_crc_1280_import.rake index 3297ccb5b389dba72c419215b387281eeb9b2ea6..fe04e5306759365c562ec76a780bc04145e8529a 100644 --- a/hyrax/lib/tasks/verify_crc_1280_import.rake +++ b/hyrax/lib/tasks/verify_crc_1280_import.rake @@ -2,17 +2,18 @@ require 'json' namespace :rdms do namespace :crc_1280_import do - desc 'Verify CRC1280 import or verify CRC1280 import of an experiment. This will compare the files in the file path with the objects in the S3 bucket"]' - task :"verify_import", [:import_id] => :environment do |task, args| - puts "Verifying import #{args.import_id}" - import_status, report_paths = VerifyCRCDatasetImport.verify_import(args.import_id) + desc 'Verify all experiments in CRC1280 import. This will compare the files in the file path with the objects in the S3 bucket' + task :verify_import, [:import_id] => :environment do |task, args| + puts "Verifying import #{args.import_id.to_i}" + import_status, report_paths = VerifyCRCDatasetImport.verify_import(args.import_id.to_i) puts "import status : #{import_status}" puts "Detailed reports are available at:" puts JSON.pretty_generate(report_paths) end - task :"verify_experiment", [:import_id, :entry_id] => :environment do |task, args| - puts "Verifying import #{args.import_id}" - experiment_status, report_path = VerifyCRCDatasetImport.verify_experiment_and_report(args.import_id, args.entry_id) + desc 'Verify experiment in CRC1280 import. This will compare the files in the file path with the objects in the S3 bucket' + task :verify_experiment, [:import_id, :entry_id] => :environment do |task, args| + puts "Verifying experiment #{args.entry_id.to_i} in import #{args.import_id.to_i}" + experiment_status, report_path = VerifyCRCDatasetImport.verify_experiment_and_report(args.import_id.to_i, args.entry_id.to_i) puts "Experiment import status : #{experiment_status}" puts "Detailed reports is available at: #{report_path}" end