Merge branch 'bug_fix/454_gather_hidden_files' into 'develop'

Gather hidden files and fix rake task

Closes FDM/rdm-system/antleaf-projectmanagement#399 and FDM/rdm-system/antleaf-projectmanagement#454

See merge request !359

hyrax/app/services/verify_crc_dataset_import.rb

 require 'json'
 require 'csv'
+require 'find'
 
 class VerifyCRCDatasetImport
   extend ComplexHelper
@@ -66,11 +67,11 @@ class VerifyCRCDatasetImport
       msg = "Error: Import entry #{entry_id} not found"
       return false, msg, paths_compared, nil
     end
-    paths_compared[source_identifier: entry.identifier]
+    paths_compared[:source_identifier] = entry.identifier
 
     # Check import id
     if import_id != entry.importerexporter_id
-      msg = "Error: Importer id #{import_id} does not match id associated with entry #{entry_id}"
+      msg = "Error: Importer id #{import_id} does not match id associated with entry #{entry.importerexporter_id}"
       return false, msg, paths_compared, nil
     end
 
@@ -124,11 +125,16 @@ class VerifyCRCDatasetImport
 
     # Get files to compare
 
-    # -- Get list of files from import directory
-    input_list = Dir.glob(File.join(import_path, '**', '*')).
-      reject {|fn| File.directory?(fn) }.
-      reject{|fn| self.restricted_file_names?(fn.split('/')[-1])}.
-      map{ |fn| fn.sub(import_path, '')}.sort
+    # -- Get list of files from import directory including hidden files
+    input_list = []
+    ::Find.find(import_path) do |fp|
+      unless FileTest.directory?(fp)
+        unless self.restricted_file_names?(fp.split('/')[-1])
+          input_list << fp.sub(import_path, '')
+        end
+      end
+    end
+    input_list.sort
     # ---- downcase folder name
     sanitised_input_list = {}
     input_list.each do |fp|

hyrax/lib/tasks/verify_crc_1280_import.rake

@@ -2,17 +2,18 @@ require 'json'
 
 namespace :rdms do
   namespace :crc_1280_import do
-    desc 'Verify CRC1280 import or verify CRC1280 import of an experiment. This will compare the files in the file path with the objects in the S3 bucket"]'
-    task :"verify_import", [:import_id] => :environment do |task, args|
-      puts "Verifying import #{args.import_id}"
-      import_status, report_paths = VerifyCRCDatasetImport.verify_import(args.import_id)
+    desc 'Verify all experiments in CRC1280 import. This will compare the files in the file path with the objects in the S3 bucket'
+    task :verify_import, [:import_id] => :environment do |task, args|
+      puts "Verifying import #{args.import_id.to_i}"
+      import_status, report_paths = VerifyCRCDatasetImport.verify_import(args.import_id.to_i)
       puts "import status : #{import_status}"
       puts "Detailed reports are available at:"
       puts JSON.pretty_generate(report_paths)
     end
-    task :"verify_experiment", [:import_id, :entry_id] => :environment do |task, args|
-      puts "Verifying import #{args.import_id}"
-      experiment_status, report_path = VerifyCRCDatasetImport.verify_experiment_and_report(args.import_id, args.entry_id)
+    desc 'Verify experiment in CRC1280 import. This will compare the files in the file path with the objects in the S3 bucket'
+    task :verify_experiment, [:import_id, :entry_id] => :environment do |task, args|
+      puts "Verifying experiment #{args.entry_id.to_i} in import #{args.import_id.to_i}"
+      experiment_status, report_path = VerifyCRCDatasetImport.verify_experiment_and_report(args.import_id.to_i, args.entry_id.to_i)
       puts "Experiment import status : #{experiment_status}"
       puts "Detailed reports is available at: #{report_path}"
     end