Merge branch 'bug_fix/341-remove-files-folder-from-s3' into 'develop'

Remove files folder from s3

See merge request !269

hyrax/app/models/file_set.rb

@@ -30,18 +30,18 @@ class FileSet < ActiveFedora::Base
     prefix = case for_complex_type
     when 'ComplexSubject'
        complex_subject = ComplexSubject.find_by(source_identifier: for_complex_identifier)
-      "#{complex_subject.s3_folder_name}/files"
+      "#{complex_subject.s3_folder_name}"
     when 'ComplexSession'
       complex_session = ComplexSession.find_by(source_identifier: for_complex_identifier)
-      "#{complex_session.complex_subject.s3_folder_name}/#{complex_session.s3_folder_name}/files"
+      "#{complex_session.complex_subject.s3_folder_name}/#{complex_session.s3_folder_name}"
     when 'ComplexModality'
       complex_modality = ComplexModality.find_by(source_identifier: for_complex_identifier)
-      "#{complex_modality.complex_subject.s3_folder_name}/#{complex_modality.complex_session.s3_folder_name}/#{complex_modality.s3_folder_name}/files"
+      "#{complex_modality.complex_subject.s3_folder_name}/#{complex_modality.complex_session.s3_folder_name}/#{complex_modality.s3_folder_name}"
     else
-      'files'
+      nil
     end
 
-    "/#{prefix}/#{title.first}"
+    prefix.present? ? "#{prefix}/#{title.first}" : "#{title.first}"
   end
 
 

hyrax/app/services/s3_storage_service.rb

@@ -155,8 +155,8 @@ class S3StorageService
     response = @s3_client.delete_bucket(bucket: bucket_name)
   end
 
-  def move_object(source_object_key, target_bucket_name, target_object_key, metadata = {})
-    file_upload_bucket = sanitise_name(ENV['S3_FILE_UPLOAD_BUCKET'])
+  def move_object(source_object_key, target_bucket_name, target_object_key, metadata = {}, source_bucket_name = nil)
+    file_upload_bucket = source_bucket_name || sanitise_name(ENV['S3_FILE_UPLOAD_BUCKET'])
     source_bucket = Aws::S3::Resource
                              .new(region: ENV['S3_REGION'])
                              .bucket(file_upload_bucket)

hyrax/lib/tasks/fix_metadata_in_s3.rake

+# frozen_string_literal: true
+namespace :rdms do
+  desc "Fix metadata in S3 for all works. usage: rdms:fix_metadata_in_s3"
+  task fix_metadata_in_s3: :environment do
+    puts "Fixing Dataset"
+    all_works = %w(Dataset CrcDataset ComplexSubjecr ComplexSession ComplexModality)
+    all_works.each do |work_type|
+      work_type.constantize.all.each do |w|
+        begin
+          w.save
+        rescue
+          puts "Error saving #{w.id}"
+        end
+      end
+    end
+  end
+end
\ No newline at end of file

hyrax/lib/tasks/remove_files_folders_from_s3.rake

+# frozen_string_literal: true
+namespace :rdms do
+  desc "Fix object key structure in S3. usage: rdms:remove_files_folders_form_s3"
+  task remove_files_folders_from_s3: :environment do
+    s3 = S3StorageService.new
+    s3.init_client
+
+    s3.list_buckets.each do |bucket|
+      if bucket[:name].start_with?("#{ENV['S3_BUCKET_PREFIX']}-".downcase.gsub(/[^0-9a-z.-]/, ''))
+        list_of_objects, total_size = s3.list_all_objects(bucket[:name])
+        list_of_objects.each do |object|
+          if object[:key].include?('/files') or object[:key].include?('//')
+            s3.move_object(object[:key], bucket[:name], object[:key].gsub('/files', '').gsub('//', '/'), {}, bucket[:name])
+          end
+        end
+      end
+    end
+  end
+end
\ No newline at end of file