Merge branch 'fix/335-relocate-files-on-name-change' into develop

hyrax/app/helpers/s3_file_handler.rb

+module S3FileHandler
+
+  def move_and_remove_files_on_s3(object_class_name, object_id, name_changes)
+    @object = object_class_name.constantize.find_by(id: object_id)
+
+    @s3 = S3StorageService.new
+    @s3.init_client
+    @bucket_name = @s3.sanitise_name(@object.crc_dataset.id)
+    @objects_prefix = objects_prefix(name_changes[0])
+
+    all_objects, size = @s3.list_all_objects(@bucket_name, @objects_prefix)
+
+    if all_objects.any?
+      all_objects.each do |object|
+        source_object_key = object[:key]
+        target_object_key = source_object_key.gsub(name_changes[0], name_changes[1])
+
+        if meta_files_keys.include?(source_object_key)
+          delete_metadata_files(source_object_key)
+        else
+          move_object(source_object_key, target_object_key)
+        end
+      end
+    end
+  end
+
+  private
+
+  def objects_prefix(previous_name)
+    case @object.class.name
+    when 'ComplexSubject'
+      "#{previous_name}/"
+    when 'ComplexSession'
+      "#{@object.complex_subject.s3_folder_name}/#{previous_name}/"
+    when 'ComplexModality'
+      "#{@object.complex_subject.s3_folder_name}/#{@object.complex_session.s3_folder_name}/#{previous_name}/"
+    end
+  end
+
+  def meta_files_keys
+    [
+      "#{@objects_prefix}metadata.json",
+      "#{@objects_prefix}system_metadata.json",
+      "#{@objects_prefix}meta.json"
+    ]
+  end
+
+  def move_object(source_object_key, target_object_key)
+    if @s3.object_exists?(@bucket_name, source_object_key)
+      @s3.move_object(source_object_key, @bucket_name, target_object_key, {}, @bucket_name)
+    end
+  end
+
+  def delete_metadata_files(object_key)
+    @s3.delete_object(@bucket_name, object_key)
+  end
+end

hyrax/app/jobs/hyrax/handle_s3_folder_name_job.rb

+# frozen_string_literal: true
+module Hyrax
+  class HandleS3FolderNameJob < ApplicationJob
+    include S3FileHandler
+
+    def perform(object_class_name, object_id, name_changes)
+      move_and_remove_files_on_s3(object_class_name, object_id, name_changes)
+    end
+  end
+end
\ No newline at end of file

hyrax/app/models/complex_modality.rb

@@ -3,6 +3,7 @@
 class ComplexModality < ActiveRecord::Base
   include ComplexHelper
   include HyraxHelper
+  include S3FileHandleable
 
   belongs_to :complex_session, foreign_key: 'parent_source_identifier', primary_key: 'source_identifier', optional: true
 
@@ -15,7 +16,7 @@ class ComplexModality < ActiveRecord::Base
   validates :s3_folder_name, uniqueness: { scope: :parent_source_identifier, message: ->(object, data) { I18n.t('rdms.errors.complex_modality.title.uniqueness', value: object.s3_folder_name) } }
 
   before_validation :set_s3_folder_name
-  after_save :save_work_meta_json_file_to_s3, :update_crc_dataset_date_modified
+  after_save :save_work_meta_json_file_to_s3, :update_crc_dataset_date_modified, :relocate_files_on_folder_name_change
 
   delegate :crc_dataset, to: :complex_session
   delegate :complex_subject, to: :complex_session

hyrax/app/models/complex_session.rb

@@ -3,6 +3,7 @@
 class ComplexSession < ActiveRecord::Base
   include ComplexHelper
   include HyraxHelper
+  include S3FileHandleable
 
   belongs_to :complex_subject, foreign_key: 'parent_source_identifier', primary_key: 'source_identifier', optional: true
 
@@ -17,7 +18,7 @@ class ComplexSession < ActiveRecord::Base
   validates :s3_folder_name, uniqueness: { scope: :parent_source_identifier,  message: ->(object, data) { I18n.t('rdms.errors.complex_session.title.uniqueness', value: object.s3_folder_name) } }
 
   before_validation :set_s3_folder_name
-  after_save :save_work_meta_json_file_to_s3, :update_crc_dataset_date_modified
+  after_save :save_work_meta_json_file_to_s3, :update_crc_dataset_date_modified, :relocate_files_on_folder_name_change
 
   delegate :crc_dataset, to: :complex_subject
 

hyrax/app/models/complex_subject.rb

@@ -3,6 +3,7 @@
 class ComplexSubject < ActiveRecord::Base
   include ComplexHelper
   include HyraxHelper
+  include S3FileHandleable
 
   has_many :complex_sessions, foreign_key: 'parent_source_identifier', primary_key: 'source_identifier', dependent: :destroy
 
@@ -15,7 +16,7 @@ class ComplexSubject < ActiveRecord::Base
   validates :s3_folder_name, uniqueness: { scope: :parent_source_identifier,  message: ->(object, data) { I18n.t('rdms.errors.complex_subject.title.uniqueness', value: object.s3_folder_name) } }
 
   before_validation :set_s3_folder_name
-  after_save :save_work_meta_json_file_to_s3, :update_crc_dataset_date_modified
+  after_save :save_work_meta_json_file_to_s3, :update_crc_dataset_date_modified, :relocate_files_on_folder_name_change
 
   scope :sort_by_title, ->(source_identifier) {
     where(parent_source_identifier: source_identifier)

hyrax/app/models/concerns/s3_file_handleable.rb

+# app/models/concerns/file_relocation_concern.rb
+module S3FileHandleable
+  extend ActiveSupport::Concern
+
+  def relocate_files_on_folder_name_change
+    return true unless saved_change_to_s3_folder_name?
+    name_changes = saved_changes[:s3_folder_name]
+    return true if name_changes[0].nil?
+
+    Hyrax::HandleS3FolderNameJob.perform_later(self.class.name, self.id, name_changes)
+  end
+end

hyrax/app/services/s3_storage_service.rb

@@ -39,6 +39,12 @@ class S3StorageService
     false
   end
 
+  def object_exists?(bucket_name, object_key)
+    return true if @s3_client.get_object(bucket: bucket_name, key: object_key)
+  rescue Aws::S3::Errors::NoSuchKey => e
+    false
+  end
+
   def list_objects(bucket_name, prefix = nil, max_keys = 1000)
     # To get the first max_keys objects in a bucket
     contents = []

hyrax/lib/tasks/remove_forward_slash_from_starting_of_s3_keys.rake

+# frozen_string_literal: true
+namespace :rdms do
+  desc "Fix object key structure in S3. usage: rdms:remove_forward_slash_from_starting_of_s3_keys"
+  task remove_forward_slash_from_starting_of_s3_keys: :environment do
+    s3 = S3StorageService.new
+    s3.init_client
+
+    s3.list_buckets.each do |bucket|
+      if bucket[:name].start_with?("#{ENV['S3_BUCKET_PREFIX']}-".downcase.gsub(/[^0-9a-z.-]/, ''))
+        list_of_objects, total_size = s3.list_all_objects(bucket[:name])
+        list_of_objects.each do |object|
+          if object[:key].start_with?('/')
+            s3.move_object(object[:key], bucket[:name], object[:key].gsub(/^\//, ''), {}, bucket[:name])
+          end
+        end
+      end
+    end
+  end
+end
\ No newline at end of file