From 78baf43f66555ba75c711b62e041e957d185e120 Mon Sep 17 00:00:00 2001 From: Pascal Ernster <pascal.ernster@rub.de> Date: Wed, 29 Jan 2025 17:21:02 +0100 Subject: [PATCH 01/16] Remove unused env vars $PORT and $VIRTUAL_PORT These variables have never made sense because of two reasons: - The "bundle exec rails server" call in hyrax/docker-entrypoint.sh has always used the hardcoded port 3000 anyway since the beginning of this project, so these variables were never actually used at all. - Even *if* these variables had been used, their existence *still* wouldn't have made any sense, because they would only set the port number *inside* the docker containers, but *not* the port number on which puma would be exposed to the host system. Signed-off-by: Pascal Ernster <pascal.ernster@rub.de> --- .env.template | 1 - .env.template.development | 1 - docker-compose.development.yml | 1 - docker-compose.yml | 1 - hyrax/config/puma.rb | 4 ---- 5 files changed, 8 deletions(-) diff --git a/.env.template b/.env.template index f089458e..265cc417 100644 --- a/.env.template +++ b/.env.template @@ -38,7 +38,6 @@ FEDORA_TEST_PORT=8080 # RAILS setting LANG=C.UTF-8 -PORT=3000 RAILS_ENV=production RAILS_LOG_TO_STDOUT=true RAILS_SERVE_STATIC_FILES=true diff --git a/.env.template.development b/.env.template.development index 07c478ce..86ec62b3 100644 --- a/.env.template.development +++ b/.env.template.development @@ -33,7 +33,6 @@ FEDORA_TEST_PORT=8080 # RAILS setting LANG=C.UTF-8 -PORT=3000 RAILS_ENV=development RAILS_LOG_TO_STDOUT=true RAILS_SERVE_STATIC_FILES=true diff --git a/docker-compose.development.yml b/docker-compose.development.yml index c8b1b3ab..14730162 100644 --- a/docker-compose.development.yml +++ b/docker-compose.development.yml @@ -114,7 +114,6 @@ services: <<: *app environment: - VIRTUAL_HOST=rdms.docker - - VIRTUAL_PORT=3000 - RAILS_FORCE_SSL=false command: bash -c "/bin/docker-entrypoint.sh" depends_on: diff --git a/docker-compose.yml b/docker-compose.yml index e4982dd5..07610571 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -111,7 +111,6 @@ services: restart: unless-stopped environment: - VIRTUAL_HOST=rdms.docker - - VIRTUAL_PORT=3000 depends_on: - appdb - solr diff --git a/hyrax/config/puma.rb b/hyrax/config/puma.rb index ed10558b..83ed4d32 100644 --- a/hyrax/config/puma.rb +++ b/hyrax/config/puma.rb @@ -7,10 +7,6 @@ threads_count = ENV.fetch("RAILS_MAX_THREADS") { 5 }.to_i threads threads_count, threads_count -# Specifies the `port` that Puma will listen on to receive requests; default is 3000. -# -port ENV.fetch("PORT") { 3000 } - # Specifies the `environment` that Puma will run in. # environment ENV.fetch("RAILS_ENV") { "development" } -- GitLab From 520f0672f21167d95327a67d68dc3b8e51925a7c Mon Sep 17 00:00:00 2001 From: Pascal Ernster <pascal.ernster@rub.de> Date: Wed, 29 Jan 2025 20:22:31 +0100 Subject: [PATCH 02/16] Remove unused environment variable $VIRTUAL_HOST from docker-compose.yml This variable is not used anywhere in the ReSeeD or Hyrax 3.5.0 codebase, therefore it can be safely removed. Signed-off-by: Pascal Ernster <pascal.ernster@rub.de> --- docker-compose.development.yml | 1 - docker-compose.yml | 2 -- 2 files changed, 3 deletions(-) diff --git a/docker-compose.development.yml b/docker-compose.development.yml index 14730162..0d087674 100644 --- a/docker-compose.development.yml +++ b/docker-compose.development.yml @@ -113,7 +113,6 @@ services: web: <<: *app environment: - - VIRTUAL_HOST=rdms.docker - RAILS_FORCE_SSL=false command: bash -c "/bin/docker-entrypoint.sh" depends_on: diff --git a/docker-compose.yml b/docker-compose.yml index 07610571..4e0cac4e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -109,8 +109,6 @@ services: <<: *app command: bash -c "/bin/docker-entrypoint.sh" restart: unless-stopped - environment: - - VIRTUAL_HOST=rdms.docker depends_on: - appdb - solr -- GitLab From 2b660f3879199c351ab900b7942b28f1ff43e898 Mon Sep 17 00:00:00 2001 From: Pascal Ernster <pascal.ernster@rub.de> Date: Wed, 29 Jan 2025 18:39:03 +0100 Subject: [PATCH 03/16] Remove useless variable $UPLOADS_PATH This variable only configures a path *inside* our containers, which is completely useless, and even creates an unecessary point of failure, because this variable breaks uploads if it is set to a different value at run time that the value it was set to at build time. Therefore, just hardcode the path to "/shared/uploads". Signed-off-by: Pascal Ernster <pascal.ernster@rub.de> --- .env.template | 2 -- .env.template.development | 2 -- docker-compose.development.yml | 5 ++--- docker-compose.yml | 3 +-- hyrax/Dockerfile | 3 +-- hyrax/config/initializers/hyrax.rb | 4 ++-- 6 files changed, 6 insertions(+), 13 deletions(-) diff --git a/.env.template b/.env.template index 265cc417..3baf1392 100644 --- a/.env.template +++ b/.env.template @@ -65,8 +65,6 @@ DERIVATIVES_PATH=/shared/derivatives/ # FITS_PATH and FITS_VERSION are used by Docker build and are therefore fixed in the resulting Docker image FITS_PATH=/fits/fits-1.5.5/fits.sh FITS_VERSION=1.5.5 -# UPLOADS_PATH is used by Docker build and is therefore fixed in the resulting Docker image -UPLOADS_PATH=/shared/uploads/ # CACHE_PATH is used by Docker build and is therefore fixed in the resulting Docker image CACHE_PATH=/shared/cache/ BRAND_PATH=/data/public/branding diff --git a/.env.template.development b/.env.template.development index 86ec62b3..f5665de2 100644 --- a/.env.template.development +++ b/.env.template.development @@ -63,8 +63,6 @@ DERIVATIVES_PATH=/shared/derivatives/ # FITS_PATH and FITS_VERSION are used by Docker build and are therefore fixed in the resulting Docker image FITS_PATH=/fits/fits-1.5.5/fits.sh FITS_VERSION=1.5.5 -# UPLOADS_PATH is used by Docker build and is therefore fixed in the resulting Docker image -UPLOADS_PATH=/shared/uploads/ # CACHE_PATH is used by Docker build and is therefore fixed in the resulting Docker image CACHE_PATH=/shared/cache/ BRAND_PATH=/data/public/branding diff --git a/docker-compose.development.yml b/docker-compose.development.yml index 0d087674..74cfe660 100644 --- a/docker-compose.development.yml +++ b/docker-compose.development.yml @@ -98,7 +98,6 @@ services: context: hyrax args: RAILS_ENV: ${RAILS_ENV} - UPLOADS_PATH: ${UPLOADS_PATH} DERIVATIVES_PATH: ${DERIVATIVES_PATH} CACHE_PATH: ${CACHE_PATH} FITS_PATH: ${FITS_PATH} @@ -129,7 +128,7 @@ services: ports: - 3000:3000 volumes: - - file_uploads:${UPLOADS_PATH} + - file_uploads:/shared/uploads - derivatives:${DERIVATIVES_PATH} - cache:${CACHE_PATH} - branding:${BRAND_PATH} @@ -147,7 +146,7 @@ services: - fcrepo - redis volumes: - - file_uploads:${UPLOADS_PATH} + - file_uploads:/shared/uploads - derivatives:${DERIVATIVES_PATH} - cache:${CACHE_PATH} - branding:${BRAND_PATH} diff --git a/docker-compose.yml b/docker-compose.yml index 4e0cac4e..e2fad435 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -89,7 +89,6 @@ services: context: hyrax args: RAILS_ENV: ${RAILS_ENV} - UPLOADS_PATH: ${UPLOADS_PATH} DERIVATIVES_PATH: ${DERIVATIVES_PATH} CACHE_PATH: ${CACHE_PATH} FITS_VERSION: ${FITS_VERSION} @@ -97,7 +96,7 @@ services: env_file: - .env volumes: - - ${DOCKER_VOLUMES_PATH_PREFIX}file_uploads:${UPLOADS_PATH} + - ${DOCKER_VOLUMES_PATH_PREFIX}file_uploads:/shared/uploads - ${DOCKER_VOLUMES_PATH_PREFIX}derivatives:${DERIVATIVES_PATH} - ${DOCKER_VOLUMES_PATH_PREFIX}cache:${CACHE_PATH} - ${CRC_FOLDER_IMPORT_PATH}:/rub-test-data diff --git a/hyrax/Dockerfile b/hyrax/Dockerfile index f007b701..5e644aa4 100644 --- a/hyrax/Dockerfile +++ b/hyrax/Dockerfile @@ -3,7 +3,6 @@ FROM ruby:2.7.8-alpine # Setup build variables ARG RAILS_ENV ARG DERIVATIVES_PATH -ARG UPLOADS_PATH ARG CACHE_PATH ARG FITS_VERSION ARG DOWNLOAD_PATH @@ -45,7 +44,7 @@ RUN cd $APP_PRODUCTION \ # create a folder to store derivatives, file uploads and cache directory RUN mkdir -p $DERIVATIVES_PATH -RUN mkdir -p $UPLOADS_PATH +RUN mkdir -p "/shared/uploads" RUN mkdir -p $CACHE_PATH RUN mkdir -p $CRC_IMPORT_FOLDER_PATH RUN mkdir -p $DOWNLOAD_PATH diff --git a/hyrax/config/initializers/hyrax.rb b/hyrax/config/initializers/hyrax.rb index 086eb2eb..e216be28 100644 --- a/hyrax/config/initializers/hyrax.rb +++ b/hyrax/config/initializers/hyrax.rb @@ -175,7 +175,7 @@ Hyrax.config do |config| # Temporary paths to hold uploads before they are ingested into FCrepo # These must be lambdas that return a Pathname. Can be configured separately - config.upload_path = ->() { ENV.fetch('UPLOADS_PATH', Rails.root.join('tmp', 'uploads')) } + config.upload_path = ->() { '/shared/uploads' } config.cache_path = ->() { ENV.fetch('CACHE_PATH', Rails.root.join('tmp', 'uploads', 'cache')) } # Location on local file system where derivatives will be stored @@ -192,7 +192,7 @@ Hyrax.config do |config| # Location on local file system where uploaded files will be staged # prior to being ingested into the repository or having derivatives generated. # If you use a multi-server architecture, this MUST be a shared volume. - config.working_path = ENV.fetch('UPLOADS_PATH', Rails.root.join('tmp', 'uploads')) + config.working_path = config.upload_path # Should the media display partial render a download link? # config.display_media_download_link = true -- GitLab From ef96b50f80ad0a1eb3598fe910727d4b58ac8c2b Mon Sep 17 00:00:00 2001 From: Pascal Ernster <pascal.ernster@rub.de> Date: Wed, 29 Jan 2025 18:48:36 +0100 Subject: [PATCH 04/16] Remove useless variable $DERIVATIVES_PATH This variable only configures a path *inside* our containers, which is completely useless, and even creates an unecessary point of failure, because this variable might breaks stuff if it is set to a different value at run time that the value it was set to at build time. Therefore, just hardcode the path to "/shared/derivatives". Signed-off-by: Pascal Ernster <pascal.ernster@rub.de> --- .env.template | 2 -- .env.template.development | 2 -- docker-compose.development.yml | 5 ++--- docker-compose.yml | 3 +-- hyrax/Dockerfile | 3 +-- hyrax/config/initializers/hyrax.rb | 2 +- 6 files changed, 5 insertions(+), 12 deletions(-) diff --git a/.env.template b/.env.template index 3baf1392..4a8127a0 100644 --- a/.env.template +++ b/.env.template @@ -60,8 +60,6 @@ MATOMO_AUTH_TOKEN= # Hyrax application setting HOST_APP_DATA_PATH=/data/ -# DERIVATIVES_PATH is used by Docker build and is therefore fixed in the resulting Docker image -DERIVATIVES_PATH=/shared/derivatives/ # FITS_PATH and FITS_VERSION are used by Docker build and are therefore fixed in the resulting Docker image FITS_PATH=/fits/fits-1.5.5/fits.sh FITS_VERSION=1.5.5 diff --git a/.env.template.development b/.env.template.development index f5665de2..19009a73 100644 --- a/.env.template.development +++ b/.env.template.development @@ -58,8 +58,6 @@ MATOMO_AUTH_TOKEN= # Hyrax application setting HOST_APP_DATA_PATH=./data/ -# DERIVATIVES_PATH is used by Docker build and is therefore fixed in the resulting Docker image -DERIVATIVES_PATH=/shared/derivatives/ # FITS_PATH and FITS_VERSION are used by Docker build and are therefore fixed in the resulting Docker image FITS_PATH=/fits/fits-1.5.5/fits.sh FITS_VERSION=1.5.5 diff --git a/docker-compose.development.yml b/docker-compose.development.yml index 74cfe660..5d3b3388 100644 --- a/docker-compose.development.yml +++ b/docker-compose.development.yml @@ -98,7 +98,6 @@ services: context: hyrax args: RAILS_ENV: ${RAILS_ENV} - DERIVATIVES_PATH: ${DERIVATIVES_PATH} CACHE_PATH: ${CACHE_PATH} FITS_PATH: ${FITS_PATH} FITS_VERSION: ${FITS_VERSION} @@ -129,7 +128,7 @@ services: - 3000:3000 volumes: - file_uploads:/shared/uploads - - derivatives:${DERIVATIVES_PATH} + - derivatives:/shared/derivatives - cache:${CACHE_PATH} - branding:${BRAND_PATH} - ${HOST_APP_DATA_PATH:-/data/}:/data/data @@ -147,7 +146,7 @@ services: - redis volumes: - file_uploads:/shared/uploads - - derivatives:${DERIVATIVES_PATH} + - derivatives:/shared/derivatives - cache:${CACHE_PATH} - branding:${BRAND_PATH} - ${HOST_APP_DATA_PATH:-/data/}:/data/data diff --git a/docker-compose.yml b/docker-compose.yml index e2fad435..7609b6e5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -89,7 +89,6 @@ services: context: hyrax args: RAILS_ENV: ${RAILS_ENV} - DERIVATIVES_PATH: ${DERIVATIVES_PATH} CACHE_PATH: ${CACHE_PATH} FITS_VERSION: ${FITS_VERSION} DOWNLOAD_PATH: ${DOWNLOAD_PATH} @@ -97,7 +96,7 @@ services: - .env volumes: - ${DOCKER_VOLUMES_PATH_PREFIX}file_uploads:/shared/uploads - - ${DOCKER_VOLUMES_PATH_PREFIX}derivatives:${DERIVATIVES_PATH} + - ${DOCKER_VOLUMES_PATH_PREFIX}derivatives:/shared/derivatives - ${DOCKER_VOLUMES_PATH_PREFIX}cache:${CACHE_PATH} - ${CRC_FOLDER_IMPORT_PATH}:/rub-test-data - ${DOCKER_VOLUMES_PATH_PREFIX}downloads:${DOWNLOAD_PATH} diff --git a/hyrax/Dockerfile b/hyrax/Dockerfile index 5e644aa4..9c1c76d4 100644 --- a/hyrax/Dockerfile +++ b/hyrax/Dockerfile @@ -2,7 +2,6 @@ FROM ruby:2.7.8-alpine # Setup build variables ARG RAILS_ENV -ARG DERIVATIVES_PATH ARG CACHE_PATH ARG FITS_VERSION ARG DOWNLOAD_PATH @@ -43,7 +42,7 @@ RUN cd $APP_PRODUCTION \ && mv Gemfile.lock Gemfile.lock.built_by_docker # create a folder to store derivatives, file uploads and cache directory -RUN mkdir -p $DERIVATIVES_PATH +RUN mkdir -p "/shared/derivatives" RUN mkdir -p "/shared/uploads" RUN mkdir -p $CACHE_PATH RUN mkdir -p $CRC_IMPORT_FOLDER_PATH diff --git a/hyrax/config/initializers/hyrax.rb b/hyrax/config/initializers/hyrax.rb index e216be28..752c606e 100644 --- a/hyrax/config/initializers/hyrax.rb +++ b/hyrax/config/initializers/hyrax.rb @@ -180,7 +180,7 @@ Hyrax.config do |config| # Location on local file system where derivatives will be stored # If you use a multi-server architecture, this MUST be a shared volume - config.derivatives_path = ENV.fetch('DERIVATIVES_PATH', Rails.root.join('tmp', 'derivatives')) + config.derivatives_path = '/shared/derivatives' # Should schema.org microdata be displayed? # config.display_microdata = true -- GitLab From f2fa85d09d8e96ff61c4f12191ec0e6d142b069f Mon Sep 17 00:00:00 2001 From: Pascal Ernster <pascal.ernster@rub.de> Date: Wed, 29 Jan 2025 18:55:29 +0100 Subject: [PATCH 05/16] Remove useless variable $CACHE_PATH This variable only configures a path *inside* our containers, which is completely useless, and even creates an unecessary point of failure, because this variable might break stuff if it is set to a different value at run time that the value it was set to at build time. Therefore, just hardcode the path to "/shared/cache". Signed-off-by: Pascal Ernster <pascal.ernster@rub.de> --- .env.template | 2 -- .env.template.development | 2 -- docker-compose.development.yml | 5 ++--- docker-compose.yml | 3 +-- hyrax/Dockerfile | 3 +-- hyrax/config/initializers/hyrax.rb | 2 +- 6 files changed, 5 insertions(+), 12 deletions(-) diff --git a/.env.template b/.env.template index 4a8127a0..745c84e0 100644 --- a/.env.template +++ b/.env.template @@ -63,8 +63,6 @@ HOST_APP_DATA_PATH=/data/ # FITS_PATH and FITS_VERSION are used by Docker build and are therefore fixed in the resulting Docker image FITS_PATH=/fits/fits-1.5.5/fits.sh FITS_VERSION=1.5.5 -# CACHE_PATH is used by Docker build and is therefore fixed in the resulting Docker image -CACHE_PATH=/shared/cache/ BRAND_PATH=/data/public/branding DEFAULT_DATE_FORMAT=%Y-%m-%d NOTIFICATIONS_EMAIL_DEFAULT_FROM_ADDRESS=no-reply@mailboxer.com diff --git a/.env.template.development b/.env.template.development index 19009a73..2054b46a 100644 --- a/.env.template.development +++ b/.env.template.development @@ -61,8 +61,6 @@ HOST_APP_DATA_PATH=./data/ # FITS_PATH and FITS_VERSION are used by Docker build and are therefore fixed in the resulting Docker image FITS_PATH=/fits/fits-1.5.5/fits.sh FITS_VERSION=1.5.5 -# CACHE_PATH is used by Docker build and is therefore fixed in the resulting Docker image -CACHE_PATH=/shared/cache/ BRAND_PATH=/data/public/branding DEFAULT_DATE_FORMAT=%Y-%m-%d NOTIFICATIONS_EMAIL_DEFAULT_FROM_ADDRESS=no-reply@mailboxer.com diff --git a/docker-compose.development.yml b/docker-compose.development.yml index 5d3b3388..5c1b660e 100644 --- a/docker-compose.development.yml +++ b/docker-compose.development.yml @@ -98,7 +98,6 @@ services: context: hyrax args: RAILS_ENV: ${RAILS_ENV} - CACHE_PATH: ${CACHE_PATH} FITS_PATH: ${FITS_PATH} FITS_VERSION: ${FITS_VERSION} CRC_FOLDER_IMPORT_PATH: ${CRC_FOLDER_IMPORT_PATH} @@ -129,7 +128,7 @@ services: volumes: - file_uploads:/shared/uploads - derivatives:/shared/derivatives - - cache:${CACHE_PATH} + - cache:/shared/cache - branding:${BRAND_PATH} - ${HOST_APP_DATA_PATH:-/data/}:/data/data - ./hyrax:/data:cached @@ -147,7 +146,7 @@ services: volumes: - file_uploads:/shared/uploads - derivatives:/shared/derivatives - - cache:${CACHE_PATH} + - cache:/shared/cache - branding:${BRAND_PATH} - ${HOST_APP_DATA_PATH:-/data/}:/data/data - ./hyrax:/data:cached diff --git a/docker-compose.yml b/docker-compose.yml index 7609b6e5..a868c643 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -89,7 +89,6 @@ services: context: hyrax args: RAILS_ENV: ${RAILS_ENV} - CACHE_PATH: ${CACHE_PATH} FITS_VERSION: ${FITS_VERSION} DOWNLOAD_PATH: ${DOWNLOAD_PATH} env_file: @@ -97,7 +96,7 @@ services: volumes: - ${DOCKER_VOLUMES_PATH_PREFIX}file_uploads:/shared/uploads - ${DOCKER_VOLUMES_PATH_PREFIX}derivatives:/shared/derivatives - - ${DOCKER_VOLUMES_PATH_PREFIX}cache:${CACHE_PATH} + - ${DOCKER_VOLUMES_PATH_PREFIX}cache:/shared/cache - ${CRC_FOLDER_IMPORT_PATH}:/rub-test-data - ${DOCKER_VOLUMES_PATH_PREFIX}downloads:${DOWNLOAD_PATH} networks: diff --git a/hyrax/Dockerfile b/hyrax/Dockerfile index 9c1c76d4..3576f47d 100644 --- a/hyrax/Dockerfile +++ b/hyrax/Dockerfile @@ -2,7 +2,6 @@ FROM ruby:2.7.8-alpine # Setup build variables ARG RAILS_ENV -ARG CACHE_PATH ARG FITS_VERSION ARG DOWNLOAD_PATH @@ -44,7 +43,7 @@ RUN cd $APP_PRODUCTION \ # create a folder to store derivatives, file uploads and cache directory RUN mkdir -p "/shared/derivatives" RUN mkdir -p "/shared/uploads" -RUN mkdir -p $CACHE_PATH +RUN mkdir -p "/shared/cache" RUN mkdir -p $CRC_IMPORT_FOLDER_PATH RUN mkdir -p $DOWNLOAD_PATH diff --git a/hyrax/config/initializers/hyrax.rb b/hyrax/config/initializers/hyrax.rb index 752c606e..6bb9dbb3 100644 --- a/hyrax/config/initializers/hyrax.rb +++ b/hyrax/config/initializers/hyrax.rb @@ -176,7 +176,7 @@ Hyrax.config do |config| # Temporary paths to hold uploads before they are ingested into FCrepo # These must be lambdas that return a Pathname. Can be configured separately config.upload_path = ->() { '/shared/uploads' } - config.cache_path = ->() { ENV.fetch('CACHE_PATH', Rails.root.join('tmp', 'uploads', 'cache')) } + config.cache_path = ->() { '/shared/cache' } # Location on local file system where derivatives will be stored # If you use a multi-server architecture, this MUST be a shared volume -- GitLab From 6aa34d971f89add4f9689983505706e406728d38 Mon Sep 17 00:00:00 2001 From: Pascal Ernster <pascal.ernster@rub.de> Date: Wed, 29 Jan 2025 19:02:57 +0100 Subject: [PATCH 06/16] Remove useless variable $DOWNLOAD_PATH This variable only configures a path *inside* our containers, which is completely useless, and even creates an unecessary point of failure, because this variable potentially breaks downloads if it is set to a different value at run time that the value it was set to at build time. Therefore, just hardcode the path to "/shared/downloads". Signed-off-by: Pascal Ernster <pascal.ernster@rub.de> --- .env.template | 2 -- .env.template.development | 2 -- docker-compose.development.yml | 5 ++--- docker-compose.yml | 3 +-- hyrax/Dockerfile | 3 +-- hyrax/app/helpers/download_helper.rb | 8 ++++---- hyrax/app/helpers/hyrax_helper.rb | 2 +- hyrax/app/helpers/search_download_helper.rb | 2 +- hyrax/app/services/verify_crc_dataset_import.rb | 2 +- 9 files changed, 11 insertions(+), 18 deletions(-) diff --git a/.env.template b/.env.template index 745c84e0..6ffa1d62 100644 --- a/.env.template +++ b/.env.template @@ -159,8 +159,6 @@ ARK_NAAN= # Download DOWNLOAD_FILES_SIZE_LIMIT=100000000 DOWNLOAD_FILES_COUNT_LIMIT=150 -# DOWNLOAD_PATH is used by Docker build and is therefore fixed in the resulting Docker image -DOWNLOAD_PATH=/shared/downloads # Admin set titles for the workflows RUB_ADMIN_SET_TITLE="RUB publication workflow" diff --git a/.env.template.development b/.env.template.development index 2054b46a..bf86b185 100644 --- a/.env.template.development +++ b/.env.template.development @@ -155,8 +155,6 @@ ARK_NAAN= # Download DOWNLOAD_FILES_SIZE_LIMIT=100000000 DOWNLOAD_FILES_COUNT_LIMIT=150 -# DOWNLOAD_PATH is used by Docker build and is therefore fixed in the resulting Docker image -DOWNLOAD_PATH=/shared/downloads # Admin set titles for the workflows RUB_ADMIN_SET_TITLE="RUB publication workflow" diff --git a/docker-compose.development.yml b/docker-compose.development.yml index 5c1b660e..1f0211ad 100644 --- a/docker-compose.development.yml +++ b/docker-compose.development.yml @@ -101,7 +101,6 @@ services: FITS_PATH: ${FITS_PATH} FITS_VERSION: ${FITS_VERSION} CRC_FOLDER_IMPORT_PATH: ${CRC_FOLDER_IMPORT_PATH} - DOWNLOAD_PATH: ${DOWNLOAD_PATH} env_file: - .env networks: @@ -133,7 +132,7 @@ services: - ${HOST_APP_DATA_PATH:-/data/}:/data/data - ./hyrax:/data:cached - ${CRC_FOLDER_IMPORT_PATH}:/rub-test-data - - downloads:${DOWNLOAD_PATH} + - downloads:/shared/downloads workers: <<: *app @@ -151,7 +150,7 @@ services: - ${HOST_APP_DATA_PATH:-/data/}:/data/data - ./hyrax:/data:cached - ${CRC_FOLDER_IMPORT_PATH}:/rub-test-data - - downloads:${DOWNLOAD_PATH} + - downloads:/shared/downloads redis: image: redis:6 diff --git a/docker-compose.yml b/docker-compose.yml index a868c643..a3ef495b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -90,7 +90,6 @@ services: args: RAILS_ENV: ${RAILS_ENV} FITS_VERSION: ${FITS_VERSION} - DOWNLOAD_PATH: ${DOWNLOAD_PATH} env_file: - .env volumes: @@ -98,7 +97,7 @@ services: - ${DOCKER_VOLUMES_PATH_PREFIX}derivatives:/shared/derivatives - ${DOCKER_VOLUMES_PATH_PREFIX}cache:/shared/cache - ${CRC_FOLDER_IMPORT_PATH}:/rub-test-data - - ${DOCKER_VOLUMES_PATH_PREFIX}downloads:${DOWNLOAD_PATH} + - ${DOCKER_VOLUMES_PATH_PREFIX}downloads:/shared/downloads networks: internal: diff --git a/hyrax/Dockerfile b/hyrax/Dockerfile index 3576f47d..68886f51 100644 --- a/hyrax/Dockerfile +++ b/hyrax/Dockerfile @@ -3,7 +3,6 @@ FROM ruby:2.7.8-alpine # Setup build variables ARG RAILS_ENV ARG FITS_VERSION -ARG DOWNLOAD_PATH ENV APP_PRODUCTION=/data/ ENV CRC_IMPORT_FOLDER_PATH=/rub-test-data @@ -45,7 +44,7 @@ RUN mkdir -p "/shared/derivatives" RUN mkdir -p "/shared/uploads" RUN mkdir -p "/shared/cache" RUN mkdir -p $CRC_IMPORT_FOLDER_PATH -RUN mkdir -p $DOWNLOAD_PATH +RUN mkdir -p "/shared/downloads" # copy the application COPY . $APP_PRODUCTION diff --git a/hyrax/app/helpers/download_helper.rb b/hyrax/app/helpers/download_helper.rb index ffd48ae3..9184ce8a 100644 --- a/hyrax/app/helpers/download_helper.rb +++ b/hyrax/app/helpers/download_helper.rb @@ -49,19 +49,19 @@ module DownloadHelper end def base_download_dir - File.join(ENV.fetch('DOWNLOAD_PATH', '/shared/downloads'), work.id) + File.join("/shared/downloads", work.id) end def zip_file_path - File.join(ENV.fetch('DOWNLOAD_PATH', "/shared/downloads"), "#{work.id}.zip") + File.join("/shared/downloads", "#{work.id}.zip") end def shell_file_path - File.join(ENV.fetch('DOWNLOAD_PATH', "/shared/downloads"), "#{work.id}.sh") + File.join("/shared/downloads", "#{work.id}.sh") end def ts_file_path - File.join(ENV.fetch('DOWNLOAD_PATH', "/shared/downloads"), "#{work.id}.txt") + File.join("/shared/downloads", "#{work.id}.txt") end def create_or_modify_ts_file diff --git a/hyrax/app/helpers/hyrax_helper.rb b/hyrax/app/helpers/hyrax_helper.rb index a56140a5..7fe27063 100644 --- a/hyrax/app/helpers/hyrax_helper.rb +++ b/hyrax/app/helpers/hyrax_helper.rb @@ -190,7 +190,7 @@ module HyraxHelper def latest_varification_report_details(entry) # Define the file pattern and search path - download_path = ENV.fetch('DOWNLOAD_PATH', '/shared/downloads') + download_path = "/shared/downloads" report_pattern = "analysis_report_#{entry.importerexporter_id}_#{entry.id}_*.csv" matching_files = Dir.glob(File.join(download_path, report_pattern)) diff --git a/hyrax/app/helpers/search_download_helper.rb b/hyrax/app/helpers/search_download_helper.rb index c432715f..b2940fdd 100644 --- a/hyrax/app/helpers/search_download_helper.rb +++ b/hyrax/app/helpers/search_download_helper.rb @@ -29,7 +29,7 @@ module SearchDownloadHelper end def shell_file_path - @shell_file_path ||= File.join(ENV.fetch('DOWNLOAD_PATH', "/shared/downloads"), "users/#{shell_file_name}") + @shell_file_path ||= File.join("/shared/downloads", "users/#{shell_file_name}") end def shell_file_name diff --git a/hyrax/app/services/verify_crc_dataset_import.rb b/hyrax/app/services/verify_crc_dataset_import.rb index f4e5eb39..a472625f 100644 --- a/hyrax/app/services/verify_crc_dataset_import.rb +++ b/hyrax/app/services/verify_crc_dataset_import.rb @@ -22,7 +22,7 @@ class VerifyCRCDatasetImport # prepare csv report report_name = "analysis_report_#{import_id}_#{entry_id}_#{Time.now.strftime("%Y-%m-%dT%H-%M-%S")}_#{ verification_status ? "success" : "failure"}.csv" - report_path = File.join(ENV.fetch('DOWNLOAD_PATH', '/shared/downloads'), report_name) + report_path = File.join("/shared/downloads", report_name) headers = %w(id status message) CSV.open(report_path, "w") do |csv| -- GitLab From 89b93b60f584c813ed7ea9af0d5666dba332fdbf Mon Sep 17 00:00:00 2001 From: Pascal Ernster <pascal.ernster@rub.de> Date: Wed, 29 Jan 2025 19:29:38 +0100 Subject: [PATCH 07/16] Clean up variable $CRC_FOLDER_IMPORT_PATH, remove $CRC_IMPORT_FOLDER_PATH and $BRAND_PATH $CRC_FOLDER_IMPORT_PATH should only be used *outside* our containers to configure the path to the CRC 1280 bulk ingest mountpoint/directory in docker-compose.yml. Inside the containers, the hardcoded path "/rub-test-data" is used (at least for now, this path should probably be changed to a less confusing name in the future, albeit this will require changes to the CRC 1280 bulk ingest documentation, and should therefore be done on all ReSeeD instances simultanously. Ideally, this path should not be configurable at all inside the CRC 1280 bulk importer configuration web interface, and should instead be removed from the web interface and hardcoded to a fixed path. Also, remove the hyrax/Dockerfile argument $CRC_IMPORT_FOLDER_PATH and replace it with the hardcoded path "/rub-test-data" as well. Finally, remove the $BRAND_PATH environment variable and the "branding" volume as both have never been used and only needlessly clutter the code base. Fixes: 15c541d94de49e6508a3fecc3a849cab2c34cc7d "Fixing mount of import folder path" Signed-off-by: Pascal Ernster <pascal.ernster@rub.de> --- .env.template | 2 -- .env.template.development | 2 -- docker-compose.development.yml | 4 ---- hyrax/Dockerfile | 3 +-- 4 files changed, 1 insertion(+), 10 deletions(-) diff --git a/.env.template b/.env.template index 6ffa1d62..33813ff5 100644 --- a/.env.template +++ b/.env.template @@ -63,7 +63,6 @@ HOST_APP_DATA_PATH=/data/ # FITS_PATH and FITS_VERSION are used by Docker build and are therefore fixed in the resulting Docker image FITS_PATH=/fits/fits-1.5.5/fits.sh FITS_VERSION=1.5.5 -BRAND_PATH=/data/public/branding DEFAULT_DATE_FORMAT=%Y-%m-%d NOTIFICATIONS_EMAIL_DEFAULT_FROM_ADDRESS=no-reply@mailboxer.com USER_MANAGEMENT_EMAIL_FROM_ADDRESS=repo-admin@example.org @@ -76,7 +75,6 @@ IIIF_SEARCH_ENDPOINT= CONFIG_IIIF_IMAGE_ENDPOINT= # If the rails server is configured to serve requests in https, set this to true IIIF_TO_SERVE_SSL_URLS=false -# CRC_FOLDER_IMPORT_PATH is used by Docker build and is therefore fixed in the resulting Docker image CRC_FOLDER_IMPORT_PATH=/mnt/rdms_data/RUB/testData # Browse Everything credentials diff --git a/.env.template.development b/.env.template.development index bf86b185..301a037e 100644 --- a/.env.template.development +++ b/.env.template.development @@ -61,7 +61,6 @@ HOST_APP_DATA_PATH=./data/ # FITS_PATH and FITS_VERSION are used by Docker build and are therefore fixed in the resulting Docker image FITS_PATH=/fits/fits-1.5.5/fits.sh FITS_VERSION=1.5.5 -BRAND_PATH=/data/public/branding DEFAULT_DATE_FORMAT=%Y-%m-%d NOTIFICATIONS_EMAIL_DEFAULT_FROM_ADDRESS=no-reply@mailboxer.com USER_MANAGEMENT_EMAIL_FROM_ADDRESS=repo-admin@example.org @@ -74,7 +73,6 @@ IIIF_SEARCH_ENDPOINT= CONFIG_IIIF_IMAGE_ENDPOINT= # If the rails server is configured to serve requests in https, set this to true IIIF_TO_SERVE_SSL_URLS=false -# CRC_FOLDER_IMPORT_PATH is used by Docker build and is therefore fixed in the resulting Docker image CRC_FOLDER_IMPORT_PATH=/mnt/rdms_data/RUB/testData # Browse Everything credentials diff --git a/docker-compose.development.yml b/docker-compose.development.yml index 1f0211ad..fd8d0187 100644 --- a/docker-compose.development.yml +++ b/docker-compose.development.yml @@ -10,7 +10,6 @@ volumes: file_uploads: derivatives: cache: - branding: downloads: networks: @@ -100,7 +99,6 @@ services: RAILS_ENV: ${RAILS_ENV} FITS_PATH: ${FITS_PATH} FITS_VERSION: ${FITS_VERSION} - CRC_FOLDER_IMPORT_PATH: ${CRC_FOLDER_IMPORT_PATH} env_file: - .env networks: @@ -128,7 +126,6 @@ services: - file_uploads:/shared/uploads - derivatives:/shared/derivatives - cache:/shared/cache - - branding:${BRAND_PATH} - ${HOST_APP_DATA_PATH:-/data/}:/data/data - ./hyrax:/data:cached - ${CRC_FOLDER_IMPORT_PATH}:/rub-test-data @@ -146,7 +143,6 @@ services: - file_uploads:/shared/uploads - derivatives:/shared/derivatives - cache:/shared/cache - - branding:${BRAND_PATH} - ${HOST_APP_DATA_PATH:-/data/}:/data/data - ./hyrax:/data:cached - ${CRC_FOLDER_IMPORT_PATH}:/rub-test-data diff --git a/hyrax/Dockerfile b/hyrax/Dockerfile index 68886f51..1d7bc7cb 100644 --- a/hyrax/Dockerfile +++ b/hyrax/Dockerfile @@ -5,7 +5,6 @@ ARG RAILS_ENV ARG FITS_VERSION ENV APP_PRODUCTION=/data/ -ENV CRC_IMPORT_FOLDER_PATH=/rub-test-data # Add backports to apt-get sources # Install libraries, dependencies, java and fits @@ -43,7 +42,7 @@ RUN cd $APP_PRODUCTION \ RUN mkdir -p "/shared/derivatives" RUN mkdir -p "/shared/uploads" RUN mkdir -p "/shared/cache" -RUN mkdir -p $CRC_IMPORT_FOLDER_PATH +RUN mkdir -p "/rub-test-data" RUN mkdir -p "/shared/downloads" # copy the application -- GitLab From b3aea9a7604ccf09b760291a98c5d0e6387ed842 Mon Sep 17 00:00:00 2001 From: Pascal Ernster <pascal.ernster@rub.de> Date: Wed, 29 Jan 2025 20:07:43 +0100 Subject: [PATCH 08/16] Remove useless variables $FITS_PATH and $FITS_VERSION These variables can only be used at build time and must not be changed at run time, therefore they do not belong into the generic .env file. Instead, hardcode their values as they are only set/used at a single place in the whole code base anyway. Signed-off-by: Pascal Ernster <pascal.ernster@rub.de> --- .env.template | 3 --- .env.template.development | 3 --- .gitlab-ci.yml | 7 +++---- docker-compose.development.yml | 2 -- docker-compose.yml | 1 - hyrax/Dockerfile | 7 +++---- hyrax/config/initializers/hyrax.rb | 2 +- 7 files changed, 7 insertions(+), 18 deletions(-) diff --git a/.env.template b/.env.template index 33813ff5..bcc4d148 100644 --- a/.env.template +++ b/.env.template @@ -60,9 +60,6 @@ MATOMO_AUTH_TOKEN= # Hyrax application setting HOST_APP_DATA_PATH=/data/ -# FITS_PATH and FITS_VERSION are used by Docker build and are therefore fixed in the resulting Docker image -FITS_PATH=/fits/fits-1.5.5/fits.sh -FITS_VERSION=1.5.5 DEFAULT_DATE_FORMAT=%Y-%m-%d NOTIFICATIONS_EMAIL_DEFAULT_FROM_ADDRESS=no-reply@mailboxer.com USER_MANAGEMENT_EMAIL_FROM_ADDRESS=repo-admin@example.org diff --git a/.env.template.development b/.env.template.development index 301a037e..601c2768 100644 --- a/.env.template.development +++ b/.env.template.development @@ -58,9 +58,6 @@ MATOMO_AUTH_TOKEN= # Hyrax application setting HOST_APP_DATA_PATH=./data/ -# FITS_PATH and FITS_VERSION are used by Docker build and are therefore fixed in the resulting Docker image -FITS_PATH=/fits/fits-1.5.5/fits.sh -FITS_VERSION=1.5.5 DEFAULT_DATE_FORMAT=%Y-%m-%d NOTIFICATIONS_EMAIL_DEFAULT_FROM_ADDRESS=no-reply@mailboxer.com USER_MANAGEMENT_EMAIL_FROM_ADDRESS=repo-admin@example.org diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8fc55fa0..fd05c872 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -25,9 +25,9 @@ build-job: - apk update && apk upgrade && apk add nodejs curl libarchive-tools build-base libpq-dev sqlite-dev libxml2-dev tzdata - cd hyrax && bundle install --jobs 4 --retry 3 script: - - mkdir -p /fits/fits-1.5.5 - - curl --fail --location "https://github.com/harvard-lts/fits/releases/download/1.5.5/fits-1.5.5.zip" | bsdtar --extract --directory /fits/fits-1.5.5 - - chmod +x "/fits/fits-1.5.5/fits.sh" "/fits/fits-1.5.5/fits-env.sh" "/fits/fits-1.5.5/fits-ngserver.sh" + - mkdir -p /fits + - curl --fail --location "https://github.com/harvard-lts/fits/releases/download/1.5.5/fits-1.5.5.zip" | bsdtar --extract --directory /fits + - chmod +x "/fits/fits.sh" "/fits/fits-env.sh" "/fits/fits-ngserver.sh" - bundle exec rake db:test:prepare test-job: @@ -38,7 +38,6 @@ test-job: - cd hyrax && bundle install --jobs 4 --retry 3 - bundle exec rake db:setup script: - - export FITS_PATH=/fits/fits-1.5.5/fits.sh - if test -f "lib/tasks/test_servers.rake"; then echo 'Running full tests' && bundle exec rake test:servers:start && bundle exec rake rdms:setup_hyrax && bundle exec rspec && bundle exec rake test:servers:stop; else echo "Running limited tests"; bundle exec rspec --exclude-pattern "**/features/*_spec.rb"; fi deploy-job: diff --git a/docker-compose.development.yml b/docker-compose.development.yml index fd8d0187..95540790 100644 --- a/docker-compose.development.yml +++ b/docker-compose.development.yml @@ -97,8 +97,6 @@ services: context: hyrax args: RAILS_ENV: ${RAILS_ENV} - FITS_PATH: ${FITS_PATH} - FITS_VERSION: ${FITS_VERSION} env_file: - .env networks: diff --git a/docker-compose.yml b/docker-compose.yml index a3ef495b..0714b2bb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -89,7 +89,6 @@ services: context: hyrax args: RAILS_ENV: ${RAILS_ENV} - FITS_VERSION: ${FITS_VERSION} env_file: - .env volumes: diff --git a/hyrax/Dockerfile b/hyrax/Dockerfile index 1d7bc7cb..b1fbc1f1 100644 --- a/hyrax/Dockerfile +++ b/hyrax/Dockerfile @@ -2,7 +2,6 @@ FROM ruby:2.7.8-alpine # Setup build variables ARG RAILS_ENV -ARG FITS_VERSION ENV APP_PRODUCTION=/data/ @@ -21,9 +20,9 @@ RUN apk update && \ COPY policy.xml /etc/ImageMagick-7/policy.xml -RUN mkdir -p /fits/fits-$FITS_VERSION \ - && curl --fail --location "https://github.com/harvard-lts/fits/releases/download/$FITS_VERSION/fits-$FITS_VERSION.zip" | bsdtar --extract --directory /fits/fits-$FITS_VERSION \ - && chmod +x "/fits/fits-$FITS_VERSION/fits.sh" "/fits/fits-$FITS_VERSION/fits-env.sh" "/fits/fits-$FITS_VERSION/fits-ngserver.sh" +RUN mkdir -p /fits \ + && curl --fail --location "https://github.com/harvard-lts/fits/releases/download/1.5.5/fits-1.5.5.zip" | bsdtar --extract --directory /fits \ + && chmod +x "/fits/fits.sh" "/fits/fits-env.sh" "/fits/fits-ngserver.sh" # copy gemfiles to production folder COPY Gemfile Gemfile.lock $APP_PRODUCTION diff --git a/hyrax/config/initializers/hyrax.rb b/hyrax/config/initializers/hyrax.rb index 6bb9dbb3..4936f467 100644 --- a/hyrax/config/initializers/hyrax.rb +++ b/hyrax/config/initializers/hyrax.rb @@ -79,7 +79,7 @@ Hyrax.config do |config| # config.redis_namespace = "hyrax" # Path to the file characterization tool - config.fits_path = ENV['FITS_PATH'] || "/fits/fits-1.5.5/fits.sh" + config.fits_path = "/fits/fits.sh" # Path to the file derivatives creation tool # config.libreoffice_path = "soffice" -- GitLab From 01e9892a60cda349b54dfbfec16d344168f6e48c Mon Sep 17 00:00:00 2001 From: Pascal Ernster <pascal.ernster@rub.de> Date: Wed, 29 Jan 2025 20:30:39 +0100 Subject: [PATCH 09/16] Remove deprecated "version" property from docker-compose.yml This has been deprecated a while ago, so remove it: https://github.com/compose-spec/compose-spec/commit/6d19079ef64dd94d203076e9de6caeb20c498efe Signed-off-by: Pascal Ernster <pascal.ernster@rub.de> --- docker-compose.development.yml | 2 -- docker-compose.yml | 2 -- 2 files changed, 4 deletions(-) diff --git a/docker-compose.development.yml b/docker-compose.development.yml index 95540790..465e6f66 100644 --- a/docker-compose.development.yml +++ b/docker-compose.development.yml @@ -1,5 +1,3 @@ -version: '3' - volumes: db: db-fcrepo: diff --git a/docker-compose.yml b/docker-compose.yml index 0714b2bb..821d26bf 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,3 @@ -version: '3' - networks: internal: -- GitLab From 080633238f4412f3cc2414e20515d67cc4f04994 Mon Sep 17 00:00:00 2001 From: Pascal Ernster <pascal.ernster@rub.de> Date: Wed, 29 Jan 2025 20:50:26 +0100 Subject: [PATCH 10/16] docker-compose.yml: Restrict port forwardings to 127.0.0.1 Remove non-essential port forwardings from docker-compose.yml and restrict all port forwardings to 127.0.0.1. Without this change, all ports are forwarded on *all* network interfaces of the host system, resulting in Postgres, SOLR and Tomcat/ fcrepo being exposed to the internet unless an additional firewall is used. Signed-off-by: Pascal Ernster <pascal.ernster@rub.de> --- docker-compose.development.yml | 8 ++++---- docker-compose.yml | 8 +------- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/docker-compose.development.yml b/docker-compose.development.yml index 465e6f66..93670821 100644 --- a/docker-compose.development.yml +++ b/docker-compose.development.yml @@ -25,7 +25,7 @@ services: expose: - 8080 ports: - - 8080:8080 + - "127.0.0.1:8080:8080/tcp" volumes: - fcrepo:/data @@ -44,7 +44,7 @@ services: expose: - 8983 ports: - - 8983:8983 + - "127.0.0.1:8983:8983/tcp" volumes: - solr:/var/solr - ./hyrax/solr/conf:/opt/solr/solr_conf @@ -117,7 +117,7 @@ services: expose: - 3000 ports: - - 3000:3000 + - "127.0.0.1:3000:3000/tcp" volumes: - file_uploads:/shared/uploads - derivatives:/shared/derivatives @@ -152,7 +152,7 @@ services: expose: - 6379 ports: - - 6379:6379 + - "127.0.0.1:6379:6379/tcp" healthcheck: test: ["CMD-SHELL", "redis-cli ping"] interval: 30s diff --git a/docker-compose.yml b/docker-compose.yml index 821d26bf..b50fdc88 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,8 +13,6 @@ services: internal: environment: - CATALINA_OPTS=-Djava.awt.headless=true -Dfile.encoding=UTF-8 -server -Xms512m -Xmx1024m -XX:NewSize=256m -XX:MaxNewSize=256m -XX:PermSize=256m -XX:MaxPermSize=256m -XX:+DisableExplicitGC -Dfcrepo.home=/data -Dfcrepo.object.directory=/data/objects -Dfcrepo.binary.directory=/data/binaries -Dfcrepo.postgresql.username=$POSTGRES_USER -Dfcrepo.postgresql.password=$POSTGRES_PASSWORD -Dfcrepo.postgresql.host=$POSTGRES_HOST_FCREPO -Dfcrepo.postgresql.port=$POSTGRES_PORT -Dfcrepo.modeshape.configuration=classpath:/config/jdbc-postgresql/repository.json - ports: - - 8080:8080 volumes: - ${DOCKER_VOLUMES_PATH_PREFIX}fcrepo:/data @@ -23,8 +21,6 @@ services: restart: unless-stopped expose: - 8983 - ports: - - 8983:8983 env_file: - .env environment: @@ -108,7 +104,7 @@ services: - fcrepo - redis ports: - - 3000:3000 + - "127.0.0.1:3000:3000/tcp" networks: internal: aliases: @@ -139,8 +135,6 @@ services: retries: 3 expose: - 6379 - ports: - - 6379:6379 volumes: - ${DOCKER_VOLUMES_PATH_PREFIX}redis:/data -- GitLab From 6f39a120d02a2d7ca6749512999337382bf88075 Mon Sep 17 00:00:00 2001 From: Pascal Ernster <pascal.ernster@rub.de> Date: Wed, 29 Jan 2025 21:07:37 +0100 Subject: [PATCH 11/16] Add $HOST_PORT to make the host-side Puma forwarding port configurable This is required to allow running multiple instances of ReSeeD simultanously on the same host, by allowing to connect to each instance through a different forwarding port from the host system. Note that this *only* changes the port on the host system, whilst Puma still uses the hardcoded port 3000 inside the web container. For further details, see https://docs.docker.com/reference/compose-file/services/#long-syntax-3 Signed-off-by: Pascal Ernster <pascal.ernster@rub.de> --- .env.template | 1 + .env.template.development | 2 ++ docker-compose.development.yml | 2 +- docker-compose.yml | 2 +- 4 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.env.template b/.env.template index bcc4d148..b8078fdf 100644 --- a/.env.template +++ b/.env.template @@ -4,6 +4,7 @@ IN_DOCKER=true # Note: This path needs to start with either "./" or "/" and needs to end with a "slash". # This is to allow for setting this to an empty string or not defining the variable at all if Docker's standard volume directory should be used. DOCKER_VOLUMES_PATH_PREFIX="./volumes/" +HOST_PORT=3000 # Solr setting SOLR_URL_SCHEME=http diff --git a/.env.template.development b/.env.template.development index 601c2768..004b16f9 100644 --- a/.env.template.development +++ b/.env.template.development @@ -1,5 +1,7 @@ IN_DOCKER=true +HOST_PORT=3000 + # Solr setting SOLR_URL_SCHEME=http SOLR_HOST=solr diff --git a/docker-compose.development.yml b/docker-compose.development.yml index 93670821..a1a891e1 100644 --- a/docker-compose.development.yml +++ b/docker-compose.development.yml @@ -117,7 +117,7 @@ services: expose: - 3000 ports: - - "127.0.0.1:3000:3000/tcp" + - "127.0.0.1:${HOST_PORT}:3000/tcp" volumes: - file_uploads:/shared/uploads - derivatives:/shared/derivatives diff --git a/docker-compose.yml b/docker-compose.yml index b50fdc88..3efa9478 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -104,7 +104,7 @@ services: - fcrepo - redis ports: - - "127.0.0.1:3000:3000/tcp" + - "127.0.0.1:${HOST_PORT}:3000/tcp" networks: internal: aliases: -- GitLab From de6a7bb198ed17479484ab4549ef9790bf50e474 Mon Sep 17 00:00:00 2001 From: Pascal Ernster <pascal.ernster@rub.de> Date: Wed, 29 Jan 2025 21:34:45 +0100 Subject: [PATCH 12/16] Add $COMPOSE_PROJECT_NAME in .env.template This allows configuring the name prefix for all containers of a given ReSeeD instance without having to rename the directory that contains the docker-compose.yml file. This can be used to avoiding container name collisions when running multiple instances of ReSeeD on the same host. For further details, see https://docs.docker.com/compose/how-tos/environment-variables/envvars/#compose_project_name Signed-off-by: Pascal Ernster <pascal.ernster@rub.de> --- .env.template | 1 + .env.template.development | 1 + 2 files changed, 2 insertions(+) diff --git a/.env.template b/.env.template index b8078fdf..150621f6 100644 --- a/.env.template +++ b/.env.template @@ -1,5 +1,6 @@ IN_DOCKER=true +COMPOSE_PROJECT_NAME="reseed-prod" # Common path prefix for all subvolume bind mounts to facilitate backup create and restoration # Note: This path needs to start with either "./" or "/" and needs to end with a "slash". # This is to allow for setting this to an empty string or not defining the variable at all if Docker's standard volume directory should be used. diff --git a/.env.template.development b/.env.template.development index 004b16f9..d0c05af7 100644 --- a/.env.template.development +++ b/.env.template.development @@ -1,5 +1,6 @@ IN_DOCKER=true +COMPOSE_PROJECT_NAME="reseed-dev" HOST_PORT=3000 # Solr setting -- GitLab From c9e0fbd5f15668c04150bfdc0ba3db7ae6f28781 Mon Sep 17 00:00:00 2001 From: Pascal Ernster <pascal.ernster@rub.de> Date: Wed, 29 Jan 2025 21:25:25 +0100 Subject: [PATCH 13/16] Add docker-compose.build.yml, remove "build" section from docker-compose.yml We use different docker-compose.yml files to build the container images and to run ReSeeD using those container images. Also add $CONTAINER_IMAGE_TAG, which is required to allow running multiple ReSeeD instances with different container image versions on the same host. Signed-off-by: Pascal Ernster <pascal.ernster@rub.de> --- .env.template | 1 + docker-compose.build.yml | 138 +++++++++++++++++++++++++++++++++++++++ docker-compose.yml | 7 +- 3 files changed, 142 insertions(+), 4 deletions(-) create mode 100644 docker-compose.build.yml diff --git a/.env.template b/.env.template index 150621f6..1f3ba453 100644 --- a/.env.template +++ b/.env.template @@ -1,6 +1,7 @@ IN_DOCKER=true COMPOSE_PROJECT_NAME="reseed-prod" +CONTAINER_IMAGE_TAG="Change_this_on_each_container_image_update" # Common path prefix for all subvolume bind mounts to facilitate backup create and restoration # Note: This path needs to start with either "./" or "/" and needs to end with a "slash". # This is to allow for setting this to an empty string or not defining the variable at all if Docker's standard volume directory should be used. diff --git a/docker-compose.build.yml b/docker-compose.build.yml new file mode 100644 index 00000000..20c6ee2b --- /dev/null +++ b/docker-compose.build.yml @@ -0,0 +1,138 @@ +networks: + internal: + +services: + fcrepo: + image: ualbertalib/docker-fcrepo4:4.7 + restart: unless-stopped + expose: + - 8080 + depends_on: + - fcrepodb + networks: + internal: + environment: + - CATALINA_OPTS=-Djava.awt.headless=true -Dfile.encoding=UTF-8 -server -Xms512m -Xmx1024m -XX:NewSize=256m -XX:MaxNewSize=256m -XX:PermSize=256m -XX:MaxPermSize=256m -XX:+DisableExplicitGC -Dfcrepo.home=/data -Dfcrepo.object.directory=/data/objects -Dfcrepo.binary.directory=/data/binaries -Dfcrepo.postgresql.username=$POSTGRES_USER -Dfcrepo.postgresql.password=$POSTGRES_PASSWORD -Dfcrepo.postgresql.host=$POSTGRES_HOST_FCREPO -Dfcrepo.postgresql.port=$POSTGRES_PORT -Dfcrepo.modeshape.configuration=classpath:/config/jdbc-postgresql/repository.json + volumes: + - ${DOCKER_VOLUMES_PATH_PREFIX}fcrepo:/data + + solr: + image: solr:8 + restart: unless-stopped + expose: + - 8983 + env_file: + - .env + environment: + - OOM=script + healthcheck: + test: ["CMD-SHELL", "wget -O /dev/null http://localhost:8983/solr/"] + interval: 30s + timeout: 5s + retries: 3 + networks: + internal: + volumes: + - ${DOCKER_VOLUMES_PATH_PREFIX}solr:/var/solr + - ./hyrax/solr/conf:/opt/solr/solr_conf + command: + - sh + - "-c" + - "precreate-core ${SOLR_TEST_CORE} /opt/solr/solr_conf; solr-precreate ${SOLR_CORE} /opt/solr/solr_conf" + + fcrepodb: + image: postgres:13-alpine + restart: unless-stopped + env_file: + - .env + environment: + - POSTGRES_DB=${POSTGRES_DB_FCREPO} + volumes: + - ${DOCKER_VOLUMES_PATH_PREFIX}db-fcrepo:/var/lib/postgresql/data + networks: + internal: + expose: + - 5432 + healthcheck: + test: ["CMD-SHELL", "POSTGRES_PASSWORD=${POSTGRES_PASSWORD} pg_isready -U ${POSTGRES_USER} -h localhost -d ${POSTGRES_DB_FCREPO}"] + interval: 30s + timeout: 5s + retries: 3 + + appdb: + image: postgres:13-alpine + restart: unless-stopped + env_file: + - .env + environment: + - POSTGRES_DB=${POSTGRES_DB_APP} + volumes: + - ${DOCKER_VOLUMES_PATH_PREFIX}db-app:/var/lib/postgresql/data + networks: + internal: + expose: + - 5432 + healthcheck: + test: ["CMD-SHELL", "POSTGRES_PASSWORD=${POSTGRES_PASSWORD} pg_isready -U ${POSTGRES_USER} -h localhost -d ${POSTGRES_DB_APP}"] + interval: 30s + timeout: 5s + retries: 3 + + app: &app + build: + context: hyrax + args: + RAILS_ENV: ${RAILS_ENV} + env_file: + - .env + volumes: + - ${DOCKER_VOLUMES_PATH_PREFIX}file_uploads:/shared/uploads + - ${DOCKER_VOLUMES_PATH_PREFIX}derivatives:/shared/derivatives + - ${DOCKER_VOLUMES_PATH_PREFIX}cache:/shared/cache + - ${CRC_FOLDER_IMPORT_PATH}:/rub-test-data + - ${DOCKER_VOLUMES_PATH_PREFIX}downloads:/shared/downloads + networks: + internal: + + web: + <<: *app + command: bash -c "/bin/docker-entrypoint.sh" + restart: unless-stopped + depends_on: + - appdb + - solr + - fcrepo + - redis + networks: + internal: + aliases: + - web + expose: + - 3000 + + workers: + <<: *app + command: bash -c "bundle install && bundle exec sidekiq" + restart: unless-stopped + depends_on: + - appdb + - solr + - fcrepo + - redis + + redis: + image: redis:6 + command: redis-server --appendonly yes + restart: unless-stopped + networks: + internal: + healthcheck: + test: ["CMD-SHELL", "redis-cli ping"] + interval: 30s + timeout: 5s + retries: 3 + expose: + - 6379 + volumes: + - ${DOCKER_VOLUMES_PATH_PREFIX}redis:/data + diff --git a/docker-compose.yml b/docker-compose.yml index 3efa9478..591d1d5d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -79,10 +79,7 @@ services: retries: 3 app: &app - build: - context: hyrax - args: - RAILS_ENV: ${RAILS_ENV} + image: reseed-app:${CONTAINER_IMAGE_TAG} env_file: - .env volumes: @@ -96,6 +93,7 @@ services: web: <<: *app + image: reseed-web:${CONTAINER_IMAGE_TAG} command: bash -c "/bin/docker-entrypoint.sh" restart: unless-stopped depends_on: @@ -114,6 +112,7 @@ services: workers: <<: *app + image: reseed-workers:${CONTAINER_IMAGE_TAG} command: bash -c "bundle install && bundle exec sidekiq" restart: unless-stopped depends_on: -- GitLab From edd2a0c92ac5e1b31d12d26e946cf8452690829b Mon Sep 17 00:00:00 2001 From: Pascal Ernster <pascal.ernster@rub.de> Date: Wed, 29 Jan 2025 21:47:54 +0100 Subject: [PATCH 14/16] Add $COMPOSE_NETWORK_BRIDGE_NAME and $COMPOSE_NETWORK_BRIDGE_IPV4_SUBNET This allows for configuring the name of the network bridge device to which the containers of a given ReSeeD instance connect, and it allows configuring the IPv4 subnet from which those containers get assigned IP addresses. Signed-off-by: Pascal Ernster <pascal.ernster@rub.de> --- .env.template | 3 +++ .env.template.development | 3 +++ docker-compose.development.yml | 6 ++++++ docker-compose.yml | 6 ++++++ 4 files changed, 18 insertions(+) diff --git a/.env.template b/.env.template index 1f3ba453..bf5062b6 100644 --- a/.env.template +++ b/.env.template @@ -7,6 +7,9 @@ CONTAINER_IMAGE_TAG="Change_this_on_each_container_image_update" # This is to allow for setting this to an empty string or not defining the variable at all if Docker's standard volume directory should be used. DOCKER_VOLUMES_PATH_PREFIX="./volumes/" HOST_PORT=3000 +# Note: ${COMPOSE_NETWORK_BRIDGE_NAME} cannot be longer than 15 characters +COMPOSE_NETWORK_BRIDGE_NAME="${COMPOSE_PROJECT_NAME}" +COMPOSE_NETWORK_BRIDGE_IPV4_SUBNET="172.17.42.0/24" # Solr setting SOLR_URL_SCHEME=http diff --git a/.env.template.development b/.env.template.development index d0c05af7..d8ce195f 100644 --- a/.env.template.development +++ b/.env.template.development @@ -2,6 +2,9 @@ IN_DOCKER=true COMPOSE_PROJECT_NAME="reseed-dev" HOST_PORT=3000 +# Note: ${COMPOSE_NETWORK_BRIDGE_NAME} cannot be longer than 15 characters +COMPOSE_NETWORK_BRIDGE_NAME="${COMPOSE_PROJECT_NAME}" +COMPOSE_NETWORK_BRIDGE_IPV4_SUBNET="172.17.23.0/24" # Solr setting SOLR_URL_SCHEME=http diff --git a/docker-compose.development.yml b/docker-compose.development.yml index a1a891e1..63123cd9 100644 --- a/docker-compose.development.yml +++ b/docker-compose.development.yml @@ -12,6 +12,12 @@ volumes: networks: internal: + driver_opts: + com.docker.network.bridge.name: ${COMPOSE_NETWORK_BRIDGE_NAME} + ipam: + driver: default + config: + - subnet: "${COMPOSE_NETWORK_BRIDGE_IPV4_SUBNET}" services: fcrepo: diff --git a/docker-compose.yml b/docker-compose.yml index 591d1d5d..7ed57a29 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,11 @@ networks: internal: + driver_opts: + com.docker.network.bridge.name: ${COMPOSE_NETWORK_BRIDGE_NAME} + ipam: + driver: default + config: + - subnet: "${COMPOSE_NETWORK_BRIDGE_IPV4_SUBNET}" services: fcrepo: -- GitLab From 36393c3bed5b71a564d8f2afe9eb6d1f0448fc2b Mon Sep 17 00:00:00 2001 From: Pascal Ernster <pascal.ernster@rub.de> Date: Wed, 29 Jan 2025 21:53:44 +0100 Subject: [PATCH 15/16] Add IPv6 support, introduce $COMPOSE_NETWORK_BRIDGE_IPV6_SUBNET Signed-off-by: Pascal Ernster <pascal.ernster@rub.de> --- .env.template | 1 + .env.template.development | 1 + docker-compose.development.yml | 6 ++++++ docker-compose.yml | 3 +++ hyrax/docker-entrypoint.sh | 2 +- 5 files changed, 12 insertions(+), 1 deletion(-) diff --git a/.env.template b/.env.template index bf5062b6..e6645710 100644 --- a/.env.template +++ b/.env.template @@ -10,6 +10,7 @@ HOST_PORT=3000 # Note: ${COMPOSE_NETWORK_BRIDGE_NAME} cannot be longer than 15 characters COMPOSE_NETWORK_BRIDGE_NAME="${COMPOSE_PROJECT_NAME}" COMPOSE_NETWORK_BRIDGE_IPV4_SUBNET="172.17.42.0/24" +COMPOSE_NETWORK_BRIDGE_IPV6_SUBNET="fd00:0:0:42::/64" # Solr setting SOLR_URL_SCHEME=http diff --git a/.env.template.development b/.env.template.development index d8ce195f..d2c7ebd8 100644 --- a/.env.template.development +++ b/.env.template.development @@ -5,6 +5,7 @@ HOST_PORT=3000 # Note: ${COMPOSE_NETWORK_BRIDGE_NAME} cannot be longer than 15 characters COMPOSE_NETWORK_BRIDGE_NAME="${COMPOSE_PROJECT_NAME}" COMPOSE_NETWORK_BRIDGE_IPV4_SUBNET="172.17.23.0/24" +COMPOSE_NETWORK_BRIDGE_IPV6_SUBNET="fd00:0:0:23::/64" # Solr setting SOLR_URL_SCHEME=http diff --git a/docker-compose.development.yml b/docker-compose.development.yml index 63123cd9..4c214e15 100644 --- a/docker-compose.development.yml +++ b/docker-compose.development.yml @@ -14,10 +14,12 @@ networks: internal: driver_opts: com.docker.network.bridge.name: ${COMPOSE_NETWORK_BRIDGE_NAME} + enable_ipv6: True ipam: driver: default config: - subnet: "${COMPOSE_NETWORK_BRIDGE_IPV4_SUBNET}" + - subnet: "${COMPOSE_NETWORK_BRIDGE_IPV6_SUBNET}" services: fcrepo: @@ -32,6 +34,7 @@ services: - 8080 ports: - "127.0.0.1:8080:8080/tcp" + - "[::1]:8080:8080/tcp" volumes: - fcrepo:/data @@ -51,6 +54,7 @@ services: - 8983 ports: - "127.0.0.1:8983:8983/tcp" + - "[::1]:8983:8983/tcp" volumes: - solr:/var/solr - ./hyrax/solr/conf:/opt/solr/solr_conf @@ -124,6 +128,7 @@ services: - 3000 ports: - "127.0.0.1:${HOST_PORT}:3000/tcp" + - "[::1]:${HOST_PORT}:3000/tcp" volumes: - file_uploads:/shared/uploads - derivatives:/shared/derivatives @@ -159,6 +164,7 @@ services: - 6379 ports: - "127.0.0.1:6379:6379/tcp" + - "[::1]:6379:6379/tcp" healthcheck: test: ["CMD-SHELL", "redis-cli ping"] interval: 30s diff --git a/docker-compose.yml b/docker-compose.yml index 7ed57a29..8aaf6580 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,10 +2,12 @@ networks: internal: driver_opts: com.docker.network.bridge.name: ${COMPOSE_NETWORK_BRIDGE_NAME} + enable_ipv6: True ipam: driver: default config: - subnet: "${COMPOSE_NETWORK_BRIDGE_IPV4_SUBNET}" + - subnet: "${COMPOSE_NETWORK_BRIDGE_IPV6_SUBNET}" services: fcrepo: @@ -109,6 +111,7 @@ services: - redis ports: - "127.0.0.1:${HOST_PORT}:3000/tcp" + - "[::1]:${HOST_PORT}:3000/tcp" networks: internal: aliases: diff --git a/hyrax/docker-entrypoint.sh b/hyrax/docker-entrypoint.sh index fcb33560..585cb2c5 100644 --- a/hyrax/docker-entrypoint.sh +++ b/hyrax/docker-entrypoint.sh @@ -68,4 +68,4 @@ bundle exec rake rdms:setup_hyrax["seed/setup.json"] # echo "--------- Starting Hyrax in ${RAILS_ENV} mode ---------" rm -f /tmp/hyrax.pid -bundle exec rails server -p 3000 -b '0.0.0.0' --pid /tmp/hyrax.pid +bundle exec rails server -p 3000 -b '[::]' --pid /tmp/hyrax.pid -- GitLab From 84e7c77d81c124c59bca2e25427f79576eae1ffb Mon Sep 17 00:00:00 2001 From: Pascal Ernster <pascal.ernster@rub.de> Date: Sat, 15 Feb 2025 19:58:50 +0100 Subject: [PATCH 16/16] docker-compose.yml: Use redis:6-alpine instead of redis:6 image Both images are functionally equivalent, but the alpine version is less than a third of the size of the non-alpine version, so it saves both disk space and RAM. Signed-off-by: Pascal Ernster <pascal.ernster@rub.de> --- docker-compose.build.yml | 2 +- docker-compose.development.yml | 2 +- docker-compose.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docker-compose.build.yml b/docker-compose.build.yml index 20c6ee2b..8184a481 100644 --- a/docker-compose.build.yml +++ b/docker-compose.build.yml @@ -121,7 +121,7 @@ services: - redis redis: - image: redis:6 + image: redis:6-alpine command: redis-server --appendonly yes restart: unless-stopped networks: diff --git a/docker-compose.development.yml b/docker-compose.development.yml index 4c214e15..ad1e02d9 100644 --- a/docker-compose.development.yml +++ b/docker-compose.development.yml @@ -156,7 +156,7 @@ services: - downloads:/shared/downloads redis: - image: redis:6 + image: redis:6-alpine command: redis-server --appendonly yes networks: internal: diff --git a/docker-compose.yml b/docker-compose.yml index 8aaf6580..4ecf99c5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -131,7 +131,7 @@ services: - redis redis: - image: redis:6 + image: redis:6-alpine command: redis-server --appendonly yes restart: unless-stopped networks: -- GitLab