diff --git a/.github/workflows/build-docker-images-and-push-to-ecr.yml b/.github/workflows/build-docker-images-and-push-to-ecr.yml new file mode 100644 index 0000000000000..e3f52d1deb9f3 --- /dev/null +++ b/.github/workflows/build-docker-images-and-push-to-ecr.yml @@ -0,0 +1,164 @@ +name: Build Docker Images and Push to AWS ECR + +on: + push: + branches: [testing, master] + # TODO remove pull request after done testing + pull_request: + branches: [testing] + +jobs: + setup-env: + runs-on: ubuntu-latest + outputs: + aws-s3-bloom-config-bucket: ${{ steps.branch_based.outputs.aws-s3-bloom-config-bucket }} + aws-ecs-task-def: ${{ steps.branch_based.outputs.aws-ecs-task-def }} + aws-ecs-service: ${{ steps.branch_based.outputs.aws-ecs-service }} + steps: + - name: Set Variables Depending on Current Branch + id: branch_based + run: | + echo "Running on branch ${{ github.ref }}" + if [ "${{ github.ref }}" = "refs/heads/master" ]; then + echo "::set-output name=aws-s3-bloom-config-bucket::bloom-config-production" + echo "::set-output name=aws-ecs-task-def::community-production" + echo "::set-output name=aws-ecs-service::community-production" + # TODO uncomment and delete next line elif [ "${{ github.ref }}" = "refs/heads/testing" ]; then + else + echo "::set-output name=aws-s3-bloom-config-bucket::bloom-config-testing" + echo "::set-output name=aws-ecs-task-def::community-testing" + echo "::set-output name=aws-ecs-service::community-testing" + fi + + build-proxy: + runs-on: ubuntu-latest + outputs: + docker-image: ${{ steps.docker-image.outputs.image }} + steps: + # - name: Configure AWS credentials + # uses: aws-actions/configure-aws-credentials@v1 + # with: + # aws-access-key-id: ${{ secrets.AWS_COMMUNITY_CI_ACCESS_KEY_ID }} + # aws-secret-access-key: ${{ secrets.AWS_COMMUNITY_CI_SECRET_ACCESS_KEY }} + # aws-region: ${{ secrets.AWS_REGION }} + + # - name: Login to Amazon ECR + # id: login-ecr + # uses: aws-actions/amazon-ecr-login@v1 + + # - uses: actions/checkout@v3 + + - name: Set docker image name and tag + id: docker-image + env: + AWS_ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + ECR_REPOSITORY: discourse-proxy + IMAGE_TAG: ${{ github.sha }} + run: | + echo "::set-output name=image::$AWS_ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG" + echo "::set-output name=untagged-image::$AWS_ECR_REGISTRY/$ECR_REPOSITORY" + + # - name: Build, tag, and push image to Amazon ECR + # id: build-image + # run: | + # docker pull ${{ steps.docker-image.outputs.untagged-image }} + # docker build -t ${{ steps.docker-image.outputs.image }} --cache-from ${{ steps.docker-image.outputs.untagged-image }} -f ./docker/nginx.dockerfile ./docker + # docker push ${{ steps.docker-image.outputs.image }} + + build-api: + runs-on: ubuntu-latest + needs: setup-env + outputs: + docker-image: ${{ steps.build-image.outputs.image }} + steps: + # - name: Configure AWS credentials + # uses: aws-actions/configure-aws-credentials@v1 + # with: + # aws-access-key-id: ${{ secrets.AWS_COMMUNITY_CI_ACCESS_KEY_ID }} + # aws-secret-access-key: ${{ secrets.AWS_COMMUNITY_CI_SECRET_ACCESS_KEY }} + # aws-region: ${{ secrets.AWS_REGION }} + + # - name: Login to Amazon ECR + # id: login-ecr + # uses: aws-actions/amazon-ecr-login@v1 + + # - uses: actions/checkout@v3 + + # - name: Pull .env from S3 + # run: aws s3 cp "s3://${{ needs.setup-env.outputs.aws-s3-bloom-config-bucket }}/community/api.env" ./.env + + # - name: Set environment variables + # run: cat .env >> $GITHUB_ENV + + # - name: Build, tag, and push image to Amazon ECR + # id: build-image + # env: + # AWS_ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + # ECR_REPOSITORY: discourse-api + # IMAGE_TAG: ${{ github.sha }} + # run: | + # docker pull $AWS_ECR_REGISTRY/$ECR_REPOSITORY + # docker build -t $AWS_ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG --cache-from $AWS_ECR_REGISTRY/$ECR_REPOSITORY -f ./docker/dockerfile --build-arg DISCOURSE_MAXMIND_LICENSE_KEY=${DISCOURSE_MAXMIND_LICENSE_KEY} . + # docker push $AWS_ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG + - name: Output + id: build-image + env: + AWS_ECR_REGISTRY: 673099801448.dkr.ecr.eu-west-1.amazonaws.com + ECR_REPOSITORY: discourse-api + IMAGE_TAG: ${{ github.sha }} + run: | + echo "::set-output name=image::$AWS_ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG" + echo "$AWS_ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG" + + # - name: Compile assets and upload them to S3 + # env: + # API_DOCKER_IMAGE: ${{ steps.build-image.outputs.image }} + # run: | + # echo "${{ steps.build-image.outputs.image }}" + # docker network create discourse + # docker run --rm -d --name discourse_db -e POSTGRES_DB=discourse -e POSTGRES_USER=discourse -e POSTGRES_PASSWORD=password --network discourse postgres:13.4-alpine + # docker run --rm -d --name discourse_cache --network discourse redis:alpine + # docker run --rm -i --name assets_compiler --env-file .env -e DISCOURSE_DB_HOST=discourse_db -e DISCOURSE_DB_PASSWORD=password -e DISCOURSE_REDIS_HOST=discourse_cache --network discourse $API_DOCKER_IMAGE sh -c 'bundle exec rake db:prepare && bundle exec rake assets:precompile && bundle exec rake s3:upload_assets' + + deploy: + runs-on: ubuntu-latest + needs: [setup-env, build-proxy, build-api] + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_COMMUNITY_CI_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_COMMUNITY_CI_SECRET_ACCESS_KEY }} + aws-region: ${{ secrets.AWS_REGION }} + + - name: Download task definition + run: | + echo "${{ needs.setup-env.outputs.aws-ecs-task-def }}" + echo "${{ needs.build-proxy.outputs.docker-image }}" + echo "${{ needs.build-api.outputs.docker-image }}" + echo "${{ needs.setup-env.outputs.aws-ecs-service }}" + aws ecs describe-task-definition --task-definition ${{ needs.setup-env.outputs.aws-ecs-task-def }} --query taskDefinition > task-definition.json + + - name: Update Task Definition Proxy Image + id: proxy-task-def + uses: aws-actions/amazon-ecs-render-task-definition@v1 + with: + task-definition: task-definition.json + container-name: proxy + image: ${{ needs.build-proxy.outputs.docker-image }} + + - name: Update Task Definition API Image + id: api-task-def + uses: aws-actions/amazon-ecs-render-task-definition@v1 + with: + task-definition: ${{ steps.proxy-task-def.outputs.task-definition }} + container-name: api + image: ${{ needs.build-api.outputs.docker-image }} + + - name: Deploy Amazon ECS task definition + uses: aws-actions/amazon-ecs-deploy-task-definition@v1 + with: + task-definition: ${{ steps.api-task-def.outputs.task-definition }} + service: ${{ needs.setup-env.outputs.aws-ecs-service }} + cluster: bloom + wait-for-service-stability: true diff --git a/docker-compose.yml b/docker-compose.yml index 52a9b109846ff..7816345a1f4ed 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -47,8 +47,8 @@ services: DISCOURSE_SECRET_KEY_BASE: "${DISCOURSE_SECRET_KEY_BASE}" #entrypoint: ["/bin/sh", "sleep", "10000"] command: ["no-compile"] - - web: + + web: build: context: ./docker dockerfile: ./nginx.dockerfile @@ -58,32 +58,32 @@ services: - ./uploads:/var/www/discourse/public/uploads ports: - 80:80 - environment: + environment: DISCOURSE_HOST: "${DISCOURSE_HOST}" depends_on: - app command: ["/bin/sh" , "/etc/nginx/nginx.sh"] - + cache: image: redis ports: - 6379:6379 - + mail: image: namshi/smtp restart: always volumes: - ./log:/var/log/exim4 - environment: + environment: SES_USER: "${DISCOURSE_SMTP_USER_NAME}" SES_PASSWORD: "${DISCOURSE_SMTP_PASSWORD}" SES_REGION: "${AWS_REGION}" - logrotate: + logrotate: image: blacklabelops/logrotate volumes: - ./log:/var/lib/discourse - environment: + environment: LOGS_DIRECTORIES: "/var/lib/discourse" LOGROTATE_SIZE: "10M" LOGROTATE_INTERVAL: "hourly" diff --git a/docker/discourse_first_run b/docker/discourse_first_run deleted file mode 100644 index cb7b6606d86ca..0000000000000 --- a/docker/discourse_first_run +++ /dev/null @@ -1 +0,0 @@ -placeholder file \ No newline at end of file diff --git a/docker/discourse_install.sh b/docker/discourse_install.sh deleted file mode 100644 index 8392c1ba3e585..0000000000000 --- a/docker/discourse_install.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash - -set -x -set -e -bundle config set --local deployment 'true' -bundle config mirror.https://rubygems.org http://localhost:9292 -bundle config mirror.https://github.com http://localhost:9292 -chown -R discourse:discourse /var/www/discourse -cd /var/www/discourse -sudo -u discourse bundle install --deployment --jobs 4 --without test -sudo -u discourse yarn install --production -sudo -u discourse yarn cache clean -bundle exec rake maxminddb:get diff --git a/docker/discourse_run.sh b/docker/discourse_run.sh deleted file mode 100644 index df5e44f3bf1f0..0000000000000 --- a/docker/discourse_run.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash -set -x -set -e -env - -OPTION=$1 -bundle config set --local deployment 'true' - -if [[ $OPTION == "compile" || -f discourse_first_run ]]; then - rm -Rf discourse_first_run - bundle exec rake plugin:pull_compatible_all RAILS_ENV=production - find /var/www/discourse/vendor/bundle -name tmp -type d -exec rm -rf {} + - bundle exec rake db:prepare - bundle exec rake assets:precompile -fi - -#PIDFILE=/var/www/discourse/pids/puma.pid bundle exec rails s - -echo $RUBY_ALLOCATOR - -rm -Rf /var/www/discourse/tmp/pids/unicorn.pid -touch /var/www/discourse/tmp/pids/unicorn.pid -chmod 0777 /var/www/discourse/tmp/pids/unicorn.pid -touch /var/www/discourse/log/unicorn.stderr.log -chmod 0777 /var/www/discourse/log/unicorn.stderr.log -rm -Rf /var/www/discourse/log/production.log -touch /var/www/discourse/log/production.log -chmod 0664 /var/www/discourse/log/production.log - -chmod -R 0777 tmp/ember-rails -runsvdir /etc/service & - -LD_PRELOAD=$RUBY_ALLOCATOR HOME=/home/discourse USER=discourse UNICORN_BIND_ALL=1 UNICORN_PORT=3000 exec thpoff chpst -u discourse:www-data -U discourse:www-data bundle exec config/unicorn_launcher -E production -c config/unicorn.conf.rb \ No newline at end of file diff --git a/docker/dockerfile b/docker/dockerfile index a6d2e3fe81c86..09898fd9bfae5 100644 --- a/docker/dockerfile +++ b/docker/dockerfile @@ -1,20 +1,22 @@ -# NAME: discourse/base -# VERSION: release -FROM debian:buster-slim +# This code comes from with minor modifications: +# - https://github.com/discourse/discourse_docker/blob/93906f14f4a2df88aea11340fb1b11efbc4a785f/image/base/slim.Dockerfile +# - https://github.com/discourse/discourse_docker/blob/93906f14f4a2df88aea11340fb1b11efbc4a785f/image/base/release.Dockerfile +FROM debian:bullseye-slim + +ARG DISCOURSE_MAXMIND_LICENSE_KEY ENV PG_MAJOR 13 ENV RUBY_ALLOCATOR /usr/lib/libjemalloc.so.1 ENV RAILS_ENV production - -#LABEL maintainer="Sam Saffron \"https://twitter.com/samsaffron\"" +ENV DISCOURSE_MAXMIND_LICENSE_KEY=$DISCOURSE_MAXMIND_LICENSE_KEY +ENV BUNDLER_VERSION 2.3.5 RUN echo 2.0.`date +%Y%m%d` > /VERSION -RUN apt update && apt install -y gnupg sudo curl +RUN echo 'deb http://deb.debian.org/debian bullseye-backports main' > /etc/apt/sources.list.d/bullseye-backports.list RUN echo "debconf debconf/frontend select Teletype" | debconf-set-selections -RUN apt update && apt -y install fping +RUN apt update && apt -y install gnupg sudo curl fping RUN sh -c "fping proxy && echo 'Acquire { Retries \"0\"; HTTP { Proxy \"http://proxy:3128\";}; };' > /etc/apt/apt.conf.d/40proxy && apt update || true" -RUN apt -y install software-properties-common RUN apt-mark hold initscripts RUN apt -y upgrade @@ -24,7 +26,7 @@ ENV LANG en_US.UTF-8 ENV LANGUAGE en_US.UTF-8 RUN curl https://apt.postgresql.org/pub/repos/apt/ACCC4CF8.asc | apt-key add - -RUN echo "deb http://apt.postgresql.org/pub/repos/apt/ buster-pgdg main" | \ +RUN echo "deb http://apt.postgresql.org/pub/repos/apt/ bullseye-pgdg main" | \ tee /etc/apt/sources.list.d/postgres.list RUN curl --silent --location https://deb.nodesource.com/setup_16.x | sudo bash - RUN curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - @@ -33,14 +35,15 @@ RUN apt -y update # install these without recommends to avoid pulling in e.g. # X11 libraries, mailutils RUN apt -y install --no-install-recommends git rsyslog logrotate cron ssh-client less -RUN apt -y install build-essential rsync \ +RUN apt -y install autoconf build-essential ca-certificates rsync \ libxslt-dev libcurl4-openssl-dev \ libssl-dev libyaml-dev libtool \ + libpcre3 libpcre3-dev zlib1g zlib1g-dev \ libxml2-dev gawk parallel \ - postgresql-client-${PG_MAJOR} \ + postgresql-${PG_MAJOR} postgresql-client-${PG_MAJOR} \ postgresql-contrib-${PG_MAJOR} libpq-dev libreadline-dev \ anacron wget \ - psmisc vim whois brotli libunwind-dev \ + psmisc whois brotli libunwind-dev \ libtcmalloc-minimal4 cmake \ pngcrush pngquant RUN sed -i -e 's/start -q anacron/anacron -s/' /etc/cron.d/anacron @@ -58,38 +61,24 @@ RUN cd / &&\ npm install -g terser &&\ npm install -g uglify-js +ADD ./docker/install-imagemagick /tmp/install-imagemagick +RUN /tmp/install-imagemagick -RUN apt -y install advancecomp jhead jpegoptim libjpeg-turbo-progs optipng - -RUN apt -y install zlib1g-dev +ADD ./docker/install-jemalloc /tmp/install-jemalloc +RUN /tmp/install-jemalloc -RUN mkdir /jemalloc-stable && cd /jemalloc-stable &&\ - wget https://github.com/jemalloc/jemalloc/releases/download/3.6.0/jemalloc-3.6.0.tar.bz2 &&\ - tar -xjf jemalloc-3.6.0.tar.bz2 && cd jemalloc-3.6.0 && ./configure --prefix=/usr && make && make install &&\ - cd / && rm -rf /jemalloc-stable +ADD ./docker/install-oxipng /tmp/install-oxipng +RUN /tmp/install-oxipng -RUN mkdir /jemalloc-new && cd /jemalloc-new &&\ - wget https://github.com/jemalloc/jemalloc/releases/download/5.2.1/jemalloc-5.2.1.tar.bz2 &&\ - tar -xjf jemalloc-5.2.1.tar.bz2 && cd jemalloc-5.2.1 && ./configure --prefix=/usr --with-install-suffix=5.2.1 && make build_lib && make install_lib &&\ - cd / && rm -rf /jemalloc-new +ADD ./docker/install-ruby /tmp/install-ruby +RUN /tmp/install-ruby RUN echo 'gem: --no-document' >> /usr/local/etc/gemrc &&\ - mkdir /src && cd /src && git clone https://github.com/sstephenson/ruby-build.git &&\ - cd /src/ruby-build && ./install.sh &&\ - cd / && rm -rf /src/ruby-build && (ruby-build 2.7.4 /usr/local) - -RUN gem update --system - -RUN gem install bundler --force &&\ - rm -rf /usr/local/share/ri/2.7.4/system &&\ - cd / && git clone https://github.com/discourse/pups.git + gem update --system - -ADD docker/install-imagemagick /tmp/install-imagemagick -RUN /tmp/install-imagemagick - -# Validate install -RUN ruby -Eutf-8 -e "v = \`convert -version\`; %w{png tiff jpeg freetype heic}.each { |f| unless v.include?(f); STDERR.puts('no ' + f + ' support in imagemagick'); exit(-1); end }" +RUN gem install "bundler:${BUNDLER_VERSION}" pups --force &&\ + mkdir -p /pups/bin/ &&\ + ln -s /usr/local/bin/pups /pups/bin/pups # This tool allows us to disable huge page support for our current process # since the flag is preserved through forks and execs it can be used on any @@ -98,45 +87,34 @@ ADD ./docker/thpoff.c /src/thpoff.c RUN gcc -o /usr/local/sbin/thpoff /src/thpoff.c && rm /src/thpoff.c # clean up for docker squash -RUN rm -fr /usr/share/man &&\ - rm -fr /usr/share/doc &&\ - rm -fr /usr/share/vim/vim74/tutor &&\ - rm -fr /usr/share/vim/vim74/doc &&\ - rm -fr /usr/share/vim/vim74/lang &&\ - rm -fr /usr/local/share/doc &&\ - rm -fr /usr/local/share/ruby-build &&\ - rm -fr /root/.gem &&\ - rm -fr /root/.npm &&\ - rm -fr /tmp/* &&\ - rm -fr /usr/share/vim/vim74/spell/en* - - -# this can probably be done, but I worry that people changing PG locales will have issues -# cd /usr/share/locale && rm -fr `ls -d */ | grep -v en` - -RUN mkdir -p /etc/runit/3.d -RUN ls -ADD ./docker/runit1 /etc/runit/1 -ADD ./docker/runit1.d-clean-pids /etc/runit/1.d/cleanup-pids -ADD ./docker/runit1.d-anacron /etc/runit/1.d/anacron -ADD ./docker/runit1.d /etc/runit/1.d/00-fix-var-logs -ADD ./docker/runit2 /etc/runit/2 -ADD ./docker/runit3 /etc/runit/3 -ADD ./docker/boot /sbin/boot - -ADD ./docker/cron /etc/service/cron/run -ADD ./docker/rsyslog /etc/service/rsyslog/run -ADD ./docker/cron.d-anacron /etc/cron.d/anacron - -RUN chmod +x /etc/service/cron/run && chmod +x /etc/service/rsyslog/run && chmod +x /etc/cron.d/anacron - +RUN rm -fr /usr/share/man &&\ + rm -fr /usr/share/doc &&\ + rm -fr /usr/share/vim/vim74/doc &&\ + rm -fr /usr/share/vim/vim74/lang &&\ + rm -fr /usr/share/vim/vim74/spell/en* &&\ + rm -fr /usr/share/vim/vim74/tutor &&\ + rm -fr /usr/local/share/doc &&\ + rm -fr /usr/local/share/ri &&\ + rm -fr /usr/local/share/ruby-build &&\ + rm -fr /var/lib/apt/lists/* &&\ + rm -fr /root/.gem &&\ + rm -fr /root/.npm &&\ + rm -fr /tmp/* + +# this is required for arch64 which uses buildx +# see https://github.com/docker/buildx/issues/150 +RUN rm -f /etc/service # Discourse specific bits RUN useradd discourse -s /bin/bash -m -U &&\ mkdir -p /var/www/discourse &&\ - cd /var/www/discourse + mkdir -p /var/www/discourse/public/javascripts &&\ + mkdir -p /var/www/discourse/log WORKDIR /var/www/discourse +COPY ./docker/etc/ /etc +COPY ./docker/sbin/ /sbin + ADD ["./config.ru", "./.ruby-version.sample","./Brewfile","./package.json","./README.md","./.rubocop.yml","./.eslintignore","./Gemfile.lock","./.prettierrc","./Gemfile","./.prettierignore","./.ruby-gemset.sample","./translator.yml","./Rakefile","./.licensed.yml","./lefthook.yml","./yarn.lock","./.eslintrc","./.template-lintrc.js","./.rspec_parallel","./CONTRIBUTING.md","./LICENSE.txt","./.rspec", "./"] . ADD "script/" "script/" ADD "app/" "app/" @@ -148,32 +126,22 @@ ADD "test/" "test/" ADD "bin/" "bin" ADD "images/" "images/" ADD "vendor/" "vendor/" -ADD "docker/discourse_install.sh" "./" -RUN mkdir pids -RUN chmod 0777 pids -RUN chmod +x discourse_install.sh -RUN ./discourse_install.sh -RUN mkdir -p tmp/sockets -RUN chmod -R 0755 tmp/ -ADD "docker/discourse_run.sh" "./" -RUN chmod +x discourse_run.sh - -RUN cd plugins && git clone https://github.com/discourse/docker_manager.git &&\ - git clone https://github.com/angusmcleod/discourse-events.git &&\ - git clone https://github.com/davidtaylorhq/discourse-whos-online.git &&\ - git clone https://github.com/discourse/discourse-checklist.git &&\ - git clone https://github.com/discourse/discourse-data-explorer.git &&\ - git clone https://github.com/BeXcellent/discourse-allowiframe.git - -RUN mkdir -p pids/ -RUN chmod -R 0777 pids/ -RUN touch pids/puma.pid -RUN chmod 0777 pids/puma.pid -RUN mkdir -p /var/www/discourse/tmp/pids -RUN chmod -R 0777 /var/www/discourse/tmp - -RUN bundle exec rails plugin:install_all_gems - -COPY docker/discourse_first_run . -ENTRYPOINT ["/bin/bash", "./discourse_run.sh"] -CMD ["compile"] \ No newline at end of file + +RUN chown -R discourse:discourse /var/www/discourse + +RUN sudo -u discourse bundle config --local deployment true &&\ + sudo -u discourse bundle config --local path ./vendor/bundle &&\ + sudo -u discourse bundle config --local without test development &&\ + sudo -u discourse bundle install --jobs 4 &&\ + sudo -u discourse yarn install --production &&\ + sudo -u discourse yarn cache clean &&\ + bundle exec rake maxminddb:get &&\ + chown -R discourse:discourse tmp public &&\ + find /var/www/discourse/vendor/bundle -name tmp -type d -exec rm -rf {} + + +RUN cd plugins && git clone https://github.com/discourse/docker_manager.git &&\ + git clone https://github.com/angusmcleod/discourse-events.git &&\ + git clone https://github.com/davidtaylorhq/discourse-whos-online.git &&\ + git clone https://github.com/discourse/discourse-checklist.git &&\ + git clone https://github.com/discourse/discourse-data-explorer.git &&\ + git clone https://github.com/BeXcellent/discourse-allowiframe.git diff --git a/docker/dockerfile.new b/docker/dockerfile.new new file mode 100644 index 0000000000000..5919a8e5b237e --- /dev/null +++ b/docker/dockerfile.new @@ -0,0 +1,182 @@ +# NAME: discourse/base +# VERSION: release +FROM debian:buster-slim + +ENV PG_MAJOR 13 +ENV RUBY_ALLOCATOR /usr/lib/libjemalloc.so.1 +ENV RAILS_ENV production +ENV BUNDLER_VERSION 2.3.5 + +#LABEL maintainer="Sam Saffron \"https://twitter.com/samsaffron\"" + +RUN echo 2.0.`date +%Y%m%d` > /VERSION + +RUN apt update && apt install -y gnupg sudo curl +RUN echo "debconf debconf/frontend select Teletype" | debconf-set-selections +RUN apt update && apt -y install fping +RUN sh -c "fping proxy && echo 'Acquire { Retries \"0\"; HTTP { Proxy \"http://proxy:3128\";}; };' > /etc/apt/apt.conf.d/40proxy && apt update || true" +RUN apt -y install software-properties-common +RUN apt-mark hold initscripts +RUN apt -y upgrade + +RUN apt install -y locales locales-all +ENV LC_ALL en_US.UTF-8 +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US.UTF-8 + +RUN curl https://apt.postgresql.org/pub/repos/apt/ACCC4CF8.asc | apt-key add - +RUN echo "deb http://apt.postgresql.org/pub/repos/apt/ buster-pgdg main" | \ + tee /etc/apt/sources.list.d/postgres.list +RUN curl --silent --location https://deb.nodesource.com/setup_16.x | sudo bash - +RUN curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - +RUN echo "deb https://dl.yarnpkg.com/debian/ stable main" > /etc/apt/sources.list.d/yarn.list +RUN apt -y update +# install these without recommends to avoid pulling in e.g. +# X11 libraries, mailutils +RUN apt -y install --no-install-recommends git rsyslog logrotate cron ssh-client less +RUN apt -y install build-essential rsync \ + libxslt-dev libcurl4-openssl-dev \ + libssl-dev libyaml-dev libtool \ + libxml2-dev gawk parallel \ + postgresql-client-${PG_MAJOR} \ + postgresql-contrib-${PG_MAJOR} libpq-dev libreadline-dev \ + anacron wget \ + psmisc vim whois brotli libunwind-dev \ + libtcmalloc-minimal4 cmake \ + pngcrush pngquant +RUN sed -i -e 's/start -q anacron/anacron -s/' /etc/cron.d/anacron +RUN sed -i.bak 's/$ModLoad imklog/#$ModLoad imklog/' /etc/rsyslog.conf +RUN sed -i.bak 's/module(load="imklog")/#module(load="imklog")/' /etc/rsyslog.conf +RUN dpkg-divert --local --rename --add /sbin/initctl +RUN sh -c "test -f /sbin/initctl || ln -s /bin/true /sbin/initctl" +RUN cd / &&\ + apt -y install runit socat &&\ + mkdir -p /etc/runit/1.d &&\ + apt clean &&\ + rm -f /etc/apt/apt.conf.d/40proxy &&\ + locale-gen en_US &&\ + apt install -y nodejs yarn &&\ + npm install -g terser &&\ + npm install -g uglify-js + + +RUN apt -y install advancecomp jhead jpegoptim libjpeg-turbo-progs optipng + +RUN apt -y install zlib1g-dev + +RUN sudo apt install python3-distutils -y # required by nodeJS https://github.com/nodejs/node/issues/30189#issuecomment-548250793 + +RUN mkdir /jemalloc-stable && cd /jemalloc-stable &&\ + wget https://github.com/jemalloc/jemalloc/releases/download/3.6.0/jemalloc-3.6.0.tar.bz2 &&\ + tar -xjf jemalloc-3.6.0.tar.bz2 && cd jemalloc-3.6.0 && ./configure --prefix=/usr && make && make install &&\ + cd / && rm -rf /jemalloc-stable + +RUN mkdir /jemalloc-new && cd /jemalloc-new &&\ + wget https://github.com/jemalloc/jemalloc/releases/download/5.2.1/jemalloc-5.2.1.tar.bz2 &&\ + tar -xjf jemalloc-5.2.1.tar.bz2 && cd jemalloc-5.2.1 && ./configure --prefix=/usr --with-install-suffix=5.2.1 && make build_lib && make install_lib &&\ + cd / && rm -rf /jemalloc-new + +RUN echo 'gem: --no-document' >> /usr/local/etc/gemrc &&\ + mkdir /src && cd /src && git clone https://github.com/sstephenson/ruby-build.git &&\ + cd /src/ruby-build && ./install.sh &&\ + cd / && rm -rf /src/ruby-build && (ruby-build 2.7.4 /usr/local) + +RUN gem update --system + +RUN gem install bundler -v ${BUNDLER_VERSION} --force &&\ + rm -rf /usr/local/share/ri/2.7.4/system &&\ + cd / && git clone https://github.com/discourse/pups.git + + +ADD docker/install-imagemagick /tmp/install-imagemagick +RUN /tmp/install-imagemagick + +# Validate install +RUN ruby -Eutf-8 -e "v = \`convert -version\`; %w{png tiff jpeg freetype heic}.each { |f| unless v.include?(f); STDERR.puts('no ' + f + ' support in imagemagick'); exit(-1); end }" + +# This tool allows us to disable huge page support for our current process +# since the flag is preserved through forks and execs it can be used on any +# process +ADD ./docker/thpoff.c /src/thpoff.c +RUN gcc -o /usr/local/sbin/thpoff /src/thpoff.c && rm /src/thpoff.c + +# clean up for docker squash +RUN rm -fr /usr/share/man &&\ + rm -fr /usr/share/doc &&\ + rm -fr /usr/share/vim/vim74/tutor &&\ + rm -fr /usr/share/vim/vim74/doc &&\ + rm -fr /usr/share/vim/vim74/lang &&\ + rm -fr /usr/local/share/doc &&\ + rm -fr /usr/local/share/ruby-build &&\ + rm -fr /root/.gem &&\ + rm -fr /root/.npm &&\ + rm -fr /tmp/* &&\ + rm -fr /usr/share/vim/vim74/spell/en* + + +# this can probably be done, but I worry that people changing PG locales will have issues +# cd /usr/share/locale && rm -fr `ls -d */ | grep -v en` + +RUN mkdir -p /etc/runit/3.d +RUN ls +ADD ./docker/runit1 /etc/runit/1 +ADD ./docker/runit1.d-clean-pids /etc/runit/1.d/cleanup-pids +ADD ./docker/runit1.d-anacron /etc/runit/1.d/anacron +ADD ./docker/runit1.d /etc/runit/1.d/00-fix-var-logs +ADD ./docker/runit2 /etc/runit/2 +ADD ./docker/runit3 /etc/runit/3 +ADD ./docker/boot /sbin/boot + +ADD ./docker/cron /etc/service/cron/run +ADD ./docker/rsyslog /etc/service/rsyslog/run +ADD ./docker/cron.d-anacron /etc/cron.d/anacron + +RUN chmod +x /etc/service/cron/run && chmod +x /etc/service/rsyslog/run && chmod +x /etc/cron.d/anacron + + +# Discourse specific bits +RUN useradd discourse -s /bin/bash -m -U &&\ + mkdir -p /var/www/discourse &&\ + cd /var/www/discourse +WORKDIR /var/www/discourse + +ADD ["./config.ru", "./.ruby-version.sample","./Brewfile","./package.json","./README.md","./.rubocop.yml","./.eslintignore","./Gemfile.lock","./.prettierrc","./Gemfile","./.prettierignore","./.ruby-gemset.sample","./translator.yml","./Rakefile","./.licensed.yml","./lefthook.yml","./yarn.lock","./.eslintrc","./.template-lintrc.js","./.rspec_parallel","./CONTRIBUTING.md","./LICENSE.txt","./.rspec", "./"] . +ADD "script/" "script/" +ADD "app/" "app/" +ADD "db/" "db/" +ADD "plugins/" "plugins/" +ADD "lib/" "lib/" +ADD "config/" "config/" +ADD "test/" "test/" +ADD "bin/" "bin" +ADD "images/" "images/" +ADD "vendor/" "vendor/" +ADD "docker/discourse_install.sh" "./" +RUN mkdir pids +RUN chmod 0777 pids +RUN chmod +x discourse_install.sh +# RUN ./discourse_install.sh +# RUN mkdir -p tmp/sockets +# RUN chmod -R 0755 tmp/ +# ADD "docker/discourse_run.sh" "./" +# RUN chmod +x discourse_run.sh + +# RUN cd plugins && git clone https://github.com/discourse/docker_manager.git &&\ + # git clone https://github.com/angusmcleod/discourse-events.git &&\ + # git clone https://github.com/davidtaylorhq/discourse-whos-online.git &&\ + # git clone https://github.com/discourse/discourse-checklist.git &&\ + # git clone https://github.com/discourse/discourse-data-explorer.git &&\ + # git clone https://github.com/BeXcellent/discourse-allowiframe.git + +# RUN mkdir -p pids/ +# RUN chmod -R 0777 pids/ +# RUN touch pids/puma.pid +# RUN chmod 0777 pids/puma.pid +# RUN mkdir -p /var/www/discourse/tmp/pids +# RUN chmod -R 0777 /var/www/discourse/tmp + +# RUN bundle exec rails plugin:install_all_gems + +# COPY docker/discourse_first_run . +# ENTRYPOINT ["/bin/bash", "./discourse_run.sh"] +# CMD ["compile"] diff --git a/docker/cron.d-anacron b/docker/etc/cron.d/cron.d_anacron similarity index 72% rename from docker/cron.d-anacron rename to docker/etc/cron.d/cron.d_anacron index c1b5b47dcedcf..0f784a94274c6 100644 --- a/docker/cron.d-anacron +++ b/docker/etc/cron.d/cron.d_anacron @@ -3,4 +3,4 @@ SHELL=/bin/sh PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin -30 7 * * * root /usr/sbin/anacron -s >/dev/null \ No newline at end of file +30 7 * * * root /usr/sbin/anacron -s >/dev/null diff --git a/docker/runit1 b/docker/etc/runit/1 old mode 100644 new mode 100755 similarity index 95% rename from docker/runit1 rename to docker/etc/runit/1 index fd935fe8d9d7f..9c5707775d628 --- a/docker/runit1 +++ b/docker/etc/runit/1 @@ -1,3 +1,3 @@ #!/bin/bash -/bin/run-parts --verbose --exit-on-error /etc/runit/1.d || exit 100 \ No newline at end of file +/bin/run-parts --verbose --exit-on-error /etc/runit/1.d || exit 100 diff --git a/docker/etc/runit/1.d/00-fix-var-logs b/docker/etc/runit/1.d/00-fix-var-logs new file mode 100755 index 0000000000000..c12065f38c3dd --- /dev/null +++ b/docker/etc/runit/1.d/00-fix-var-logs @@ -0,0 +1,8 @@ +#!/bin/bash +mkdir -p /var/log/nginx +chown -R www-data:www-data /var/log/nginx +chmod -R 644 /var/log/nginx +chmod 755 /var/log/nginx +touch /var/log/syslog && chown -f root:adm /var/log/syslog* +touch /var/log/auth.log && chown -f root:adm /var/log/auth.log* +touch /var/log/kern.log && chown -f root:adm /var/log/kern.log* diff --git a/docker/etc/runit/1.d/anacron b/docker/etc/runit/1.d/anacron new file mode 100755 index 0000000000000..5080bfc50287e --- /dev/null +++ b/docker/etc/runit/1.d/anacron @@ -0,0 +1,2 @@ +#!/bin/bash +/usr/sbin/anacron -s diff --git a/docker/runit1.d-clean-pids b/docker/etc/runit/1.d/cleanup-pids old mode 100644 new mode 100755 similarity index 65% rename from docker/runit1.d-clean-pids rename to docker/etc/runit/1.d/cleanup-pids index 811749cf46e23..cf5c3da6f4d13 --- a/docker/runit1.d-clean-pids +++ b/docker/etc/runit/1.d/cleanup-pids @@ -1,3 +1,3 @@ #!/bin/bash /bin/echo "Cleaning stale PID files" -/bin/rm -f /var/run/*.pid \ No newline at end of file +/bin/rm -f /var/run/*.pid diff --git a/docker/etc/runit/2 b/docker/etc/runit/2 new file mode 100755 index 0000000000000..ba3cfb2d9e05c --- /dev/null +++ b/docker/etc/runit/2 @@ -0,0 +1,2 @@ +#!/bin/bash +exec /usr/bin/runsvdir -P /etc/service diff --git a/docker/etc/runit/3 b/docker/etc/runit/3 new file mode 100755 index 0000000000000..40e510cff6752 --- /dev/null +++ b/docker/etc/runit/3 @@ -0,0 +1,3 @@ +#!/bin/bash + +/bin/run-parts --verbose /etc/runit/3.d diff --git a/docker/cron b/docker/etc/service/cron/run old mode 100644 new mode 100755 similarity index 67% rename from docker/cron rename to docker/etc/service/cron/run index a11b4b98f1467..4170299393445 --- a/docker/cron +++ b/docker/etc/service/cron/run @@ -1,4 +1,4 @@ #!/bin/bash exec 2>&1 cd / -exec cron -f \ No newline at end of file +exec cron -f diff --git a/docker/etc/service/rsyslog/run b/docker/etc/service/rsyslog/run new file mode 100755 index 0000000000000..ecc41d43d1677 --- /dev/null +++ b/docker/etc/service/rsyslog/run @@ -0,0 +1,7 @@ +#!/bin/bash +exec 2>&1 +cd / +chgrp syslog /var/log +chmod g+w /var/log +rm -f /var/run/rsyslogd.pid +exec rsyslogd -n diff --git a/docker/install-imagemagick b/docker/install-imagemagick index 908c028bb13bb..71700e159ba3a 100755 --- a/docker/install-imagemagick +++ b/docker/install-imagemagick @@ -2,15 +2,8 @@ set -e # version check: https://github.com/ImageMagick/ImageMagick/releases -IMAGE_MAGICK_VERSION="7.0.11-13" -IMAGE_MAGICK_HASH="fc454be622724c6224fa6c8230bb9c50191a05fbf05b9c9c25aa3e5497090b83" - -# version check: https://github.com/strukturag/libheif/releases -LIBHEIF_VERSION="1.12.0" -LIBHEIF_HASH="086145b0d990182a033b0011caadb1b642da84f39ab83aa66d005610650b3c65" - -# version check: https://aomedia.googlesource.com/aom -LIB_AOM_VERSION="3.1.0" +IMAGE_MAGICK_VERSION="7.1.0-20" +IMAGE_MAGICK_HASH="c7526ca8b341e42f026f8157632b1722555f0d015c281861d9c4d314c0c8f78c" # We use debian, but GitHub CI is stuck on Ubuntu Bionic, so this must be compatible with both LIBJPEGTURBO=$(cat /etc/issue | grep -qi Debian && echo 'libjpeg62-turbo libjpeg62-turbo-dev' || echo 'libjpeg-turbo8 libjpeg-turbo8-dev') @@ -20,39 +13,20 @@ WDIR=/tmp/imagemagick # Install build deps apt -y -q remove imagemagick -apt -y -q install git make gcc pkg-config autoconf curl g++ \ - yasm cmake \ +apt -y -q install git make gcc pkg-config autoconf curl g++ yasm cmake \ libde265-0 libde265-dev ${LIBJPEGTURBO} x265 libx265-dev libtool \ - libpng16-16 libpng-dev ${LIBJPEGTURBO} libwebp6 libwebp-dev libgomp1 libwebpmux3 libwebpdemux2 ghostscript libxml2-dev libxml2-utils \ - libbz2-dev gsfonts libtiff-dev libfreetype6-dev libjpeg-dev + libpng16-16 libpng-dev ${LIBJPEGTURBO} libwebp6 libwebp-dev libgomp1 \ + libwebpmux3 libwebpdemux2 ghostscript libxml2-dev libxml2-utils \ + libltdl7-dev libbz2-dev gsfonts libtiff-dev libfreetype6-dev libjpeg-dev -mkdir -p $WDIR -cd $WDIR +# Use backports instead of compiling it +apt -y -q install -t bullseye-backports libheif1 libaom-dev libheif-dev -# Building libaom -git clone https://aomedia.googlesource.com/aom -cd aom && git checkout v${LIB_AOM_VERSION} && cd .. -mkdir build_aom -cd build_aom -cmake ../aom/ -DENABLE_TESTS=0 -DBUILD_SHARED_LIBS=1 && make && make install -ldconfig /usr/local/lib -cd .. -rm -rf aom -rm -rf build_aom - -# Build and install libheif +mkdir -p $WDIR cd $WDIR -wget -O $WDIR/libheif.tar.gz "https://github.com/strukturag/libheif/archive/v$LIBHEIF_VERSION.tar.gz" -sha256sum $WDIR/libheif.tar.gz -echo "$LIBHEIF_HASH $WDIR/libheif.tar.gz" | sha256sum -c -tar -xzvf $WDIR/libheif.tar.gz -cd libheif-$LIBHEIF_VERSION -./autogen.sh -./configure -make && make install # Build and install ImageMagick -wget -O $WDIR/ImageMagick.tar.gz "https://github.com/ImageMagick/ImageMagick/archive/$IMAGE_MAGICK_VERSION.tar.gz" +wget -q -O $WDIR/ImageMagick.tar.gz "https://github.com/ImageMagick/ImageMagick/archive/$IMAGE_MAGICK_VERSION.tar.gz" sha256sum $WDIR/ImageMagick.tar.gz echo "$IMAGE_MAGICK_HASH $WDIR/ImageMagick.tar.gz" | sha256sum -c IMDIR=$WDIR/$(tar tzf $WDIR/ImageMagick.tar.gz --wildcards "ImageMagick-*/configure" |cut -d/ -f1) @@ -83,4 +57,7 @@ make all && make install cd $HOME rm -rf $WDIR -ldconfig /usr/local/lib \ No newline at end of file +ldconfig /usr/local/lib + +# Validate ImageMagick install +test $(convert -version | grep -o -e png -e tiff -e jpeg -e freetype -e heic -e webp | wc -l) -eq 6 diff --git a/docker/install-jemalloc b/docker/install-jemalloc new file mode 100755 index 0000000000000..dab487af2b2d0 --- /dev/null +++ b/docker/install-jemalloc @@ -0,0 +1,26 @@ +#!/bin/bash +set -e + +# version check: https://github.com/jemalloc/jemalloc/releases + +# jemalloc stable +mkdir /jemalloc-stable +cd /jemalloc-stable + +wget -q https://github.com/jemalloc/jemalloc/releases/download/3.6.0/jemalloc-3.6.0.tar.bz2 +sha256sum jemalloc-3.6.0.tar.bz2 +echo "e16c2159dd3c81ca2dc3b5c9ef0d43e1f2f45b04548f42db12e7c12d7bdf84fe jemalloc-3.6.0.tar.bz2" | sha256sum -c +tar --strip-components=1 -xjf jemalloc-3.6.0.tar.bz2 +./configure --prefix=/usr && make && make install +cd / && rm -rf /jemalloc-stable + +# jemalloc new +mkdir /jemalloc-new +cd /jemalloc-new + +wget -q https://github.com/jemalloc/jemalloc/releases/download/5.2.1/jemalloc-5.2.1.tar.bz2 +sha256sum jemalloc-5.2.1.tar.bz2 +echo "34330e5ce276099e2e8950d9335db5a875689a4c6a56751ef3b1d8c537f887f6 jemalloc-5.2.1.tar.bz2" | sha256sum -c +tar --strip-components=1 -xjf jemalloc-5.2.1.tar.bz2 +./configure --prefix=/usr --with-install-suffix=5.2.1 && make build_lib && make install_lib +cd / && rm -rf /jemalloc-new diff --git a/docker/install-oxipng b/docker/install-oxipng new file mode 100755 index 0000000000000..93b30b222ceb7 --- /dev/null +++ b/docker/install-oxipng @@ -0,0 +1,21 @@ +#!/bin/bash +set -e + +# version check: https://github.com/shssoichiro/oxipng/releases +OXIPNG_VERSION="5.0.1" +OXIPNG_FILE="oxipng-${OXIPNG_VERSION}-x86_64-unknown-linux-musl.tar.gz" +OXIPNG_HASH="89240cfd863f8007ab3ad95d88dc2ce15fc003a0421508728d73fec1375f19b6" + +# Install other deps +apt -y -q install advancecomp jhead jpegoptim libjpeg-turbo-progs optipng + +mkdir /oxipng-install +cd /oxipng-install + +wget -q https://github.com/shssoichiro/oxipng/releases/download/v${OXIPNG_VERSION}/${OXIPNG_FILE} +sha256sum ${OXIPNG_FILE} +echo "${OXIPNG_HASH} ${OXIPNG_FILE}" | sha256sum -c + +tar --strip-components=1 -xzf $OXIPNG_FILE +cp -v ./oxipng /usr/local/bin +cd / && rm -fr /oxipng-install diff --git a/docker/install-ruby b/docker/install-ruby new file mode 100755 index 0000000000000..0c77c181e6782 --- /dev/null +++ b/docker/install-ruby @@ -0,0 +1,11 @@ +#!/bin/bash +set -e + +RUBY_VERSION="2.7.6" + +mkdir /src +git -C /src clone https://github.com/rbenv/ruby-build.git +cd /src/ruby-build && ./install.sh +cd / && rm -fr /src + +ruby-build ${RUBY_VERSION} /usr/local diff --git a/docker/rsyslog b/docker/rsyslog deleted file mode 100644 index e5faf5a6d2144..0000000000000 --- a/docker/rsyslog +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash -exec 2>&1 -cd / -rm -f /var/run/rsyslogd.pid -exec rsyslogd -n \ No newline at end of file diff --git a/docker/runit1.d b/docker/runit1.d deleted file mode 100644 index 40e752b9185b9..0000000000000 --- a/docker/runit1.d +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash -touch /var/log/syslog && chown -f root:adm /var/log/syslog* -touch /var/log/auth.log && chown -f root:adm /var/log/auth.log* -touch /var/log/kern.log && chown -f root:adm /var/log/kern.log* \ No newline at end of file diff --git a/docker/runit1.d-anacron b/docker/runit1.d-anacron deleted file mode 100644 index 9b072fe389c56..0000000000000 --- a/docker/runit1.d-anacron +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -/usr/sbin/anacron -s \ No newline at end of file diff --git a/docker/runit2 b/docker/runit2 deleted file mode 100644 index c75051373a60d..0000000000000 --- a/docker/runit2 +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -exec /usr/bin/runsvdir -P /etc/service \ No newline at end of file diff --git a/docker/runit3 b/docker/runit3 deleted file mode 100644 index 6e506094f0e20..0000000000000 --- a/docker/runit3 +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -/bin/run-parts --verbose /etc/runit/3.d \ No newline at end of file diff --git a/docker/boot b/docker/sbin/boot old mode 100644 new mode 100755 similarity index 98% rename from docker/boot rename to docker/sbin/boot index a5b78288d4cf0..67fa5aa12351a --- a/docker/boot +++ b/docker/sbin/boot @@ -24,4 +24,4 @@ echo "Started runsvdir, PID is $RUNSVDIR" trap shutdown SIGTERM SIGHUP wait $RUNSVDIR -shutdown \ No newline at end of file +shutdown diff --git a/docker/site.conf.t b/docker/site.conf.t index 3e08504c81642..6f3803507f14d 100644 --- a/docker/site.conf.t +++ b/docker/site.conf.t @@ -4,7 +4,7 @@ types { application/wasm wasm; } - + # inactive means we keep stuff around for 1440m minutes regardless of last access (1 week) # levels means it is a 2 deep hierarchy cause we can have lots of files @@ -40,7 +40,7 @@ server { resolver 127.0.0.11 valid=30s; - set $upstream app:3000; + set $upstream localhost:3000; access_log /var/log/nginx/access.log log_discourse; @@ -87,7 +87,7 @@ server { internal; } - + location / { root $public; add_header ETag ""; @@ -204,7 +204,7 @@ server { add_header Access-Control-Allow-Origin *; try_files $uri =404; } - + try_files $uri @discourse; break; } @@ -279,4 +279,4 @@ server { proxy_pass http://$upstream; } -} \ No newline at end of file +}