From 723f3459bf2ad503ba230f20925ad1774b6c3bf5 Mon Sep 17 00:00:00 2001 From: Joseph Yu Date: Thu, 6 Jun 2019 17:01:02 +0100 Subject: [PATCH 1/5] Cherry picked files from wwfx.0.4.0 --- Dockerfile | 90 ++++++++++++++----------- preview_docs.sh | 142 ++++++++++++++++++++++++++++++++++++++-- scripts/build_sphinx.py | 1 + scripts/serve_docs.sh | 132 +++++++++++++++++++++++++++++++++++++ 4 files changed, 319 insertions(+), 46 deletions(-) create mode 100755 scripts/serve_docs.sh diff --git a/Dockerfile b/Dockerfile index 1052adf..ef018cb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,6 +6,8 @@ # # dockerfile to spin up a container suitable for generation of documentation +# - Gem/pip files used for setup are located in /usr/local/src +# - The default build_docs.sh is in /usr/local/src/scripts FROM centos:7 LABEL maintainer="toolkit@shotgunsoftware.com" @@ -18,58 +20,66 @@ LABEL maintainer="toolkit@shotgunsoftware.com" RUN yum update -y && \ yum install -y epel-release && \ yum install -y \ - # Generic build packages - autoconf \ - automake \ - gcc \ - gcc-c++ \ - git \ - libtool \ - make \ - nasm \ - perl-devel \ - zlib-devel \ - tar \ - nc \ - xz \ - # sphinx - pandoc \ - # Python libs - python-pip \ - # Ruby - libyaml-devel \ - openssl-devel \ - libreadline-dev \ - zlib-devel \ - python-pyside \ + # Generic build packages + # autoconf, automake, gcc will be installed as libtool dependencies + gcc-c++ \ + git \ + libtool \ + make \ + nasm \ + nc \ + perl-devel \ + # sphinx + pandoc \ + # Python libs + python-pip \ + python-pyside \ + # Ruby + libreadline-dev \ + libyaml-devel \ + openssl-devel \ + zlib-devel && \ yum clean all # Ruby ENV RUBY_MAJOR_VER=2.5 \ - RUBY_VER=2.5.3 \ - PATH=/opt/ruby-2.5/bin:$PATH + RUBY_VER=2.5.3 +ENV PATH=/opt/ruby-${RUBY_MAJOR_VER}/bin:$PATH RUN DIR=$(mktemp -d) && cd ${DIR} && \ - curl -OLs http://ftp.ruby-lang.org/pub/ruby/ruby-$RUBY_VER.tar.gz && \ - tar xzf ruby-$RUBY_VER.tar.gz && \ - cd ruby-$RUBY_VER && \ - ./configure --prefix=/opt/ruby-${RUBY_VER} --bindir=/opt/ruby-${RUBY_VER}/bin --disable-install-doc && \ - make -j 8 && \ + curl -Ls http://ftp.ruby-lang.org/pub/ruby/ruby-${RUBY_VER}.tar.gz | \ + tar --strip-components=1 -xz && \ + ./configure \ + --prefix=/opt/ruby-${RUBY_VER} \ + --bindir=/opt/ruby-${RUBY_VER}/bin \ + --disable-install-doc && \ + make -j $(nproc) && \ make install && \ make distclean && \ rm -rf ${DIR} && \ cd /opt/ && ln -sfn ruby-${RUBY_VER} ruby-${RUBY_MAJOR_VER} -RUN mkdir -p /app -WORKDIR /app - -COPY ./Gemfile /app -COPY ./Gemfile.lock /app -COPY ./requirements.txt /app +# Copy in any Gem/pip related files +WORKDIR /usr/local/src +COPY ./Gemfile . +COPY ./Gemfile.lock . +COPY ./requirements.txt . # Install any needed packages specified in requirements.txt RUN pip install -r requirements.txt # note: stay on bundler 1.17 to be travis compatible -RUN gem install bundler -v 1.17.2 --no-document -RUN bundle config --global jobs 7 -RUN bundle install +RUN gem install bundler -v 1.17.2 --no-document && \ + bundle config --global jobs $(( $(nproc) - 1 )) && \ + bundle install + +# Copy in and set "build_docs.sh" as default image executable for "docker run" +COPY ./jekyll jekyll +COPY ./scripts scripts +COPY ./sphinx sphinx +ENV PATH=${PATH}:/usr/local/src/scripts + +# Install tk-core (python only) +RUN get-tk-core-packages.sh /usr/local/lib64/python +ENV PYTHONPATH=${PYTHONPATH}:/usr/local/lib64/python + +ENTRYPOINT [ "serve_docs.sh" ] \ No newline at end of file diff --git a/preview_docs.sh b/preview_docs.sh index 453ec2b..306b607 100755 --- a/preview_docs.sh +++ b/preview_docs.sh @@ -7,12 +7,142 @@ # this software in either electronic or hard copy form. # -# Builds a local preview of the documentation in the parent repository that is -# including tk-doc-generator as a submodule. +set -eu -o pipefail + +ARG1="${1:-}" +case "${ARG1}" in + -h|--help) + cat << EOF +Usage: preview_docs.sh [--rebuild|-h] [URL_PATH] + +Builds a local preview of the documentation in the parent repository that +includes tk-doc-generator as a sub-module. + +Options: + --rebuild Force re-build docker image. By default (for speed), + it will only build a new Docker image if one doesn't exist. + -h, --help Print this help and exit 0. + URL_PATH Explicit URL sub folder/path to host under. + By default, it's automatically calculated in order: + 1. DOC_PATH=/* in .travis.yml + 2. Current folder name + +Environment variables: + IMAGE_TAG: + Tag for the image Docker builds. + Defaults "tk-doc-generator" if not set + + MOUNT_TO: + Destination directory which the parent directory this script, + the folder above tk-doc-generator, is mounted to in the container. + Defaults "/app" if not set + +Examples: + Typically, this command is run in the parent repository that includes + tk-doc-generator as a sub-module. + + To force an image rebuild by running before creating documentation: + + tk-doc-generator/preview_docs.sh --rebuild + + Some environment variables will also influence this script's behaviour. + For example, you can run the script like this to build/run the docker image + tagged "my-doc-generator", you can run: + + IMAGE_TAG="my-doc-generator" tk-doc-generator/preview_docs.sh + +EOF + exit + ;; +esac + +# Setup useful directory variables for current script's folder/parent folder THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -docker build ${THIS_DIR}/../tk-doc-generator -t tk-doc-generator && \ -docker run --mount type=bind,source="${THIS_DIR}/..",target=/app tk-doc-generator \ -/app/tk-doc-generator/scripts/build_docs.sh --url=http://localhost --url-path=${THIS_DIR}/../_build --source=/app/docs --output=/app/_build && \ -echo "Documentation built in ${THIS_DIR}/../_build/index.html" + +# Build if image does not exist OR first argument is --rebuild +IMAGE_TAG="${IMAGE_TAG:-tk-doc-generator}" +if [ -z "$(docker images --quiet ${IMAGE_TAG})" -o "${ARG1}" == "--rebuild" ] +then + docker build --tag "${IMAGE_TAG}" ${THIS_DIR} + shift # Process remaining arg as explicit URL_PATH +else + echo "'${IMAGE_TAG}' docker image already built" + echo "To force a re-build, try running this script with --rebuild flag" + echo +fi + + +# Setup mount/volume flags depending on which one current docker supports +MOUNT_TO="${MOUNT_TO:-/app}" +if ( docker run --help | grep --quiet -- --mount ) +then + MOUNT_FLAGS='--mount '"type=bind,source=$(pwd),target=${MOUNT_TO}" +else + MOUNT_FLAGS='-v '"$(pwd):${MOUNT_TO}" +fi + + +# Finally, perform the document generation in container +EXPOSED_PORT="4000" +URL_PATH="${1:-$( + grep -oP '(?<=DOC_PATH=/)[a-zA-Z0-9][a-zA-Z0-9_.-]*' $(pwd)/.travis.yml \ + || basename $(pwd) +)}" + +if (echo "${URL_PATH}" | grep -qP '^[a-zA-Z0-9][a-zA-Z0-9_.-]*$') +then + CONTAINER_NAME="${URL_PATH}-${EXPOSED_PORT}" + until docker run --rm -d \ + --hostname="${CONTAINER_NAME}" \ + --name="${CONTAINER_NAME}" \ + -p "${EXPOSED_PORT}:${EXPOSED_PORT}" \ + -e EXPOSED_PORT="${EXPOSED_PORT}" \ + -e URLPATH="/${URL_PATH}" \ + -e SOURCE="${MOUNT_TO}/docs" \ + -e OUTPUT="${MOUNT_TO}/_build" \ + ${MOUNT_FLAGS} \ + ${IMAGE_TAG} + do + EXPOSED_PORT="$(( EXPOSED_PORT + 1 ))" + CONTAINER_NAME="${URL_PATH}-${EXPOSED_PORT}" + done +else + echo "Invalid URL_PATH to host as: '${URL_PATH}'" + echo 'It must only contain these characters: [a-zA-Z0-9_.-]' + echo + echo 'Please either:' + echo '1. Set a valid DOC_PATH=/something in $(pwd)/.travis.yml, where that' + echo ' something must only contain those valid characters: e.g.' + echo + echo " env:" + echo " global:" + echo " - DOC_PATH=/tk-amazing-tool" + echo + echo "2. Rename current folder to only contain those valid characters." + echo + echo "3. Explicitly provide a valid URL_PATH to create under, e.g." + echo + echo " preview_docs.sh tk-amazing-tool" + echo + echo + URL_PATH='' + while true + do + CONTAINER_ID=$(docker run --rm -d \ + -p "${EXPOSED_PORT}:${EXPOSED_PORT}" \ + -e EXPOSED_PORT="${EXPOSED_PORT}" \ + -e URLPATH="/" \ + -e SOURCE="${MOUNT_TO}/docs" \ + -e OUTPUT="${MOUNT_TO}/_build" \ + ${MOUNT_FLAGS} \ + ${IMAGE_TAG}) \ + && break \ + || EXPOSED_PORT="$(( EXPOSED_PORT + 1 ))" + done + CONTAINER_NAME=$(docker ps -f id=${CONTAINER_ID} --format '{{.Names}}') +fi + +# Directly calling log follow here to show any errors in serve_docs.sh +docker logs --follow ${CONTAINER_NAME} \ No newline at end of file diff --git a/scripts/build_sphinx.py b/scripts/build_sphinx.py index 34c0dea..3127cad 100755 --- a/scripts/build_sphinx.py +++ b/scripts/build_sphinx.py @@ -1,3 +1,4 @@ +#!/bin/python2 # Copyright 2019 Autodesk, Inc. All rights reserved. # # Use of this software is subject to the terms of the Autodesk license agreement diff --git a/scripts/serve_docs.sh b/scripts/serve_docs.sh new file mode 100755 index 0000000..25bb05a --- /dev/null +++ b/scripts/serve_docs.sh @@ -0,0 +1,132 @@ +#!/bin/bash +# Copyright 2019 Autodesk, Inc. All rights reserved. +# +# Use of this software is subject to the terms of the Autodesk license agreement +# provided at the time of installation or download, or which otherwise accompanies +# this software in either electronic or hard copy form. +# + +# Usage: serve_docs.sh +# +# Build and serve documentation using "jekyll serve". +# +# Although this script does not take in arguments directly, certain environment +# variables do change certain settings and paths. +# +# Environment Variables: +# +# SOURCE: Directory to look for documentation to generate from. +# Defaults to current working directory +# (e.g. from "docker run -w DIR DOCKER_IMAGE") +# +# OUTPUT: Directory to output generated HTML and assets in. +# Defaults to "_site" folder under current working directory. +# +# EXPOSED_PORT: Port on host to expose website from. +# Defaults to 4000. +# +# URLPATH: Base directory (under root/domain) to host site on. +# Defaults to /, which resolves to domain root +# e.g. "localhost:4000/". +# + +# exit on error +set -eu -o pipefail + +SOURCE=${SOURCE:-$(pwd)} +OUTPUT=${OUTPUT:-$(pwd)/_site} +EXPOSED_PORT=${EXPOSED_PORT:-4000} +URLPATH=${URLPATH:-/} +echo "---------------------------------------------------" +echo "Shotgun Ecosystem Documentation Build Process" +echo "---------------------------------------------------" +echo " Source: ${SOURCE}" +echo " Output: ${OUTPUT}" +echo " URL (localhost): $(hostname -I):${EXPOSED_PORT}/${URLPATH}" +echo "---------------------------------------------------" + +# contains Gemfile, jekyll, etc +TK_DOC_GEN_SRC="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/.. >/dev/null 2>&1 && pwd )" +OUR_REPO_ROOT="$(readlink -m ${SOURCE}/..)" # Typically user's project root folder + +TMP_FOLDER="${TK_DOC_GEN_SRC}/_doc_generator_tmp" +TMP_BUILD_FOLDER="${TMP_FOLDER}/markdown_src" + +echo +echo "Cleaning out and creating new internal build location '${TMP_FOLDER}'..." +rm -rf ${TMP_FOLDER} +mkdir -p ${TMP_FOLDER} + +echo "Cleaning out final build location '${OUTPUT}'..." +rm -rf ${OUTPUT} + +echo "Synlinking source files into '${TMP_BUILD_FOLDER}'..." +# In case build_sphinx.py generates additional junk that's not part of SOURCE/* +for SRC_PATH in ${SOURCE}/* +do + ln -rsf ${SRC_PATH} ${TMP_BUILD_FOLDER} +done + +echo "Running Sphinx RST -> Markdown build process..." +build_sphinx.py ${TMP_BUILD_FOLDER} + +# Setup additional plugin paths +PLUGINS="${TK_DOC_GEN_SRC}/jekyll/_plugins" +EXTRA_PLUGIN=${TMP_BUILD_FOLDER}/_plugins +if [ -d "${EXTRA_PLUGIN}" ] +then + echo "Using additional plugins from ${EXTRA_PLUGIN}..." + PLUGINS="${PLUGINS},${EXTRA_PLUGIN}" +fi + +# Setup additional config overrides +CONFIGS="${TK_DOC_GEN_SRC}/jekyll/_config.yml" +OVERRIDE_CONFIG=${OUR_REPO_ROOT}/jekyll_config.yml +if [ -e "${OVERRIDE_CONFIG}" ] +then + echo "Using override config from ${OVERRIDE_CONFIG}..." + CONFIGS="${CONFIGS},${OVERRIDE_CONFIG}" +fi + +echo "Running Jekyll to generate html from markdown..." +echo "---------------------------------------------------" +umask 0000 +BUNDLE_GEMFILE=${TK_DOC_GEN_SRC}/Gemfile JEKYLL_ENV=production \ + # --detach \ +bundle exec jekyll serve \ + --detach \ + --baseurl "${URLPATH}" \ + --host $(hostname -I) --port "${EXPOSED_PORT}" \ + --trace \ + --config "${CONFIGS}" \ + --plugins "${PLUGINS}" \ + --source "${TMP_BUILD_FOLDER}" \ + --destination "${OUTPUT}" + +# Setup URLPATH for localhost formatting ("docker logs" strips empty lines) +URLPATH="${URLPATH#/}" +cat << EOF +########################################################################### +# +# Documentation built. Open in web-browser: +# http://localhost:${EXPOSED_PORT}/${URLPATH}${URLPATH:+/} +# +########################################################################### +| +|>> This terminal is currently viewing the live log. +| CTRL+C to exit viewing this log. +| +|>> You may also want to run this every now and then if you have rst files: +| docker exec $(hostname) build_sphinx.py ${TMP_BUILD_FOLDER} +| +--------------------------------------------------------------------------- +| +| To ACTUALLY stop the jekyll server running in Docker: +| docker stop $(hostname) +| +--------------------------------------------------------------------------- +EOF +while pgrep -f jekyll &> /dev/null +do + sleep 2s +done \ No newline at end of file From 14868992afdab63d02d52451b5b2810b1ed9df3b Mon Sep 17 00:00:00 2001 From: Joseph Yu Date: Thu, 6 Jun 2019 19:45:54 +0100 Subject: [PATCH 2/5] Added missing get-tk-core-packages.sh --- scripts/get-tk-core-packages.sh | 85 +++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100755 scripts/get-tk-core-packages.sh diff --git a/scripts/get-tk-core-packages.sh b/scripts/get-tk-core-packages.sh new file mode 100755 index 0000000..b966142 --- /dev/null +++ b/scripts/get-tk-core-packages.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +# Copyright 2019 Autodesk, Inc. All rights reserved. +# +# Use of this software is subject to the terms of the Autodesk license agreement +# provided at the time of installation or download, or which otherwise accompanies +# this software in either electronic or hard copy form. +# + +set -eu -o pipefail + +usage() { +cat << EOF +Usage: get-tk-core-packages.sh [-h] FOLDER + +Downloads the latest (tagged) version of tk-core into a given FOLDER. +If FOLDER does not exist, it will try create it. + +Options: + -h, --help Print this help and exit 0 + +Example: + To download the latest sgtk, tank and tank_vendor package folders into + /usr/local/lib64/python, run: + + get-tk-core-packages.sh /usr/local/lib64/python + + This should result in a sgtk, tank and tank_vendor directly under + /usr/local/lib64/python. +EOF +} + +if [ $# -ne 1 ] +then + echo 'ERROR: Please provide a folder to download/install into!' + echo + usage + exit 1 +fi + +case $1 in + -h|--help) + usage + exit + ;; +esac + +# Try create folder if it does not currently exist +FOLDER="$1" +[ -d "${FOLDER}" ] || mkdir -vp "${FOLDER}" + +curl_untar() { + # GitHub API to get all tags (seems to be latest first/at top of file) + TAGS_FILE="$(mktemp)" + curl -s "https://api.github.com/repos/shotgunsoftware/tk-core/tags" \ + | tee ${TAGS_FILE} + + # Grab first version (tag name), extract sgtk, tank and tank_vendor packages + VERSION=$(grep --color=never -m 1 -o -P '(?<="name": ")[^"]+' ${TAGS_FILE}) + + echo "Downloading and extracting tk-core ${VERSION} into: ${FOLDER}" + curl -Ls https://github.com/shotgunsoftware/tk-core/archive/${VERSION}.tar.gz \ + | tar --directory "${FOLDER}" \ + --extract --gzip --file=- \ + --exclude='*tests*' --wildcards --wildcards-match-slash \ + --strip-components=2 '*python*' +} + +temp_repo() { + # Clone into a temporary directory + TEMP_REPO="$(mktemp -d)" + pushd "${TEMP_REPO}" + git clone --no-checkout https://github.com/shotgunsoftware/tk-core.git . + + # Checkout latest version, grab the files/folders under python folder + VERSION="$(git for-each-ref --sort=taggerdate --format '%(tag)' | tail -1)" + echo "Cloned and extracting tk-core ${VERSION} into: ${FOLDER}" + git checkout "${VERSION}" + mv python/* "${FOLDER}" + popd +} + +# Try faster curl_untar first before falling back to temporary clone method +curl_untar || temp_repo +ls -lh "${FOLDER}" \ No newline at end of file From 6f035a41d34c810111c06620e2ff57a0b3db9284 Mon Sep 17 00:00:00 2001 From: Joseph Yu Date: Thu, 6 Jun 2019 19:46:07 +0100 Subject: [PATCH 3/5] It works again! --- Dockerfile | 3 ++ preview_docs.sh | 50 ++++++++++++++++++++++++------ scripts/serve_docs.sh | 71 +++++++++---------------------------------- 3 files changed, 58 insertions(+), 66 deletions(-) diff --git a/Dockerfile b/Dockerfile index ef018cb..026b11e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -82,4 +82,7 @@ ENV PATH=${PATH}:/usr/local/src/scripts RUN get-tk-core-packages.sh /usr/local/lib64/python ENV PYTHONPATH=${PYTHONPATH}:/usr/local/lib64/python +RUN mkdir -vp _doc_generator_tmp/markdown_src +VOLUME [ "$(pwd)/_doc_generator_tmp/markdown_src" ] + ENTRYPOINT [ "serve_docs.sh" ] \ No newline at end of file diff --git a/preview_docs.sh b/preview_docs.sh index 306b607..c918a33 100755 --- a/preview_docs.sh +++ b/preview_docs.sh @@ -93,8 +93,10 @@ URL_PATH="${1:-$( if (echo "${URL_PATH}" | grep -qP '^[a-zA-Z0-9][a-zA-Z0-9_.-]*$') then - CONTAINER_NAME="${URL_PATH}-${EXPOSED_PORT}" - until docker run --rm -d \ + while true + do + CONTAINER_NAME="${URL_PATH}-${EXPOSED_PORT}" + CONTAINER_ID=$(docker run --rm -d \ --hostname="${CONTAINER_NAME}" \ --name="${CONTAINER_NAME}" \ -p "${EXPOSED_PORT}:${EXPOSED_PORT}" \ @@ -103,10 +105,9 @@ then -e SOURCE="${MOUNT_TO}/docs" \ -e OUTPUT="${MOUNT_TO}/_build" \ ${MOUNT_FLAGS} \ - ${IMAGE_TAG} - do - EXPOSED_PORT="$(( EXPOSED_PORT + 1 ))" - CONTAINER_NAME="${URL_PATH}-${EXPOSED_PORT}" + ${IMAGE_TAG}) \ + && break \ + || EXPOSED_PORT="$(( EXPOSED_PORT + 1 ))" done else echo "Invalid URL_PATH to host as: '${URL_PATH}'" @@ -133,7 +134,6 @@ else CONTAINER_ID=$(docker run --rm -d \ -p "${EXPOSED_PORT}:${EXPOSED_PORT}" \ -e EXPOSED_PORT="${EXPOSED_PORT}" \ - -e URLPATH="/" \ -e SOURCE="${MOUNT_TO}/docs" \ -e OUTPUT="${MOUNT_TO}/_build" \ ${MOUNT_FLAGS} \ @@ -143,6 +143,36 @@ else done CONTAINER_NAME=$(docker ps -f id=${CONTAINER_ID} --format '{{.Names}}') fi - -# Directly calling log follow here to show any errors in serve_docs.sh -docker logs --follow ${CONTAINER_NAME} \ No newline at end of file +echo "Waiting for container to startup: ${CONTAINER_NAME}" +echo +echo + +# To prevent error in "docker logs" writing out blank logs file +# we use a buffer file so: docker logs -> buffer -> log file +TMP_LOG=$(mktemp) +TMP_LOG_BUFFER=$(mktemp) +while + docker inspect ${CONTAINER_ID} &> /dev/null \ + && docker logs ${CONTAINER_ID} > ${TMP_LOG_BUFFER} +do + cp -f ${TMP_LOG_BUFFER} ${TMP_LOG} + grep -q "Server running... press ctrl-c to stop." ${TMP_LOG} && break +done + +# Output logs so users can see if it failed during startup +cat ${TMP_LOG} + +# Output "Open in web-browser" if the serve script actually succeeded +docker inspect ${CONTAINER_ID} &> /dev/null && cat << EOF +############################################################################### + + Documentation built. Open in web-browser: + + http://localhost:${EXPOSED_PORT}/${URL_PATH}${URL_PATH:+/} + + VIEW logs: docker logs -f ${CONTAINER_NAME} +CONVERT sphinx rst: docker exec ${CONTAINER_NAME} build_sphinx.py ${MOUNT_TO}/docs + CHECK if running: docker ps --filter name=${CONTAINER_NAME} + STOP container: docker stop ${CONTAINER_NAME} + +EOF \ No newline at end of file diff --git a/scripts/serve_docs.sh b/scripts/serve_docs.sh index 25bb05a..ad7c834 100755 --- a/scripts/serve_docs.sh +++ b/scripts/serve_docs.sh @@ -33,46 +33,25 @@ # exit on error set -eu -o pipefail +# Set defaults if no env vars detected SOURCE=${SOURCE:-$(pwd)} OUTPUT=${OUTPUT:-$(pwd)/_site} EXPOSED_PORT=${EXPOSED_PORT:-4000} URLPATH=${URLPATH:-/} -echo "---------------------------------------------------" -echo "Shotgun Ecosystem Documentation Build Process" -echo "---------------------------------------------------" -echo " Source: ${SOURCE}" -echo " Output: ${OUTPUT}" -echo " URL (localhost): $(hostname -I):${EXPOSED_PORT}/${URLPATH}" -echo "---------------------------------------------------" # contains Gemfile, jekyll, etc TK_DOC_GEN_SRC="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/.. >/dev/null 2>&1 && pwd )" OUR_REPO_ROOT="$(readlink -m ${SOURCE}/..)" # Typically user's project root folder -TMP_FOLDER="${TK_DOC_GEN_SRC}/_doc_generator_tmp" -TMP_BUILD_FOLDER="${TMP_FOLDER}/markdown_src" - -echo -echo "Cleaning out and creating new internal build location '${TMP_FOLDER}'..." -rm -rf ${TMP_FOLDER} -mkdir -p ${TMP_FOLDER} - echo "Cleaning out final build location '${OUTPUT}'..." rm -rf ${OUTPUT} -echo "Synlinking source files into '${TMP_BUILD_FOLDER}'..." -# In case build_sphinx.py generates additional junk that's not part of SOURCE/* -for SRC_PATH in ${SOURCE}/* -do - ln -rsf ${SRC_PATH} ${TMP_BUILD_FOLDER} -done - echo "Running Sphinx RST -> Markdown build process..." -build_sphinx.py ${TMP_BUILD_FOLDER} +build_sphinx.py ${SOURCE} # Setup additional plugin paths PLUGINS="${TK_DOC_GEN_SRC}/jekyll/_plugins" -EXTRA_PLUGIN=${TMP_BUILD_FOLDER}/_plugins +EXTRA_PLUGIN=${SOURCE}/_plugins if [ -d "${EXTRA_PLUGIN}" ] then echo "Using additional plugins from ${EXTRA_PLUGIN}..." @@ -90,43 +69,23 @@ fi echo "Running Jekyll to generate html from markdown..." echo "---------------------------------------------------" +echo " Source: ${SOURCE}" +echo " Output: ${OUTPUT}" +echo " Plugins: ${PLUGINS}" +echo " Configurations: ${CONFIGS}" +echo " URL (localhost): localhost:${EXPOSED_PORT}${URLPATH}" +echo "---------------------------------------------------" umask 0000 +echo "$(hostname -I)localhost" >> /etc/hosts + +# Auto-regeneration will be disabled if running server detached. + # --verbose \ BUNDLE_GEMFILE=${TK_DOC_GEN_SRC}/Gemfile JEKYLL_ENV=production \ - # --detach \ bundle exec jekyll serve \ - --detach \ --baseurl "${URLPATH}" \ - --host $(hostname -I) --port "${EXPOSED_PORT}" \ + --host localhost --port "${EXPOSED_PORT}" \ --trace \ --config "${CONFIGS}" \ --plugins "${PLUGINS}" \ - --source "${TMP_BUILD_FOLDER}" \ + --source "${SOURCE}" \ --destination "${OUTPUT}" - -# Setup URLPATH for localhost formatting ("docker logs" strips empty lines) -URLPATH="${URLPATH#/}" -cat << EOF -########################################################################### -# -# Documentation built. Open in web-browser: -# http://localhost:${EXPOSED_PORT}/${URLPATH}${URLPATH:+/} -# -########################################################################### -| -|>> This terminal is currently viewing the live log. -| CTRL+C to exit viewing this log. -| -|>> You may also want to run this every now and then if you have rst files: -| docker exec $(hostname) build_sphinx.py ${TMP_BUILD_FOLDER} -| ---------------------------------------------------------------------------- -| -| To ACTUALLY stop the jekyll server running in Docker: -| docker stop $(hostname) -| ---------------------------------------------------------------------------- -EOF -while pgrep -f jekyll &> /dev/null -do - sleep 2s -done \ No newline at end of file From 1c0af8ff906634c6cc83e947a3d272ff752ba0e6 Mon Sep 17 00:00:00 2001 From: Joseph Yu Date: Fri, 7 Jun 2019 17:54:23 +0100 Subject: [PATCH 4/5] Better S3 detection --- travis-generate-docs.py | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/travis-generate-docs.py b/travis-generate-docs.py index f7869e3..c3b7ccd 100755 --- a/travis-generate-docs.py +++ b/travis-generate-docs.py @@ -124,27 +124,30 @@ def main(): current_branch = os.environ.get("TRAVIS_BRANCH") inside_pr = os.environ.get("TRAVIS_PULL_REQUEST") != "false" - # first figure out i we are on master or in a PR. + # first figure out if we are on master or in a PR. if current_branch != "master" or inside_pr: - # we are in a PR. - log.info("Inside a pull request.") + log.info("Inside a pull request (or not master branch).") - # see if we have access to an AWS bucket - if "S3_BUCKET" in os.environ and "S3_WEB_URL" in os.environ: + # See if we have access to an AWS bucket + s3_bucket = os.getenv("S3_BUCKET", default=None) + target_url = os.getenv("S3_WEB_URL", default=None) + if s3_bucket and target_url: log.info("Detected AWS S3 bucket for preview workflow.") - - s3_bucket = os.environ["S3_BUCKET"] - target_url = os.environ["S3_WEB_URL"] - target_url_path = "/tk-doc-generator/{commit}".format( - commit=os.environ["TRAVIS_COMMIT"] - ) - + target_url_path = "/tk-doc-generator/{env[TRAVIS_COMMIT]}" + target_url_path = target_url_path.format(env=os.environ) else: log.warning("No S3_BUCKET and S3_WEB_URL detected in environment. " "No S3 preview will be generated") - s3_bucket = None - # enter dummy paths so we can at least build - # the docs to check for errors + + # Then try use DOC_* for target + target_url = os.getenv("DOC_URL", default=None) + target_url_path = os.getenv("DOC_PATH", default=None) + if target_url and target_url_path: + log.info("Using DOC_URL/PATH.") + else: + # Fall back to using root directly on dummy domain url + log.warning("Using dummy paths so we can at least build the docs " + "to check for errors") target_url = "https://dummy.url.com" target_url_path = "/" From 684d1daad6f4fe31b5452c90268fe9c53d8e06b8 Mon Sep 17 00:00:00 2001 From: Joseph Yu Date: Fri, 7 Jun 2019 18:06:47 +0100 Subject: [PATCH 5/5] Better S3 access detection --- travis-generate-docs.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/travis-generate-docs.py b/travis-generate-docs.py index c3b7ccd..ce40006 100755 --- a/travis-generate-docs.py +++ b/travis-generate-docs.py @@ -163,12 +163,14 @@ def main(): ) execute_external_command(doc_command) - if s3_bucket: + aws_access_key = os.getenv("AWS_S3_ACCESS_KEY", default=None) + aws_access_token = os.getenv("AWS_S3_ACCESS_TOKEN", default=None) + if s3_bucket and aws_access_key and aws_access_token: log.info("Uploading build result to S3...") s3_client = boto3.client( "s3", - aws_access_key_id=os.environ["AWS_S3_ACCESS_KEY"], - aws_secret_access_key=os.environ["AWS_S3_ACCESS_TOKEN"] + aws_access_key=aws_access_key, + aws_access_token=aws_access_token ) # note: skip the first slash when uploading to S3