Merge pull request #5468 from mishaschwartz/v1.13.0

v1.13.0

.dockerfiles/.env

@@ -7,7 +7,6 @@ BOOTSNAP_CACHE_DIR=/bundle/bootsnap
 WEBPACKER_DEV_SERVER_HOST=webpacker
 WEB_CONCURRENCY=1
 EDITOR=vi
-MARKUS__AUTOTEST__SERVER_HOST=autotest
-MARKUS__AUTOTEST__SERVER_USERNAME=docker
+MARKUS__AUTOTEST__URL=http://autotest-client:5000
 TZ=America/Toronto
 NBCONVERT=/app/nbconvertvenv/bin/jupyter-nbconvert

.dockerfiles/Dockerfile

@@ -7,45 +7,18 @@ ARG PG_MAJOR
 ARG NODE_MAJOR
 ARG BUNDLER_VERSION
 ARG YARN_VERSION
-ARG NBCONVERTVENV
+ARG PYTHON_VERSION
 ARG USER
 
-RUN apt-get update -qq && DEBIAN_FRONTEND=noninteractive apt-get -yq dist-upgrade && \
-    DEBIAN_FRONTEND=noninteractive apt-get install -yq --no-install-recommends curl \
-                                                                               ca-certificates \
-                                                                               gnupg2 \
-                                                                               software-properties-common
-
-# Add NodeJS to sources list
-RUN APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=1 \
-    curl -vsL https://deb.nodesource.com/setup_$NODE_MAJOR.x | bash -
-
-# Remove cmdtest which come pre-installed and has a package called yarn. This prevents us from installing
-# the yarn package we want otherwise.
-RUN DEBIAN_FRONTEND=noninteractive apt-get remove -yq cmdtest
-
-# Add Yarn to the sources list
-RUN APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=1 \
-  curl -vsL https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - \
-  && echo 'deb https://dl.yarnpkg.com/debian/ stable main' > /etc/apt/sources.list.d/yarn.list
-
-# Add ppa so that we can select different ruby versions
-RUN DEBIAN_FRONTEND=noninteractive apt-add-repository -y ppa:brightbox/ruby-ng
-
-# Add ppa so that we can get the right python version
-RUN DEBIAN_FRONTEND=noninteractive apt-add-repository -y ppa:deadsnakes/ppa
-
 # install packages
-RUN DEBIAN_FRONTEND=noninteractive apt-get install -yq --no-install-recommends \
+RUN apt-get update -qq && \
+    DEBIAN_FRONTEND=noninteractive apt-get install -yq --no-install-recommends \
     build-essential \
+    software-properties-common \
     postgresql-client-$PG_MAJOR \
     tzdata \
     libpq-dev \
-    nodejs \
-    yarn \
     libv8-dev \
-    ruby$RUBY_VERSION \
-    ruby${RUBY_VERSION}-dev \
     ruby-svn \
     ghostscript \
     imagemagick \
@@ -57,13 +30,33 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get install -yq --no-install-recommends \
     libssl-dev \
     swig \
     graphviz \
-    yarn \
     rsync \
     git \
-    python3.8 \
-    python3.8-venv \
-    pandoc && \
-    apt-get clean && \
+    python3-venv \
+    python3-dev \
+    pandoc
+
+# Install node
+ADD https://deb.nodesource.com/setup_$NODE_MAJOR.x /tmp/setup_node.sh
+RUN bash /tmp/setup_node.sh && DEBIAN_FRONTEND=noninteractive apt-get install -yq nodejs && rm /tmp/setup_node.sh
+
+# Install yarn
+RUN npm install --global yarn@${YARN_VERSION}
+
+# Add ppa so that we can select different ruby versions and install ruby
+RUN apt-add-repository -y ppa:brightbox/ruby-ng && \
+    apt-get update -qq && \
+    apt-get install -yq ruby$RUBY_VERSION ruby${RUBY_VERSION}-dev rubygems-integration && \
+    apt-add-repository -y --remove ppa:brightbox/ruby-ng
+
+# Add ppa so that we can get the right python version and install python
+RUN apt-add-repository -y ppa:deadsnakes/ppa && \
+    apt-get update -qq && apt-get install -yq --no-install-recommends python${PYTHON_VERSION} \
+                                                python${PYTHON_VERSION}-venv \
+                                                python${PYTHON_VERSION}-dev && \
+    apt-add-repository -y --remove ppa:deadsnakes/ppa
+
+RUN apt-get clean && \
     rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \
     truncate -s 0 /var/log/*log
 
@@ -72,8 +65,7 @@ RUN sed -ri 's/(rights=")none("\s+pattern="PDF")/\1read\2/' /etc/ImageMagick-6/p
 
 RUN useradd -m -s /bin/bash $USER
 
-# Install nbconvert for rendering .ipynb files
-RUN python3.8 -m venv $NBCONVERTVENV && $NBCONVERTVENV/bin/pip install wheel nbconvert
+RUN ln -fs $(which python${PYTHON_VERSION}) /usr/bin/python3
 
 ENV GEM_HOME="/bundle"
 ENV PATH="$GEM_HOME/bin:$GEM_HOME/gems/bin:$PATH"
@@ -94,7 +86,8 @@ RUN mkdir -p /bundle \
              /app/node_modules \
              /app/public/packs \
              /app/public/packs-test \
-             /app/log && \
+             /app/log \
+             /app/venv && \
              chown -R $USER /bundle /ssh_pub_key /app
 
 USER $USER
@@ -119,12 +112,17 @@ USER $USER
 COPY --chown=${USER}:${USER} . /app
 COPY --chown=${USER}:${USER} .dockerfiles/database.yml.postgresql /app/config/database.yml
 
+# install python packages
+RUN python${PYTHON_VERSION} -m venv /app/venv && \
+    /app/venv/bin/pip install nbconvert==6.1.0
+
 ENV RAILS_ENV=production
 ENV NODE_ENV=production
 
 # install bundler
 RUN gem$RUBY_VERSION install bundler -v $BUNDLER_VERSION
 
-RUN SECRET_KEY_BASE=1 bundle install --deployment --path $GEM_HOME --without development test offline mysql sqlite && \
-    SECRET_KEY_BASE=1 PGDATABASE=dummy bundle exec rails i18n:js:export && \
-    SECRET_KEY_BASE=1 PGDATABASE=dummy bundle exec rails assets:precompile
+RUN SECRET_KEY_BASE=1 bundle install --deployment --path $GEM_HOME --without development test offline mysql sqlite
+
+RUN SECRET_KEY_BASE=1 NO_SCHEMA_VALIDATE=true PGDATABASE=dummy bundle exec rails i18n:js:export && \
+    SECRET_KEY_BASE=1 NO_SCHEMA_VALIDATE=true PGDATABASE=dummy bundle exec rails assets:precompile

.dockerfiles/entrypoint-dev-rails.sh

@@ -6,6 +6,10 @@ bundle check 2>/dev/null || bundle install --without mysql sqlite unicorn
 # install yarn packages if not up to date with yarn.lock file
 yarn check --integrity 2>/dev/null || yarn install --check-files
 
+# install python packages
+python3 -m venv ./venv
+./venv/bin/pip install -r requirements.txt
+
 # setup the database (checks for db existence first)
 cp .dockerfiles/database.yml.postgresql config/database.yml
 until psql "postgres://${PGUSER}:${PGPASSWORD}@${PGHOST}:${PGPORT}/postgres" -lqt &>/dev/null; do
@@ -16,5 +20,7 @@ psql "postgres://${PGUSER}:${PGPASSWORD}@${PGHOST}:${PGPORT}" -lqt 2> /dev/null
 
 rm -f ./tmp/pids/server.pid
 
+bundle exec rails markus:setup_autotest
+
 # Then exec the container's main process (what's set as CMD in the Dockerfile or docker-compose.yml).
 exec "$@"

.dockerfiles/entrypoint-dev-resque.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+THISDIR="$(dirname "${BASH_SOURCE[0]}")"
+"${THISDIR}/entrypoint-dev-wait-for-install.sh"
+
+BACKGROUND=true bundle exec rails resque:scheduler
+exec "$@"

.dockerignore

@@ -9,6 +9,7 @@ public/
 
 config/database.yml
 config/local_environment_override.rb
+config/autotest.api_key
 
 # Files created by sass
 .sass-cache
@@ -70,9 +71,9 @@ coverage/
 # byebug gem
 .byebug_history
 
-# scanned exam python files
+# python files
 **/venv/
 **/__pycache__/
 
-# nbconvert python files
-**/nbconvertvenv/
+# If this file is pre-generated, for Windows and WSL 2 (see Wiki docker setup instructions)
+app/javascript/routes.js

.github/PULL_REQUEST_TEMPLATE.md

@@ -40,7 +40,7 @@
 
 - [ ] I have performed a self-review of my own code.
 - [ ] I have fixed any Hound bot comments. <!-- (check after opening pull request) -->
-- [ ] I have verified that the TravisCI tests have passed. <!-- (check after opening pull request) -->
+- [ ] I have verified that the CI tests have passed. <!-- (check after opening pull request) -->
 - [ ] I have reviewed the test coverage changes reported on Coveralls. <!-- (check after opening pull request) -->
 - [ ] I have added tests for my changes. <!-- (delete this checklist item if not applicable) -->
 - [ ] I have updated the Changelog.md file. <!-- (delete this checklist item if not applicable) -->

.github/dependabot.yml

@@ -0,0 +1,22 @@
+version: 2
+updates:
+- package-ecosystem: bundler
+  directory: "/"
+  schedule:
+    interval: monthly
+    time: "10:00"
+  open-pull-requests-limit: 10
+- package-ecosystem: npm
+  directory: "/"
+  schedule:
+    interval: monthly
+    time: "10:00"
+  open-pull-requests-limit: 10
+  ignore:
+  - dependency-name: react-table
+    versions:
+    - "> 6.9.2"
+  - dependency-name: webpack
+    versions:
+    - 5.19.0
+    - 5.24.2

.github/workflows/test_ci.yml

@@ -0,0 +1,90 @@
+name: MarkUs Tests
+on:
+  pull_request:
+    types: [opened, synchronize, reopened, ready_for_review]
+jobs:
+  test:
+    if: github.event.pull_request.draft == false
+    runs-on: ubuntu-20.04
+    services:
+      postgres:
+        image: postgres:12
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: markus_test
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+      redis:
+        image: redis
+        ports:
+          - 6379:6379
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+          --entrypoint redis-server
+    env:
+      BUNDLE_WITHOUT: development:production:console:mysql:sqlite:unicorn
+      RAILS_ENV: test
+      NODE_ENV: test
+      RSPEC_RENDER_VIEWS: true
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v2
+      - name: Install system dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get -yqq install libpq-dev cmake ghostscript pandoc imagemagick libmagickwand-dev git
+      - name: Set up ruby and cache gems
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ruby-2.7
+          bundler-cache: true
+      - name: Set up node/yarn and cache packages
+        uses: actions/setup-node@v2
+        with:
+          node-version: 12
+          cache: yarn
+      - name: Install yarn packages
+        run: yarn install
+      - name: Install python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.9
+      - name: Cache pip
+        uses: actions/cache@v2
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+      - name: Install python packages
+        run: |
+          python3.9 -m venv venv
+          ./venv/bin/pip install nbconvert==6.1.0 # install rest of requirements if needed for tests later
+      - name: Configure server
+        run: |
+          sudo rm -f /etc/localtime
+          sudo ln -s /usr/share/zoneinfo/US/Eastern /etc/localtime
+          sudo sed -ri 's/(rights=")none("\s+pattern="PDF")/\1read\2/' /etc/ImageMagick-6/policy.xml
+          cp config/database.yml.ci config/database.yml
+      - name: Build assets
+        run: |
+          bundle exec rake i18n:js:export &&
+          chmod u+x bin/webpack &&
+          bundle exec rake webpacker:compile
+      - name: Set up database
+        run: bundle exec rails db:migrate
+      - name: Run rspec tests
+        run: bundle exec rspec
+      - name: Coveralls
+        uses: coverallsapp/[email protected]
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -74,22 +74,22 @@ dump.rdb
 yarn-debug.log*
 .yarn-integrity
 yarn-error.log
+# If this file is pre-generated, for Windows and WSL 2 (see Wiki docker setup instructions)
+app/javascript/routes.js
 
 # simplecov gem
 coverage/
 
 # byebug gem
 .byebug_history
 
-# scanned exam python files
+# python files
 venv/
 __pycache__/
 
-# nbconvert python files
-nbconvertvenv/
-
 config/settings.local.yml
 config/settings/*.local.yml
 config/environments/*.local.yml
+config/autotest.api_key
 
 lib/repo/git_hooks/max_file_size