diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 03144b48..4fbfb834 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -5,8 +5,7 @@
stages:
- lint
- test
- - prod-deployment
- - dev-deployment
+ - build-and-deploy
- release
variables:
diff --git a/.gitlab/docker-build.yml b/.gitlab/docker-build.yml
index aa578e97..ad83f135 100644
--- a/.gitlab/docker-build.yml
+++ b/.gitlab/docker-build.yml
@@ -1,81 +1,42 @@
# yamllint disable rule:line-length
---
-docker-build-dev:
+docker-build:
image: docker:latest
services:
- name: docker:dind
- stage: dev-deployment
+ stage: build-and-deploy
tags:
- docker_priv
- rules:
- - if: $CI_COMMIT_BRANCH == 'dev'
- environment:
- name: development
- before_script:
- - |
- apk --no-cache add jq
- MAJOR=$(cat app/config/version.json | jq '.major' )
- MINOR=$(cat app/config/version.json | jq '.minor' )
- SUB=$(cat app/config/version.json | jq '.sub' )
- - |
- apk --no-cache add curl
- latest_tag=$(curl -s https://api.github.com/repos/docker/buildx/releases/latest | sed -Ene '/^ *"tag_name": *"(v.+)",$/s//\1/p')
- echo "Using buildx version $latest_tag"
- curl -sSLo docker-buildx "https://github.com/docker/buildx/releases/download/$latest_tag/buildx-$latest_tag.linux-amd64"
- chmod a+x docker-buildx
- mkdir -p ~/.docker/cli-plugins
- mv docker-buildx ~/.docker/cli-plugins/docker-buildx
- docker version
- - docker run --rm --privileged aptman/qus -- -r
- - docker run --rm --privileged aptman/qus -s -- -p aarch64 x86_64
- - echo $CI_JOB_TOKEN | docker login -u "$CI_REGISTRY_USER" --password-stdin $CI_REGISTRY
- - echo $DOCKERHUB_TOKEN | docker login -u "$DOCKERHUB_USER" --password-stdin $DOCKERHUB_REGISTRY
- script:
- - |
- tag=":$CI_COMMIT_REF_SLUG"
- VERSION="${MAJOR}.${MINOR}.${SUB}"
- - |
- echo "Running on branch '$CI_COMMIT_BRANCH': tag = $tag"
- echo "Crafty Version: $VERSION"
- - docker context create tls-environment
- - docker buildx create --name zedBuilder --use tls-environment
- - docker buildx build
- --cache-from type=registry,ref="$CI_REGISTRY_IMAGE${tag}"
- --build-arg BUILDKIT_INLINE_CACHE=1
- --build-arg "BUILD_DATE=$(date +"%Y-%m-%dT%H:%M:%SZ")"
- --build-arg "BUILD_REF=${CI_COMMIT_SHA}"
- --build-arg "CRAFTY_VER=${VERSION}"
- --provenance false
- --tag "$CI_REGISTRY_IMAGE${tag}"
- --tag "arcadiatechnology/crafty-4${tag}"
- --platform linux/arm64/v8,linux/amd64
- --push .
- after_script:
- - |
- docker buildx rm zedBuilder && echo "Successfully Stopped builder instance" || echo "Failed to stop builder instance."
- docker context rm tls-environment || true
- echo "Please review multi-arch manifests are present:"
- docker buildx imagetools inspect "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG"
-docker-build-prod:
- image: docker:latest
- services:
- - name: docker:dind
- stage: prod-deployment
- tags:
- - docker_priv
rules:
- - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+ # Development branch
+ - if: $CI_COMMIT_BRANCH == 'dev'
+ variables:
+ ENVIRONMENT_NAME: "development"
+ DOCKER_TAGS: "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG arcadiatechnology/crafty-4:$CI_COMMIT_REF_SLUG"
+ # Production branch (main) when not scheduled
+ - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_PIPELINE_SOURCE != "schedule"
+ variables:
+ ENVIRONMENT_NAME: "production"
+ # We'll set the version in the script and append stable tags
+ DOCKER_TAGS: "" # We'll determine in script
+ # Scheduled nightly builds on main
+ - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+ variables:
+ ENVIRONMENT_NAME: "nightly"
+ DOCKER_TAGS: "$CI_REGISTRY_IMAGE:nightly arcadiatechnology/crafty-4:nightly"
+
environment:
- name: production
+ name: $ENVIRONMENT_NAME
+
before_script:
- |
- apk --no-cache add jq
- MAJOR=$(cat app/config/version.json | jq '.major' )
- MINOR=$(cat app/config/version.json | jq '.minor' )
- SUB=$(cat app/config/version.json | jq '.sub' )
+ apk --no-cache add jq curl
+ MAJOR=$(cat app/config/version.json | jq -r '.major')
+ MINOR=$(cat app/config/version.json | jq -r '.minor')
+ SUB=$(cat app/config/version.json | jq -r '.sub')
+ VERSION="${MAJOR}.${MINOR}.${SUB}"
- |
- apk --no-cache add curl
latest_tag=$(curl -s https://api.github.com/repos/docker/buildx/releases/latest | sed -Ene '/^ *"tag_name": *"(v.+)",$/s//\1/p')
echo "Using buildx version $latest_tag"
curl -sSLo docker-buildx "https://github.com/docker/buildx/releases/download/$latest_tag/buildx-$latest_tag.linux-amd64"
@@ -87,30 +48,61 @@ docker-build-prod:
- docker run --rm --privileged aptman/qus -s -- -p aarch64 x86_64
- echo $CI_JOB_TOKEN | docker login -u "$CI_REGISTRY_USER" --password-stdin $CI_REGISTRY
- echo $DOCKERHUB_TOKEN | docker login -u "$DOCKERHUB_USER" --password-stdin $DOCKERHUB_REGISTRY
+ - docker context create tls-environment-$CI_JOB_ID
+ - docker buildx create --name zedBuilder-$CI_JOB_ID --use tls-environment-$CI_JOB_ID
+
script:
+ - echo "Running on branch '$CI_COMMIT_BRANCH', environment- $ENVIRONMENT_NAME"
+ - echo "Crafty Version- $VERSION"
+
+ # If DOCKER_TAGS is empty (production default branch and not schedule), we set stable tags:
+ # This scenario adds both VERSION and 'latest' tags.
- |
- VERSION="${MAJOR}.${MINOR}.${SUB}"
+ if [ "$ENVIRONMENT_NAME" = "production" ] && [ -z "$DOCKER_TAGS" ]; then
+ DOCKER_TAGS="$CI_REGISTRY_IMAGE:${VERSION} \
+ $CI_REGISTRY_IMAGE:latest \
+ arcadiatechnology/crafty-4:${VERSION} \
+ arcadiatechnology/crafty-4:latest"
+ fi
+
+ - echo "Using the following tags- $DOCKER_TAGS"
+
+ # Prepare build command
+ # We break DOCKER_TAGS into separate --tag arguments
+ - TARGS=""
+ - for t in $DOCKER_TAGS; do TARGS="$TARGS --tag $t"; done
+ - echo "Tag arguments- $TARGS"
+
+ # Conditional build command: omit cache if schedule
- |
- echo "Running on branch '$CI_COMMIT_BRANCH'"
- echo "Crafty Version: $VERSION"
- - docker context create tls-environment
- - docker buildx create --name zedBuilder --use tls-environment
- - docker buildx build
- --cache-from type=registry,ref="$CI_REGISTRY_IMAGE:latest"
- --build-arg BUILDKIT_INLINE_CACHE=1
- --build-arg "BUILD_DATE=$(date +"%Y-%m-%dT%H:%M:%SZ")"
- --build-arg "BUILD_REF=${CI_COMMIT_SHA}"
- --build-arg "CRAFTY_VER=${VERSION}"
- --provenance false
- --tag "$CI_REGISTRY_IMAGE:$VERSION"
- --tag "$CI_REGISTRY_IMAGE:latest"
- --tag "arcadiatechnology/crafty-4:$VERSION"
- --tag "arcadiatechnology/crafty-4:latest"
- --platform linux/arm64/v8,linux/amd64
- --push .
+ if [ "$CI_PIPELINE_SOURCE" = "schedule" ]; then
+ echo "Omitting cache for nightly build."
+ docker buildx build \
+ --build-arg "BUILD_DATE=$(date +"%Y-%m-%dT%H:%M:%SZ")" \
+ --build-arg "BUILD_REF=${CI_COMMIT_SHA}" \
+ --build-arg "CRAFTY_VER=${VERSION}" \
+ --provenance false \
+ $TARGS \
+ --platform linux/arm64/v8,linux/amd64 \
+ --push .
+ else
+ echo "Using cache for build."
+ docker buildx build \
+ --cache-from type=registry,ref="$CI_REGISTRY_IMAGE:dev" \
+ --build-arg BUILDKIT_INLINE_CACHE=1 \
+ --build-arg "BUILD_DATE=$(date +"%Y-%m-%dT%H:%M:%SZ")" \
+ --build-arg "BUILD_REF=${CI_COMMIT_SHA}" \
+ --build-arg "CRAFTY_VER=${VERSION}" \
+ --provenance false \
+ $TARGS \
+ --platform linux/arm64/v8,linux/amd64 \
+ --push .
+ fi
+
after_script:
- - |
- docker buildx rm zedBuilder && echo "Successfully Stopped builder instance" || echo "Failed to stop builder instance."
- docker context rm tls-environment || true
- echo "Please review multi-arch manifests are present:"
- docker buildx imagetools inspect "$CI_REGISTRY_IMAGE${tag}"
+ - docker buildx rm zedBuilder-$CI_JOB_ID && echo "Successfully Stopped builder instance" || echo "Failed to stop builder instance."
+ - docker context rm tls-environment-$CI_JOB_ID || true
+ - echo "Please review multi-arch manifests are present:"
+ - if [ "$ENVIRONMENT_NAME" = "development" ]; then docker buildx imagetools inspect "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG"; fi
+ - if [ "$ENVIRONMENT_NAME" = "production" ] && [ -n "$VERSION" ]; then docker buildx imagetools inspect "$CI_REGISTRY_IMAGE:$VERSION"; fi
+ - if [ "$ENVIRONMENT_NAME" = "nightly" ]; then docker buildx imagetools inspect "$CI_REGISTRY_IMAGE:nightly"; fi
diff --git a/.gitlab/lint.yml b/.gitlab/lint.yml
index 5cfae504..db223801 100644
--- a/.gitlab/lint.yml
+++ b/.gitlab/lint.yml
@@ -9,6 +9,8 @@ yamllint:
rules:
- if: "$CODE_QUALITY_DISABLED"
when: never
+ - if: $CI_PIPELINE_SOURCE == "schedule"
+ when: never
- if: "$CI_COMMIT_TAG || $CI_COMMIT_BRANCH"
script:
- yamllint .
@@ -22,6 +24,8 @@ jsonlint:
rules:
- if: "$CODE_QUALITY_DISABLED"
when: never
+ - if: $CI_PIPELINE_SOURCE == "schedule"
+ when: never
- if: "$CI_COMMIT_TAG || $CI_COMMIT_BRANCH"
script:
- |
@@ -37,6 +41,8 @@ black:
rules:
- if: "$CODE_QUALITY_DISABLED"
when: never
+ - if: $CI_PIPELINE_SOURCE == "schedule"
+ when: never
- if: "$CI_COMMIT_TAG || $CI_COMMIT_BRANCH"
script:
- black --check --verbose -- .
@@ -50,6 +56,8 @@ pylint:
rules:
- if: "$CODE_QUALITY_DISABLED"
when: never
+ - if: $CI_PIPELINE_SOURCE == "schedule"
+ when: never
- if: "$CI_COMMIT_TAG || $CI_COMMIT_BRANCH"
before_script:
- apk update
@@ -75,6 +83,8 @@ sonarcloud-check:
when: never
- if: "$CODE_QUALITY_DISABLED"
when: never
+ - if: $CI_PIPELINE_SOURCE == "schedule"
+ when: never
- if: "$CI_COMMIT_TAG || $CI_COMMIT_BRANCH"
variables:
SONAR_USER_HOME: "${CI_PROJECT_DIR}/.sonar" # Defines the location of the analysis task cache
@@ -94,6 +104,8 @@ sonarcloud-check:
# rules:
# - if: "$CODE_QUALITY_DISABLED"
# when: never
+# - if: $CI_PIPELINE_SOURCE == "schedule"
+# when: never
# - if: "$CI_COMMIT_TAG || $CI_COMMIT_BRANCH"
# allow_failure: true
# before_script:
diff --git a/.gitlab/release.yml b/.gitlab/release.yml
index f1ed61da..d9c05824 100644
--- a/.gitlab/release.yml
+++ b/.gitlab/release.yml
@@ -4,6 +4,8 @@ release:
stage: release
image: registry.gitlab.com/gitlab-org/release-cli:latest
rules:
+ - if: $CI_PIPELINE_SOURCE == "schedule"
+ when: never
- if: $CI_COMMIT_TAG
needs:
- job: win-prod-build
diff --git a/.gitlab/windows-build.yml b/.gitlab/windows-build.yml
index 2dd6fb50..a883e115 100644
--- a/.gitlab/windows-build.yml
+++ b/.gitlab/windows-build.yml
@@ -1,13 +1,15 @@
# yamllint disable rule:line-length
---
win-dev-build:
- stage: dev-deployment
+ stage: build-and-deploy
tags:
- win64
cache:
paths:
- .venv/
rules:
+ - if: $CI_PIPELINE_SOURCE == "schedule"
+ when: never
- if: "$CI_COMMIT_BRANCH == 'dev'"
environment:
name: development
@@ -48,13 +50,15 @@ win-dev-build:
- app\classes\**\*
win-prod-build:
- stage: prod-deployment
+ stage: build-and-deploy
tags:
- win64
cache:
paths:
- .venv/
rules:
+ - if: $CI_PIPELINE_SOURCE == "schedule"
+ when: never
- if: "$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH"
- if: $CI_COMMIT_TAG
environment:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 477fb3fa..68261b23 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -4,13 +4,18 @@
TBD
### Refactor
- Refactor and standardize all JSON validator errors returning human readable translations ([Merge Request](https://gitlab.com/crafty-controller/crafty-4/-/merge_requests/786))
+- Improve docker-build CI/CD, supporting nightly builds ([Merge Request](https://gitlab.com/crafty-controller/crafty-4/-/merge_requests/813))
### Bug fixes
- Bump requests to resolve yank for CVE-2024-35195 ([Merge Request](https://gitlab.com/crafty-controller/crafty-4/-/merge_requests/808))
### Tweaks
-TBD
+- Bump Docker base image `22.04` -> `24.04` ([Merge Request](https://gitlab.com/crafty-controller/crafty-4/-/merge_requests/812))
+- Bump python pip `2.0.3` -> `24.3.1` ([Merge Request](https://gitlab.com/crafty-controller/crafty-4/-/merge_requests/812))
+- Bump python setuptools `50.3.2` -> `75.6.0` ([Merge Request](https://gitlab.com/crafty-controller/crafty-4/-/merge_requests/812))
+- Bump tornado for CVE-2024-52804 ([Merge Request](https://gitlab.com/crafty-controller/crafty-4/-/merge_requests/814))
### Lang
- Weblate Translation Platform Integration
- Remove incomplete labels from translation files to better support new translation workflow ([Merge Request](https://gitlab.com/crafty-controller/crafty-4/-/merge_requests/810))
+- New langs added `ja_JP`, `ko_KR` ([Merge Request](https://gitlab.com/crafty-controller/crafty-4/-/merge_requests/811))
## --- [4.4.4] - 2024/10/03
diff --git a/Dockerfile b/Dockerfile
index d05cf3d0..8c16e3a3 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,4 +1,4 @@
-FROM ubuntu:22.04
+FROM ubuntu:24.04
ENV DEBIAN_FRONTEND="noninteractive"
@@ -36,7 +36,7 @@ WORKDIR /crafty
COPY --chown=crafty:root requirements.txt ./
RUN python3 -m venv ./.venv \
&& . .venv/bin/activate \
- && pip3 install --no-cache-dir --upgrade setuptools==50.3.2 pip==22.0.3 \
+ && pip3 install --no-cache-dir --upgrade setuptools==75.6.0 pip==24.3.1 \
&& pip3 install --no-cache-dir -r requirements.txt \
&& deactivate
USER root
diff --git a/requirements.txt b/requirements.txt
index 7bc5ae87..b01144c8 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -15,7 +15,7 @@ pyjwt==2.8.0
PyYAML==6.0.1
requests==2.32.3
termcolor==1.1
-tornado==6.4.1
+tornado==6.4.2
tzlocal==5.1
jsonschema==4.19.1
orjson==3.9.15