NussNougate/crafty-4
2
0
Fork 0
mirror of https://gitlab.com/crafty-controller/crafty-4.git synced 2025-01-19 17:55:29 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'devops/update-master-pipes' into 'master'

Browse Source 
Devops/update master pipes

See merge request crafty-controller/crafty-4!275
This commit is contained in:
Iain Powrie 2022-05-25 01:01:35 +00:00
commit 1bbe6fbae3
1 changed files with 167 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

240
.gitlab-ci.yml
View File

@ -1,35 +1,77 @@
# Crafty Controller 4.0 - Lint & Build Pipes
# [Maintainer: Zedifus(https://gitlab.com/Zedifus)]
###################################################
# yamllint disable rule:line-length
---
stages: stages:
- test - lint
- prod-deployment - test
- dev-deployment - prod-deployment
- dev-deployment
variables: variables:
DOCKER_HOST: tcp://docker:2376 DOCKER_HOST: tcp://docker:2376
DOCKER_TLS_CERTDIR: "/certs" DOCKER_TLS_CERTDIR: "/certs"
pylint: yamllint:
stage: test stage: lint
image: python:3.7-slim image: registry.gitlab.com/pipeline-components/yamllint:latest
services:
- name: docker:dind
tags: tags:
- 'docker_testers' - docker
rules: rules:
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"' - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
- if: '$CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS' - if: "$CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS"
when: never
script:
- yamllint .
jsonlint:
stage: lint
image: registry.gitlab.com/pipeline-components/jsonlint:latest
tags:
- docker
rules:
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
- if: "$CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS"
when: never when: never
before_script:
- mkdir -p public/badges public/lint
- echo undefined > public/badges/$CI_JOB_NAME.score
- pip install pylint-gitlab
script: script:
- pylint --exit-zero --output-format=text $(find -type f -name "*.py" ! -path "**/.venv/**" ! -path "**/app/migrations/**") | tee /tmp/pylint.txt
- sed -n 's/^Your code has been rated at \([-0-9.]*\)\/.*/\1/p' /tmp/pylint.txt > public/badges/$CI_JOB_NAME.score
- pylint --exit-zero --output-format=pylint_gitlab.GitlabCodeClimateReporter $(find -type f -name "*.py" ! -path "**/.venv/**" ! -path "**/app/migrations/**") > codeclimate.json
after_script:
- anybadge --overwrite --label $CI_JOB_NAME --value=$(cat public/badges/$CI_JOB_NAME.score) --file=public/badges/$CI_JOB_NAME.svg 4=red 6=orange 8=yellow 10=green
- | - |
echo "Your score is: $(cat public/badges/$CI_JOB_NAME.score)" find . -not -path './.git/*' -name '*.json' -type f -print0 |
parallel --will-cite -k -0 -n1 jsonlint -q
black:
stage: lint
image: registry.gitlab.com/pipeline-components/black:latest
tags:
- docker
rules:
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
- if: "$CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS"
when: never
script:
- black --check --verbose -- .
pylint:
stage: lint
image: registry.gitlab.com/pipeline-components/pylint:latest
tags:
- docker
rules:
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
- if: "$CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS"
when: never
# before_script:
# - mkdir -p public/badges public/lint
# - echo undefined > public/badges/$CI_JOB_NAME.score
script:
# - pylint --exit-zero --output-format=text $(find -type f -name "*.py" ! -path "**/.venv/**" ! -path "**/app/migrations/**") | tee /tmp/pylint.txt
# - sed -n 's/^Your code has been rated at \([-0-9.]*\)\/.*/\1/p' /tmp/pylint.txt > public/badges/$CI_JOB_NAME.score
- pylint --exit-zero --output-format=pylint_gitlab.GitlabCodeClimateReporter $(find -type f -name "*.py" ! -path "**/.venv/**" ! -path "**/app/migrations/**") > codeclimate.json
# after_script:
# - anybadge --overwrite --label $CI_JOB_NAME --value=$(cat public/badges/$CI_JOB_NAME.score) --file=public/badges/$CI_JOB_NAME.svg 4=red 6=orange 8=yellow 10=green
# - |
# echo "Your score is: $(cat public/badges/$CI_JOB_NAME.score)"
# Removed lint badge generation until public release
artifacts: artifacts:
paths: paths:
- public - public
@ -43,12 +85,18 @@ docker-build-dev:
- name: docker:dind - name: docker:dind
stage: dev-deployment stage: dev-deployment
tags: tags:
- docker - docker_priv
rules: rules:
- if: $CI_COMMIT_BRANCH == 'dev' - if: $CI_COMMIT_BRANCH == 'dev'
environment: environment:
name: development name: development
before_script: before_script:
- |
apk --no-cache add jq
MAJOR=$(cat app/config/version.json | jq '.major' )
MINOR=$(cat app/config/version.json | jq '.minor' )
SUB=$(cat app/config/version.json | jq '.sub' )
META=$(cat app/config/version.json | jq -r '.meta' )
- | - |
apk --no-cache add curl apk --no-cache add curl
latest_tag=$(curl -s https://api.github.com/repos/docker/buildx/releases/latest | sed -Ene '/^ *"tag_name": *"(v.+)",$/s//\1/p') latest_tag=$(curl -s https://api.github.com/repos/docker/buildx/releases/latest | sed -Ene '/^ *"tag_name": *"(v.+)",$/s//\1/p')
@ -64,12 +112,18 @@ docker-build-dev:
script: script:
- | - |
tag=":$CI_COMMIT_REF_SLUG" tag=":$CI_COMMIT_REF_SLUG"
VERSION="${MAJOR}.${MINOR}.${SUB}-${META}"
- |
echo "Running on branch '$CI_COMMIT_BRANCH': tag = $tag" echo "Running on branch '$CI_COMMIT_BRANCH': tag = $tag"
echo "Crafty Version: $VERSION"
- docker context create tls-environment - docker context create tls-environment
- docker buildx create --name zedBuilder --use tls-environment - docker buildx create --name zedBuilder --use tls-environment
- docker buildx build - docker buildx build
--cache-from type=registry,ref="$CI_REGISTRY_IMAGE${tag}" --cache-from type=registry,ref="$CI_REGISTRY_IMAGE${tag}"
--build-arg BUILDKIT_INLINE_CACHE=1 --build-arg BUILDKIT_INLINE_CACHE=1
--build-arg "BUILD_DATE=$(date +"%Y-%m-%dT%H:%M:%SZ")"
--build-arg "BUILD_REF=${CI_COMMIT_SHA}"
--build-arg "CRAFTY_VER=${VERSION}"
--tag "$CI_REGISTRY_IMAGE${tag}" --tag "$CI_REGISTRY_IMAGE${tag}"
--platform linux/arm64/v8,linux/amd64 --platform linux/arm64/v8,linux/amd64
--push . --push .
@ -86,12 +140,18 @@ docker-build-prod:
- name: docker:dind - name: docker:dind
stage: prod-deployment stage: prod-deployment
tags: tags:
- docker - docker_priv
rules: rules:
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
environment: environment:
name: production name: production
before_script: before_script:
- |
apk --no-cache add jq
MAJOR=$(cat app/config/version.json | jq '.major' )
MINOR=$(cat app/config/version.json | jq '.minor' )
SUB=$(cat app/config/version.json | jq '.sub' )
META=$(cat app/config/version.json | jq -r '.meta' )
- | - |
apk --no-cache add curl apk --no-cache add curl
latest_tag=$(curl -s https://api.github.com/repos/docker/buildx/releases/latest | sed -Ene '/^ *"tag_name": *"(v.+)",$/s//\1/p') latest_tag=$(curl -s https://api.github.com/repos/docker/buildx/releases/latest | sed -Ene '/^ *"tag_name": *"(v.+)",$/s//\1/p')
@ -107,12 +167,18 @@ docker-build-prod:
script: script:
- | - |
tag="" tag=""
echo "Running on default branch '$CI_DEFAULT_BRANCH': tag = 'latest'" VERSION="${MAJOR}.${MINOR}.${SUB}-${META}"
- |
echo "Running on branch '$CI_COMMIT_BRANCH': tag = $tag"
echo "Crafty Version: $VERSION"
- docker context create tls-environment - docker context create tls-environment
- docker buildx create --name zedBuilder --use tls-environment - docker buildx create --name zedBuilder --use tls-environment
- docker buildx build - docker buildx build
--cache-from type=registry,ref="$CI_REGISTRY_IMAGE${tag}" --cache-from type=registry,ref="$CI_REGISTRY_IMAGE${tag}"
--build-arg BUILDKIT_INLINE_CACHE=1 --build-arg BUILDKIT_INLINE_CACHE=1
--build-arg "BUILD_DATE=$(date +"%Y-%m-%dT%H:%M:%SZ")"
--build-arg "BUILD_REF=${CI_COMMIT_SHA}"
--build-arg "CRAFTY_VER=${VERSION}"
--tag "$CI_REGISTRY_IMAGE${tag}" --tag "$CI_REGISTRY_IMAGE${tag}"
--platform linux/arm64/v8,linux/amd64 --platform linux/arm64/v8,linux/amd64
--push . --push .
@ -126,81 +192,109 @@ docker-build-prod:
win-dev-build: win-dev-build:
stage: dev-deployment stage: dev-deployment
tags: tags:
- win64 - win64
cache: cache:
paths: paths:
- .venv/ - .venv/
rules: rules:
- if: "$CI_COMMIT_BRANCH == 'dev'" - if: "$CI_COMMIT_BRANCH == 'dev'"
environment: environment:
name: development name: development
script: script:
- | - |
$ErrorActionPreference = "Stop" $ErrorActionPreference = "Stop"
py -m venv .venv py -m venv .venv
.venv\Scripts\activate.ps1 .venv\Scripts\activate.ps1
pip install pyinstaller pip install pyinstaller
pip install -r requirements.txt pip install -r requirements.txt
- pyinstaller -F main.py - pyinstaller -F main.py
--distpath . --distpath .
--icon app\frontend\static\assets\images\Crafty_4-0_Logo_square.ico --icon app\frontend\static\assets\images\Crafty_4-0_Logo_square.ico
--name "crafty_commander" --name "crafty_commander"
--paths .venv\Lib\site-packages --paths .venv\Lib\site-packages
--hidden-import cryptography --hidden-import cryptography
--hidden-import cffi --hidden-import cffi
--hidden-import apscheduler --hidden-import apscheduler
--collect-all tzlocal --collect-all tzlocal
--collect-all tzdata --collect-all tzdata
--collect-all pytz --collect-all pytz
--collect-all six --collect-all six
# Download latest:
# | https://gitlab.com/crafty-controller/crafty-4/-/jobs/artifacts/dev/download?job=win-dev-build
artifacts: artifacts:
name: "crafty-${CI_RUNNER_TAGS}-${CI_COMMIT_BRANCH}_${CI_COMMIT_SHORT_SHA}" name: "crafty-${CI_RUNNER_TAGS}-${CI_COMMIT_BRANCH}_${CI_COMMIT_SHORT_SHA}"
paths: paths:
- app\ - app\
- .\crafty_commander.exe - .\crafty_commander.exe
exclude: exclude:
- app\classes\**\* - app\classes\**\*
# Download latest:
# | https://gitlab.com/crafty-controller/crafty-commander/-/jobs/artifacts/dev/download?job=win-dev-build
win-prod-build: win-prod-build:
stage: prod-deployment stage: prod-deployment
tags: tags:
- win64 - win64
cache: cache:
paths: paths:
- .venv/ - .venv/
rules: rules:
- if: "$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH" - if: "$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH"
environment: environment:
name: production name: production
script: script:
- | - |
$ErrorActionPreference = "Stop" $ErrorActionPreference = "Stop"
py -m venv .venv py -m venv .venv
.venv\Scripts\activate.ps1 .venv\Scripts\activate.ps1
pip install pyinstaller pip install pyinstaller
pip install -r requirements.txt pip install -r requirements.txt
- pyinstaller -F main.py - pyinstaller -F main.py
--distpath . --distpath .
--icon app\frontend\static\assets\images\Crafty_4-0_Logo_square.ico --icon app\frontend\static\assets\images\Crafty_4-0_Logo_square.ico
--name "crafty_commander" --name "crafty_commander"
--paths .venv\Lib\site-packages --paths .venv\Lib\site-packages
--hidden-import cryptography --hidden-import cryptography
--hidden-import cffi --hidden-import cffi
--hidden-import apscheduler --hidden-import apscheduler
--collect-all tzlocal --collect-all tzlocal
--collect-all tzdata --collect-all tzdata
--collect-all pytz --collect-all pytz
--collect-all six --collect-all six
# Download latest:
# | https://gitlab.com/crafty-controller/crafty-4/-/jobs/artifacts/master/download?job=win-prod-build
artifacts: artifacts:
name: "crafty-${CI_RUNNER_TAGS}-${CI_COMMIT_BRANCH}_${CI_COMMIT_SHORT_SHA}" name: "crafty-${CI_RUNNER_TAGS}-${CI_COMMIT_BRANCH}_${CI_COMMIT_SHORT_SHA}"
paths: paths:
- app\ - app\
- .\crafty_commander.exe - .\crafty_commander.exe
exclude: exclude:
- app\classes\**\* - app\classes\**\*
# Download latest:
# | https://gitlab.com/crafty-controller/crafty-commander/-/jobs/artifacts/master/download?job=win-prod-build sast:
variables:
SAST_EXCLUDED_PATHS: spec, test, tests, tmp, migrations, vendors
SAST_BANDIT_EXCLUDED_PATHS: "'*/migrations/*, */vendors/*'"
SAST_EXCLUDED_ANALYZERS: semgrep
stage: test
tags:
- docker
secret_detection:
variables:
SECRET_DETECTION_EXCLUDED_PATHS: migrations, vendors
tags:
- docker
gemnasium-dependency_scanning:
tags:
- docker
gemnasium-python-dependency_scanning:
tags:
- docker
include:
- template: Security/Dependency-Scanning.gitlab-ci.yml
- template: Security/SAST.gitlab-ci.yml
- template: Security/Secret-Detection.gitlab-ci.yml