gitlab-ci-universal-pipeline.yml

gitlab-ci-universal-pipeline.yml

---
# Example manual devops image prepare
# export CI_REGISTRY_USER=alexgluck CI_JOB_TOKEN=<INSERT Personal Access Token with r\w registry permission>

stages:
  - "prepare environment"
  - "lint"
  - "get tokens"
  - "filter token"
  - "create token"
  - "publish token"
  - "build"
  - "publish"
  - "deploy"
  - "cleanup"

variables:
  BUILDAH_FORMAT: "docker"
  STORAGE_DRIVER: "vfs"
  USE_BUILDAH: "true"
  BUILDAH_ISOLATION: "chroot"
  BUILDAH_LAYERS: "true"
  USE_SKOPEO: "true"

.prepare_devops_images: &prepare_devops_images
  image:
    name: gitlab.example.com/devops/ci-templates/skopeo:latest
    entrypoint: [""]
  variables:
    GIT_STRATEGY: none
  script:
    - env | sort
    - command -v skopeo &> /dev/null && test -n "${USE_SKOPEO}" && { shopt -s expand_aliases && alias docker=skopeo ; }
    - echo "${CI_JOB_TOKEN}" | docker login -u "${CI_REGISTRY_USER:-gitlab-ci-token}" --password-stdin "${CI_REGISTRY:-registry.example.com}"
    - skopeo copy docker://quay.io/buildah/stable:latest docker://${CI_REGISTRY:-registry.example.com}/${CI_PROJECT_PATH:-devops/ci-templates}/buildah:latest
    - skopeo copy docker://registry.gitlab.com/gitlab-org/release-cli:latest docker://${CI_REGISTRY:-registry.example.com}/${CI_PROJECT_PATH:-devops/ci-templates}/release-cli:latest
    - skopeo copy docker://docker.io/alpine/helm:latest docker://${CI_REGISTRY:-registry.example.com}/${CI_PROJECT_PATH:-devops/ci-templates}/helm:latest
    - skopeo copy docker://docker.io/curlimages/curl:latest docker://${CI_REGISTRY:-registry.example.com}/${CI_PROJECT_PATH:-devops/ci-templates}/curl:latest
    - skopeo copy docker://quay.io/skopeo/stable:latest docker://${CI_REGISTRY:-registry.example.com}/${CI_PROJECT_PATH:-devops/ci-templates}/skopeo:latest
    - skopeo copy docker://registry.gitlab.com/gitlab-ci-utils/curl-jq:latest docker://${CI_REGISTRY:-registry.example.com}/${CI_PROJECT_PATH:-devops/ci-templates}/jq:latest
    - skopeo copy docker://gcr.io/kaniko-project/executor:v1.12.1-debug docker://${CI_REGISTRY:-registry.example.com}/${CI_PROJECT_PATH:-devops/ci-templates}/kaniko:v1.12.1-debug
    - skopeo copy docker://gcr.io/go-containerregistry/crane:latest docker://${CI_REGISTRY:-registry.example.com}/${CI_PROJECT_PATH:-devops/ci-templates}/crane:latest
  allow_failure: true

.prepare_devops_images_fallback:
  extends: .prepare_devops_images
  <<: *prepare_devops_images
  image:
    name: quay.io/skopeo/stable:latest
    entrypoint: [""]
  allow_failure: false

.build-image: &build-image
  stage: build
  image:
    name: gitlab.example.com/devops/ci-templates/buildah:latest
    entrypoint: [""]
  # variables:
  #   KUBERNETES_POD_ANNOTATIONS_1: "container.apparmor.security.beta.kubernetes.io/build=unconfined" # https://docs.gitlab.com/runner/executors/kubernetes.html#overwrite-pod-annotations workaround from https://github.com/containers/buildah/issues/4920#issuecomment-1633910481
  #   KUBERNETES_CPU_LIMIT: "3.8"
  #   KUBERNETES_MEMORY_LIMIT: 9Gi
  artifacts:
    expire_in: 2 hours
    name: Env's files
    paths:
      - "*.env"
      - ".env"
      - "${SERVICES_FOLDER:-services}/**/*.env"
      - "${SERVICES_FOLDER:-services}/**/.env"
  script:
    - test -f ./.env && source .env
    - env | sort
    - export IMAGE_CACHE_TAG=":latest"
    - mkdir -p /etc/containers && echo 'unqualified-search-registries=["docker.io"]' >> /etc/containers/registries.conf
    - test -f /kaniko/executor && test -n "${USE_KANIKO}" && export USE_KANIKO_READY=true
    - command -v buildah &> /dev/null && test -n "${USE_BUILDAH}" && { unset IMAGE_CACHE_TAG ; shopt -s expand_aliases && alias docker=buildah ; }
    - command -v podman  &> /dev/null && test -n "${USE_PODMAN}"  && { unset IMAGE_CACHE_TAG ; shopt -s expand_aliases && alias docker=podman  ; }
    - |
      function docker_build() {
        docker pull --policy always ${SERVICE:-$CI_REGISTRY_IMAGE}:latest || true
        docker build --cache-from "${SERVICE:-${CI_REGISTRY_IMAGE}}${IMAGE_CACHE_TAG}" -f ${REQUIRED_FILE_PATH:-${REQUIRED_FILE:-Dockerfile}} --tag "${SERVICE:-${CI_REGISTRY_IMAGE}}:${CI_COMMIT_SHA}" .
      }

      function image_push() {
        docker push ${SERVICE:-$CI_REGISTRY_IMAGE}:${CI_COMMIT_SHA} ${SERVICE:-${CI_REGISTRY_IMAGE}}:latest
        docker push ${SERVICE:-$CI_REGISTRY_IMAGE}:${CI_COMMIT_SHA} ${SERVICE:-${CI_REGISTRY_IMAGE}}:${CI_COMMIT_SHA}
        docker push ${SERVICE:-$CI_REGISTRY_IMAGE}:${CI_COMMIT_SHA} ${SERVICE:-${CI_REGISTRY_IMAGE}}:${CI_COMMIT_SHORT_SHA}
      }
    - |
      if [ -f ${REQUIRED_FILE:-Dockerfile} ] ; then
        if [[ -n ${USE_KANIKO_READY} ]] ; then
          echo "{\"auths\":{\"${CI_REGISTRY:-registry.example.com}\":{\"username\":\"${CI_REGISTRY_USER:-gitlab-ci-token}\",\"password\":\"${CI_JOB_TOKEN}\"}}}" > /kaniko/.docker/config.json
          /kaniko/executor --context . --dockerfile ${REQUIRED_FILE_PATH:-${REQUIRED_FILE:-Dockerfile}} --cache=true --cache-copy-layers --cache-repo "${SERVICE:-${CI_REGISTRY_IMAGE}}" --destination "${SERVICE:-${CI_REGISTRY_IMAGE}}:${CI_COMMIT_SHA}" --destination ${SERVICE:-${CI_REGISTRY_IMAGE}}:${CI_COMMIT_SHORT_SHA} --destination ${SERVICE:-${CI_REGISTRY_IMAGE}}:latest
        else
          echo "${CI_JOB_TOKEN}" | docker login -u "${CI_REGISTRY_USER:-gitlab-ci-token}" --password-stdin "${CI_REGISTRY:-registry.example.com}"
          docker_build
          image_push
        fi
      else
        export REQUIRED_FILE_PATHS=$(ls -1 ${SERVICES_FOLDER:-services}/*/${REQUIRED_FILE:-Dockerfile} || { echo "Required file(s) ${SERVICES_FOLDER:-services}/*/${REQUIRED_FILE:-Dockerfile} not found" ; exit 1 ; })
        test -n "${REQUIRED_FILE_PATHS}" || { echo "Required file(s) ${SERVICES_FOLDER:-services}/*/${REQUIRED_FILE:-Dockerfile} not found" ; exit 1 ; }
        for REQUIRED_FILE_PATH in ${REQUIRED_FILE_PATHS:-.} ; do
          export SERVICE="${CI_REGISTRY_IMAGE}/$(basename "${REQUIRED_FILE_PATH//${REQUIRED_FILE:-Dockerfile}/}")"
          if [[ -n ${USE_KANIKO_READY} ]] ; then
            echo "{\"auths\":{\"${CI_REGISTRY:-registry.example.com}\":{\"username\":\"${CI_REGISTRY_USER:-gitlab-ci-token}\",\"password\":\"${CI_JOB_TOKEN}\"}}}" > /kaniko/.docker/config.json
            /kaniko/executor --context . --dockerfile ${REQUIRED_FILE_PATH:-${REQUIRED_FILE:-Dockerfile}} --cache=true --cache-copy-layers --cache-repo "${SERVICE:-${CI_REGISTRY_IMAGE}}" --destination "${SERVICE:-${CI_REGISTRY_IMAGE}}:${CI_COMMIT_SHA}" --destination ${SERVICE:-${CI_REGISTRY_IMAGE}}:${CI_COMMIT_SHORT_SHA} --destination ${SERVICE:-${CI_REGISTRY_IMAGE}}:latest
          else
            echo "${CI_JOB_TOKEN}" | docker login -u "${CI_REGISTRY_USER:-gitlab-ci-token}" --password-stdin "${CI_REGISTRY:-registry.example.com}"
            docker_build
            image_push
          fi
        done
      fi

.build-image-release:
  extends: .build-image
  <<: *build-image
  after_script:
    - test -f /kaniko/executor && test -n "${USE_KANIKO}" && { echo "error this job not supported kaniko yet" ; exit 1 ; }
    - command -v buildah &> /dev/null && test -n "${USE_BUILDAH}" && { shopt -s expand_aliases && alias docker=buildah ; }
    - command -v podman  &> /dev/null && test -n "${USE_PODMAN}"  && { shopt -s expand_aliases && alias docker=podman  ; }
    - test -f ./.env && source .env
    - docker login -u ${CI_REGISTRY_USER:-gitlab-ci-token} -p ${CI_JOB_TOKEN} ${CI_REGISTRY:-registry.example.com}
    - |
      function image_commit_hash() {
        docker push ${SERVICE:-$CI_REGISTRY_IMAGE}:${CI_COMMIT_SHA} ${SERVICE:-${CI_REGISTRY_IMAGE}}:${CI_COMMIT_SHA}
        docker push ${SERVICE:-$CI_REGISTRY_IMAGE}:${CI_COMMIT_SHA} ${SERVICE:-${CI_REGISTRY_IMAGE}}:${CI_COMMIT_SHORT_SHA}
      }

      # calver and calver+git-commit
      function image_calver_tags() {
        docker push ${SERVICE:-$CI_REGISTRY_IMAGE}:${CI_COMMIT_SHA} ${SERVICE:-$CI_REGISTRY_IMAGE}:$(date -u +%FT%H.%M.%SZ)
        docker push ${SERVICE:-$CI_REGISTRY_IMAGE}:${CI_COMMIT_SHA} ${SERVICE:-$CI_REGISTRY_IMAGE}:$(date -u +%FT%H.%M.%SZ)-${CI_COMMIT_SHA}
        docker push ${SERVICE:-$CI_REGISTRY_IMAGE}:${CI_COMMIT_SHA} ${SERVICE:-$CI_REGISTRY_IMAGE}:$(date -u +%FT%H.%M.%SZ)-${CI_COMMIT_SHORT_SHA}
      }

      # git tag versioning if exist; else auto increment semver; git-tag/auto-semver+git-commit
      function image_semver_tags() {
        docker push ${SERVICE:-$CI_REGISTRY_IMAGE}:${CI_COMMIT_SHA} ${SERVICE:-$CI_REGISTRY_IMAGE}:${CI_COMMIT_TAG:-v${VERSION_MAJOR:-0}.${VERSION_MINOR:-0}.${VERSION_PATCH:-${CI_PIPELINE_IID}}}
        docker push ${SERVICE:-$CI_REGISTRY_IMAGE}:${CI_COMMIT_SHA} ${SERVICE:-$CI_REGISTRY_IMAGE}:${CI_COMMIT_TAG:-v${VERSION_MAJOR:-0}.${VERSION_MINOR:-0}.${VERSION_PATCH:-${CI_PIPELINE_IID}}}-${CI_COMMIT_SHA}
        docker push ${SERVICE:-$CI_REGISTRY_IMAGE}:${CI_COMMIT_SHA} ${SERVICE:-$CI_REGISTRY_IMAGE}:${CI_COMMIT_TAG:-v${VERSION_MAJOR:-0}.${VERSION_MINOR:-0}.${VERSION_PATCH:-${CI_PIPELINE_IID}}}-${CI_COMMIT_SHORT_SHA}
      }
    - |
      if [ -f ${REQUIRED_FILE:-Dockerfile} ] ; then
        image_commit_hash
        image_calver_tags
        image_semver_tags
      else
        export REQUIRED_FILE_PATHS=$(ls -1 ${SERVICES_FOLDER:-services}/*/${REQUIRED_FILE:-Dockerfile} || { echo "Required file(s) ${SERVICES_FOLDER:-services}/*/${REQUIRED_FILE:-Dockerfile} not found" ; exit 1 ; })
        test -n "${REQUIRED_FILE_PATHS}" || { echo "Required file(s) ${SERVICES_FOLDER:-services}/*/${REQUIRED_FILE:-Dockerfile} not found" ; exit 1 ; }
        for REQUIRED_FILE_PATH in ${REQUIRED_FILE_PATHS:-.} ; do
          export SERVICE="${CI_REGISTRY_IMAGE}/$($basename "${REQUIRED_FILE_PATH//${REQUIRED_FILE:-Dockerfile}/}")"
          image_commit_hash
          image_calver_tags
          image_semver_tags
        done
      fi

.tag-image:
  stage: build
  image:
    name: gitlab.example.com/devops/ci-templates/skopeo:latest
    entrypoint: [""]
  # variables:
  #   GIT_STRATEGY: none
  script:
    - test -f ./.env && source .env
    - env | sort
    - command -v skopeo &> /dev/null && test -n "${USE_SKOPEO}" && { shopt -s expand_aliases && alias docker=skopeo ; }
    - command -v crane  &> /dev/null && test -n "${USE_CRANE}"  && { shopt -s expand_aliases && alias docker=crane  ; }
    - echo "${CI_JOB_TOKEN}" | docker login -u "${CI_REGISTRY_USER:-gitlab-ci-token}" --password-stdin "${CI_REGISTRY:-registry.example.com}"
    - |
      # calver and calver+git-commit-hash
      function image_calver_tags() {
        skopeo copy --multi-arch=index-only docker://${SERVICE:-$CI_REGISTRY_IMAGE}:${CI_COMMIT_SHA} \
        docker://${SERVICE:-$CI_REGISTRY_IMAGE}:$(date -u +%FT%H.%M.%SZ) && \
        skopeo copy --multi-arch=index-only docker://${SERVICE:-$CI_REGISTRY_IMAGE}:${CI_COMMIT_SHA} \
        docker://${SERVICE:-$CI_REGISTRY_IMAGE}:$(date -u +%FT%H.%M.%SZ)-${CI_COMMIT_SHA} && \
        skopeo copy --multi-arch=index-only docker://${SERVICE:-$CI_REGISTRY_IMAGE}:${CI_COMMIT_SHA} \
        docker://${SERVICE:-$CI_REGISTRY_IMAGE}:$(date -u +%FT%H.%M.%SZ)-${CI_COMMIT_SHORT_SHA}
      }

      # git tag versioning if exist; else auto increment semver; git-tag/auto-semver+git-commit
      function image_semver_tags() {
        skopeo copy --multi-arch=index-only docker://${SERVICE:-$CI_REGISTRY_IMAGE}:${CI_COMMIT_SHA} \
        docker://${SERVICE:-$CI_REGISTRY_IMAGE}:${CI_COMMIT_TAG:-v${VERSION_MAJOR:-0}.${VERSION_MINOR:-0}.${VERSION_PATCH:-${CI_PIPELINE_IID}}} && \
        skopeo copy --multi-arch=index-only docker://${SERVICE:-$CI_REGISTRY_IMAGE}:${CI_COMMIT_SHA} \
        docker://${SERVICE:-$CI_REGISTRY_IMAGE}:${CI_COMMIT_TAG:-v${VERSION_MAJOR:-0}.${VERSION_MINOR:-0}.${VERSION_PATCH:-${CI_PIPELINE_IID}}}-${CI_COMMIT_SHA} && \
        skopeo copy --multi-arch=index-only docker://${SERVICE:-$CI_REGISTRY_IMAGE}:${CI_COMMIT_SHA} \
        docker://${SERVICE:-$CI_REGISTRY_IMAGE}:${CI_COMMIT_TAG:-v${VERSION_MAJOR:-0}.${VERSION_MINOR:-0}.${VERSION_PATCH:-${CI_PIPELINE_IID}}}-${CI_COMMIT_SHORT_SHA}
      }
    - |
      if [[ -f "${REQUIRED_FILE:-Dockerfile}" ]] || [[ "${GIT_STRATEGY:-none}" = "none" ]] ; then
        image_calver_tags
        image_semver_tags
      else
        export REQUIRED_FILE_PATHS=$(ls -1 ${SERVICES_FOLDER:-services}/*/${REQUIRED_FILE:-Dockerfile} || { echo "Required file(s) ${SERVICES_FOLDER:-services}/*/${REQUIRED_FILE:-Dockerfile} not found" ; exit 1 ; })
        test -n "${REQUIRED_FILE_PATHS}" || { echo "Required file(s) ${SERVICES_FOLDER:-services}/*/${REQUIRED_FILE:-Dockerfile} not found" ; exit 1 ; }
        for REQUIRED_FILE_PATH in ${REQUIRED_FILE_PATHS:-.} ; do
          export SERVICE="${CI_REGISTRY_IMAGE}/$($basename "${REQUIRED_FILE_PATH//${REQUIRED_FILE:-Dockerfile}/}")"
          cd ${REQUIRED_FILE_PATH} && { test -f ./.env && source .env ; } || true
          image_calver_tags
          image_semver_tags
        done
      fi
  rules:
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH