diff --git a/.mch-ci.yml b/.mch-ci.yml index 7fc6cb84..fe45d446 100644 --- a/.mch-ci.yml +++ b/.mch-ci.yml @@ -6,75 +6,67 @@ default: gitCalculateSemanticVersion: - getImageName: containerConstructImageName: - - imageTester: - containerBuildImage: - fullImageName: ${var.image}-tester - target: tester - extraArgs: - - --build-arg - - VERSION=${var.version} - - imageAwsRunner: - containerBuildImage: - fullImageName: ${var.image} - target: runner - extraArgs: - - --build-arg - - VERSION=${var.version} - pullAlways: false + - artifacts: + - containerBuildImage: + imageName: ${var.containerImageName}-tester + imageTag: ${var.semanticVersion} + target: tester + extraArgs: + - --build-arg + - VERSION=${var.semanticVersion} + - containerBuildImage: + imageName: ${var.containerImageName}-aws + imageTag: ${var.semanticVersion} + target: runner + extraArgs: + - --build-arg + - VERSION=${var.semanticVersion} + pullAlways: false - test: - unit: - script: mkdir -p test_reports - - pythonTest: - fullImageName: ${var.image}-tester + - pythonContainerTest: + imageName: ${var.containerImageName}-tester + imageTag: ${var.semanticVersion} testDirectory: tests packageManager: '' pullAlways: false - - pythonCoverage: - fullImageName: ${var.image}-tester - packageManager: '' - pullAlways: false + withCoverage: true + removeLocalImage: false - lint: - script: mkdir -p test_reports - - pythonLint: - fullImageName: ${var.image}-tester + - pythonContainerLint: + imageName: ${var.containerImageName}-tester + imageTag: ${var.semanticVersion} packageManager: '' pullAlways: false - - pythonTypeChecker: - fullImageName: ${var.image}-tester + removeLocalImage: false + - pythonContainerTypeCheck: + imageName: ${var.containerImageName}-tester + imageTag: ${var.semanticVersion} packageManager: '' pullAlways: false + removeLocalImage: false - verify: - - securityScan: - - securityReport: - format: [ 'html', 'sbom', 'table', 'print' ] - severity: [ 'CRITICAL', 'HIGH' ] - target: file://poetry.lock - qualityGate: - threshold: 5 - criticalFactor: 5 - highFactor: 1 - publishSbom: - securityPublishSbom: - publish: - - image: - containerPublishImage: - - deploy: - - pypi: - pythonPublishPackage: - setPackageVersion: False - pythonImageName: '3.10' - pullAlways: false + - artifacts: + - containerPublishImage: + imageName: ${var.containerImageName}-aws + imageTag: ${var.semanticVersion} + removeLocalImage: false - clean: - images: script: | - if test -n "${var.version}"; then + if test -n "${var.semanticVersion}"; then podman image rm -f $(podman image ls -q \ - -f "label=ch.meteoswiss.project=${var.project}-${var.version}") || : + -f "label=ch.meteoswiss.project=${var.project}-${var.semanticVersion}") || : fi variables: project: pyflexplot # default full image name, useful when running tasks locally - image: ${var.containerImageName}:${var.version} + image: ${var.containerImageName}:${var.semanticVersion} solution: dispersionmodelling notifyOnNightlyFailure: p_dispersionmodelling@meteoswiss.ch diff --git a/Jenkinsfile b/Jenkinsfile index 640a1787..b7a15db2 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,7 +1,6 @@ class Globals { - // Pin mchbuild to stable version to avoid breaking changes - static String mchbuildVersion = '0.8.0' + static String mchbuildPipPackage = 'mchbuild>=0.11.3,<0.13.0' // sets to abort the pipeline if the Sonarqube QualityGate fails static boolean qualityGateAbortPipeline = false @@ -9,12 +8,13 @@ class Globals { // the default python version static String pythonVersion = '3.10' - // the reference (image name + tag) of the container image - static String imageReference = '' + // Name of the container image + static String containerImageName= '' - // the service version - static String version = '' + // Semantic version of the artifact + static String semanticVersion = '' + static String PIP_INDEX_URL = 'https://hub.meteoswiss.ch/nexus/repository/python-all/simple' } String rebuild_cron = env.BRANCH_NAME == "main" ? "@midnight" : "" @@ -52,9 +52,10 @@ pipeline { echo '---- INSTALL MCHBUILD ----' sh """ python -m venv .venv-mchbuild - PIP_INDEX_URL=https://hub.meteoswiss.ch/nexus/repository/python-all/simple \ - .venv-mchbuild/bin/pip install mchbuild==${Globals.mchbuildVersion} + PIP_INDEX_URL=${Globals.PIP_INDEX_URL} \ + .venv-mchbuild/bin/pip install --upgrade "${Globals.mchbuildPipPackage}" """ + echo '---- INITIALIZE PARAMETERS ----' if (env.TAG_NAME) { @@ -67,31 +68,35 @@ pipeline { currentBuild.result = 'ABORTED' error('Build aborted because release builds are only triggered for tags of the form ...') } - Globals.version = env.TAG_NAME - } else - { + Globals.semanticVersion = env.TAG_NAME + } else { echo "Detected development build triggered from branch." - Globals.version= sh( + Globals.semanticVersion = sh( script: 'mchbuild -g semanticVersion build.getSemanticVersion', returnStdout: true ) } - def imageName = sh( + Globals.containerImageName = sh( script: 'mchbuild -g containerImageName build.getImageName', returnStdout: true ) - Globals.imageReference = imageName + ':' + Globals.version - echo "Using version ${Globals.version} and image reference ${Globals.imageReference}" + echo "Using semantic version: ${Globals.semanticVersion}" + echo "Using container image name: ${Globals.containerImageName}" } } } stage('Build') { steps { - echo '---- BUILD IMAGE ----' + echo '---- BUILDING CONTAINER IMAGES ----' + sh """ + mchbuild -s semanticVersion=${Globals.semanticVersion} -s containerImageName=${Globals.containerImageName} build.artifacts + """ + + echo("---- RUNNING UNIT TESTS & COLLECTING COVERAGE ----") sh """ - mchbuild -s version=${Globals.version} -s image=${Globals.imageReference} build.imageTester test.unit + mchbuild -s semanticVersion=${Globals.semanticVersion} -s containerImageName=${Globals.containerImageName} test.unit """ } post { @@ -103,12 +108,8 @@ pipeline { stage('Scan') { steps { - - echo("---- DEPENDENCIES SECURITY SCAN ----") - sh "mchbuild verify.securityScan" - echo '---- LINT & TYPE CHECK ----' - sh "mchbuild -s image=${Globals.imageReference} test.lint" + sh "mchbuild -s semanticVersion=${Globals.semanticVersion} -s containerImageName=${Globals.containerImageName} test.lint" script { try { recordIssues(qualityGates: [[threshold: 10, type: 'TOTAL', unstable: false]], tools: [myPy(pattern: 'test_reports/mypy.log')]) @@ -136,27 +137,18 @@ pipeline { } } - stage('Create Artifacts') { - steps { - script { - echo '---- CREATE IMAGE ----' - sh """ - mchbuild -s version=${Globals.version} -s image=${Globals.imageReference} build.imageAwsRunner - """ - } - } - } - stage('Publish Artifacts') { environment { REGISTRY_AUTH_FILE = "$workspace/.containers/auth.json" } steps { - echo "---- PUBLISH IMAGE ----" + echo "---- PUBLISHING CONTAINER IMAGES ----" withCredentials([usernamePassword(credentialsId: 'openshift-nexus', passwordVariable: 'NXPASS', usernameVariable: 'NXUSER')]) { - sh "mchbuild publish.image -s fullImageName=${Globals.imageReference}" + sh """ + mchbuild -s semanticVersion=${Globals.semanticVersion} -s containerImageName=${Globals.containerImageName} publish.artifacts + """ } } } @@ -168,27 +160,12 @@ pipeline { } steps { script { - echo "---- PUBLISH PYPI ----" - withCredentials([ - usernamePassword( - credentialsId: 'github app credential for the meteoswiss-apn github organization', - passwordVariable: 'GITHUB_ACCESS_TOKEN', - usernameVariable: 'GITHUB_APP'), - string(credentialsId: 'python-mch-nexus-secret', - variable: 'PYPIPASS') - ]) { - sh 'PYPIUSER=python-mch mchbuild deploy.pypi' - - sh "git remote set-url origin https://${GITHUB_APP}:${GITHUB_ACCESS_TOKEN}@github.com/MeteoSwiss-APN/pyflexplot" - Globals.version = sh(script: 'git describe --tags --abbrev=0', returnStdout: true).trim() - } - - echo("---- PUBLISH DEPENDENCIES TO DEPENDENCY REGISTRY ----") + echo("---- PUBLISH SBOM ----") withCredentials([string( credentialsId: 'dependency-track-token-prod', variable: 'DTRACK_TOKEN')]) { catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { - sh "mchbuild verify.publishSbom -s version=${Globals.version}" + sh "mchbuild verify.publishSbom -s version=${Globals.semanticVersion}" } } } @@ -199,7 +176,7 @@ pipeline { post { cleanup { sh """ - mchbuild -s version=${Globals.version} clean + mchbuild -s semanticVersion=${Globals.semanticVersion} clean """ cleanWs() }