Skip to content

Commit 55e527a

Browse files
HartornHartorn
authored
[GSK-852] Change Tensorflow to latest version, to enable arm64 test again (#1409)
* Rewrite Dockerfile and CI workflow to build images * Handle missing tensorflow for integration tests * Splitting tests into two docker steps * [GSK-852] Change Tensorflow to latest version, to enable arm64 test again * [GSK-1763] Adding python 3.11 support Contributes to GSK-1762 * Fix lock file (weird changes at regeneration) * Using setup tools as build backend (not working with pdm) * Minor changes to Github installed package
1 parent 41df70b commit 55e527a

26 files changed

Lines changed: 1299 additions & 698 deletions

‎.github/workflows/build-images.yml‎

Lines changed: 224 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -1,59 +1,252 @@
11
name: Build & Push dev images
22

33
on:
4-
# schedule:
5-
# - cron: '0 5 * * *'
4+
push:
5+
branches:
6+
- main
7+
tags:
8+
- "v*.*.*"
69
workflow_dispatch:
10+
inputs:
11+
run-tests:
12+
description: 'If all python tests should be run during build'
13+
required: true
14+
type: boolean
15+
default: true
16+
build_only:
17+
description: 'If images should be built and not push'
18+
required: true
19+
type: boolean
20+
default: false
721

22+
workflow_call:
23+
inputs:
24+
run-tests:
25+
description: 'If all python tests should be run during build'
26+
required: true
27+
type: boolean
28+
default: false
29+
build_only:
30+
description: 'If images should be built and not push'
31+
required: true
32+
type: boolean
33+
default: true
34+
35+
pull_request: # This will allow to trigger on PR only with a specific label
36+
types: [opened, reopened, synchronize, labeled, unlabeled]
37+
# Concurrency : auto-cancel "old" jobs ie when pushing again
38+
# https://docs.github.com/fr/actions/using-jobs/using-concurrency
39+
concurrency:
40+
group: ${{ github.workflow }}-${{ github.ref || github.run_id }}
41+
cancel-in-progress: true
42+
env:
43+
RUN_TESTS: false
44+
BUILD_ONLY: false
45+
REGISTRY_IMAGE: giskardai/giskard
46+
DOCKERHUB_USER: giskardai
47+
# https://docs.docker.com/build/ci/github-actions/multi-platform/#distribute-build-across-multiple-runners
848
jobs:
949
build-images:
50+
# Debug
51+
strategy:
52+
matrix:
53+
platform: ["linux/amd64", "linux/arm64"]
54+
fail-fast: false
55+
if: ${{ !github.event.pull_request || contains( github.event.pull_request.labels.*.name, 'Docker') }}
1056
runs-on: ubuntu-latest
1157
steps:
58+
- name: Check if python tests should be run
59+
if: ${{ github.event_name == 'push' || github.event_name == 'pull_request' || inputs.run-tests }}
60+
run: echo 'RUN_TESTS=true' >> $GITHUB_ENV
61+
62+
- name: Check if it should be build only (if yes, do not push the images)
63+
if: ${{ github.event_name == 'pull_request' || inputs.build_only }}
64+
run: echo 'BUILD_ONLY=true' >> $GITHUB_ENV
65+
66+
- name: Check disk space
67+
if: ${{ env.RUN_TESTS }}
68+
run: |
69+
df -h
70+
- name: List installed packages
71+
run: |
72+
dpkg-query --show --showformat='${Installed-Size}\t${Package}\n' | sort -rh | head -25 | awk '{print $1/1024, $2}'
73+
apt list --installed | wc -l
74+
75+
- name: Free disk space
76+
if: ${{ env.RUN_TESTS }}
77+
run: |
78+
sudo swapoff -a
79+
sudo rm -f /swapfile
80+
sudo rm -rf /usr/local/lib/android
81+
docker rmi $(docker image ls -aq)
82+
docker system prune -f
83+
sudo apt remove -y \
84+
alsa-topology-conf alsa-ucm-conf \
85+
google-cloud-cli azure-cli microsoft-edge-stable dotnet-sdk-7.0 dotnet-sdk-6.0 temurin-17-jdk \
86+
google-chrome-stable temurin-11-jdk llvm-14-dev llvm-13-dev llvm-12-dev firefox temurin-8-jdk \
87+
powershell mysql-server-core-8.0 libllvm15 libllvm14 libllvm13 libllvm12 libclang-common-13-dev
88+
sudo apt autoremove -y
89+
sudo apt autoclean -y
90+
91+
- name: Check new disk space
92+
if: ${{ env.RUN_TESTS }}
93+
run: |
94+
df -h
95+
- name: List remaning installed packages
96+
if: ${{ env.RUN_TESTS }}
97+
run: |
98+
dpkg-query --show --showformat='${Installed-Size}\t${Package}\n' | sort -rh | head -25 | awk '{print $1/1024, $2}'
99+
apt list --installed | wc -l
100+
12101
- name: Checkout code
13-
uses: actions/checkout@v3.3.0
102+
uses: actions/checkout@v4
103+
with:
104+
fetch-depth: 1
14105

15106
- name: Set up Docker Buildx
16107
id: builder
17-
uses: docker/setup-buildx-action@master
108+
uses: docker/setup-buildx-action@v3
18109

19110
- name: "Append ARM buildx builder from Oracle cloud"
111+
if: ${{ matrix.platform == 'linux/arm64'}}
20112
uses: baschny/append-buildx-action@v1
21113
with:
22114
builder: ${{ steps.builder.outputs.name }}
23115
endpoint: ssh://${{ secrets.ARM_SSH_CONNECTION_STRING }}
24116
ssh_private_key: ${{ secrets.ARM_SSH_PRIVATE_KEY }}
25117

26-
- name: Extract branch name
27-
shell: bash
28-
run: |
29-
BRANCH=${GITHUB_REF#refs/heads/}
30-
SANITIZED_BRANCH=${BRANCH//"/"/-}
31-
32-
echo $BRANCH
33-
echo $SANITIZED_BRANCH
34-
35-
echo "branch=$BRANCH" >> $GITHUB_OUTPUT
36-
echo "tag=$SANITIZED_BRANCH" >> $GITHUB_OUTPUT
37-
id: extract_branch
38-
118+
- name: Docker meta
119+
id: meta
120+
uses: docker/metadata-action@v5
121+
with:
122+
images: |
123+
${{ env.REGISTRY_IMAGE }}
124+
tags: |
125+
type=edge,branch=main
126+
type=schedule
127+
type=ref,event=branch
128+
type=ref,event=tag
129+
type=ref,event=pr
130+
39131
- name: Login to Docker registry
40-
uses: docker/login-action@v2
132+
if: ${{ env.BUILD_ONLY != 'true' }}
133+
uses: docker/login-action@v3
41134
with:
42-
username: giskardai
135+
username: ${{ env.DOCKERHUB_USER }}
43136
password: ${{ secrets.DOCKERHUB_TOKEN }}
44137

45-
- name: Build and push image based on the current branch
46-
env:
47-
IMAGE: docker.io/giskardai/giskard
138+
- name: Install deps inside docker
139+
if: ${{ env.RUN_TESTS }}
140+
uses: docker/build-push-action@v5
141+
with:
142+
context: .
143+
target: full-install-python
144+
push: false
145+
load: true
146+
tags: ${{ steps.meta.outputs.tags }}
147+
labels: ${{ steps.meta.outputs.labels }}
148+
builder: ${{ steps.builder.outputs.name }}
149+
platforms: |
150+
${{ matrix.platform}}
151+
cache-from: type=gha
152+
cache-to: type=gha,mode=max
153+
154+
- name: Run python test inside docker
155+
if: ${{ env.RUN_TESTS }}
156+
uses: docker/build-push-action@v5
157+
with:
158+
context: .
159+
target: test-python
160+
push: false
161+
load: false
162+
tags: ${{ steps.meta.outputs.tags }}
163+
labels: ${{ steps.meta.outputs.labels }}
164+
builder: ${{ steps.builder.outputs.name }}
165+
platforms: |
166+
${{ matrix.platform}}
167+
cache-from: type=gha
168+
169+
# - name: Run python integration test inside docker
170+
# if: ${{ env.RUN_TESTS }}
171+
# uses: docker/build-push-action@v5
172+
# with:
173+
# context: .
174+
# target: integration-test-python
175+
# push: false
176+
# load: false
177+
# tags: ${{ steps.meta.outputs.tags }}
178+
# labels: ${{ steps.meta.outputs.labels }}
179+
# builder: ${{ steps.builder.outputs.name }}
180+
# platforms: |
181+
# ${{ matrix.platform}}
182+
# cache-from: type=gha
183+
184+
- name: Build and push
185+
id: build
186+
uses: docker/build-push-action@v5
187+
with:
188+
context: .
189+
target: prod
190+
push: ${{ env.BUILD_ONLY != 'true' }}
191+
tags: ${{ steps.meta.outputs.tags }}
192+
labels: ${{ steps.meta.outputs.labels }}
193+
builder: ${{ steps.builder.outputs.name }}
194+
platforms: |
195+
${{ matrix.platform}}
196+
cache-from: type=gha
197+
cache-to: type=gha,mode=max
198+
outputs: type=image,name=${{ env.REGISTRY_IMAGE }},push-by-digest=true,name-canonical=true,push=${{ env.BUILD_ONLY != 'true' }}
199+
200+
# For details, see link below
201+
# https://docs.docker.com/build/ci/github-actions/multi-platform/
202+
- name: Export digest
48203
run: |
49-
docker buildx build \
50-
--platform linux/amd64,linux/arm64 \
51-
--tag $IMAGE:${{ steps.extract_branch.outputs.tag }} \
52-
--file Dockerfile \
53-
--push \
54-
.
204+
mkdir -p /tmp/digests
205+
digest="${{ steps.build.outputs.digest }}"
206+
touch "/tmp/digests/${digest#sha256:}"
207+
208+
- name: Upload digest
209+
uses: actions/upload-artifact@v3
210+
with:
211+
name: digests
212+
path: /tmp/digests/*
213+
if-no-files-found: error
214+
retention-days: 1
55215

56-
- name: Invoke deployment hook
57-
if: ${{ steps.extract_branch.outputs.branch == 'main' }}
216+
merge:
217+
runs-on: ubuntu-latest
218+
if: ${{ github.event_name != 'pull_request' && !inputs.build_only }}
219+
needs:
220+
- build-images
221+
steps:
222+
- name: Download digests
223+
uses: actions/download-artifact@v3
224+
with:
225+
name: digests
226+
path: /tmp/digests
227+
- name: Set up Docker Buildx
228+
uses: docker/setup-buildx-action@v3
229+
- name: Docker meta
230+
id: meta
231+
uses: docker/metadata-action@v5
232+
with:
233+
images: ${{ env.REGISTRY_IMAGE }}
234+
- name: Login to Docker Hub
235+
uses: docker/login-action@v3
236+
with:
237+
username: ${{ env.DOCKERHUB_USER }}
238+
password: ${{ secrets.DOCKERHUB_TOKEN }}
239+
- name: Create manifest list and push
240+
working-directory: /tmp/digests
58241
run: |
59-
curl -L --silent --max-time 900 ${{ secrets.TEST_SERVER_WEBHOOK_URL_BASE }}/redeploy-dev
242+
docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
243+
$(printf '${{ env.REGISTRY_IMAGE }}@sha256:%s ' *)
244+
- name: Inspect image
245+
run: |
246+
docker buildx imagetools inspect ${{ env.REGISTRY_IMAGE }}:${{ steps.meta.outputs.version }}
247+
248+
249+
# - name: Invoke deployment hook
250+
# if: ${{ steps.extract_branch.outputs.branch == 'main' && env.BUILD_ONLY != 'true' }}
251+
# run: |
252+
# curl -L --silent --max-time 900 ${{ secrets.TEST_SERVER_WEBHOOK_URL_BASE }}/redeploy-dev

‎.github/workflows/build-release-images.yml‎

Lines changed: 0 additions & 72 deletions
This file was deleted.

0 commit comments

Comments
 (0)