Skip to content

Commit e9721a9

Browse files
pennampennam
authored
Merge pull request #216 from pillo79/pr-multi-package
ci: add multi-artifact build
2 parents 68ee7e1 + efe6e9c commit e9721a9

15 files changed

+375
-198
lines changed

‎.github/workflows/package_core.yml‎

Lines changed: 122 additions & 67 deletions
Original file line numberDiff line numberDiff line change
@@ -7,18 +7,21 @@ on:
77
jobs:
88

99
package-core:
10-
name: Build and package core
10+
name: Build and package cores
1111
runs-on: ubuntu-latest
1212
env:
1313
CCACHE_IGNOREOPTIONS: -specs=*
1414
outputs:
1515
CORE_TAG: ${{ env.CORE_TAG }}
16-
CORE_ARTIFACT: ${{ env.CORE_ARTIFACT }}
17-
BOARD_VARIANTS: ${{ env.BOARD_VARIANTS }}
16+
CORE_HASH: ${{ env.CORE_HASH }}
17+
ALL_BOARD_DATA: ${{ env.ALL_BOARD_DATA }}
18+
ARTIFACTS: ${{ env.ARTIFACTS }}
19+
SUB_ARCHES: ${{ env.SUB_ARCHES }}
1820
steps:
1921
- name: Install OS dependencies
2022
working-directory: /opt
2123
run: |
24+
sudo apt-get remove --purge man-db -y # skips the mandb triggers
2225
sudo apt-get update
2326
sudo apt-get install -y --no-install-recommends git cmake wget python3-pip ninja-build ccache
2427
@@ -32,9 +35,16 @@ jobs:
3235
- name: Initialize Zephyr environment
3336
run: |
3437
yes | ./extra/bootstrap.sh -o=--filter=tree:0
35-
echo "CORE_TAG=$(git describe --tags --exact-match || git describe --always)" >> "$GITHUB_ENV"
36-
echo "CORE_ARTIFACT=ArduinoCore-zephyr-$(git describe --always)" >> "$GITHUB_ENV"
37-
echo "BOARD_VARIANTS=$(extra/get_board_details.sh | jq -cr 'sort_by(.variant)')" >> "$GITHUB_ENV"
38+
echo "CORE_HASH=$(git describe --always)" >> "$GITHUB_ENV"
39+
echo "ALL_BOARD_DATA=$(extra/get_board_details.sh | jq -c 'sort_by(.variant)')" >> "$GITHUB_ENV"
40+
echo "## Building \`$(extra/get_core_version.sh)\`" >> "$GITHUB_STEP_SUMMARY"
41+
42+
- name: Map output packages
43+
# needs the above env vars to be usable
44+
run: |
45+
echo "CORE_TAG=$(git describe --tags --exact-match 2>/dev/null || echo $CORE_HASH)" >> "$GITHUB_ENV"
46+
echo "ARTIFACTS=$(jq -c '["zephyr"] + (map(.artifact) | unique)' <<< ${ALL_BOARD_DATA})" >> "$GITHUB_ENV"
47+
echo "SUB_ARCHES=$(jq -c 'map(.subarch) | unique' <<< ${ALL_BOARD_DATA})" >> "$GITHUB_ENV"
3848
3949
- name: ccache
4050
uses: hendrikmuhs/ccache-action@v1.2
@@ -46,36 +56,63 @@ jobs:
4656
run: |
4757
./extra/build_all.sh -f
4858
49-
- name: Package core
59+
- name: Package cores
5060
run: |
51-
./extra/package_core.sh ${{ env.CORE_TAG }} distrib/${{ env.CORE_ARTIFACT }}.tar.bz2
61+
jq -cr '.[]' <<< ${ARTIFACTS} | while read -r artifact; do
62+
ARTIFACT_NAME=ArduinoCore-${artifact}-${CORE_HASH}
63+
./extra/package_core.sh ${artifact} ${CORE_TAG} distrib/${ARTIFACT_NAME}.tar.bz2
64+
done
5265
53-
- name: Archive core
66+
- name: Archive cores
5467
uses: actions/upload-artifact@v4
68+
with:
69+
name: ArduinoCore-archives-${{ env.CORE_HASH }}
70+
path: distrib/*.tar.bz2
71+
72+
split-core:
73+
name: Split off ${{ matrix.artifact }}
74+
runs-on: ubuntu-latest
75+
needs: package-core
76+
env:
77+
ALL_CORES_ARTIFACT: ArduinoCore-archives-${{ needs.package-core.outputs.CORE_HASH }}
78+
CORE_ARTIFACT: ArduinoCore-${{ matrix.artifact }}-${{ needs.package-core.outputs.CORE_HASH }}
79+
strategy:
80+
matrix:
81+
artifact: ${{ fromJSON( needs.package-core.outputs.ARTIFACTS ) }}
82+
steps:
83+
- uses: actions/download-artifact@v4
84+
with:
85+
name: ${{ env.ALL_CORES_ARTIFACT }}
86+
87+
- uses: actions/upload-artifact@v4
5588
with:
5689
name: ${{ env.CORE_ARTIFACT }}
57-
path: distrib/${{ env.CORE_ARTIFACT }}.tar.bz2
90+
path: ${{ env.CORE_ARTIFACT }}.tar.bz2
5891

5992
test-core:
60-
name: Test on ${{ matrix.board }}
93+
name: Test ${{ matrix.subarch }}:${{ matrix.board }}
6194
runs-on: ubuntu-latest
62-
needs: package-core
95+
needs:
96+
- package-core
97+
- split-core
6398
strategy:
6499
matrix:
65100
include:
66-
${{ fromJSON( needs.package-core.outputs.BOARD_VARIANTS ) }}
101+
${{ fromJSON( needs.package-core.outputs.ALL_BOARD_DATA ) }}
67102
fail-fast: false
68103
env:
69-
FQBN: arduino:zephyr:${{ matrix.board }}
70-
REPORT_FILE: arduino-zephyr-${{ matrix.board }}.json
104+
PLAT: arduino:${{ matrix.subarch }}
105+
FQBN: arduino:${{ matrix.subarch }}:${{ matrix.board }}
106+
CORE_ARTIFACT: ArduinoCore-${{ matrix.artifact }}-${{ needs.package-core.outputs.CORE_HASH }}
71107
steps:
72108
- uses: actions/download-artifact@v4
73109
with:
74-
name: ${{ needs.package-core.outputs.CORE_ARTIFACT }}
110+
name: ${{ env.CORE_ARTIFACT }}
75111

76112
- name: Set up core
77113
run: |
78-
tar xf ${{ needs.package-core.outputs.CORE_ARTIFACT }}.tar.bz2
114+
tar xf ${CORE_ARTIFACT}.tar.bz2 # will create ArduinoCore-zephyr/
115+
echo "REPORT_FILE=$(echo ${FQBN} | tr ':' '-').json" >> $GITHUB_ENV
79116
80117
- name: Create Blink sketch
81118
run: |
@@ -90,7 +127,7 @@ jobs:
90127
# Use Board Manager to install the latest release of Arduino Zephyr Boards to get the toolchain
91128
- name: "arduino:zephyr"
92129
source-url: "https://downloads.arduino.cc/packages/package_zephyr_index.json"
93-
- name: "arduino:zephyr"
130+
- name: ${{ env.PLAT }}
94131
source-path: "ArduinoCore-zephyr"
95132
sketch-paths: Blink
96133
cli-compile-flags: |
@@ -114,7 +151,7 @@ jobs:
114151
repo: context.repo.repo,
115152
run_id: context.runId
116153
});
117-
const job_name = `Test on ${{ matrix.board }}`
154+
const job_name = `Test ${{ matrix.subarch }}:${{ matrix.board }}`
118155
return workflow_run.jobs.find((job) => job.name === job_name).id;
119156
120157
- name: Prepare log
@@ -137,7 +174,7 @@ jobs:
137174
- test-core
138175
if: ${{ !cancelled() && needs.package-core.result == 'success' }}
139176
env:
140-
BOARD_VARIANTS: ${{ needs.package-core.outputs.BOARD_VARIANTS }}
177+
ALL_BOARD_DATA: ${{ needs.package-core.outputs.ALL_BOARD_DATA }}
141178
steps:
142179
- uses: actions/download-artifact@v4
143180
with:
@@ -146,37 +183,41 @@ jobs:
146183
merge-multiple: true
147184

148185
- run: |
149-
echo "### Core test results:" >> "$GITHUB_STEP_SUMMARY"
150-
jq -c '.[]' <<< "$BOARD_VARIANTS" | while read -r BOARD_VARIANT; do
151-
BOARD=$(echo $BOARD_VARIANT | jq -cr '.board')
152-
VARIANT=$(echo $BOARD_VARIANT | jq -cr '.variant')
153-
FQBN="arduino:zephyr:$BOARD"
154-
REPORT_FILE="arduino-zephyr-$BOARD.json"
155-
if [ ! -f $REPORT_FILE ]; then
156-
echo "* :x: $BOARD (`$VARIANT`) - No report found?" >> "$GITHUB_STEP_SUMMARY"
157-
else
158-
REPORT=$(jq -cr '.boards[0].sketches[0]' $REPORT_FILE)
159-
JOB_ID=$(echo $REPORT | jq -cr '.job_id')
160-
JOB_URL="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}/job/${JOB_ID}#step:5:2"
161-
if ! $(echo $REPORT | jq -cr '.compilation_success') ; then
162-
echo "* :x: [$BOARD]($JOB_URL) (\`$VARIANT\`) - Build failed" >> "$GITHUB_STEP_SUMMARY"
186+
ARTIFACTS=$(jq -cr 'map(.artifact) | unique | .[]' <<< ${ALL_BOARD_DATA}) # this avoids the 'zephyr' artifact
187+
for artifact in $ARTIFACTS ; do
188+
echo "### \`$artifact\` test results:" >> "$GITHUB_STEP_SUMMARY"
189+
jq -c "map(select(.artifact == \"$artifact\")) | .[]" <<< ${ALL_BOARD_DATA} | while read -r BOARD_DATA; do
190+
BOARD=$(echo $BOARD_DATA | jq -cr '.board')
191+
VARIANT=$(echo $BOARD_DATA | jq -cr '.variant')
192+
SUBARCH=$(echo $BOARD_DATA | jq -cr '.subarch')
193+
FQBN="arduino:$SUBARCH:$BOARD"
194+
REPORT_FILE="$(echo $FQBN | tr ':' '-').json"
195+
if [ ! -f $REPORT_FILE ]; then
196+
echo "* :x: $BOARD (\`$VARIANT\`) - No report found?" >> "$GITHUB_STEP_SUMMARY"
163197
else
164-
WARNINGS=$(echo $REPORT | jq -cr '.warnings.current.absolute // 0')
165-
if [ $WARNINGS -eq 0 ]; then
166-
echo "* :white_check_mark: $BOARD (\`$VARIANT\`) - Build successful" >> "$GITHUB_STEP_SUMMARY"
198+
REPORT=$(jq -cr '.boards[0].sketches[0]' $REPORT_FILE)
199+
JOB_ID=$(echo $REPORT | jq -cr '.job_id')
200+
JOB_URL="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}/job/${JOB_ID}#step:5:2"
201+
if ! $(echo $REPORT | jq -cr '.compilation_success') ; then
202+
echo "* :x: [$BOARD]($JOB_URL) (\`$VARIANT\`) - Build failed" >> "$GITHUB_STEP_SUMMARY"
167203
else
168-
echo >> "$GITHUB_STEP_SUMMARY"
169-
echo "<details><summary>&nbsp;&nbsp; :warning: <a href=\"$JOB_URL\">$BOARD</a> (<tt>$VARIANT</tt>) - $WARNINGS warnings:</summary>" >> "$GITHUB_STEP_SUMMARY"
170-
echo >> "$GITHUB_STEP_SUMMARY"
171-
echo "\`\`\`" >> "$GITHUB_STEP_SUMMARY"
172-
echo $REPORT | jq -cr '.warnings_log[]' >> "$GITHUB_STEP_SUMMARY"
173-
echo "\`\`\`" >> "$GITHUB_STEP_SUMMARY"
174-
echo >> "$GITHUB_STEP_SUMMARY"
175-
echo "</details>" >> "$GITHUB_STEP_SUMMARY"
176-
echo >> "$GITHUB_STEP_SUMMARY"
204+
WARNINGS=$(echo $REPORT | jq -cr '.warnings.current.absolute // 0')
205+
if [ $WARNINGS -eq 0 ]; then
206+
echo "* :white_check_mark: $BOARD (\`$VARIANT\`) - Build successful" >> "$GITHUB_STEP_SUMMARY"
207+
else
208+
echo >> "$GITHUB_STEP_SUMMARY"
209+
echo "<details><summary>&nbsp;&nbsp; :warning: <a href=\"$JOB_URL\">$BOARD</a> (<tt>$VARIANT</tt>) - $WARNINGS warnings:</summary>" >> "$GITHUB_STEP_SUMMARY"
210+
echo >> "$GITHUB_STEP_SUMMARY"
211+
echo "\`\`\`" >> "$GITHUB_STEP_SUMMARY"
212+
echo $REPORT | jq -cr '.warnings_log[]' >> "$GITHUB_STEP_SUMMARY"
213+
echo "\`\`\`" >> "$GITHUB_STEP_SUMMARY"
214+
echo >> "$GITHUB_STEP_SUMMARY"
215+
echo "</details>" >> "$GITHUB_STEP_SUMMARY"
216+
echo >> "$GITHUB_STEP_SUMMARY"
217+
fi
177218
fi
178219
fi
179-
fi
220+
done
180221
done
181222
182223
- name: Clean up intermediate artifacts
@@ -207,12 +248,11 @@ jobs:
207248
id-token: write
208249
contents: read
209250
env:
210-
CORE_ARTIFACT: ${{ needs.package-core.outputs.CORE_ARTIFACT }}
211-
ARTIFACT_FILE: ${{ needs.package-core.outputs.CORE_ARTIFACT }}.tar.bz2
251+
ALL_CORES_ARTIFACT: ArduinoCore-archives-${{ needs.package-core.outputs.CORE_HASH }}
212252
steps:
213253
- uses: actions/download-artifact@v4
214254
with:
215-
name: ${{ env.CORE_ARTIFACT }}
255+
name: ${{ env.ALL_CORES_ARTIFACT }}
216256

217257
- name: Configure AWS credentials
218258
uses: aws-actions/configure-aws-credentials@v4
@@ -221,40 +261,55 @@ jobs:
221261
aws-region: ${{ secrets.AWS_REGION }}
222262

223263
- name: Upload artifact
224-
run: aws s3 cp ${{ env.ARTIFACT_FILE }} s3://${{ secrets.S3_BUCKET }}/
264+
run: aws s3 cp ArduinoCore-*.tar.bz2 s3://${{ secrets.S3_BUCKET }}/
225265

226266
publish-json:
227-
name: Publish json
267+
name: Publish jsons
228268
runs-on: ubuntu-latest
229-
if: ${{ github.event_name == 'push' && github.repository == 'arduino/ArduinoCore-zephyr' }}
269+
if: ${{ github.repository == 'arduino/ArduinoCore-zephyr' }}
230270
needs:
231271
- package-core
232272
- test-core
233-
- publish-core
234273
env:
235-
CORE_ARTIFACT: ${{ needs.package-core.outputs.CORE_ARTIFACT }}
236-
ARTIFACT_FILE: ${{ needs.package-core.outputs.CORE_ARTIFACT }}.tar.bz2
274+
ALL_CORES_ARTIFACT: ArduinoCore-archives-${{ needs.package-core.outputs.CORE_HASH }}
237275
CORE_TAG: ${{ needs.package-core.outputs.CORE_TAG }}
238-
PACKAGE_INDEX_JSON: zephyr-core-${{ needs.package-core.outputs.CORE_TAG }}.json
276+
CORE_HASH: ${{ needs.package-core.outputs.CORE_HASH }}
277+
ARTIFACTS: ${{ needs.package-core.outputs.ARTIFACTS }}
239278
steps:
240279
- uses: actions/checkout@v4
241280
with:
242281
fetch-depth: 0
243282
persist-credentials: false
244-
sparse-checkout: |
245-
extra/gen_package_index_json.sh
246-
extra/zephyr-core-template.json
283+
fetch-tags: true
247284

248285
- uses: actions/download-artifact@v4
249286
with:
250-
name: ${{ env.CORE_ARTIFACT }}
287+
name: ${{ env.ALL_CORES_ARTIFACT }}
251288

252-
# uses: ARTIFACT_FILE CORE_TAG PACKAGE_INDEX_JSON
253-
- name: Prepare package index snippet
254-
run: ./extra/gen_package_index_json.sh
289+
- name: Prepare package index snippets
290+
run: |
291+
jq -cr '.[]' <<< ${ARTIFACTS} | while read -r artifact; do
292+
ARTIFACT_FILE=ArduinoCore-${artifact}-${CORE_HASH}.tar.bz2
293+
PACKAGE_JSON=ArduinoCore-${artifact}-${CORE_TAG}.json
294+
./extra/gen_package_index_json.sh ${artifact} ${ARTIFACT_FILE} ${PACKAGE_JSON}
295+
done
255296
256-
- name: Archive package index snippet
297+
- name: Archive package index snippets
257298
uses: actions/upload-artifact@v4
258299
with:
259-
name: ${{ env.PACKAGE_INDEX_JSON }}
260-
path: ${{ env.PACKAGE_INDEX_JSON }}
300+
name: ArduinoCore-zephyr-${{ env.CORE_TAG }}-jsons
301+
path: ArduinoCore-*-${{ env.CORE_TAG }}.json
302+
303+
cleanup-artifacts:
304+
runs-on: ubuntu-latest
305+
needs:
306+
- package-core
307+
- publish-core
308+
- publish-json
309+
if: ${{ !cancelled() }}
310+
steps:
311+
- name: Clean up intermediate artifacts
312+
uses: geekyeggo/delete-artifact@v5.1.0
313+
with:
314+
name: ArduinoCore-archives-${{ needs.package-core.outputs.CORE_HASH }}
315+
failOnError: false

‎boards.txt‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ giga.build.zephyr_target=arduino_giga_r1//m7
1717
giga.build.zephyr_args=--shield arduino_giga_display_shield
1818
giga.build.zephyr_hals=hal_stm32 hal_infineon
1919
giga.build.variant=arduino_giga_r1_stm32h747xx_m7
20+
giga.build.artifact=zephyr_main
2021
giga.build.mcu=cortex-m7
2122
giga.build.fpu=-mfpu=fpv5-d16
2223
giga.build.architecture=cortex-m7
@@ -78,6 +79,7 @@ nano33ble.menu.debug.true.build.zsk_args.debug=-debug
7879
nano33ble.build.zephyr_target=arduino_nano_33_ble//sense
7980
nano33ble.build.zephyr_args=
8081
nano33ble.build.zephyr_hals=hal_nordic
82+
nano33ble.build.artifact=zephyr_main
8183
nano33ble.build.variant=arduino_nano_33_ble_nrf52840_sense
8284
nano33ble.build.mcu=cortex-m4
8385
nano33ble.build.fpu=-mfpu=fpv4-sp-d16
@@ -126,7 +128,7 @@ nano33ble.debug.cortex-debug.custom.request=attach
126128

127129
##############################################################################################################
128130

129-
ek_ra8d1.name=EK_RA8D1
131+
ek_ra8d1.name=Renesas RA8D1 EK
130132
ek_ra8d1.build.core=arduino
131133
ek_ra8d1.build.crossprefix=arm-zephyr-eabi-
132134
ek_ra8d1.build.compiler_path={runtime.tools.arm-zephyr-eabi-0.16.8.path}/bin/
@@ -180,7 +182,7 @@ ek_ra8d1.bootloader.target=R7FA8D1AH
180182

181183
##############################################################################################################
182184

183-
frdm_mcxn947.name=MCXN947
185+
frdm_mcxn947.name=NXP FRDM MCXN947
184186
frdm_mcxn947.build.core=arduino
185187
frdm_mcxn947.build.crossprefix=arm-zephyr-eabi-
186188
frdm_mcxn947.build.compiler_path={runtime.tools.arm-zephyr-eabi-0.16.8.path}/bin/
@@ -241,6 +243,7 @@ portentah7.menu.debug.true.build.zsk_args.debug=-debug
241243
portentah7.build.zephyr_target=arduino_portenta_h7@1.0.0//m7
242244
portentah7.build.zephyr_args=
243245
portentah7.build.zephyr_hals=hal_stm32 hal_infineon
246+
portentah7.build.artifact=zephyr_main
244247
portentah7.build.variant=arduino_portenta_h7_stm32h747xx_m7
245248
portentah7.build.mcu=cortex-m7
246249
portentah7.build.fpu=-mfpu=fpv5-d16
@@ -291,7 +294,7 @@ portentah7.debug.svd_file={runtime.platform.path}/svd/STM32H747_CM7.svd
291294

292295
##############################################################################################################
293296

294-
frdm_rw612.name=RW612
297+
frdm_rw612.name=NXP FRDM RW612
295298
frdm_rw612.build.core=arduino
296299
frdm_rw612.build.crossprefix=arm-zephyr-eabi-
297300
frdm_rw612.build.compiler_path={runtime.tools.arm-zephyr-eabi-0.16.8.path}/bin/
@@ -351,6 +354,7 @@ niclasense.menu.debug.true.build.zsk_args.debug=-debug
351354
niclasense.build.zephyr_target=arduino_nicla_sense_me
352355
niclasense.build.zephyr_args=
353356
niclasense.build.zephyr_hals=hal_nordic
357+
niclasense.build.artifact=zephyr_main
354358
niclasense.build.variant=arduino_nicla_sense_me_nrf52832
355359
niclasense.build.mcu=cortex-m4
356360
niclasense.build.fpu=-mfpu=fpv4-sp-d16
@@ -405,7 +409,7 @@ niclasense.debug.cortex-debug.custom.request=attach
405409

406410
##########################################################################################
407411

408-
portentac33.name=Portenta C33
412+
portentac33.name=Arduino Portenta C33
409413
portentac33.build.core=arduino
410414
portentac33.build.crossprefix=arm-zephyr-eabi-
411415
portentac33.build.compiler_path={runtime.tools.arm-zephyr-eabi-0.16.8.path}/bin/
@@ -422,6 +426,7 @@ portentac33.menu.link_mode.static.upload.extension=bin-zsk.bin
422426
portentac33.build.zephyr_target=arduino_portenta_c33
423427
portentac33.build.zephyr_args=
424428
portentac33.build.zephyr_hals=hal_renesas nanopb
429+
portentac33.build.artifact=zephyr_main
425430
portentac33.build.variant=arduino_portenta_c33_r7fa6m5bh3cfc
426431
portentac33.build.mcu=cortex-m33
427432
portentac33.build.fpu=-mfpu=fpv5-sp-d16
@@ -473,6 +478,7 @@ opta.menu.debug.true.build.zsk_args.debug=-debug
473478
opta.build.zephyr_target=arduino_opta//m7
474479
opta.build.zephyr_args=
475480
opta.build.zephyr_hals=hal_stm32 hal_infineon
481+
opta.build.artifact=zephyr_main
476482
opta.build.variant=arduino_opta_stm32h747xx_m7
477483
opta.build.mcu=cortex-m7
478484
opta.build.fpu=-mfpu=fpv5-d16
@@ -541,6 +547,8 @@ unoq.menu.flash_mode.ram.openocd_cfg=flash_sketch_ram.cfg
541547
unoq.build.zephyr_target=arduino_uno_q
542548
unoq.build.zephyr_args=
543549
unoq.build.variant=arduino_uno_q_stm32u585xx
550+
unoq.build.artifact=zephyr_unoq
551+
unoq.build.subarch=zephyr
544552
unoq.build.mcu=cortex-m33
545553
unoq.build.fpu=-mfpu=fpv5-sp-d16
546554
unoq.build.architecture=cortex-m33

‎extra/package_core.exc‎ renamed to ‎extra/artifacts/_common.exc‎

File renamed without changes.

0 commit comments

Comments
 (0)