Use buf.build to manage our protobuf related activities (#2736)

* Introduce `buf.build` files

* protobuf: remove google, and use buf dep

* Taskfile: use `buf` instead of `protoc`

* rpc: regenerate all with `buf`

* Taskfile: use buf to generate docs

* github: update protobuf related actions

* docs: update CONTRIBUTING

* buf: add COMMENT linter

* Added missing comments (for linter)

* Fixed punctation and capitalization of grpc comments

---------

Co-authored-by: Cristian Maglie <c.maglie@arduino.cc>

Author: alessio-perugini

.github/workflows/check-markdown-task.yml

@@ -116,10 +116,9 @@ jobs:
       - name: Install Go dependencies
         run: go install github.com/pseudomuto/protoc-gen-doc/cmd/protoc-gen-doc@v1.4.1
 
-      - name: Install protoc compiler
-        uses: arduino/setup-protoc@v3
+      - uses: bufbuild/buf-action@v1
         with:
-          repo-token: ${{ secrets.GITHUB_TOKEN }}
+          setup_only: true
 
       - name: Install Task
         uses: arduino/setup-task@v2

.github/workflows/check-mkdocs-task.yml

@@ -44,10 +44,9 @@ jobs:
       - name: Install Go dependencies
         run: go install github.com/pseudomuto/protoc-gen-doc/cmd/protoc-gen-doc@v1.4.1
 
-      - name: Install protoc compiler
-        uses: arduino/setup-protoc@v3
+      - uses: bufbuild/buf-action@v1
         with:
-          repo-token: ${{ secrets.GITHUB_TOKEN }}
+          setup_only: true
 
       - name: Install Python
         uses: actions/setup-python@v5

.github/workflows/check-protobuf-task.yml

@@ -53,21 +53,9 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v4
 
-      - name: Install Go
-        uses: actions/setup-go@v5
+      - uses: bufbuild/buf-action@v1
         with:
-          go-version: ${{ env.GO_VERSION }}
-
-      - name: Install protoc compiler
-        uses: arduino/setup-protoc@v3
-        with:
-          version: v26.1
-          repo-token: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Install Go deps
-        run: |
-          go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.34.2
-          go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.3.0
+          setup_only: true
 
       - name: Install Task
         uses: arduino/setup-task@v2
@@ -90,43 +78,15 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v4
 
-      - name: Install Go
-        uses: actions/setup-go@v5
-        with:
-          go-version: ${{ env.GO_VERSION }}
-
-      - name: Install buf (protoc linter)
+      # used by the protobuf breaking change detector
+      - name: Fetch main branch
         run: |
-          go install github.com/bufbuild/buf/cmd/buf@v1.20.0
-          go install github.com/bufbuild/buf/cmd/protoc-gen-buf-breaking@v1.20.0
-          go install github.com/bufbuild/buf/cmd/protoc-gen-buf-lint@v1.20.0
-
-      - name: Install Task
-        uses: arduino/setup-task@v2
-        with:
-          repo-token: ${{ secrets.GITHUB_TOKEN }}
-          version: 3.x
-
-      - name: Lint protocol buffers
-        run: task protoc:check
+          git fetch origin master
 
-  check-formatting:
-    needs: run-determination
-    if: needs.run-determination.outputs.result == 'true'
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-
-      - name: Install Task
-        uses: arduino/setup-task@v2
+      - uses: bufbuild/buf-action@v1
         with:
-          repo-token: ${{ secrets.GITHUB_TOKEN }}
-          version: 3.x
-
-      - name: Format protocol buffers
-        run: task protoc:format
-
-      - name: Check formatting
-        run: git diff --color --exit-code
+          pr_comment: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository }}
+          lint: ${{ github.event_name == 'pull_request' }}
+          format: ${{ github.event_name == 'pull_request' }}
+          breaking: ${{ github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'buf skip breaking') }}
+          breaking_against: ".git#branch=origin/master,subdir=rpc"

.github/workflows/deploy-cobra-mkdocs-versioned-poetry.yml

@@ -64,10 +64,9 @@ jobs:
       - name: Install Go dependencies
         run: go install github.com/pseudomuto/protoc-gen-doc/cmd/protoc-gen-doc@v1.4.1
 
-      - name: Install protoc compiler
-        uses: arduino/setup-protoc@v3
+      - uses: bufbuild/buf-action@v1
         with:
-          repo-token: ${{ secrets.GITHUB_TOKEN }}
+          setup_only: true
 
       - name: Install Python
         uses: actions/setup-python@v5

Taskfile.yml

@@ -232,16 +232,19 @@ tasks:
       - protoc:check
       - protoc:format
       - protoc:compile
+      - protoc:breaking-change-detection
 
   protoc:compile:
     desc: Compile protobuf definitions
     cmds:
-      - '{{ default "protoc" .PROTOC_BINARY }} --proto_path=rpc --go_out=./rpc --go_opt=paths=source_relative --go-grpc_out=./rpc --go-grpc_opt=paths=source_relative ./rpc/cc/arduino/cli/commands/v1/*.proto'
+      - buf dep update
+      - buf generate
 
   protoc:docs:
     desc: Generate docs for protobuf definitions
     cmds:
-      - '{{ default "protoc" .PROTOC_BINARY }} --doc_out=./docs/rpc --doc_opt=markdown,commands.md --proto_path=rpc ./rpc/cc/arduino/cli/commands/v1/*.proto'
+      - |
+        buf generate --template buf.doc.gen.yaml
 
   docs:include-configuration-json-schema:
     desc: Copy configuration JSON schema to make it available in documentation
@@ -251,7 +254,7 @@ tasks:
   protoc:check:
     desc: Perform linting of the protobuf definitions
     cmds:
-      - buf lint rpc
+      - buf lint
 
   protoc:collect:
     desc: Create a zip file containing all .proto files in DIST_DIR
@@ -263,7 +266,12 @@ tasks:
   protoc:format:
     desc: Perform formatting of the protobuf definitions
     cmds:
-      - clang-format -i rpc/cc/arduino/cli/*/*/*.proto
+      - buf format --write --exit-code
+
+  protoc:breaking-change-detection:
+    desc: Detect protobuf breaking changes
+    cmds:
+      - buf breaking --against '.git#branch=origin/master,subdir=rpc'
 
   build:
     desc: Build the project

buf.doc.gen.yaml

@@ -0,0 +1,10 @@
+version: v2
+plugins:
+  # Local plugin used to generate docs
+  # go install github.com/pseudomuto/protoc-gen-doc/cmd/protoc-gen-doc@v1.4.1
+  - local: protoc-gen-doc
+    out: ./docs/rpc
+    opt:
+      - markdown,commands.md
+inputs:
+  - directory: ./rpc

buf.gen.yaml

@@ -0,0 +1,14 @@
+version: v2
+plugins:
+  # Use protoc-gen-go
+  - remote: buf.build/protocolbuffers/go:v1.34.2
+    out: ./rpc
+    opt:
+      - paths=source_relative
+  # Use of protoc-gen-go-grpc
+  - remote: buf.build/grpc/go:v1.5.1
+    out: ./rpc
+    opt:
+      - paths=source_relative
+inputs:
+  - directory: ./rpc

buf.lock

@@ -0,0 +1,6 @@
+# Generated by buf. DO NOT EDIT.
+version: v2
+deps:
+  - name: buf.build/googleapis/googleapis
+    commit: e7f8d366f5264595bcc4cd4139af9973
+    digest: b5:0cd69a689ee320ed815663d57d1bc3a1d6823224a7a717d46fee3a68197c25a6f5f932c0b0e49f8370c70c247a6635969a6a54af5345cafd51e0667298768aca

buf.yaml

@@ -0,0 +1,19 @@
+version: v2
+deps:
+  - buf.build/googleapis/googleapis:e7f8d366f5264595bcc4cd4139af9973
+breaking:
+  use:
+    - FILE
+lint:
+  use:
+    - STANDARD
+    - COMMENT_ENUM
+    - COMMENT_ENUM_VALUE
+    - COMMENT_FIELD
+    - COMMENT_RPC
+    - COMMENT_SERVICE
+  ignore_only:
+    ENUM_ZERO_VALUE_SUFFIX:
+      - rpc/cc/arduino/cli/commands/v1/lib.proto
+modules:
+  - path: rpc

docs/CONTRIBUTING.md

@@ -79,13 +79,7 @@ If you want to run integration tests you will also need:
 
 If you're working on the gRPC interface you will also have to:
 
-- download and install the [protoc][6] compiler (use the version required to match the generated code, please note that
-  the latest releases does not follow semantic versioning anymore so, for example, the version 5.26.1 must be searched
-  as 26.1 dropping the major number)
-- install `protoc-gen-go` using: `go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.34.2` (use the version
-  required to match the generated code)
-- install `protoc-gen-go-grpc` using: `go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.3.0` (use the version
-  required to match the generated code)
+- download and install [buf][6] our tool to compile and generate proto files
 
 ### Building the source code
 
@@ -337,7 +331,7 @@ If your PR doesn't need to be included in the changelog, please start the commit
 [1]: https://go.dev/doc/install
 [2]: https://taskfile.dev/#/installation
 [3]: https://www.python.org/downloads/
-[6]: https://github.com/protocolbuffers/protobuf/releases
+[6]: https://buf.build/docs/installation/
 [7]: https://pages.github.com/
 [9]: https://www.mkdocs.org/
 [11]: https://github.com/arduino/arduino-cli/blob/master/.github/workflows/deploy-cobra-mkdocs-versioned-poetry.yml