test: add containerd e2e testings to API v2(rust client) (#3203)

* feat: add e2e to rust client Signed-off-by: Gaius <[email protected]> * test: add containerd e2e testings for API v2 Signed-off-by: Gaius <[email protected]> --------- Signed-off-by: Gaius <[email protected]>
dragonflyoss · Apr 19, 2024 · 11ff4d4 · 11ff4d4
1 parent d1d3bc2
commit 11ff4d4
Show file tree

Hide file tree

Showing 29 changed files with 876 additions and 119 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -5,7 +5,7 @@ on:
     branches: [main, release-*]
     paths-ignore: ['**.md', '**.png', '**.jpg', '**.svg', '**/docs/**']
   pull_request:
-    branches: [ main, release-* ]
+    branches: [main, release-*]
     paths-ignore: ['**.md', '**.png', '**.jpg', '**.svg', '**/docs/**']
   schedule:
     - cron: '0 4 * * *'

diff --git a/...lows/compatibility-e2e-with-client-go.yml → .github/workflows/compatibility-e2e-v1.yml b/...lows/compatibility-e2e-with-client-go.yml → .github/workflows/compatibility-e2e-v1.yml
@@ -1,4 +1,4 @@
-name: Compatibility E2E Test(Golang Client)
+name: Compatibility E2E Test(API v1 - Golang Client)
 
 on:
   push:
@@ -21,8 +21,7 @@ env:
   DRAGONFLY_PROXY_SERVER_PATH: test/testdata/k8s/proxy.yaml
 
 jobs:
-  compatibility_e2e_tests_with_client_go:
-    name: e2e_tests_with_${{ matrix.module }}
+  compatibility_e2e_tests:
     timeout-minutes: 60
     runs-on: ubuntu-latest
     strategy:
@@ -162,7 +161,9 @@ jobs:
           DRAGONFLY_COMPATIBILITY_E2E_TEST_MODE: ${{ matrix.module }}
           DRAGONFLY_COMPATIBILITY_E2E_TEST_IMAGE: ${{ matrix.image }}
           DRAGONFLY_STABLE_IMAGE_TAG: ${{ matrix.image-tag }}
-        run: make actions-e2e-test-coverage
+        run: |
+          ginkgo -v -r --race --fail-fast --cover --trace --show-node-events test/e2e/v1
+          cat coverprofile.out >> coverage.txt
 
       - name: Move cache
         run: |

diff --git a/.github/workflows/e2e-with-client-go.yml → .github/workflows/e2e-v1.yml b/.github/workflows/e2e-with-client-go.yml → .github/workflows/e2e-v1.yml
@@ -1,4 +1,4 @@
-name: E2E Test(Golang Client)
+name: E2E Test(API v1 - Golang Client)
 
 on:
   push:
@@ -21,7 +21,7 @@ env:
   DRAGONFLY_MINIO_SERVER_PATH: test/testdata/k8s/minio.yaml
 
 jobs:
-  e2e_tests_with_client_go:
+  e2e_tests:
     runs-on: ubuntu-latest
     timeout-minutes: 60
     strategy:
@@ -184,7 +184,7 @@ jobs:
           make build-e2e-download-grpc-test
           # generate an empty file
           docker exec kind-control-plane touch /tmp/empty-file
-          ginkgo -v -r --race --fail-fast --cover --trace --show-node-events --skip=${{ matrix.skip }} test/e2e -- \
+          ginkgo -v -r --race --fail-fast --cover --trace --show-node-events --skip=${{ matrix.skip }} test/e2e/v1 -- \
               --feature-gates=dfget-range=true,dfget-open-range=true,dfget-empty-file=true,dfget-recursive=true
           cat coverprofile.out >> coverage.txt
 

diff --git a/.github/workflows/e2e-with-client-rs.yml → .github/workflows/e2e-v2.yml b/.github/workflows/e2e-with-client-rs.yml → .github/workflows/e2e-v2.yml
@@ -1,4 +1,4 @@
-name: E2E Test(Rust Client)
+name: E2E Test(API v2 - Rust Client)
 
 on:
   push:
@@ -13,12 +13,12 @@ on:
 env:
   KIND_VERSION: v0.12.0
   CONTAINERD_VERSION: v1.5.2
-  KIND_CONFIG_PATH: test/testdata/kind/config-rs.yaml
+  KIND_CONFIG_PATH: test/testdata/kind/config-v2.yaml
   DRAGONFLY_CHARTS_PATH: deploy/helm-charts/charts/dragonfly
   DRAGONFLY_FILE_SERVER_PATH: test/testdata/k8s/file-server.yaml
 
 jobs:
-  e2e_tests_with_client_rs:
+  e2e_tests:
     runs-on: ubuntu-latest
     timeout-minutes: 60
     strategy:
@@ -27,7 +27,7 @@ jobs:
           - "normal"
         include:
           - module: normal
-            charts-config: test/testdata/charts/config-rs.yaml
+            charts-config: test/testdata/charts/config-v2.yaml
             skip: ""
     steps:
       - name: Free Disk Space (Ubuntu)
@@ -74,11 +74,11 @@ jobs:
       - name: Pull Rust Client Image
         run: |
           cd client-rs
-          # TODO Use the latest tag.
-          # CLIENT_TAG=$(git describe --tags $(git rev-list --tags --max-count=1))
-          CLIENT_TAG=latest
+          CLIENT_TAG=$(git describe --tags $(git rev-parse HEAD))
           docker pull dragonflyoss/client:$CLIENT_TAG
           docker tag dragonflyoss/client:$CLIENT_TAG dragonflyoss/client:latest
+          docker pull dragonflyoss/dfinit:$CLIENT_TAG
+          docker tag dragonflyoss/dfinit:$CLIENT_TAG dragonflyoss/dfinit:latest
 
       - name: Build Scheduler Image
         uses: docker/build-push-action@v5
@@ -114,20 +114,19 @@ jobs:
           kind load docker-image dragonflyoss/manager:latest
           kind load docker-image dragonflyoss/scheduler:latest
           kind load docker-image dragonflyoss/client:latest
+          kind load docker-image dragonflyoss/dfinit:latest
 
-      # TODO Setup dragonfly
-      # - name: Setup dragonfly
-        # run: |
-          # helm install --wait --timeout 10m --dependency-update --create-namespace --namespace dragonfly-system -f ${{ matrix.charts-config }} dragonfly ${{ env.DRAGONFLY_CHARTS_PATH }}
-          # kubectl apply -f ${{ env.DRAGONFLY_FILE_SERVER_PATH }}
-          # kubectl wait po file-server-0 --namespace dragonfly-e2e --for=condition=ready --timeout=10m
-
-      # TODO Run tests
-      # - name: Run E2E test
-        # run: |
-          # ginkgo -v -r --race --fail-fast --cover --trace --show-node-events --skip=${{ matrix.skip }} test/e2e -- \
-              # --feature-gates=dfget-range=true,dfget-open-range=true,dfget-empty-file=true,dfget-recursive=true
-          # cat coverprofile.out >> coverage.txt
+      - name: Setup dragonfly
+        run: |
+          helm install --wait --timeout 10m --dependency-update --create-namespace --namespace dragonfly-system -f ${{ matrix.charts-config }} dragonfly ${{ env.DRAGONFLY_CHARTS_PATH }}
+          kubectl apply -f ${{ env.DRAGONFLY_FILE_SERVER_PATH }}
+          kubectl wait po file-server-0 --namespace dragonfly-e2e --for=condition=ready --timeout=10m
+          kubectl get po -n dragonfly-system
+
+      - name: Run E2E test
+        run: |
+          ginkgo -v -r --race --fail-fast --cover --trace --show-node-events --skip=${{ matrix.skip }} test/e2e/v2
+          cat coverprofile.out >> coverage.txt
 
       - name: Move cache
         run: |

diff --git a/.gitignore b/.gitignore
@@ -67,3 +67,5 @@ artifacts
 .run
 deploy/docker-compose/log
 deploy/docker-compose/config
+test/e2e/v1/*.log
+test/e2e/v2/*.log
diff --git a/client-rs b/client-rs
diff --git a/test/e2e/concurrency_test.go → test/e2e/v1/concurrency_test.go b/test/e2e/concurrency_test.go → test/e2e/v1/concurrency_test.go
@@ -22,7 +22,7 @@ import (
 	. "github.com/onsi/ginkgo/v2" //nolint
 	. "github.com/onsi/gomega"    //nolint
 
-	"d7y.io/dragonfly/v2/test/e2e/util"
+	"d7y.io/dragonfly/v2/test/e2e/v1/util"
 )
 
 var _ = Describe("Download concurrency", func() {

diff --git a/test/e2e/constants.go → test/e2e/v1/constants.go b/test/e2e/constants.go → test/e2e/v1/constants.go
diff --git a/test/e2e/containerd_test.go → test/e2e/v1/containerd_test.go b/test/e2e/containerd_test.go → test/e2e/v1/containerd_test.go
@@ -22,7 +22,7 @@ import (
 	. "github.com/onsi/ginkgo/v2" //nolint
 	. "github.com/onsi/gomega"    //nolint
 
-	"d7y.io/dragonfly/v2/test/e2e/util"
+	"d7y.io/dragonfly/v2/test/e2e/v1/util"
 )
 
 var _ = Describe("Containerd with CRI support", func() {

diff --git a/test/e2e/dfget_test.go → test/e2e/v1/dfget_test.go b/test/e2e/dfget_test.go → test/e2e/v1/dfget_test.go
@@ -27,7 +27,7 @@ import (
 	. "github.com/onsi/gomega"    //nolint
 
 	"d7y.io/dragonfly/v2/pkg/net/http"
-	"d7y.io/dragonfly/v2/test/e2e/util"
+	"d7y.io/dragonfly/v2/test/e2e/v1/util"
 )
 
 var _ = Describe("Download with dfget and proxy", func() {

diff --git a/test/e2e/e2e_test.go → test/e2e/v1/e2e_test.go b/test/e2e/e2e_test.go → test/e2e/v1/e2e_test.go
@@ -28,8 +28,8 @@ import (
 	. "github.com/onsi/ginkgo/v2" //nolint
 	. "github.com/onsi/gomega"    //nolint
 
-	_ "d7y.io/dragonfly/v2/test/e2e/manager"
-	"d7y.io/dragonfly/v2/test/e2e/util"
+	_ "d7y.io/dragonfly/v2/test/e2e/v1/manager"
+	"d7y.io/dragonfly/v2/test/e2e/v1/util"
 )
 
 var featureGatesFlag string

diff --git a/test/e2e/manager/constants.go → test/e2e/v1/manager/constants.go b/test/e2e/manager/constants.go → test/e2e/v1/manager/constants.go
+85 −3		.github/workflows/docker.yml
+21 −1		.github/workflows/release.yml
+1 −1		CONTRIBUTING.md
+890 −122		Cargo.lock
+40 −17		Cargo.toml
+1 −1		README.md
+7 −0		dragonfly-client-backend/Cargo.toml
+19 −0		dragonfly-client-backend/examples/plugin/Cargo.toml
+27 −0		dragonfly-client-backend/examples/plugin/README.md
+51 −0		dragonfly-client-backend/examples/plugin/src/lib.rs
+153 −92		dragonfly-client-backend/src/http.rs
+219 −0		dragonfly-client-backend/src/lib.rs
+2 −1		dragonfly-client-config/Cargo.toml
+20 −14		dragonfly-client-config/src/dfdaemon.rs
+1 −1		dragonfly-client-config/src/dfget.rs
+405 −0		dragonfly-client-config/src/dfinit.rs
+1 −1		dragonfly-client-config/src/dfstore.rs
+1 −0		dragonfly-client-config/src/lib.rs
+1 −14		dragonfly-client-core/Cargo.toml
+237 −0		dragonfly-client-core/src/error/errors.rs
+59 −0		dragonfly-client-core/src/error/message.rs
+187 −0		dragonfly-client-core/src/error/mod.rs
+4 −247		dragonfly-client-core/src/lib.rs
+19 −0		dragonfly-client-core/src/result.rs
+27 −0		dragonfly-client-init/Cargo.toml
+42 −0		dragonfly-client-init/Dockerfile
+101 −0		dragonfly-client-init/src/bin/main.rs
+220 −0		dragonfly-client-init/src/container_runtime/containerd.rs
+111 −0		dragonfly-client-init/src/container_runtime/crio.rs
+53 −0		dragonfly-client-init/src/container_runtime/docker.rs
+125 −0		dragonfly-client-init/src/container_runtime/mod.rs
+17 −0		dragonfly-client-init/src/lib.rs
+3 −1		dragonfly-client-storage/src/content.rs
+1 −1		dragonfly-client-storage/src/lib.rs
+257 −301		dragonfly-client-storage/src/metadata.rs
+15 −9		dragonfly-client-util/src/http/mod.rs
+5 −2		dragonfly-client-util/src/id_generator/mod.rs
+15 −7		dragonfly-client-util/src/tls/mod.rs
+2 −2		dragonfly-client/Cargo.toml
+8 −2		dragonfly-client/Dockerfile
+10 −10		dragonfly-client/src/bin/dfdaemon/main.rs
+2 −1		dragonfly-client/src/bin/dfget/main.rs
+1 −1		dragonfly-client/src/dynconfig/mod.rs
+63 −23		dragonfly-client/src/grpc/dfdaemon_download.rs
+214 −56		dragonfly-client/src/grpc/dfdaemon_upload.rs
+8 −3		dragonfly-client/src/grpc/health.rs
+7 −3		dragonfly-client/src/grpc/manager.rs
+56 −0		dragonfly-client/src/grpc/mod.rs
+32 −11		dragonfly-client/src/grpc/scheduler.rs
+6 −2		dragonfly-client/src/grpc/security.rs
+17 −0		dragonfly-client/src/proxy/header.rs
+56 −19		dragonfly-client/src/proxy/mod.rs
+252 −202		dragonfly-client/src/task/mod.rs
+78 −57		dragonfly-client/src/task/piece.rs
+21 −15		dragonfly-client/src/task/piece_collector.rs