Skip to content

Commit

Permalink
feat: add e2e to rust client
Browse files Browse the repository at this point in the history
Signed-off-by: Gaius <[email protected]>
  • Loading branch information
gaius-qi committed Apr 18, 2024
1 parent 4abb179 commit 90f4fca
Show file tree
Hide file tree
Showing 39 changed files with 1,325 additions and 53 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/check-size.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ on:
push:
branches: [main, release-*]
pull_request:
branches: [main, release-*]
branches: [amain, release-*]

env:
GO_VERSION: '1.21'
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ on:
branches: [main, release-*]
paths-ignore: ['**.md', '**.png', '**.jpg', '**.svg', '**/docs/**']
pull_request:
branches: [ main, release-* ]
branches: [amain, release-*]
paths-ignore: ['**.md', '**.png', '**.jpg', '**.svg', '**/docs/**']
schedule:
- cron: '0 4 * * *'
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/codeql-analysis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ on:
branches: [main, release-*]
paths-ignore: ['**.md', '**.png', '**.jpg', '**.svg', '**/docs/**']
pull_request:
branches: [main, release-*]
branches: [amain, release-*]
paths-ignore: ['**.md', '**.png', '**.jpg', '**.svg', '**/docs/**']
schedule:
- cron: '0 4 * * *'
Expand Down
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
name: Compatibility E2E Test(Golang Client)
name: Compatibility E2E Test(API v1)

on:
push:
branches: [main, release-*]
paths-ignore: ["**.md", "**.png", "**.jpg", "**.svg", "**/docs/**"]
pull_request:
branches: [main, release-*]
branches: [amain, release-*]
paths-ignore: ["**.md", "**.png", "**.jpg", "**.svg", "**/docs/**"]
schedule:
- cron: '0 4 * * *'
Expand Down Expand Up @@ -162,7 +162,9 @@ jobs:
DRAGONFLY_COMPATIBILITY_E2E_TEST_MODE: ${{ matrix.module }}
DRAGONFLY_COMPATIBILITY_E2E_TEST_IMAGE: ${{ matrix.image }}
DRAGONFLY_STABLE_IMAGE_TAG: ${{ matrix.image-tag }}
run: make actions-e2e-test-coverage
run: |
ginkgo -v -r --race --fail-fast --cover --trace --show-node-events test/e2e/v1
cat coverprofile.out >> coverage.txt
- name: Move cache
run: |
Expand Down
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
name: E2E Test(Golang Client)
name: E2E Test(API v1)

on:
push:
branches: [main, release-*]
paths-ignore: ["**.md", "**.png", "**.jpg", "**.svg", "**/docs/**"]
pull_request:
branches: [main, release-*]
branches: [amain, release-*]
paths-ignore: ["**.md", "**.png", "**.jpg", "**.svg", "**/docs/**"]
schedule:
- cron: '0 4 * * *'
Expand Down Expand Up @@ -184,7 +184,7 @@ jobs:
make build-e2e-download-grpc-test
# generate an empty file
docker exec kind-control-plane touch /tmp/empty-file
ginkgo -v -r --race --fail-fast --cover --trace --show-node-events --skip=${{ matrix.skip }} test/e2e -- \
ginkgo -v -r --race --fail-fast --cover --trace --show-node-events --skip=${{ matrix.skip }} --skip=client-rs test/e2e/v1 -- \
--feature-gates=dfget-range=true,dfget-open-range=true,dfget-empty-file=true,dfget-recursive=true
cat coverprofile.out >> coverage.txt
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
name: E2E Test(Rust Client)
name: E2E Test(API v2)

on:
push:
Expand All @@ -13,7 +13,7 @@ on:
env:
KIND_VERSION: v0.12.0
CONTAINERD_VERSION: v1.5.2
KIND_CONFIG_PATH: test/testdata/kind/config-rs.yaml
KIND_CONFIG_PATH: test/testdata/kind/config-v2.yaml
DRAGONFLY_CHARTS_PATH: deploy/helm-charts/charts/dragonfly
DRAGONFLY_FILE_SERVER_PATH: test/testdata/k8s/file-server.yaml

Expand All @@ -27,7 +27,7 @@ jobs:
- "normal"
include:
- module: normal
charts-config: test/testdata/charts/config-rs.yaml
charts-config: test/testdata/charts/config-v2.yaml
skip: ""
steps:
- name: Free Disk Space (Ubuntu)
Expand Down Expand Up @@ -74,11 +74,11 @@ jobs:
- name: Pull Rust Client Image
run: |
cd client-rs
# TODO Use the latest tag.
# CLIENT_TAG=$(git describe --tags $(git rev-list --tags --max-count=1))
CLIENT_TAG=latest
CLIENT_TAG=$(git describe --tags $(git rev-parse HEAD))
docker pull dragonflyoss/client:$CLIENT_TAG
docker tag dragonflyoss/client:$CLIENT_TAG dragonflyoss/client:latest
docker pull dragonflyoss/dfinit:$CLIENT_TAG
docker tag dragonflyoss/dfinit:$CLIENT_TAG dragonflyoss/dfinit:latest
- name: Build Scheduler Image
uses: docker/build-push-action@v5
Expand Down Expand Up @@ -114,20 +114,19 @@ jobs:
kind load docker-image dragonflyoss/manager:latest
kind load docker-image dragonflyoss/scheduler:latest
kind load docker-image dragonflyoss/client:latest
kind load docker-image dragonflyoss/dfinit:latest
# TODO Setup dragonfly
# - name: Setup dragonfly
# run: |
# helm install --wait --timeout 10m --dependency-update --create-namespace --namespace dragonfly-system -f ${{ matrix.charts-config }} dragonfly ${{ env.DRAGONFLY_CHARTS_PATH }}
# kubectl apply -f ${{ env.DRAGONFLY_FILE_SERVER_PATH }}
# kubectl wait po file-server-0 --namespace dragonfly-e2e --for=condition=ready --timeout=10m

# TODO Run tests
# - name: Run E2E test
# run: |
# ginkgo -v -r --race --fail-fast --cover --trace --show-node-events --skip=${{ matrix.skip }} test/e2e -- \
# --feature-gates=dfget-range=true,dfget-open-range=true,dfget-empty-file=true,dfget-recursive=true
# cat coverprofile.out >> coverage.txt
- name: Setup dragonfly
run: |
helm install --wait --timeout 10m --dependency-update --create-namespace --namespace dragonfly-system -f ${{ matrix.charts-config }} dragonfly ${{ env.DRAGONFLY_CHARTS_PATH }}
kubectl apply -f ${{ env.DRAGONFLY_FILE_SERVER_PATH }}
kubectl wait po file-server-0 --namespace dragonfly-e2e --for=condition=ready --timeout=10m
kubectl get po -n dragonfly-system
- name: Run E2E test
run: |
ginkgo -v -r --race --fail-fast --cover --trace --show-node-events --skip=${{ matrix.skip }} --skip=client-go test/e2e/v2
cat coverprofile.out >> coverage.txt
- name: Move cache
run: |
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/lint.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ on:
push:
branches: [main, release-*]
pull_request:
branches: [main, release-*]
branches: [amain, release-*]

env:
GO_VERSION: '1.21'
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/nydus-e2e.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ on:
branches: [main, release-*]
paths-ignore: ["**.md", "**.png", "**.jpg", "**.svg", "**/docs/**"]
pull_request:
branches: [main, release-*]
branches: [amain, release-*]
paths-ignore: ["**.md", "**.png", "**.jpg", "**.svg", "**/docs/**"]
schedule:
- cron: '0 4 * * *'
Expand Down
2 changes: 1 addition & 1 deletion client-rs
Submodule client-rs updated 55 files
+85 −3 .github/workflows/docker.yml
+21 −1 .github/workflows/release.yml
+1 −1 CONTRIBUTING.md
+890 −122 Cargo.lock
+40 −17 Cargo.toml
+1 −1 README.md
+7 −0 dragonfly-client-backend/Cargo.toml
+19 −0 dragonfly-client-backend/examples/plugin/Cargo.toml
+27 −0 dragonfly-client-backend/examples/plugin/README.md
+51 −0 dragonfly-client-backend/examples/plugin/src/lib.rs
+153 −92 dragonfly-client-backend/src/http.rs
+219 −0 dragonfly-client-backend/src/lib.rs
+2 −1 dragonfly-client-config/Cargo.toml
+20 −14 dragonfly-client-config/src/dfdaemon.rs
+1 −1 dragonfly-client-config/src/dfget.rs
+405 −0 dragonfly-client-config/src/dfinit.rs
+1 −1 dragonfly-client-config/src/dfstore.rs
+1 −0 dragonfly-client-config/src/lib.rs
+1 −14 dragonfly-client-core/Cargo.toml
+237 −0 dragonfly-client-core/src/error/errors.rs
+59 −0 dragonfly-client-core/src/error/message.rs
+187 −0 dragonfly-client-core/src/error/mod.rs
+4 −247 dragonfly-client-core/src/lib.rs
+19 −0 dragonfly-client-core/src/result.rs
+27 −0 dragonfly-client-init/Cargo.toml
+42 −0 dragonfly-client-init/Dockerfile
+101 −0 dragonfly-client-init/src/bin/main.rs
+220 −0 dragonfly-client-init/src/container_runtime/containerd.rs
+111 −0 dragonfly-client-init/src/container_runtime/crio.rs
+53 −0 dragonfly-client-init/src/container_runtime/docker.rs
+125 −0 dragonfly-client-init/src/container_runtime/mod.rs
+17 −0 dragonfly-client-init/src/lib.rs
+3 −1 dragonfly-client-storage/src/content.rs
+1 −1 dragonfly-client-storage/src/lib.rs
+257 −301 dragonfly-client-storage/src/metadata.rs
+15 −9 dragonfly-client-util/src/http/mod.rs
+5 −2 dragonfly-client-util/src/id_generator/mod.rs
+15 −7 dragonfly-client-util/src/tls/mod.rs
+2 −2 dragonfly-client/Cargo.toml
+8 −2 dragonfly-client/Dockerfile
+10 −10 dragonfly-client/src/bin/dfdaemon/main.rs
+2 −1 dragonfly-client/src/bin/dfget/main.rs
+1 −1 dragonfly-client/src/dynconfig/mod.rs
+63 −23 dragonfly-client/src/grpc/dfdaemon_download.rs
+214 −56 dragonfly-client/src/grpc/dfdaemon_upload.rs
+8 −3 dragonfly-client/src/grpc/health.rs
+7 −3 dragonfly-client/src/grpc/manager.rs
+56 −0 dragonfly-client/src/grpc/mod.rs
+32 −11 dragonfly-client/src/grpc/scheduler.rs
+6 −2 dragonfly-client/src/grpc/security.rs
+17 −0 dragonfly-client/src/proxy/header.rs
+56 −19 dragonfly-client/src/proxy/mod.rs
+252 −202 dragonfly-client/src/task/mod.rs
+78 −57 dragonfly-client/src/task/piece.rs
+21 −15 dragonfly-client/src/task/piece_collector.rs
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ import (
. "github.com/onsi/ginkgo/v2" //nolint
. "github.com/onsi/gomega" //nolint

"d7y.io/dragonfly/v2/test/e2e/util"
"d7y.io/dragonfly/v2/test/e2e/v1/util"
)

var _ = Describe("Download concurrency", func() {
Expand Down
File renamed without changes.
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ import (
. "github.com/onsi/ginkgo/v2" //nolint
. "github.com/onsi/gomega" //nolint

"d7y.io/dragonfly/v2/test/e2e/util"
"d7y.io/dragonfly/v2/test/e2e/v1/util"
)

var _ = Describe("Containerd with CRI support", func() {
Expand Down
2 changes: 1 addition & 1 deletion test/e2e/dfget_test.go → test/e2e/v1/dfget_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ import (
. "github.com/onsi/gomega" //nolint

"d7y.io/dragonfly/v2/pkg/net/http"
"d7y.io/dragonfly/v2/test/e2e/util"
"d7y.io/dragonfly/v2/test/e2e/v1/util"
)

var _ = Describe("Download with dfget and proxy", func() {
Expand Down
4 changes: 2 additions & 2 deletions test/e2e/e2e_test.go → test/e2e/v1/e2e_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,8 @@ import (
. "github.com/onsi/ginkgo/v2" //nolint
. "github.com/onsi/gomega" //nolint

_ "d7y.io/dragonfly/v2/test/e2e/manager"
"d7y.io/dragonfly/v2/test/e2e/util"
_ "d7y.io/dragonfly/v2/test/e2e/v1/manager"
"d7y.io/dragonfly/v2/test/e2e/v1/util"
)

var featureGatesFlag string
Expand Down
Empty file added test/e2e/v1/manager-stdout.log
Empty file.
File renamed without changes.
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ import (
"d7y.io/dragonfly/v2/manager/types"
"d7y.io/dragonfly/v2/pkg/idgen"
"d7y.io/dragonfly/v2/pkg/structure"
"d7y.io/dragonfly/v2/test/e2e/util"
"d7y.io/dragonfly/v2/test/e2e/v1/util"
)

var _ = Describe("Preheat with manager", func() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ import (
. "github.com/onsi/ginkgo/v2" //nolint
. "github.com/onsi/gomega" //nolint

"d7y.io/dragonfly/v2/test/e2e/util"
"d7y.io/dragonfly/v2/test/e2e/v1/util"
)

var _ = Describe("Evaluator with networkTopology", func() {
Expand Down
Empty file.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
66 changes: 66 additions & 0 deletions test/e2e/v2/concurrency_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
/*
* Copyright 2020 The Dragonfly Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package e2e

import (
"fmt"

. "github.com/onsi/ginkgo/v2" //nolint
. "github.com/onsi/gomega" //nolint

"d7y.io/dragonfly/v2/test/e2e/v2/util"
)

var _ = Describe("Download concurrency", func() {
Context("ab", func() {
It("concurrent 100 should be ok", Label("concurrent", "100"), func() {
url := util.GetFileURL(hostnameFilePath)
fmt.Println("download url " + url)

out, err := util.ABCommand("-c", "100", "-n", "200", "-X", "localhost:4001", url).CombinedOutput()
fmt.Println(string(out))
Expect(err).NotTo(HaveOccurred())
})

It("concurrent 200 should be ok", Label("concurrent", "200"), func() {
url := util.GetFileURL(hostnameFilePath)
fmt.Println("download url " + url)

out, err := util.ABCommand("-c", "200", "-n", "400", "-X", "localhost:4001", url).CombinedOutput()
fmt.Println(string(out))
Expect(err).NotTo(HaveOccurred())
})

It("concurrent 500 should be ok", Label("concurrent", "500"), func() {
url := util.GetFileURL(hostnameFilePath)
fmt.Println("download url " + url)

out, err := util.ABCommand("-c", "500", "-n", "1000", "-X", "localhost:4001", url).CombinedOutput()
fmt.Println(string(out))
Expect(err).NotTo(HaveOccurred())
})

It("concurrent 1000 should be ok", Label("concurrent", "1000"), func() {
url := util.GetFileURL(hostnameFilePath)
fmt.Println("download url " + url)

out, err := util.ABCommand("-c", "1000", "-n", "2000", "-X", "localhost:4001", url).CombinedOutput()
fmt.Println(string(out))
Expect(err).NotTo(HaveOccurred())
})
})
})
69 changes: 69 additions & 0 deletions test/e2e/v2/constants.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
/*
* Copyright 2020 The Dragonfly Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package e2e

const (
hostnameFilePath = "/etc/hostname"
dragonflyNamespace = "dragonfly-system"
dragonflyE2ENamespace = "dragonfly-e2e"
)

const (
dfdaemonCompatibilityTestMode = "dfdaemon"
schedulerCompatibilityTestMode = "scheduler"
)

const (
managerServerName = "manager"
schedulerServerName = "scheduler"
seedClientServerName = "seed-client"
clientServerName = "client"
)

type server struct {
name string
namespace string
logDirName string
replicas int
}

var servers = map[string]server{
managerServerName: {
name: managerServerName,
namespace: dragonflyNamespace,
logDirName: managerServerName,
replicas: 1,
},
schedulerServerName: {
name: schedulerServerName,
namespace: dragonflyNamespace,
logDirName: schedulerServerName,
replicas: 3,
},
seedClientServerName: {
name: seedClientServerName,
namespace: dragonflyNamespace,
logDirName: "dfdaemon",
replicas: 3,
},
clientServerName: {
name: clientServerName,
namespace: dragonflyNamespace,
logDirName: "dfdaemon",
replicas: 1,
},
}
Loading

0 comments on commit 90f4fca

Please sign in to comment.