From 350ed360556fc64edc8c33c95e57b49a07360577 Mon Sep 17 00:00:00 2001 From: erda-bot <81558540+erda-bot@users.noreply.github.com> Date: Tue, 28 Jun 2022 21:21:06 +0800 Subject: [PATCH] feat: delete invalid package and package api (#5141) (#5142) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: remove useless data * feat: comment provider.Run * feat: projectID * feat: hepa openapi * feat: permission * feat: runtime cli * feat: fatal * feat: typo * feat: more info * feat: do clearing async * fix: NotFoundError Co-authored-by: 悟空 --- api/proto/common/status.proto | 10 + .../core/hepa/endpoint_api/endpoint_api.proto | 37 ++- api/proto/core/project/core-project.proto | 78 +++++ api/proto/orchestrator/runtime/runtime.proto | 12 + cmd/core-services/main.go | 19 -- cmd/core-services/register.go | 39 +++ cmd/hepa/bootstrap.yaml | 8 + internal/core/project/dao/option.go | 194 ++++++++++++ internal/core/project/dao/query.go | 26 ++ internal/core/project/dao/tx.go | 159 ++++++++++ internal/core/project/dao/tx_test.go | 276 ++++++++++++++++++ internal/core/project/model/base.go | 26 ++ internal/core/project/model/erda_project.go | 54 ++++ internal/core/project/project.service.go | 70 +++++ internal/core/project/provider.go | 117 ++++++++ internal/core/project/register.go | 19 ++ .../components/runtime/runtime.service.go | 9 + .../endpoint_api/endpoint.api.service.go | 199 ++++++++++++- .../hepa/providers/endpoint_api/provider.go | 50 +++- .../hepa/services/endpoint_api/impl/impl.go | 8 + .../hepa/services/endpoint_api/interface.go | 2 + pkg/common/errors/data.go | 5 + pkg/time/ticker/ticker.go | 18 +- 23 files changed, 1402 insertions(+), 33 deletions(-) create mode 100644 api/proto/common/status.proto create mode 100644 api/proto/core/project/core-project.proto create mode 100644 cmd/core-services/register.go create mode 100644 internal/core/project/dao/option.go create mode 100644 internal/core/project/dao/query.go create mode 100644 internal/core/project/dao/tx.go create mode 100644 internal/core/project/dao/tx_test.go create mode 100644 internal/core/project/model/base.go create mode 100644 internal/core/project/model/erda_project.go create mode 100644 internal/core/project/project.service.go create mode 100644 internal/core/project/provider.go create mode 100644 internal/core/project/register.go diff --git a/api/proto/common/status.proto b/api/proto/common/status.proto new file mode 100644 index 00000000000..0f9899c14a1 --- /dev/null +++ b/api/proto/common/status.proto @@ -0,0 +1,10 @@ +syntax = "proto3"; + +package erda.common; + +option go_package = "github.com/erda-project/erda-proto-go/common/pb"; + +enum StatusEnum { + unknown_status = 0; + not_found = 404; +} diff --git a/api/proto/core/hepa/endpoint_api/endpoint_api.proto b/api/proto/core/hepa/endpoint_api/endpoint_api.proto index a4d01cac14f..109ce1424c6 100644 --- a/api/proto/core/hepa/endpoint_api/endpoint_api.proto +++ b/api/proto/core/hepa/endpoint_api/endpoint_api.proto @@ -5,6 +5,8 @@ option go_package = "github.com/erda-project/erda-proto-go/core/hepa/endpoint_ap import "google/api/annotations.proto"; import "google/protobuf/struct.proto"; import "core/hepa/hepa.proto"; +import "common/http.proto"; +import "common/openapi.proto"; // +publish service:"hepa" service EndpointApiService { @@ -90,6 +92,23 @@ service EndpointApiService { }; } + rpc ListInvalidEndpointApi(common.VoidRequest) returns(ListInvalidEndpointApiResp) { + option(google.api.http) = { + get: "/api/gateway/openapi/invalid-endpoints", + }; + option(erda.common.openapi) = { + path: "/api/gateway/openapi/invalid-endpoints", + }; + } + + rpc ClearInvalidEndpointApi(common.VoidRequest) returns (common.VoidResponse) { + option(google.api.http) = { + delete: "/api/gateway/openapi/invalid-endpoints", + }; + option(erda.common.openapi) = { + path: "/api/gateway/openapi/invalid-endpoints", + }; + } } message ChangeEndpointRootResponse { @@ -129,7 +148,7 @@ message EndpointApi { string redirectType = 2; string redirectAddr = 3; string redirectPath = 4; - string redirectApp = 5; + string redirectApp = 5; string redirectService = 6; string redirectRuntimeId = 7; string redirectRuntimeName = 8; @@ -234,4 +253,20 @@ message GetEndpointRequest { message GetEndpointResponse { Endpoint data = 1; +} + +message ListInvalidEndpointApiResp { + repeated ListInvalidEndpointApiItem list = 1; +} + +message ListInvalidEndpointApiItem { + string invalidReason = 1; + string type = 2; + string projectID = 3; + string packageID = 4; + string packageApiID = 5; + string upstreamApiID = 6; + string upstreamID = 7; + string upstreamName = 8; + string runtimeID = 9; } \ No newline at end of file diff --git a/api/proto/core/project/core-project.proto b/api/proto/core/project/core-project.proto new file mode 100644 index 00000000000..2c925178617 --- /dev/null +++ b/api/proto/core/project/core-project.proto @@ -0,0 +1,78 @@ +syntax = "proto3"; + +package erda.core.project; + +import "google/protobuf/timestamp.proto"; +import "common/status.proto"; + +option go_package = "github.com/erda-project/erda-proto-go/core/project/pb"; + +service Project { + rpc CheckProjectExist(CheckProjectExistReq)returns(CheckProjectExistResp){} + rpc GetProjectByID(GetProjectByIDReq)returns(ProjectDto){} +} + +message CheckProjectExistReq { + uint64 id = 1; +} + +message CheckProjectExistResp { + bool ok = 1; +} + +message GetProjectByIDReq { + uint64 id = 1; + optional string userID = 2; +} + +message ProjectDto { + // project primary key + uint64 id = 1; + // project identifies + string name = 2; + // project display name + string displayName = 3; + // for string ddHook + reserved 4; + // org primary key + uint64 orgID = 5; + // the project creator userID + string creatorID = 6; + // the project logo url + string logo = 7; + // the project description + string desc = 8; + // the owners' userIDs of the project + // for owners + reserved 9; + // project active time + google.protobuf.Timestamp activeTime = 10; + // for string joined + reserved 11; + // for bool canUnblock + reserved 12; + // for string blockStatus + reserved 13; + // for bool CanManage + reserved 14; + // is the project public + bool isPublic = 15; + // for ProjectStats stats + reserved 16; + // for ProjectResourceUsage + reserved 17, 18, 19, 20; + // for clusterConfig + reserved 21; + // for ResourceConfigsInfo resourceConfig + reserved 22; + // for map RollbackConfig + reserved 23; + // for float64 cpuQuota + reserved 24; + // for float64 memQuota + reserved 25; + google.protobuf.Timestamp createdTime = 26; + google.protobuf.Timestamp updatedTime = 27; + // project type + string type = 28; +} diff --git a/api/proto/orchestrator/runtime/runtime.proto b/api/proto/orchestrator/runtime/runtime.proto index 0718ba446e6..8b0f8fe47a1 100644 --- a/api/proto/orchestrator/runtime/runtime.proto +++ b/api/proto/orchestrator/runtime/runtime.proto @@ -6,6 +6,7 @@ import "google/protobuf/struct.proto"; import "google/protobuf/any.proto"; import "google/protobuf/timestamp.proto"; import "common/http.proto"; +import "common/status.proto"; option go_package = "github.com/erda-project/erda-proto-go/orchestrator/runtime/pb"; @@ -98,12 +99,23 @@ message Runtime { repeated ErrorResponse errors = 13; } +message CheckRuntimeExistReq { + uint64 id = 1; +} + +message CheckRuntimeExistResp { + bool ok = 1; +} + service RuntimeService { rpc GetRuntime (GetRuntimeRequest) returns (RuntimeInspect) { option (google.api.http) = { get: "/api/runtimes/{nameOrID}?applicationId={appID}&workspace={workspace}", }; } + + rpc CheckRuntimeExist (CheckRuntimeExistReq) returns (CheckRuntimeExistResp) {} + rpc DelRuntime (DelRuntimeRequest) returns (Runtime) { option (google.api.http) = { delete: "/api/runtimes/{id}", diff --git a/cmd/core-services/main.go b/cmd/core-services/main.go index 96ebcf87b49..233744e5880 100644 --- a/cmd/core-services/main.go +++ b/cmd/core-services/main.go @@ -19,25 +19,6 @@ import ( "github.com/erda-project/erda-infra/base/servicehub" "github.com/erda-project/erda/pkg/common" - - // providers - _ "github.com/erda-project/erda-infra/providers/grpcclient" - _ "github.com/erda-project/erda-infra/providers/pprof" - _ "github.com/erda-project/erda-infra/providers/redis" - _ "github.com/erda-project/erda-proto-go/core/pipeline/cms/client" - _ "github.com/erda-project/erda/internal/core/legacy" - _ "github.com/erda-project/erda/internal/core/legacy/providers/token" - _ "github.com/erda-project/erda/internal/core/legacy/services/dingtalk/api" - _ "github.com/erda-project/erda/internal/core/messenger/eventbox" - _ "github.com/erda-project/erda/internal/core/messenger/notify" - _ "github.com/erda-project/erda/internal/core/messenger/notify-channel" - _ "github.com/erda-project/erda/internal/core/messenger/notifygroup" - - // infra - _ "github.com/erda-project/erda-infra/providers/grpcserver" - _ "github.com/erda-project/erda-infra/providers/httpserver" - _ "github.com/erda-project/erda-infra/providers/mysql" - _ "github.com/erda-project/erda-infra/providers/serviceregister" ) //go:embed bootstrap.yaml diff --git a/cmd/core-services/register.go b/cmd/core-services/register.go new file mode 100644 index 00000000000..d80b405f9f4 --- /dev/null +++ b/cmd/core-services/register.go @@ -0,0 +1,39 @@ +// Copyright (c) 2021 Terminus, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + _ "embed" + + // providers + _ "github.com/erda-project/erda-infra/providers/grpcclient" + _ "github.com/erda-project/erda-infra/providers/pprof" + _ "github.com/erda-project/erda-infra/providers/redis" + _ "github.com/erda-project/erda-proto-go/core/pipeline/cms/client" + _ "github.com/erda-project/erda/internal/core/legacy" + _ "github.com/erda-project/erda/internal/core/legacy/providers/token" + _ "github.com/erda-project/erda/internal/core/legacy/services/dingtalk/api" + _ "github.com/erda-project/erda/internal/core/messenger/eventbox" + _ "github.com/erda-project/erda/internal/core/messenger/notify" + _ "github.com/erda-project/erda/internal/core/messenger/notify-channel" + _ "github.com/erda-project/erda/internal/core/messenger/notifygroup" + _ "github.com/erda-project/erda/internal/core/project" + + // infra + _ "github.com/erda-project/erda-infra/providers/grpcserver" + _ "github.com/erda-project/erda-infra/providers/httpserver" + _ "github.com/erda-project/erda-infra/providers/mysql" + _ "github.com/erda-project/erda-infra/providers/serviceregister" +) diff --git a/cmd/hepa/bootstrap.yaml b/cmd/hepa/bootstrap.yaml index d2a3db61f84..edb399abd46 100644 --- a/cmd/hepa/bootstrap.yaml +++ b/cmd/hepa/bootstrap.yaml @@ -37,6 +37,14 @@ grpc-client@erda.core.clustermanager.cluster: addr: "${CLUSTER_MANAGER_GRPC_ADDR:cluster-manager:9095}" erda.core.clustermanager.cluster-client: {} +grpc-client@erda.core.project: + addr: "${CORE_SERVICES_GRPC_ADDR:core-services:9537}" +erda.core.project-client: {} + +grpc-client@erda.orchestrator.runtime: + addr: "${ORCHESTRATOR_GRPC_ADDR:orchestrator:7080}" +erda.orchestrator.runtime-client: {} + grpc-client@erda.msp.tenant: addr: "${MSP_GRPC_ADDR:msp:7080}" erda.msp.tenant-client: {} diff --git a/internal/core/project/dao/option.go b/internal/core/project/dao/option.go new file mode 100644 index 00000000000..be45bfac410 --- /dev/null +++ b/internal/core/project/dao/option.go @@ -0,0 +1,194 @@ +// Copyright (c) 2021 Terminus, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dao + +import ( + "fmt" + "strings" + + "gorm.io/gorm" +) + +var ( + DESC Order = "DESC" + ASC Order = "ASC" +) + +type Option func(db *gorm.DB) *gorm.DB + +type Column interface { + WhereColumn + OrderColumn + SetColumn +} + +type WhereColumn interface { + Is(value interface{}) Option + In(values ...interface{}) Option + InMap(values map[interface{}]struct{}) Option + Like(value interface{}) Option + GreaterThan(value interface{}) Option + EqGreaterThan(value interface{}) Option + LessThan(value interface{}) Option + EqLessThan(value interface{}) Option +} + +type OrderColumn interface { + DESC() Option + ASC() Option +} + +type SetColumn interface { + Set(value interface{}) Option +} + +func Where(format string, args ...interface{}) Option { + return func(db *gorm.DB) *gorm.DB { + return db.Where(format, args...) + } +} + +func Wheres(m interface{}) Option { + return func(db *gorm.DB) *gorm.DB { + return db.Where(m) + } +} + +func Col(col string) Column { + return column{col: col} +} + +type column struct { + col string +} + +func (w column) Is(value interface{}) Option { + if value == nil { + return func(db *gorm.DB) *gorm.DB { + return db.Where(w.col + " IS NULL") + } + } + return func(db *gorm.DB) *gorm.DB { + return db.Where(w.col+" = ?", value) + } +} + +func (w column) In(values ...interface{}) Option { + return func(db *gorm.DB) *gorm.DB { + return db.Where(w.col+" IN ?", values) + } +} + +func (w column) InMap(values map[interface{}]struct{}) Option { + var values_ []interface{} + for key := range values { + values_ = append(values_, key) + } + return w.In(values_...) +} + +func (w column) Like(value interface{}) Option { + return func(db *gorm.DB) *gorm.DB { + return db.Where(w.col+" LIKE ?", value) + } +} + +func (w column) GreaterThan(value interface{}) Option { + return func(db *gorm.DB) *gorm.DB { + return db.Where(w.col+" > ?", value) + } +} + +func (w column) EqGreaterThan(value interface{}) Option { + return func(db *gorm.DB) *gorm.DB { + return db.Where(w.col+" >= ?", value) + } +} + +func (w column) LessThan(value interface{}) Option { + return func(db *gorm.DB) *gorm.DB { + return db.Where(w.col+" < ?", value) + } +} + +func (w column) EqLessThan(value interface{}) Option { + return func(db *gorm.DB) *gorm.DB { + return db.Where(w.col+" <= ?", value) + } +} + +func (w column) DESC() Option { + return func(db *gorm.DB) *gorm.DB { + return db.Order(w.col + " DESC") + } +} + +func (w column) ASC() Option { + return func(db *gorm.DB) *gorm.DB { + return db.Order(w.col + " ASC") + } +} + +func (w column) Set(value interface{}) Option { + return func(db *gorm.DB) *gorm.DB { + return db.Update(w.col, value) + } +} + +type WhereValue interface { + In(cols ...string) Option +} + +func Value(value interface{}) whereValue { + return whereValue{value: value} +} + +type whereValue struct { + value interface{} +} + +func (w whereValue) In(cols ...string) Option { + return func(db *gorm.DB) *gorm.DB { + return db.Where(fmt.Sprintf("? IN (%s)", strings.Join(cols, ",")), w.value) + } +} + +func Paging(size, no int) Option { + if size < 0 { + size = 0 + } + if no < 1 { + no = 1 + } + return func(db *gorm.DB) *gorm.DB { + return db.Limit(size).Offset((no - 1) * size) + } +} + +type Order string + +func OrderBy(col string, order Order) Option { + if !strings.EqualFold(string(order), string(DESC)) && + !strings.EqualFold(string(order), string(ASC)) { + order = "DESC" + } + return func(db *gorm.DB) *gorm.DB { + return db.Order(col + " " + strings.ToUpper(string(order))) + } +} + +func NotSoftDeleted(db *gorm.DB) *gorm.DB { + return db.Where("soft_deleted_at = ?", 0) +} diff --git a/internal/core/project/dao/query.go b/internal/core/project/dao/query.go new file mode 100644 index 00000000000..435ce8c0eeb --- /dev/null +++ b/internal/core/project/dao/query.go @@ -0,0 +1,26 @@ +// Copyright (c) 2021 Terminus, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dao + +import ( + "github.com/erda-project/erda/internal/core/project/model" +) + +func GetProject(tx *TX, options ...Option) (*model.ErdaProject, bool, error) { + var project model.ErdaProject + options = append(options, NotSoftDeleted) + ok, err := tx.Get(&project, options...) + return &project, ok, err +} diff --git a/internal/core/project/dao/tx.go b/internal/core/project/dao/tx.go new file mode 100644 index 00000000000..d38e1e8429c --- /dev/null +++ b/internal/core/project/dao/tx.go @@ -0,0 +1,159 @@ +// Copyright (c) 2021 Terminus, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dao + +import ( + "errors" + + "gorm.io/gorm" +) + +var q *TX + +var InvalidTransactionError = errors.New("invalid transaction, it is already committed or roll backed") + +type TX struct { + Error error + + tx *gorm.DB + inTx bool + valid bool +} + +func Init(db *gorm.DB) { + if q == nil { + q = &TX{ + tx: db, + valid: true, + } + } +} + +func Q() *TX { + if q == nil || q.tx == nil { + panic("q is not init") + } + return &TX{ + tx: q.tx, + valid: true, + } +} + +func Begin() *TX { + return &TX{ + tx: Q().DB().Begin(), + valid: true, + inTx: true, + } +} + +func (tx *TX) Create(i interface{}) error { + if tx.inTx && !tx.valid { + return InvalidTransactionError + } + tx.Error = tx.tx.Create(i).Error + return tx.Error +} + +func (tx *TX) CreateInBatches(i interface{}, size int) error { + if tx.inTx && !tx.valid { + return InvalidTransactionError + } + tx.Error = tx.tx.CreateInBatches(i, size).Error + return tx.Error +} + +func (tx *TX) Delete(i interface{}, options ...Option) error { + if tx.inTx && !tx.valid { + return InvalidTransactionError + } + var db = tx.DB() + for _, opt := range options { + db = opt(db) + } + return db.Delete(i).Error +} + +func (tx *TX) Updates(i, v interface{}, options ...Option) error { + if tx.inTx && !tx.valid { + return InvalidTransactionError + } + var db = tx.DB() + for _, opt := range options { + db = opt(db) + } + return db.Model(i).Updates(v).Error +} + +func (tx *TX) UpdateColumns(i interface{}, options ...Option) error { + if tx.inTx && !tx.valid { + return InvalidTransactionError + } + var db = tx.DB() + db = db.Model(i) + for _, opt := range options { + db = opt(db) + } + return db.Error +} + +func (tx *TX) List(i interface{}, options ...Option) (int64, error) { + var total int64 + var db = tx.DB() + for _, opt := range options { + db = opt(db) + } + + err := db.Find(i).Count(&total).Error + if err == nil { + return total, nil + } + if errors.Is(err, gorm.ErrRecordNotFound) { + return 0, nil + } + return 0, err +} + +func (tx *TX) Get(i interface{}, options ...Option) (bool, error) { + var db = tx.DB() + for _, opt := range options { + db = opt(db) + } + + err := db.First(i).Error + if err == nil { + return true, nil + } + if errors.Is(err, gorm.ErrRecordNotFound) { + return false, nil + } + return false, err +} + +func (tx *TX) CommitOrRollback() { + if tx.inTx && !tx.valid { + return + } + if tx.Error == nil { + tx.tx.Commit() + } else { + tx.tx.Rollback() + } + tx.valid = false +} + +func (tx *TX) DB() *gorm.DB { + return tx.tx +} diff --git a/internal/core/project/dao/tx_test.go b/internal/core/project/dao/tx_test.go new file mode 100644 index 00000000000..111243d5e2e --- /dev/null +++ b/internal/core/project/dao/tx_test.go @@ -0,0 +1,276 @@ +// Copyright (c) 2021 Terminus, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dao_test + +import ( + "os" + "path/filepath" + "strconv" + "testing" + + "gorm.io/driver/sqlite" + "gorm.io/gorm" + + "github.com/erda-project/erda/internal/core/project/dao" +) + +var dsn = filepath.Join(os.TempDir(), "gorm.db") + +type User struct { + Age int64 `gorm:"type:BIGINT"` + Name string +} + +func TestTX_Create(t *testing.T) { + openDB(t) + defer closeDB() + + var users []User + for i := 0; i < 10; i++ { + users = append(users, User{Age: int64(i), Name: "dspo-" + strconv.Itoa(i)}) + } + + if err := dao.Q().DB().AutoMigrate(new(User)); err != nil { + t.Fatalf("failed to migrate user: %v", err) + } + + // test create + if err := dao.Q().Create(&User{Age: 10, Name: "dspo-10"}); err != nil { + t.Fatalf("failed to Create: %v", err) + } + total, err := dao.Q().List(new([]User)) + if err != nil { + t.Fatalf("failed to List") + } + if total != 1 { + t.Fatalf("total should be %v", 1) + } + + // test create in batches + if err := dao.Q().CreateInBatches(users, 2); err != nil { + t.Fatalf("failed to CreateInBatches: %v", err) + } + total, err = dao.Q().List(new([]User)) + if err != nil { + t.Fatalf("failed to List") + } + if total != 11 { + t.Fatalf("total should be %v", 11) + } + + // test Option Where + var user User + ok, err := dao.Q().Get(&user, dao.Where("name = ?", "dspo-0")) + if err != nil { + t.Fatalf("failed to Get: %v", err) + } + if !ok { + t.Fatalf("ok should be true") + } + ok, err = dao.Q().Get(&user, dao.Where("age = ? AND name = ?", 0, "dspo-0")) + if err != nil { + t.Fatalf("failed to Get: %v", err) + } + if !ok { + t.Fatalf("ok shoule be true") + } + ok, err = dao.Q().Get(&user, dao.Where("age > ?", 15)) + if err != nil { + t.Fatalf("failed to Get: %v", err) + } + if ok { + t.Fatalf("ok should be false") + } + + // test Option Wheres + ok, err = dao.Q().Get(&user, dao.Wheres(map[string]interface{}{"name": "dspo-0"})) + if err != nil { + t.Fatalf("failed to Get: %v", err) + } + if !ok { + t.Fatalf("ok should be true") + } + if user.Age != 0 || user.Name != "dspo-0" { + t.Fatalf("user record error") + } + ok, err = dao.Q().Get(&user, dao.Wheres(User{Name: "dspo-1"})) + if err != nil { + t.Fatalf("failed to Get: %v", err) + } + if !ok { + t.Fatal("ok should be true") + } + if user.Age != 1 || user.Name != "dspo-1" { + t.Fatal("user record error") + } + + // test Option WhereColum.IS + var name = dao.Col("name") + ok, err = dao.Q().Get(&user, name.Is(nil)) + if err != nil { + t.Fatalf("failed to Get: %v", err) + } + if ok { + t.Fatal("ok should be true") + } + + ok, err = dao.Q().Get(&user, name.Is("dspo-2")) + if err != nil { + t.Fatalf("failed to Get: %v", err) + } + if !ok { + t.Fatalf("ok should be true") + } + + // test Option WhereColumn.In + var users2 []User + total, err = dao.Q().List(&users2, name.In("dspo-0", "dspo-1", "dspo-3", "dspo-15")) + if err != nil { + t.Fatalf("failed to List: %v", err) + } + if total != 3 { + t.Fatalf("total is expected to be %v, got %v", 3, total) + } + + // test Option WhereColumn.InMap + total, err = dao.Q().List(&users2, name.InMap(map[interface{}]struct{}{ + "dspo-4": {}, + "dspo-5": {}, + "dspo-15": {}, + })) + if err != nil { + t.Fatalf("failed to List: %v", err) + } + if total != 2 { + t.Fatalf("total is expected to be: %v, got: %v", 2, total) + } + + // test Option WhereColumn.Like + total, err = dao.Q().List(&users2, name.Like("dspo-%")) + if err != nil { + t.Fatalf("failed to List: %v", err) + } + if total != 11 { + t.Fatalf("total is expected: %v, got: %v", 11, total) + } + + // test Option WhereColumn.GreaterThan + var age = dao.Col("age") + total, err = dao.Q().List(&users2, age.GreaterThan(8)) + if err != nil { + t.Fatalf("failed to List: %v", err) + } + if total != 2 { + t.Fatalf("total is expected: %v, got: %v", 1, total) + } + + // test Option WhereColumn.EqGreaterThan + total, err = dao.Q().List(&users2, age.EqGreaterThan(8)) + if err != nil { + t.Fatalf("failed to List: %v", err) + } + if total != 3 { + t.Fatalf("total is expected: %v, got: %v", 2, total) + } + + // test Option WhereColumn.LessThan + total, err = dao.Q().List(&users2, age.LessThan(8)) + if err != nil { + t.Fatalf("failed to List: %v", err) + } + if total != 8 { + t.Fatalf("total is expected: %v, got: %v", 8, total) + } + + // test Option WhereColumn.EqLessThan + total, err = dao.Q().List(&users2, age.EqLessThan(8)) + if err != nil { + t.Fatalf("failed to List: %v", err) + } + if total != 9 { + t.Fatalf("total is expected to be: %v, got: %v", 9, total) + } + + // test Option Column.DESC + _, err = dao.Q().List(&users2, age.DESC()) + if err != nil { + t.Fatalf("failed to List: %v", err) + } + if users2[0].Name != "dspo-10" { + t.Fatalf("the first user's name is expected to be: %s, got: %s", "dspo-10", users2[0].Name) + } + + // test Option WhereValue.In + total, err = dao.Q().List(&users2, dao.Value("dspo-1").In("name", "age")) + if err != nil { + t.Fatalf("failed to List: %v", err) + } + if total != 1 { + t.Fatalf("total is expected to be: %v, got: %v", 1, total) + } + + // test Option Paging + total, err = dao.Q().List(&users2, age.GreaterThan(0), dao.Paging(-1, 0)) + if err != nil { + t.Fatalf("failed to List: %v", err) + } + if total != 10 { + t.Fatalf("total is expected: %v, got: %v", 10, total) + } + if len(users2) != 10 { + t.Fatalf("length of users2 is expected to be: %v, got: %v", 0, len(users2)) + } + + total, err = dao.Q().List(&users2, age.GreaterThan(0), dao.Paging(5, 1)) + if err != nil { + t.Fatalf("failed to List: %v", err) + } + if total != 10 { + t.Fatalf("failed to List: %v", err) + } + if len(users2) != 5 { + t.Fatalf("length of users2 is expected to be: %v, got: %v", 5, len(users2)) + } + + // test Option OrderBy + total, err = dao.Q().List(&users2, dao.OrderBy("age", dao.DESC)) + if err != nil { + t.Fatalf("failed to List: %v", err) + } + if total != 11 { + t.Fatalf("total is expected to be: %v, got: %v", 11, total) + } + if users2[0].Name != "dspo-10" { + t.Fatalf("the first record's Name is expected to be: %s, got: %s", "dspo-10", users2[0].Name) + } + + // test update + if err := dao.Q().Updates(&user, map[string]interface{}{"name": "cmc-10"}, name.Is(1)); err != nil { + t.Fatalf("failed to Updates: %v", err) + } + t.Logf("user: %+v", user) +} + +func openDB(t *testing.T) { + db, err := gorm.Open(sqlite.Open(dsn), &gorm.Config{}) + if err != nil { + t.Fatalf("failed to open %s: %v", dsn, err) + } + dao.Init(db.Debug()) +} + +func closeDB() { + os.Remove(dsn) +} diff --git a/internal/core/project/model/base.go b/internal/core/project/model/base.go new file mode 100644 index 00000000000..44adca1fea4 --- /dev/null +++ b/internal/core/project/model/base.go @@ -0,0 +1,26 @@ +// Copyright (c) 2021 Terminus, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package model + +import ( + "time" +) + +// BaseModel contains base fields for all models +type BaseModel struct { + ID int64 `json:"id" gorm:"primary_key"` + CreatedAt time.Time `json:"createdAt" gorm:"created_at"` + UpdatedAt time.Time `json:"updatedAt" gorm:"updated_at"` +} diff --git a/internal/core/project/model/erda_project.go b/internal/core/project/model/erda_project.go new file mode 100644 index 00000000000..6c9b522071b --- /dev/null +++ b/internal/core/project/model/erda_project.go @@ -0,0 +1,54 @@ +// Copyright (c) 2021 Terminus, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package model + +import ( + "time" +) + +// ErdaProject is the model erda_project +type ErdaProject struct { + BaseModel + + Name string + DisplayName string + Logo string + Desc string + // Cluster configuration eg: {"DEV":"terminus-y","TEST":"terminus-y","STAGING":"terminus-y","PROD":"terminus-y"} + ClusterConfig string + CpuQuota float64 + MemQuota float64 + Creator string + OrgID int64 + Version string + // DingTalk Hook + DDHook string `gorm:"column:dd_hook"` + Email string + Functions string + ActiveTime time.Time + // Rollback configuration: {"DEV": 1,"TEST": 2,"STAGING": 3,"PROD": 4} + RollbackConfig string + // Whether to open the project-level namespace + EnableNS bool `gorm:"column:enable_ns"` + // Is it a public project + IsPublic bool + // project type + Type string + SoftDeletedAt uint +} + +func (ErdaProject) TableName() string { + return "erda_project" +} diff --git a/internal/core/project/project.service.go b/internal/core/project/project.service.go new file mode 100644 index 00000000000..7521e6b379c --- /dev/null +++ b/internal/core/project/project.service.go @@ -0,0 +1,70 @@ +// Copyright (c) 2021 Terminus, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package project + +import ( + "context" + + "github.com/sirupsen/logrus" + "google.golang.org/protobuf/types/known/timestamppb" + + "github.com/erda-project/erda-proto-go/core/project/pb" + "github.com/erda-project/erda/internal/core/project/dao" + "github.com/erda-project/erda/pkg/common/errors" +) + +// project implements pb.ProjectServer +type project struct { + l *logrus.Entry +} + +func (p *project) CheckProjectExist(ctx context.Context, req *pb.CheckProjectExistReq) (*pb.CheckProjectExistResp, error) { + l := p.l.WithField("projectID", req.GetId()) + _, ok, err := dao.GetProject(dao.Q(), dao.Col("id").Is(req.GetId())) + if err != nil { + l.WithError(err).Errorln("failed to dao.GetProject") + return nil, err + } + return &pb.CheckProjectExistResp{Ok: ok}, nil +} + +func (p *project) GetProjectByID(ctx context.Context, req *pb.GetProjectByIDReq) (*pb.ProjectDto, error) { + l := p.l.WithField("projectID", req.GetId()) + proj, ok, err := dao.GetProject(dao.Q(), dao.Col("id").Is(req.GetId())) + if err != nil { + l.Errorf("failed to dao.GetProject: %v", err) + return nil, err + } + if !ok { + l.Warnln("project not found") + return nil, errors.NewNotFoundError("project") + } + + var dto = pb.ProjectDto{ + Id: uint64(proj.ID), + Name: proj.Name, + DisplayName: proj.DisplayName, + OrgID: uint64(proj.OrgID), + CreatorID: proj.Creator, + Logo: proj.Logo, + Desc: proj.Desc, + ActiveTime: timestamppb.New(proj.ActiveTime), + IsPublic: proj.IsPublic, + CreatedTime: timestamppb.New(proj.CreatedAt), + UpdatedTime: timestamppb.New(proj.UpdatedAt), + Type: proj.Type, + } + return &dto, nil +} diff --git a/internal/core/project/provider.go b/internal/core/project/provider.go new file mode 100644 index 00000000000..7dafff5695b --- /dev/null +++ b/internal/core/project/provider.go @@ -0,0 +1,117 @@ +// Copyright (c) 2021 Terminus, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package project + +import ( + "os" + "strings" + + "github.com/sirupsen/logrus" + "gorm.io/gorm" + + "github.com/erda-project/erda-infra/base/servicehub" + "github.com/erda-project/erda-infra/pkg/transport" + "github.com/erda-project/erda-proto-go/core/project/pb" + "github.com/erda-project/erda/internal/core/project/dao" +) + +var ( + name = "erda.core.project" + spec = &servicehub.Spec{ + Services: pb.ServiceNames(), + OptionalDependencies: []string{"service-register"}, + ConfigFunc: func() interface{} { + return new(config) + }, + Types: pb.Types(), + Creator: func() servicehub.Provider { + return new(provider) + }, + } +) + +func init() { + servicehub.Register(name, spec) +} + +// +provider +type provider struct { + R transport.Register `autowired:"service-register" required:"true"` + DB *gorm.DB `autowired:"mysql-gorm.v2-client"` + Cfg *config + + proj *project + l *logrus.Entry +} + +func (p *provider) Init(ctx servicehub.Context) error { + p.initLogrus() + p.initDB() + p.proj = &project{l: p.l.WithField("handler", "Project")} + + if p.R != nil { + p.l.Infoln("register Project") + pb.RegisterProjectImp(p.R, p.proj) + } + + return nil +} + +func (p *provider) Provide(ctx servicehub.DependencyContext, args ...interface{}) interface{} { + if ctx.Service() == "erda.core.project.Project" || + ctx.Type() == pb.ProjectServerType() || + ctx.Type() == pb.ProjectHandlerType() { + return p.proj + } + return p +} + +func (p *provider) initLogrus() { + switch strings.ToLower(p.Cfg.LogLevel) { + case "panic": + logrus.SetLevel(logrus.PanicLevel) + case "fatal": + logrus.SetLevel(logrus.FatalLevel) + case "error": + logrus.SetLevel(logrus.ErrorLevel) + case "warn": + logrus.SetLevel(logrus.WarnLevel) + case "info": + logrus.SetLevel(logrus.InfoLevel) + case "debug": + logrus.SetLevel(logrus.DebugLevel) + case "trace": + logrus.SetLevel(logrus.TraceLevel) + default: + logrus.SetLevel(logrus.InfoLevel) + } + p.l = logrus.WithField("provider", name) +} + +func (p *provider) initDB() { + switch strings.ToLower(p.Cfg.LogLevel) { + case "debug", "trace": + p.DB = p.DB.Debug() + default: + if os.Getenv("GORM_DEBUG") == "true" { + p.DB = p.DB.Debug() + } + } + dao.Init(p.DB) +} + +type config struct { + LogLevel string `json:"log-level" yaml:"log-level"` +} diff --git a/internal/core/project/register.go b/internal/core/project/register.go new file mode 100644 index 00000000000..359efd59756 --- /dev/null +++ b/internal/core/project/register.go @@ -0,0 +1,19 @@ +// Copyright (c) 2021 Terminus, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package project + +import ( + _ "github.com/erda-project/erda-infra/providers/mysql/v2" +) diff --git a/internal/tools/orchestrator/components/runtime/runtime.service.go b/internal/tools/orchestrator/components/runtime/runtime.service.go index 958beaa5477..c97486a027f 100644 --- a/internal/tools/orchestrator/components/runtime/runtime.service.go +++ b/internal/tools/orchestrator/components/runtime/runtime.service.go @@ -42,6 +42,7 @@ import ( "github.com/erda-project/erda/pkg/strutil" ) +// Service implements pb.RuntimeServiceServer type Service struct { logger logs.Logger @@ -293,6 +294,14 @@ func (r *Service) GetRuntime(ctx context.Context, request *pb.GetRuntimeRequest) return ri, nil } +func (r *Service) CheckRuntimeExist(ctx context.Context, req *pb.CheckRuntimeExistReq) (*pb.CheckRuntimeExistResp, error) { + runtime, err := r.db.GetRuntimeAllowNil(req.GetId()) + if err != nil { + return nil, err + } + return &pb.CheckRuntimeExistResp{Ok: runtime != nil}, nil +} + func updateStatusWhenDeploying(runtime *pb.RuntimeInspect) { if runtime == nil { return diff --git a/internal/tools/orchestrator/hepa/providers/endpoint_api/endpoint.api.service.go b/internal/tools/orchestrator/hepa/providers/endpoint_api/endpoint.api.service.go index 7716ebb589f..3ceb16d88ac 100644 --- a/internal/tools/orchestrator/hepa/providers/endpoint_api/endpoint.api.service.go +++ b/internal/tools/orchestrator/hepa/providers/endpoint_api/endpoint.api.service.go @@ -16,19 +16,39 @@ package endpoint_api import ( "context" + "path/filepath" + "strconv" + "time" "github.com/pkg/errors" + "github.com/sirupsen/logrus" + commonPb "github.com/erda-project/erda-proto-go/common/pb" "github.com/erda-project/erda-proto-go/core/hepa/endpoint_api/pb" + projPb "github.com/erda-project/erda-proto-go/core/project/pb" + runtimePb "github.com/erda-project/erda-proto-go/orchestrator/runtime/pb" "github.com/erda-project/erda/internal/tools/orchestrator/hepa/common/vars" "github.com/erda-project/erda/internal/tools/orchestrator/hepa/gateway/dto" + "github.com/erda-project/erda/internal/tools/orchestrator/hepa/repository/orm" + repositoryService "github.com/erda-project/erda/internal/tools/orchestrator/hepa/repository/service" "github.com/erda-project/erda/internal/tools/orchestrator/hepa/services/endpoint_api" "github.com/erda-project/erda/pkg/common/apis" erdaErr "github.com/erda-project/erda/pkg/common/errors" ) +var ( + invalidProject = "project not found" + invalidRuntime = "runtime not found" + clearC = make(chan struct{}, 1) +) + +// endpointApiService implements pb.EndpointApiServiceServer type endpointApiService struct { - p *provider + projCli projPb.ProjectServer + runtimeCli runtimePb.RuntimeServiceServer + gatewayApiService repositoryService.GatewayApiService + upstreamApiService repositoryService.GatewayUpstreamApiService + upstreamService repositoryService.GatewayUpstreamService } func (s *endpointApiService) GetEndpointsName(ctx context.Context, req *pb.GetEndpointsNameRequest) (resp *pb.GetEndpointsNameResponse, err error) { @@ -101,6 +121,7 @@ func (s *endpointApiService) GetEndpoint(ctx context.Context, req *pb.GetEndpoin } return } + func (s *endpointApiService) CreateEndpoint(ctx context.Context, req *pb.CreateEndpointRequest) (resp *pb.CreateEndpointResponse, err error) { service := endpoint_api.Service.Clone(ctx) if req.Endpoint == nil { @@ -249,3 +270,179 @@ func (s *endpointApiService) ChangeEndpointRoot(ctx context.Context, req *pb.Cha } return } + +func (s *endpointApiService) ListInvalidEndpointApi(ctx context.Context, _ *commonPb.VoidRequest) (*pb.ListInvalidEndpointApiResp, error) { + l := logrus.WithField("func", "ListInvalidEndpointApi") + // list all packages + eas := endpoint_api.Service.Clone(ctx) + packages, err := eas.ListAllPackages() + if err != nil { + l.Warnln("failed to ListAllPackages") + return nil, err + } + if len(packages) == 0 { + l.Warnln("no packages found") + return new(pb.ListInvalidEndpointApiResp), nil + } + + var result pb.ListInvalidEndpointApiResp + var projectPackages = make(map[string][]orm.GatewayPackage) + for _, package_ := range packages { + if package_.DiceProjectId != "" { + projectPackages[package_.DiceProjectId] = append(projectPackages[package_.DiceProjectId], package_) + } + } + for projectID := range projectPackages { + l = l.WithField("projectID", projectID) + id, err := strconv.ParseUint(projectID, 10, 32) + if err != nil { + l.WithError(err).Warnln("projectI can not be parsed to uint") + continue + } + + packages := projectPackages[projectID] + // collect the package if it's project is invalid + resp, err := s.projCli.CheckProjectExist(ctx, &projPb.CheckProjectExistReq{Id: id}) + if err == nil && !resp.GetOk() { + for _, package_ := range packages { + item := &pb.ListInvalidEndpointApiItem{ + InvalidReason: invalidProject, + Type: "package", + ProjectID: projectID, + PackageID: package_.Id, + } + result.List = append(result.List, item) + } + continue + } + + // collect the package_api if it's relation runtime is invalid + // + // join on tb_gateway_package_api.dice_api_id = tb_gateway_api.id + // tb_gateway_api.upstream_api_id = tb_gateway_upstream_api.id + // tb_gateway_upstream_api.upstream_id = tb_gateway_upstream.id + // runtimeID = filepath.base(tb_gateway_api.upstream_name) + for _, package_ := range packages { + l := l.WithField("tb_gateway_package_api.id", package_.Id) + packageApis, err := eas.ListPackageAllApis(package_.Id) + if err != nil { + l.WithError(err).Warnln("failed to ListPackageAllApis") + continue + } + for _, packageApi := range packageApis { + l := l.WithField("tb_gateway_package_api.dice_api_id", packageApi.DiceApiId) + if packageApi.DiceApiId == "" { + l.Warnln("packageApi.DiceApiId is empty") + continue + } + l = l.WithField("tb_gateway_api.id", packageApi.DiceApiId) + gatewayApi, err := s.gatewayApiService.GetById(packageApi.DiceApiId) + if err != nil { + l.WithError(err).Warnf("failed to gatewayApiService.GetById(%s)", packageApi.DiceApiId) + continue + } + // todo: if gatewayApi.redirect_addr is invalid inner address, collect the package_api + if gatewayApi.UpstreamApiId == "" { + l.Warnln("gatewayApi.UpstreamApiID is empty") + continue + } + l = l.WithField("tb_gateway_upstream_api.id", gatewayApi.UpstreamApiId) + upstreamApi, err := s.upstreamApiService.GetById(gatewayApi.UpstreamApiId) + if err != nil { + l.WithError(err).Warnln("failed to upstreamApiService.GetById") + continue + } + l = l.WithField("tb_gateway_upstream.id", upstreamApi.UpstreamId) + var cond orm.GatewayUpstream + cond.Id = upstreamApi.UpstreamId + upstreams, err := s.upstreamService.SelectByAny(&cond) + if err != nil { + l.WithError(err).Warnf("failed to upstreamService.SelectByAny(%+v)", cond) + continue + } + if len(upstreams) == 0 { + l.Warnln("not found any upstreams") + continue + } + for _, upstream := range upstreams { + l := l.WithField("upstreamName", upstream.UpstreamName) + runtimeID, err := strconv.ParseUint(filepath.Base(upstream.UpstreamName), 10, 32) + if err != nil { + l.WithError(err).Warnln("failed to parse runtime id from upstream name") + continue + } + if runtimeID == 0 { + l.Warnln("runtime id is 0 parsed from upstream name") + continue + } + l = l.WithField("runtimeID", runtimeID) + resp, err := s.runtimeCli.CheckRuntimeExist(ctx, &runtimePb.CheckRuntimeExistReq{Id: runtimeID}) + if err == nil && !resp.GetOk() { + item := &pb.ListInvalidEndpointApiItem{ + InvalidReason: invalidRuntime, + Type: "package_api", + ProjectID: projectID, + PackageID: package_.Id, + PackageApiID: packageApi.Id, + UpstreamApiID: upstreamApi.Id, + UpstreamID: upstream.Id, + UpstreamName: upstream.UpstreamName, + RuntimeID: filepath.Base(upstream.UpstreamName), + } + result.List = append(result.List, item) + } + } + } + } + } + + return &result, nil +} + +func (s *endpointApiService) ClearInvalidEndpointApi(ctx context.Context, req *commonPb.VoidRequest) (*commonPb.VoidResponse, error) { + timer := time.NewTimer(time.Second * 2) + defer timer.Stop() + select { + case <-timer.C: + return nil, errors.New("task in process") + case clearC <- struct{}{}: + go func() { + s.clearInvalidEndpointApi(ctx, req) + <-clearC + }() + + } + return new(commonPb.VoidResponse), nil +} + +func (s *endpointApiService) clearInvalidEndpointApi(ctx context.Context, _ *commonPb.VoidRequest) (*commonPb.VoidResponse, error) { + l := logrus.WithField("func", "*endpointApiService.ClearInvalidEndpointApi") + resp, err := s.ListInvalidEndpointApi(ctx, nil) + if err != nil { + return nil, err + } + for _, item := range resp.List { + if item.GetType() == "package" { + l.Infof("delete package: %+v", item) + if _, err := s.DeleteEndpoint(ctx, &pb.DeleteEndpointRequest{ + PackageId: item.GetPackageID(), + }); err != nil { + l.WithError(err).WithField("package id", item.GetPackageID()).Warnln("failed to DeleteEndpoint") + } + } + if item.GetType() == "package_api" { + l.Infof("delete package api: %+v", item) + if _, err := s.DeleteEndpointApi(ctx, &pb.DeleteEndpointApiRequest{ + PackageId: item.GetPackageID(), + ApiId: item.GetPackageApiID(), + }); err != nil { + l.WithError(err). + WithField("package id", item.GetPackageID()). + WithField("package api id", item.GetPackageApiID()). + Warnln("failed to DeleteEndpointApi") + } + } + } + + return new(commonPb.VoidResponse), nil +} diff --git a/internal/tools/orchestrator/hepa/providers/endpoint_api/provider.go b/internal/tools/orchestrator/hepa/providers/endpoint_api/provider.go index 6107406b821..c7021fd2b07 100644 --- a/internal/tools/orchestrator/hepa/providers/endpoint_api/provider.go +++ b/internal/tools/orchestrator/hepa/providers/endpoint_api/provider.go @@ -15,11 +15,18 @@ package endpoint_api import ( + "github.com/pkg/errors" + "github.com/erda-project/erda-infra/base/logs" "github.com/erda-project/erda-infra/base/servicehub" "github.com/erda-project/erda-infra/pkg/transport" "github.com/erda-project/erda-proto-go/core/hepa/endpoint_api/pb" + _ "github.com/erda-project/erda-proto-go/core/project/client" + projPb "github.com/erda-project/erda-proto-go/core/project/pb" + _ "github.com/erda-project/erda-proto-go/orchestrator/runtime/client" + runtimePb "github.com/erda-project/erda-proto-go/orchestrator/runtime/pb" "github.com/erda-project/erda/internal/tools/orchestrator/hepa/common" + repositoryService "github.com/erda-project/erda/internal/tools/orchestrator/hepa/repository/service" "github.com/erda-project/erda/internal/tools/orchestrator/hepa/services/endpoint_api/impl" zoneI "github.com/erda-project/erda/internal/tools/orchestrator/hepa/services/zone/impl" "github.com/erda-project/erda/pkg/common/apis" @@ -35,12 +42,38 @@ type provider struct { Log logs.Logger Register transport.Register endpointApiService *endpointApiService - Perm perm.Interface `autowired:"permission"` + Perm perm.Interface `autowired:"permission"` + ProjCli projPb.ProjectServer `autowired:"erda.core.project.Project"` + RuntimeCli runtimePb.RuntimeServiceServer `autowired:"erda.orchestrator.runtime.RuntimeService"` } func (p *provider) Init(ctx servicehub.Context) error { - p.endpointApiService = &endpointApiService{p} - err := zoneI.NewGatewayZoneServiceImpl() + gatewayApiService, err := repositoryService.NewGatewayApiServiceImpl() + if err != nil { + return errors.Wrap(err, "failed to NewGatewayApiServiceImpl") + } + upstreamApiService, err := repositoryService.NewGatewayUpstreamApiServiceImpl() + if err != nil { + return errors.Wrap(err, "failed to NewGatewayUpstreamApiServiceImpl") + } + upstreamService, err := repositoryService.NewGatewayUpstreamServiceImpl() + if err != nil { + return errors.Wrap(err, "failed to NewGatewayUpstreamServiceImpl") + } + if p.ProjCli == nil { + p.Log.Fatal("projCli is nil") + } + if p.RuntimeCli == nil { + p.Log.Fatal("runtimeCli is nil") + } + p.endpointApiService = &endpointApiService{ + projCli: p.ProjCli, + runtimeCli: p.RuntimeCli, + gatewayApiService: gatewayApiService, + upstreamApiService: upstreamApiService, + upstreamService: upstreamService, + } + err = zoneI.NewGatewayZoneServiceImpl() if err != nil { return err } @@ -62,11 +95,20 @@ func (p *provider) Init(ctx servicehub.Context) error { perm.Method(apiService.GetEndpointsName, perm.ScopeProject, "project", perm.ActionGet, perm.FieldValue("ProjectId")), perm.Method(apiService.UpdateEndpoint, perm.ScopeOrg, "org", perm.ActionGet, perm.OrgIDValue()), perm.Method(apiService.UpdateEndpointApi, perm.ScopeOrg, "org", perm.ActionGet, perm.OrgIDValue()), + perm.Method(apiService.ListInvalidEndpointApi, perm.ScopeOrg, "org", perm.ActionGet, perm.OrgIDValue()), ), common.AccessLogWrap(common.AccessLog)) } return nil } +//func (p *provider) Run(ctx context.Context) error { +// go ticker.New(time.Hour*24, func() (bool, error) { +// _, err := p.endpointApiService.ClearInvalidEndpointApi(ctx, new(commonPb.VoidRequest)) +// return false, err +// }).Run() +// return nil +//} + func (p *provider) Provide(ctx servicehub.DependencyContext, args ...interface{}) interface{} { switch { case ctx.Service() == "erda.core.hepa.endpoint_api.EndpointApiService" || ctx.Type() == pb.EndpointApiServiceServerType() || ctx.Type() == pb.EndpointApiServiceHandlerType(): @@ -88,6 +130,8 @@ func init() { "erda.core.hepa.api_policy.ApiPolicyService", "erda.core.hepa.domain.DomainService", "erda.core.hepa.global.GlobalService", + "erda.core.project.Project", + "erda.orchestrator.runtime.RuntimeService", }, Description: "", ConfigFunc: func() interface{} { diff --git a/internal/tools/orchestrator/hepa/services/endpoint_api/impl/impl.go b/internal/tools/orchestrator/hepa/services/endpoint_api/impl/impl.go index ef7103f886e..ad1a2595e69 100644 --- a/internal/tools/orchestrator/hepa/services/endpoint_api/impl/impl.go +++ b/internal/tools/orchestrator/hepa/services/endpoint_api/impl/impl.go @@ -468,6 +468,10 @@ func (impl GatewayOpenapiServiceImpl) GetPackages(args *gw.GetPackagesDto) (res return } +func (impl GatewayOpenapiServiceImpl) ListAllPackages() ([]orm.GatewayPackage, error) { + return impl.packageDb.SelectByAny(new(orm.GatewayPackage)) +} + func (impl GatewayOpenapiServiceImpl) packageDto(dao *orm.GatewayPackage, domains []string) *gw.PackageInfoDto { res := &gw.PackageInfoDto{ Id: dao.Id, @@ -1703,6 +1707,10 @@ func (impl GatewayOpenapiServiceImpl) GetPackageApis(id string, args *gw.GetOpen return } +func (impl GatewayOpenapiServiceImpl) ListPackageAllApis(id string) ([]orm.GatewayPackageApi, error) { + return impl.packageApiDb.SelectByAny(&orm.GatewayPackageApi{PackageId: id}) +} + func (impl GatewayOpenapiServiceImpl) UpdatePackageApi(packageId, apiId string, dto *gw.OpenapiDto) (result *gw.OpenapiInfoDto, exist bool, err error) { var dao, updateDao *orm.GatewayPackageApi var pack *orm.GatewayPackage diff --git a/internal/tools/orchestrator/hepa/services/endpoint_api/interface.go b/internal/tools/orchestrator/hepa/services/endpoint_api/interface.go index e50f6138f86..e74eae75054 100644 --- a/internal/tools/orchestrator/hepa/services/endpoint_api/interface.go +++ b/internal/tools/orchestrator/hepa/services/endpoint_api/interface.go @@ -47,12 +47,14 @@ type GatewayOpenapiService interface { CreateTenantPackage(string, *service.SessionHelper) error CreatePackage(*dto.DiceArgsDto, *dto.PackageDto) (*dto.PackageInfoDto, string, error) GetPackages(*dto.GetPackagesDto) (common.NewPageQuery, error) + ListAllPackages() ([]orm.GatewayPackage, error) GetPackage(string) (*dto.PackageInfoDto, error) GetPackagesName(*dto.GetPackagesDto) ([]dto.PackageInfoDto, error) UpdatePackage(string, string, *dto.PackageDto) (*dto.PackageInfoDto, error) DeletePackage(string) (bool, error) CreatePackageApi(string, *dto.OpenapiDto) (string, bool, error) GetPackageApis(string, *dto.GetOpenapiDto) (common.NewPageQuery, error) + ListPackageAllApis(id string) ([]orm.GatewayPackageApi, error) UpdatePackageApi(string, string, *dto.OpenapiDto) (*dto.OpenapiInfoDto, bool, error) DeletePackageApi(string, string) (bool, error) TouchPackageApiZone(info PackageApiInfo, session ...*service.SessionHelper) (string, error) diff --git a/pkg/common/errors/data.go b/pkg/common/errors/data.go index eb4f30e51f5..a88e53a89c4 100644 --- a/pkg/common/errors/data.go +++ b/pkg/common/errors/data.go @@ -17,6 +17,7 @@ package errors import ( "fmt" "net/http" + "strings" "github.com/erda-project/erda-infra/providers/i18n" ) @@ -33,6 +34,10 @@ func NewNotFoundError(resource string) *NotFoundError { return &NotFoundError{Resource: resource} } +func IsNotFoundError(err error) bool { + return err != nil && strings.HasSuffix(err.Error(), "not found") +} + func (e *NotFoundError) Error() string { if len(e.Resource) > 0 { return fmt.Sprintf("%s not found", e.Resource) diff --git a/pkg/time/ticker/ticker.go b/pkg/time/ticker/ticker.go index 380b3da2b33..86c0c143925 100644 --- a/pkg/time/ticker/ticker.go +++ b/pkg/time/ticker/ticker.go @@ -27,7 +27,7 @@ func (e ExitError) Error() string { return e.Message } -type Task func() error +type Task func() (finished bool, err error) type Ticker struct { Name string @@ -36,7 +36,7 @@ type Ticker struct { done chan bool } -func New(interval time.Duration, task func() (bool, error)) *Ticker { +func New(interval time.Duration, task Task) *Ticker { return &Ticker{ Interval: interval, Task: task, @@ -49,13 +49,13 @@ func (d *Ticker) Run() error { defer tick.Stop() var ( - err error - done bool + err error + finished bool ) fmt.Printf("the interval task %s is running right now: %s\n", d.Name, time.Now().Format(time.RFC3339)) - done, err = d.Task() + finished, err = d.Task() fmt.Printf("the interval task %s is complete this time, err: %v\n", d.Name, err) - if done { + if finished { d.Close() return err } @@ -63,13 +63,13 @@ func (d *Ticker) Run() error { for { select { case <-d.done: - fmt.Printf("the interval task %s is done!\n", d.Name) + fmt.Printf("the interval task %s is finished!\n", d.Name) return err case t := <-tick.C: fmt.Printf("the interval task %s is running at: %s\n", d.Name, t.Format(time.RFC3339)) - done, err = d.Task() + finished, err = d.Task() fmt.Printf("the interval task %s is complete this time, err: %v\n", d.Name, err) - if done { + if finished { d.Close() } }