From 239ae8f10f39dfdb78ec26906b8a8d2805cbe204 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Ga=C4=87e=C5=A1a?= Date: Wed, 6 Sep 2023 16:41:37 +0200 Subject: [PATCH 01/17] add repo import api --- cmd/gitness/wire.go | 2 + cmd/gitness/wire_gen.go | 34 +-- gitrpc/interface.go | 2 + gitrpc/internal/service/repo.go | 4 +- gitrpc/proto/repo.proto | 2 + gitrpc/repo.go | 7 +- gitrpc/rpc/repo.pb.go | 218 +++++++++--------- go.mod | 2 +- go.sum | 1 + internal/api/controller/pullreq/controller.go | 4 + internal/api/controller/repo/controller.go | 15 ++ internal/api/controller/repo/create.go | 7 +- internal/api/controller/repo/import.go | 202 ++++++++++++++++ .../api/controller/repo/import_progress.go | 31 +++ internal/api/controller/repo/wire.go | 11 +- internal/api/handler/repo/import.go | 36 +++ internal/api/handler/repo/import_progress.go | 33 +++ internal/router/api.go | 3 + internal/services/importer/provider.go | 142 ++++++++++++ internal/services/importer/repository.go | 165 +++++++++++++ internal/services/importer/wire.go | 40 ++++ .../0024_alter_repo_add_importing.down.sql | 4 + .../0024_alter_repo_add_importing.up.sql | 8 + .../0024_alter_repo_add_importing.down.sql | 2 + .../0024_alter_repo_add_importing.up.sql | 2 + internal/store/database/repo.go | 19 +- types/repo.go | 3 + 27 files changed, 862 insertions(+), 137 deletions(-) create mode 100644 internal/api/controller/repo/import.go create mode 100644 internal/api/controller/repo/import_progress.go create mode 100644 internal/api/handler/repo/import.go create mode 100644 internal/api/handler/repo/import_progress.go create mode 100644 internal/services/importer/provider.go create mode 100644 internal/services/importer/repository.go create mode 100644 internal/services/importer/wire.go create mode 100644 internal/store/database/migrate/postgres/0024_alter_repo_add_importing.down.sql create mode 100644 internal/store/database/migrate/postgres/0024_alter_repo_add_importing.up.sql create mode 100644 internal/store/database/migrate/sqlite/0024_alter_repo_add_importing.down.sql create mode 100644 internal/store/database/migrate/sqlite/0024_alter_repo_add_importing.up.sql diff --git a/cmd/gitness/wire.go b/cmd/gitness/wire.go index 09ed2eabc..cf8070c30 100644 --- a/cmd/gitness/wire.go +++ b/cmd/gitness/wire.go @@ -50,6 +50,7 @@ import ( "github.com/harness/gitness/internal/server" "github.com/harness/gitness/internal/services" "github.com/harness/gitness/internal/services/codecomments" + "github.com/harness/gitness/internal/services/importer" "github.com/harness/gitness/internal/services/job" pullreqservice "github.com/harness/gitness/internal/services/pullreq" "github.com/harness/gitness/internal/services/webhook" @@ -128,6 +129,7 @@ func initSystem(ctx context.Context, config *types.Config) (*cliserver.System, e commit.WireSet, trigger.WireSet, plugin.WireSet, + importer.WireSet, ) return &cliserver.System{}, nil } diff --git a/cmd/gitness/wire_gen.go b/cmd/gitness/wire_gen.go index 47870e02f..a46fe2c07 100644 --- a/cmd/gitness/wire_gen.go +++ b/cmd/gitness/wire_gen.go @@ -8,7 +8,6 @@ package main import ( "context" - "github.com/harness/gitness/cli/server" "github.com/harness/gitness/encrypt" "github.com/harness/gitness/events" @@ -49,6 +48,7 @@ import ( server2 "github.com/harness/gitness/internal/server" "github.com/harness/gitness/internal/services" "github.com/harness/gitness/internal/services/codecomments" + "github.com/harness/gitness/internal/services/importer" "github.com/harness/gitness/internal/services/job" pullreq2 "github.com/harness/gitness/internal/services/pullreq" "github.com/harness/gitness/internal/services/webhook" @@ -104,17 +104,29 @@ func initSystem(ctx context.Context, config *types.Config) (*server.System, erro if err != nil { return nil, err } - repoController := repo.ProvideController(config, db, provider, pathUID, authorizer, pathStore, repoStore, spaceStore, pipelineStore, principalStore, gitrpcInterface) - executionStore := database.ProvideExecutionStore(db) - commitService := commit.ProvideCommitService(gitrpcInterface) - stageStore := database.ProvideStageStore(db) - fileService := file.ProvideFileService(gitrpcInterface) - lockConfig := server.ProvideLockConfig(config) + jobStore := database.ProvideJobStore(db) + pubsubConfig := pubsub.ProvideConfig(config) universalClient, err := server.ProvideRedis(config) if err != nil { return nil, err } + pubSub := pubsub.ProvidePubSub(pubsubConfig, universalClient) + executor := job.ProvideExecutor(jobStore, pubSub) + lockConfig := server.ProvideLockConfig(config) mutexManager := lock.ProvideMutexManager(lockConfig, universalClient) + jobScheduler, err := job.ProvideScheduler(jobStore, executor, mutexManager, pubSub, config) + if err != nil { + return nil, err + } + repository, err := importer.ProvideRepoImporter(provider, gitrpcInterface, repoStore, jobScheduler, executor) + if err != nil { + return nil, err + } + repoController := repo.ProvideController(config, db, provider, pathUID, authorizer, pathStore, repoStore, spaceStore, pipelineStore, principalStore, gitrpcInterface, repository) + executionStore := database.ProvideExecutionStore(db) + commitService := commit.ProvideCommitService(gitrpcInterface) + stageStore := database.ProvideStageStore(db) + fileService := file.ProvideFileService(gitrpcInterface) schedulerScheduler, err := scheduler.ProvideScheduler(stageStore, mutexManager) if err != nil { return nil, err @@ -218,18 +230,10 @@ func initSystem(ctx context.Context, config *types.Config) (*server.System, erro cronManager := cron.ProvideManager(serverConfig) repoGitInfoView := database.ProvideRepoGitInfoView(db) repoGitInfoCache := cache.ProvideRepoGitInfoCache(repoGitInfoView) - pubsubConfig := pubsub.ProvideConfig(config) - pubSub := pubsub.ProvidePubSub(pubsubConfig, universalClient) pullreqService, err := pullreq2.ProvideService(ctx, config, readerFactory, eventsReaderFactory, reporter, gitrpcInterface, db, repoGitInfoCache, repoStore, pullReqStore, pullReqActivityStore, codeCommentView, migrator, pubSub, provider) if err != nil { return nil, err } - jobStore := database.ProvideJobStore(db) - executor := job.ProvideExecutor(jobStore, pubSub) - jobScheduler, err := job.ProvideScheduler(jobStore, executor, mutexManager, pubSub, config) - if err != nil { - return nil, err - } servicesServices := services.ProvideServices(webhookService, pullreqService, executor, jobScheduler) serverSystem := server.NewSystem(bootstrapBootstrap, serverServer, poller, grpcServer, cronManager, servicesServices) return serverSystem, nil diff --git a/gitrpc/interface.go b/gitrpc/interface.go index 0f1b609a1..9929281c4 100644 --- a/gitrpc/interface.go +++ b/gitrpc/interface.go @@ -30,6 +30,8 @@ type Interface interface { // not update of an exiting one, set the zero ref as the OldValue. UpdateRef(ctx context.Context, params UpdateRefParams) error + SyncRepository(ctx context.Context, params *SyncRepositoryParams) (*SyncRepositoryOutput, error) + /* * Commits service */ diff --git a/gitrpc/internal/service/repo.go b/gitrpc/internal/service/repo.go index e11c6a266..b84daf4e0 100644 --- a/gitrpc/internal/service/repo.go +++ b/gitrpc/internal/service/repo.go @@ -354,7 +354,9 @@ func (s RepositoryService) SyncRepository( return nil, processGitErrorf(err, "failed to set default branch of repo") } - return &rpc.SyncRepositoryResponse{}, nil + return &rpc.SyncRepositoryResponse{ + DefaultBranch: defaultBranch, + }, nil } func (s RepositoryService) HashRepository( diff --git a/gitrpc/proto/repo.proto b/gitrpc/proto/repo.proto index d24861a38..c6f3380af 100644 --- a/gitrpc/proto/repo.proto +++ b/gitrpc/proto/repo.proto @@ -203,7 +203,9 @@ message SyncRepositoryRequest { } message SyncRepositoryResponse { + string default_branch = 1; } + enum HashType { HashTypeSHA256 = 0; } diff --git a/gitrpc/repo.go b/gitrpc/repo.go index 07285f291..ecbe61f12 100644 --- a/gitrpc/repo.go +++ b/gitrpc/repo.go @@ -63,6 +63,7 @@ type SyncRepositoryParams struct { } type SyncRepositoryOutput struct { + DefaultBranch string } type HashRepositoryParams struct { @@ -164,7 +165,7 @@ func (c *Client) DeleteRepository(ctx context.Context, params *DeleteRepositoryP } func (c *Client) SyncRepository(ctx context.Context, params *SyncRepositoryParams) (*SyncRepositoryOutput, error) { - _, err := c.repoService.SyncRepository(ctx, &rpc.SyncRepositoryRequest{ + result, err := c.repoService.SyncRepository(ctx, &rpc.SyncRepositoryRequest{ Base: mapToRPCWriteRequest(params.WriteParams), Source: params.Source, CreateIfNotExists: params.CreateIfNotExists, @@ -173,7 +174,9 @@ func (c *Client) SyncRepository(ctx context.Context, params *SyncRepositoryParam return nil, processRPCErrorf(err, "failed to sync repository on server to match provided source") } - return &SyncRepositoryOutput{}, nil + return &SyncRepositoryOutput{ + DefaultBranch: result.DefaultBranch, + }, nil } func (c *Client) HashRepository(ctx context.Context, params *HashRepositoryParams) (*HashRepositoryOutput, error) { diff --git a/gitrpc/rpc/repo.pb.go b/gitrpc/rpc/repo.pb.go index ba0029a59..6d762da1a 100644 --- a/gitrpc/rpc/repo.pb.go +++ b/gitrpc/rpc/repo.pb.go @@ -1987,6 +1987,8 @@ type SyncRepositoryResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields + + DefaultBranch string `protobuf:"bytes,1,opt,name=default_branch,json=defaultBranch,proto3" json:"default_branch,omitempty"` } func (x *SyncRepositoryResponse) Reset() { @@ -2021,6 +2023,13 @@ func (*SyncRepositoryResponse) Descriptor() ([]byte, []int) { return file_repo_proto_rawDescGZIP(), []int{29} } +func (x *SyncRepositoryResponse) GetDefaultBranch() string { + if x != nil { + return x.DefaultBranch + } + return "" +} + type HashRepositoryRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -2438,114 +2447,117 @@ var file_repo_proto_rawDesc = []byte{ 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x12, 0x2f, 0x0a, 0x14, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x5f, 0x69, 0x66, 0x5f, 0x6e, 0x6f, 0x74, 0x5f, 0x65, 0x78, 0x69, 0x73, 0x74, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x11, 0x63, 0x72, 0x65, - 0x61, 0x74, 0x65, 0x49, 0x66, 0x4e, 0x6f, 0x74, 0x45, 0x78, 0x69, 0x73, 0x74, 0x73, 0x22, 0x18, + 0x61, 0x74, 0x65, 0x49, 0x66, 0x4e, 0x6f, 0x74, 0x45, 0x78, 0x69, 0x73, 0x74, 0x73, 0x22, 0x3f, 0x0a, 0x16, 0x53, 0x79, 0x6e, 0x63, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0xae, 0x01, 0x0a, 0x15, 0x48, 0x61, 0x73, - 0x68, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x25, 0x0a, 0x0e, 0x64, 0x65, 0x66, 0x61, + 0x75, 0x6c, 0x74, 0x5f, 0x62, 0x72, 0x61, 0x6e, 0x63, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x0d, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x42, 0x72, 0x61, 0x6e, 0x63, 0x68, 0x22, + 0xae, 0x01, 0x0a, 0x15, 0x48, 0x61, 0x73, 0x68, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, + 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x24, 0x0a, 0x04, 0x62, 0x61, 0x73, + 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, + 0x61, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x52, 0x04, 0x62, 0x61, 0x73, 0x65, 0x12, + 0x2a, 0x0a, 0x09, 0x68, 0x61, 0x73, 0x68, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x0e, 0x32, 0x0d, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x48, 0x61, 0x73, 0x68, 0x54, 0x79, 0x70, + 0x65, 0x52, 0x08, 0x68, 0x61, 0x73, 0x68, 0x54, 0x79, 0x70, 0x65, 0x12, 0x43, 0x0a, 0x10, 0x61, + 0x67, 0x67, 0x72, 0x65, 0x67, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x18, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x48, 0x61, 0x73, 0x68, + 0x41, 0x67, 0x67, 0x72, 0x65, 0x67, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x52, + 0x0f, 0x61, 0x67, 0x67, 0x72, 0x65, 0x67, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, + 0x22, 0x2c, 0x0a, 0x16, 0x48, 0x61, 0x73, 0x68, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, + 0x72, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x68, 0x61, + 0x73, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x68, 0x61, 0x73, 0x68, 0x22, 0x60, + 0x0a, 0x10, 0x4d, 0x65, 0x72, 0x67, 0x65, 0x42, 0x61, 0x73, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x24, 0x0a, 0x04, 0x62, 0x61, 0x73, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x61, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x52, 0x04, 0x62, 0x61, 0x73, 0x65, 0x12, 0x2a, 0x0a, 0x09, 0x68, 0x61, 0x73, 0x68, - 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x0d, 0x2e, 0x72, 0x70, - 0x63, 0x2e, 0x48, 0x61, 0x73, 0x68, 0x54, 0x79, 0x70, 0x65, 0x52, 0x08, 0x68, 0x61, 0x73, 0x68, - 0x54, 0x79, 0x70, 0x65, 0x12, 0x43, 0x0a, 0x10, 0x61, 0x67, 0x67, 0x72, 0x65, 0x67, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x18, - 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x48, 0x61, 0x73, 0x68, 0x41, 0x67, 0x67, 0x72, 0x65, 0x67, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0f, 0x61, 0x67, 0x67, 0x72, 0x65, 0x67, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x22, 0x2c, 0x0a, 0x16, 0x48, 0x61, 0x73, - 0x68, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x68, 0x61, 0x73, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x0c, 0x52, 0x04, 0x68, 0x61, 0x73, 0x68, 0x22, 0x60, 0x0a, 0x10, 0x4d, 0x65, 0x72, 0x67, 0x65, - 0x42, 0x61, 0x73, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x24, 0x0a, 0x04, 0x62, - 0x61, 0x73, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x72, 0x70, 0x63, 0x2e, - 0x52, 0x65, 0x61, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x52, 0x04, 0x62, 0x61, 0x73, - 0x65, 0x12, 0x12, 0x0a, 0x04, 0x72, 0x65, 0x66, 0x31, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x04, 0x72, 0x65, 0x66, 0x31, 0x12, 0x12, 0x0a, 0x04, 0x72, 0x65, 0x66, 0x32, 0x18, 0x03, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x04, 0x72, 0x65, 0x66, 0x32, 0x22, 0x39, 0x0a, 0x11, 0x4d, 0x65, 0x72, - 0x67, 0x65, 0x42, 0x61, 0x73, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x24, - 0x0a, 0x0e, 0x6d, 0x65, 0x72, 0x67, 0x65, 0x5f, 0x62, 0x61, 0x73, 0x65, 0x5f, 0x73, 0x68, 0x61, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x6d, 0x65, 0x72, 0x67, 0x65, 0x42, 0x61, 0x73, - 0x65, 0x53, 0x68, 0x61, 0x2a, 0x52, 0x0a, 0x0c, 0x54, 0x72, 0x65, 0x65, 0x4e, 0x6f, 0x64, 0x65, - 0x54, 0x79, 0x70, 0x65, 0x12, 0x14, 0x0a, 0x10, 0x54, 0x72, 0x65, 0x65, 0x4e, 0x6f, 0x64, 0x65, - 0x54, 0x79, 0x70, 0x65, 0x54, 0x72, 0x65, 0x65, 0x10, 0x00, 0x12, 0x14, 0x0a, 0x10, 0x54, 0x72, - 0x65, 0x65, 0x4e, 0x6f, 0x64, 0x65, 0x54, 0x79, 0x70, 0x65, 0x42, 0x6c, 0x6f, 0x62, 0x10, 0x01, - 0x12, 0x16, 0x0a, 0x12, 0x54, 0x72, 0x65, 0x65, 0x4e, 0x6f, 0x64, 0x65, 0x54, 0x79, 0x70, 0x65, - 0x43, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x10, 0x02, 0x2a, 0x81, 0x01, 0x0a, 0x0c, 0x54, 0x72, 0x65, - 0x65, 0x4e, 0x6f, 0x64, 0x65, 0x4d, 0x6f, 0x64, 0x65, 0x12, 0x14, 0x0a, 0x10, 0x54, 0x72, 0x65, - 0x65, 0x4e, 0x6f, 0x64, 0x65, 0x4d, 0x6f, 0x64, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x10, 0x00, 0x12, - 0x17, 0x0a, 0x13, 0x54, 0x72, 0x65, 0x65, 0x4e, 0x6f, 0x64, 0x65, 0x4d, 0x6f, 0x64, 0x65, 0x53, - 0x79, 0x6d, 0x6c, 0x69, 0x6e, 0x6b, 0x10, 0x01, 0x12, 0x14, 0x0a, 0x10, 0x54, 0x72, 0x65, 0x65, - 0x4e, 0x6f, 0x64, 0x65, 0x4d, 0x6f, 0x64, 0x65, 0x45, 0x78, 0x65, 0x63, 0x10, 0x02, 0x12, 0x14, - 0x0a, 0x10, 0x54, 0x72, 0x65, 0x65, 0x4e, 0x6f, 0x64, 0x65, 0x4d, 0x6f, 0x64, 0x65, 0x54, 0x72, - 0x65, 0x65, 0x10, 0x03, 0x12, 0x16, 0x0a, 0x12, 0x54, 0x72, 0x65, 0x65, 0x4e, 0x6f, 0x64, 0x65, - 0x4d, 0x6f, 0x64, 0x65, 0x43, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x10, 0x04, 0x2a, 0x1e, 0x0a, 0x08, - 0x48, 0x61, 0x73, 0x68, 0x54, 0x79, 0x70, 0x65, 0x12, 0x12, 0x0a, 0x0e, 0x48, 0x61, 0x73, 0x68, - 0x54, 0x79, 0x70, 0x65, 0x53, 0x48, 0x41, 0x32, 0x35, 0x36, 0x10, 0x00, 0x2a, 0x31, 0x0a, 0x13, - 0x48, 0x61, 0x73, 0x68, 0x41, 0x67, 0x67, 0x72, 0x65, 0x67, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, - 0x79, 0x70, 0x65, 0x12, 0x1a, 0x0a, 0x16, 0x48, 0x61, 0x73, 0x68, 0x41, 0x67, 0x67, 0x72, 0x65, - 0x67, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x58, 0x4f, 0x52, 0x10, 0x00, 0x32, - 0xb8, 0x07, 0x0a, 0x11, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x53, 0x65, - 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x51, 0x0a, 0x10, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x52, - 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x12, 0x1c, 0x2e, 0x72, 0x70, 0x63, 0x2e, + 0x73, 0x74, 0x52, 0x04, 0x62, 0x61, 0x73, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x72, 0x65, 0x66, 0x31, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x72, 0x65, 0x66, 0x31, 0x12, 0x12, 0x0a, 0x04, + 0x72, 0x65, 0x66, 0x32, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x72, 0x65, 0x66, 0x32, + 0x22, 0x39, 0x0a, 0x11, 0x4d, 0x65, 0x72, 0x67, 0x65, 0x42, 0x61, 0x73, 0x65, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x24, 0x0a, 0x0e, 0x6d, 0x65, 0x72, 0x67, 0x65, 0x5f, 0x62, + 0x61, 0x73, 0x65, 0x5f, 0x73, 0x68, 0x61, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x6d, + 0x65, 0x72, 0x67, 0x65, 0x42, 0x61, 0x73, 0x65, 0x53, 0x68, 0x61, 0x2a, 0x52, 0x0a, 0x0c, 0x54, + 0x72, 0x65, 0x65, 0x4e, 0x6f, 0x64, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x14, 0x0a, 0x10, 0x54, + 0x72, 0x65, 0x65, 0x4e, 0x6f, 0x64, 0x65, 0x54, 0x79, 0x70, 0x65, 0x54, 0x72, 0x65, 0x65, 0x10, + 0x00, 0x12, 0x14, 0x0a, 0x10, 0x54, 0x72, 0x65, 0x65, 0x4e, 0x6f, 0x64, 0x65, 0x54, 0x79, 0x70, + 0x65, 0x42, 0x6c, 0x6f, 0x62, 0x10, 0x01, 0x12, 0x16, 0x0a, 0x12, 0x54, 0x72, 0x65, 0x65, 0x4e, + 0x6f, 0x64, 0x65, 0x54, 0x79, 0x70, 0x65, 0x43, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x10, 0x02, 0x2a, + 0x81, 0x01, 0x0a, 0x0c, 0x54, 0x72, 0x65, 0x65, 0x4e, 0x6f, 0x64, 0x65, 0x4d, 0x6f, 0x64, 0x65, + 0x12, 0x14, 0x0a, 0x10, 0x54, 0x72, 0x65, 0x65, 0x4e, 0x6f, 0x64, 0x65, 0x4d, 0x6f, 0x64, 0x65, + 0x46, 0x69, 0x6c, 0x65, 0x10, 0x00, 0x12, 0x17, 0x0a, 0x13, 0x54, 0x72, 0x65, 0x65, 0x4e, 0x6f, + 0x64, 0x65, 0x4d, 0x6f, 0x64, 0x65, 0x53, 0x79, 0x6d, 0x6c, 0x69, 0x6e, 0x6b, 0x10, 0x01, 0x12, + 0x14, 0x0a, 0x10, 0x54, 0x72, 0x65, 0x65, 0x4e, 0x6f, 0x64, 0x65, 0x4d, 0x6f, 0x64, 0x65, 0x45, + 0x78, 0x65, 0x63, 0x10, 0x02, 0x12, 0x14, 0x0a, 0x10, 0x54, 0x72, 0x65, 0x65, 0x4e, 0x6f, 0x64, + 0x65, 0x4d, 0x6f, 0x64, 0x65, 0x54, 0x72, 0x65, 0x65, 0x10, 0x03, 0x12, 0x16, 0x0a, 0x12, 0x54, + 0x72, 0x65, 0x65, 0x4e, 0x6f, 0x64, 0x65, 0x4d, 0x6f, 0x64, 0x65, 0x43, 0x6f, 0x6d, 0x6d, 0x69, + 0x74, 0x10, 0x04, 0x2a, 0x1e, 0x0a, 0x08, 0x48, 0x61, 0x73, 0x68, 0x54, 0x79, 0x70, 0x65, 0x12, + 0x12, 0x0a, 0x0e, 0x48, 0x61, 0x73, 0x68, 0x54, 0x79, 0x70, 0x65, 0x53, 0x48, 0x41, 0x32, 0x35, + 0x36, 0x10, 0x00, 0x2a, 0x31, 0x0a, 0x13, 0x48, 0x61, 0x73, 0x68, 0x41, 0x67, 0x67, 0x72, 0x65, + 0x67, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x12, 0x1a, 0x0a, 0x16, 0x48, 0x61, + 0x73, 0x68, 0x41, 0x67, 0x67, 0x72, 0x65, 0x67, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, + 0x65, 0x58, 0x4f, 0x52, 0x10, 0x00, 0x32, 0xb8, 0x07, 0x0a, 0x11, 0x52, 0x65, 0x70, 0x6f, 0x73, + 0x69, 0x74, 0x6f, 0x72, 0x79, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x51, 0x0a, 0x10, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x43, 0x72, - 0x65, 0x61, 0x74, 0x65, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28, 0x01, 0x12, 0x40, 0x0a, 0x0b, 0x47, 0x65, 0x74, 0x54, - 0x72, 0x65, 0x65, 0x4e, 0x6f, 0x64, 0x65, 0x12, 0x17, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x47, 0x65, - 0x74, 0x54, 0x72, 0x65, 0x65, 0x4e, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x18, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x47, 0x65, 0x74, 0x54, 0x72, 0x65, 0x65, 0x4e, 0x6f, - 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x48, 0x0a, 0x0d, 0x4c, 0x69, - 0x73, 0x74, 0x54, 0x72, 0x65, 0x65, 0x4e, 0x6f, 0x64, 0x65, 0x73, 0x12, 0x19, 0x2e, 0x72, 0x70, - 0x63, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x72, 0x65, 0x65, 0x4e, 0x6f, 0x64, 0x65, 0x73, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1a, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x4c, 0x69, 0x73, - 0x74, 0x54, 0x72, 0x65, 0x65, 0x4e, 0x6f, 0x64, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x30, 0x01, 0x12, 0x43, 0x0a, 0x0c, 0x50, 0x61, 0x74, 0x68, 0x73, 0x44, 0x65, 0x74, - 0x61, 0x69, 0x6c, 0x73, 0x12, 0x18, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x50, 0x61, 0x74, 0x68, 0x73, - 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, - 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x50, 0x61, 0x74, 0x68, 0x73, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, - 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x43, 0x0a, 0x0c, 0x47, 0x65, 0x74, - 0x53, 0x75, 0x62, 0x6d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x12, 0x18, 0x2e, 0x72, 0x70, 0x63, 0x2e, - 0x47, 0x65, 0x74, 0x53, 0x75, 0x62, 0x6d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x75, 0x62, - 0x6d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x36, - 0x0a, 0x07, 0x47, 0x65, 0x74, 0x42, 0x6c, 0x6f, 0x62, 0x12, 0x13, 0x2e, 0x72, 0x70, 0x63, 0x2e, - 0x47, 0x65, 0x74, 0x42, 0x6c, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x14, - 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x47, 0x65, 0x74, 0x42, 0x6c, 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x30, 0x01, 0x12, 0x42, 0x0a, 0x0b, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x6f, - 0x6d, 0x6d, 0x69, 0x74, 0x73, 0x12, 0x17, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x4c, 0x69, 0x73, 0x74, - 0x43, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x18, - 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x73, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x30, 0x01, 0x12, 0x3a, 0x0a, 0x09, 0x47, 0x65, - 0x74, 0x43, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x12, 0x15, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x47, 0x65, - 0x74, 0x43, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, + 0x12, 0x1c, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x52, 0x65, 0x70, + 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, + 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x52, 0x65, 0x70, 0x6f, 0x73, + 0x69, 0x74, 0x6f, 0x72, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28, 0x01, 0x12, + 0x40, 0x0a, 0x0b, 0x47, 0x65, 0x74, 0x54, 0x72, 0x65, 0x65, 0x4e, 0x6f, 0x64, 0x65, 0x12, 0x17, + 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x47, 0x65, 0x74, 0x54, 0x72, 0x65, 0x65, 0x4e, 0x6f, 0x64, 0x65, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x18, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x47, 0x65, + 0x74, 0x54, 0x72, 0x65, 0x65, 0x4e, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x48, 0x0a, 0x0d, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x72, 0x65, 0x65, 0x4e, 0x6f, 0x64, + 0x65, 0x73, 0x12, 0x19, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x72, 0x65, + 0x65, 0x4e, 0x6f, 0x64, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1a, 0x2e, + 0x72, 0x70, 0x63, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x72, 0x65, 0x65, 0x4e, 0x6f, 0x64, 0x65, + 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x30, 0x01, 0x12, 0x43, 0x0a, 0x0c, 0x50, + 0x61, 0x74, 0x68, 0x73, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x12, 0x18, 0x2e, 0x72, 0x70, + 0x63, 0x2e, 0x50, 0x61, 0x74, 0x68, 0x73, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x50, 0x61, 0x74, 0x68, + 0x73, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x43, 0x0a, 0x0c, 0x47, 0x65, 0x74, 0x53, 0x75, 0x62, 0x6d, 0x6f, 0x64, 0x75, 0x6c, 0x65, + 0x12, 0x18, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x75, 0x62, 0x6d, 0x6f, 0x64, + 0x75, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x72, 0x70, 0x63, + 0x2e, 0x47, 0x65, 0x74, 0x53, 0x75, 0x62, 0x6d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x36, 0x0a, 0x07, 0x47, 0x65, 0x74, 0x42, 0x6c, 0x6f, 0x62, + 0x12, 0x13, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x47, 0x65, 0x74, 0x42, 0x6c, 0x6f, 0x62, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x47, 0x65, 0x74, 0x42, + 0x6c, 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x30, 0x01, 0x12, 0x42, 0x0a, + 0x0b, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x73, 0x12, 0x17, 0x2e, 0x72, + 0x70, 0x63, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x73, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x18, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x4c, 0x69, 0x73, 0x74, + 0x43, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x30, + 0x01, 0x12, 0x3a, 0x0a, 0x09, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x12, 0x15, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x5b, 0x0a, 0x14, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6d, - 0x6d, 0x69, 0x74, 0x44, 0x69, 0x76, 0x65, 0x72, 0x67, 0x65, 0x6e, 0x63, 0x65, 0x73, 0x12, 0x20, - 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x44, 0x69, - 0x76, 0x65, 0x72, 0x67, 0x65, 0x6e, 0x63, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x21, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6d, 0x6d, 0x69, 0x74, - 0x44, 0x69, 0x76, 0x65, 0x72, 0x67, 0x65, 0x6e, 0x63, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x4f, 0x0a, 0x10, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x70, - 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x12, 0x1c, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x44, 0x65, - 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x44, 0x65, 0x6c, 0x65, - 0x74, 0x65, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4b, 0x0a, 0x0e, 0x53, 0x79, 0x6e, 0x63, 0x52, 0x65, 0x70, 0x6f, - 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x12, 0x1a, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x53, 0x79, 0x6e, - 0x63, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x53, 0x79, 0x6e, 0x63, 0x52, 0x65, 0x70, - 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, - 0x00, 0x12, 0x4b, 0x0a, 0x0e, 0x48, 0x61, 0x73, 0x68, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, - 0x6f, 0x72, 0x79, 0x12, 0x1a, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x48, 0x61, 0x73, 0x68, 0x52, 0x65, - 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x1b, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x48, 0x61, 0x73, 0x68, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x69, - 0x74, 0x6f, 0x72, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x3a, - 0x0a, 0x09, 0x4d, 0x65, 0x72, 0x67, 0x65, 0x42, 0x61, 0x73, 0x65, 0x12, 0x15, 0x2e, 0x72, 0x70, - 0x63, 0x2e, 0x4d, 0x65, 0x72, 0x67, 0x65, 0x42, 0x61, 0x73, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x4d, 0x65, 0x72, 0x67, 0x65, 0x42, 0x61, - 0x73, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x27, 0x5a, 0x25, 0x67, 0x69, - 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x68, 0x61, 0x72, 0x6e, 0x65, 0x73, 0x73, - 0x2f, 0x67, 0x69, 0x74, 0x6e, 0x65, 0x73, 0x73, 0x2f, 0x67, 0x69, 0x74, 0x72, 0x70, 0x63, 0x2f, - 0x72, 0x70, 0x63, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x47, 0x65, 0x74, 0x43, + 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x5b, 0x0a, + 0x14, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x44, 0x69, 0x76, 0x65, 0x72, 0x67, + 0x65, 0x6e, 0x63, 0x65, 0x73, 0x12, 0x20, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x47, 0x65, 0x74, 0x43, + 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x44, 0x69, 0x76, 0x65, 0x72, 0x67, 0x65, 0x6e, 0x63, 0x65, 0x73, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x21, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x47, 0x65, + 0x74, 0x43, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x44, 0x69, 0x76, 0x65, 0x72, 0x67, 0x65, 0x6e, 0x63, + 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4f, 0x0a, 0x10, 0x44, 0x65, + 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x12, 0x1c, + 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x70, 0x6f, 0x73, + 0x69, 0x74, 0x6f, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x72, + 0x70, 0x63, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, + 0x6f, 0x72, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4b, 0x0a, 0x0e, 0x53, + 0x79, 0x6e, 0x63, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x12, 0x1a, 0x2e, + 0x72, 0x70, 0x63, 0x2e, 0x53, 0x79, 0x6e, 0x63, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, + 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x72, 0x70, 0x63, 0x2e, + 0x53, 0x79, 0x6e, 0x63, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4b, 0x0a, 0x0e, 0x48, 0x61, 0x73, 0x68, + 0x52, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x12, 0x1a, 0x2e, 0x72, 0x70, 0x63, + 0x2e, 0x48, 0x61, 0x73, 0x68, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x48, 0x61, 0x73, + 0x68, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x3a, 0x0a, 0x09, 0x4d, 0x65, 0x72, 0x67, 0x65, 0x42, 0x61, + 0x73, 0x65, 0x12, 0x15, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x4d, 0x65, 0x72, 0x67, 0x65, 0x42, 0x61, + 0x73, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x72, 0x70, 0x63, 0x2e, + 0x4d, 0x65, 0x72, 0x67, 0x65, 0x42, 0x61, 0x73, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x42, 0x27, 0x5a, 0x25, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, + 0x68, 0x61, 0x72, 0x6e, 0x65, 0x73, 0x73, 0x2f, 0x67, 0x69, 0x74, 0x6e, 0x65, 0x73, 0x73, 0x2f, + 0x67, 0x69, 0x74, 0x72, 0x70, 0x63, 0x2f, 0x72, 0x70, 0x63, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x33, } var ( diff --git a/go.mod b/go.mod index 0e45cc729..753842e91 100644 --- a/go.mod +++ b/go.mod @@ -25,7 +25,6 @@ require ( github.com/golang-jwt/jwt v3.2.2+incompatible github.com/golang/mock v1.6.0 github.com/google/go-cmp v0.5.9 - github.com/google/uuid v1.3.1 github.com/google/wire v0.5.0 github.com/gorhill/cronexpr v0.0.0-20180427100037-88b0669f7d75 github.com/gotidy/ptr v1.3.0 @@ -81,6 +80,7 @@ require ( github.com/ghodss/yaml v1.0.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/google/uuid v1.3.1 // indirect github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect github.com/googleapis/gax-go/v2 v2.7.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect diff --git a/go.sum b/go.sum index b03ebae09..c0039037b 100644 --- a/go.sum +++ b/go.sum @@ -317,6 +317,7 @@ github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgf github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/guregu/null v4.0.0+incompatible h1:4zw0ckM7ECd6FNNddc3Fu4aty9nTlpkkzH7dPn4/4Gw= github.com/guregu/null v4.0.0+incompatible/go.mod h1:ePGpQaN9cw0tj45IR5E5ehMvsFlLlQZAkkOXZurJ3NM= +github.com/h2non/gock v1.0.9 h1:17gCehSo8ZOgEsFKpQgqHiR7VLyjxdAG3lkhVvO9QZU= github.com/h2non/gock v1.0.9/go.mod h1:CZMcB0Lg5IWnr9bF79pPMg9WeV6WumxQiUJ1UvdO1iE= github.com/harness/go-rbac v0.0.0-20230829014129-c9b217856ea2 h1:M1Jd2uEKl4YW9g/6vzN1qo06d5dshYYdwxlhOTUSnh4= github.com/harness/go-rbac v0.0.0-20230829014129-c9b217856ea2/go.mod h1:uGgBgSZPgyygG5rWzoYsKIQ8TM4zt5yQq9nreznWvOI= diff --git a/internal/api/controller/pullreq/controller.go b/internal/api/controller/pullreq/controller.go index 0f6cd3658..e96c9c761 100644 --- a/internal/api/controller/pullreq/controller.go +++ b/internal/api/controller/pullreq/controller.go @@ -114,6 +114,10 @@ func (c *Controller) getRepoCheckAccess(ctx context.Context, return nil, fmt.Errorf("failed to find repository: %w", err) } + if repo.Importing { + return nil, usererror.BadRequest("Repository import is in progress.") + } + if err = apiauth.CheckRepo(ctx, c.authorizer, session, repo, reqPermission, false); err != nil { return nil, fmt.Errorf("access check failed: %w", err) } diff --git a/internal/api/controller/repo/controller.go b/internal/api/controller/repo/controller.go index 6bae1d018..684f11e31 100644 --- a/internal/api/controller/repo/controller.go +++ b/internal/api/controller/repo/controller.go @@ -7,11 +7,14 @@ package repo import ( "context" "fmt" + "strconv" + "strings" "github.com/harness/gitness/gitrpc" "github.com/harness/gitness/internal/auth" "github.com/harness/gitness/internal/auth/authz" "github.com/harness/gitness/internal/githook" + "github.com/harness/gitness/internal/services/importer" "github.com/harness/gitness/internal/store" "github.com/harness/gitness/internal/url" "github.com/harness/gitness/types" @@ -32,6 +35,7 @@ type Controller struct { pipelineStore store.PipelineStore principalStore store.PrincipalStore gitRPCClient gitrpc.Interface + importer *importer.Repository } func NewController( @@ -46,6 +50,7 @@ func NewController( pipelineStore store.PipelineStore, principalStore store.PrincipalStore, gitRPCClient gitrpc.Interface, + importer *importer.Repository, ) *Controller { return &Controller{ defaultBranch: defaultBranch, @@ -59,6 +64,7 @@ func NewController( pipelineStore: pipelineStore, principalStore: principalStore, gitRPCClient: gitRPCClient, + importer: importer, } } @@ -95,3 +101,12 @@ func CreateRPCReadParams(repo *types.Repository) gitrpc.ReadParams { RepoUID: repo.GitUID, } } + +func (c *Controller) validateParentRef(parentRef string) error { + parentRefAsID, err := strconv.ParseInt(parentRef, 10, 64) + if (err == nil && parentRefAsID <= 0) || (len(strings.TrimSpace(parentRef)) == 0) { + return errRepositoryRequiresParent + } + + return nil +} diff --git a/internal/api/controller/repo/create.go b/internal/api/controller/repo/create.go index c7f7a60cf..724ca934a 100644 --- a/internal/api/controller/repo/create.go +++ b/internal/api/controller/repo/create.go @@ -8,7 +8,6 @@ import ( "bytes" "context" "fmt" - "strconv" "strings" "time" @@ -151,10 +150,8 @@ func (c *Controller) getSpaceCheckAuthRepoCreation( } func (c *Controller) sanitizeCreateInput(in *CreateInput) error { - parentRefAsID, err := strconv.ParseInt(in.ParentRef, 10, 64) - - if (err == nil && parentRefAsID <= 0) || (len(strings.TrimSpace(in.ParentRef)) == 0) { - return errRepositoryRequiresParent + if err := c.validateParentRef(in.ParentRef); err != nil { + return err } if err := c.uidCheck(in.UID, false); err != nil { diff --git a/internal/api/controller/repo/import.go b/internal/api/controller/repo/import.go new file mode 100644 index 000000000..72a710ad1 --- /dev/null +++ b/internal/api/controller/repo/import.go @@ -0,0 +1,202 @@ +// Copyright 2022 Harness Inc. All rights reserved. +// Use of this source code is governed by the Polyform Free Trial License +// that can be found in the LICENSE.md file for this repository. + +package repo + +import ( + "context" + "fmt" + "time" + + "github.com/harness/gitness/gitrpc" + "github.com/harness/gitness/internal/api/usererror" + "github.com/harness/gitness/internal/auth" + "github.com/harness/gitness/internal/bootstrap" + "github.com/harness/gitness/internal/githook" + "github.com/harness/gitness/internal/paths" + "github.com/harness/gitness/internal/services/importer" + "github.com/harness/gitness/internal/services/job" + "github.com/harness/gitness/store/database/dbtx" + "github.com/harness/gitness/types" + "github.com/harness/gitness/types/enum" + + "github.com/rs/zerolog/log" +) + +type ImportInput struct { + ParentRef string `json:"parent_ref"` + UID string `json:"uid"` + + Provider importer.ProviderType `json:"provider"` + ProviderURL string `json:"provider_url"` + RepoSlug string `json:"repo_slug"` + Username string `json:"username"` + Password string `json:"password"` + + Description string `json:"description"` +} + +// Import creates a new empty repository and starts git import to it from a remote repository. +func (c *Controller) Import(ctx context.Context, session *auth.Session, in *ImportInput) (*types.Repository, error) { + parentSpace, err := c.getSpaceCheckAuthRepoCreation(ctx, session, in.ParentRef) + if err != nil { + return nil, err + } + + err = c.sanitizeImportInput(in) + if err != nil { + return nil, fmt.Errorf("failed to sanitize input: %w", err) + } + + providerInfo := importer.ProviderInfo{ + Type: in.Provider, + Host: in.ProviderURL, + User: in.Username, + Pass: in.Password, + } + + repoInfo, err := importer.Repo(ctx, providerInfo, in.RepoSlug) + if err != nil { + return nil, err + } + + jobUID, err := job.UID() + if err != nil { + return nil, fmt.Errorf("error creating job UID: %w", err) + } + + gitRPCResp, err := c.createEmptyGitRepository(ctx, session) + if err != nil { + return nil, fmt.Errorf("error creating repository on GitRPC: %w", err) + } + + now := time.Now().UnixMilli() + repo := &types.Repository{ + Version: 0, + ParentID: parentSpace.ID, + UID: in.UID, + GitUID: gitRPCResp.UID, + Path: "", // the path is set in the DB transaction below + Description: in.Description, + IsPublic: repoInfo.IsPublic, + CreatedBy: session.Principal.ID, + Created: now, + Updated: now, + ForkID: 0, + DefaultBranch: repoInfo.DefaultBranch, + Importing: true, + ImportingJobUID: &jobUID, + } + + err = dbtx.New(c.db).WithTx(ctx, func(ctx context.Context) error { + // lock parent space path to ensure it doesn't get updated while we setup new repo + spacePath, err := c.pathStore.FindPrimaryWithLock(ctx, enum.PathTargetTypeSpace, parentSpace.ID) + if err != nil { + return usererror.BadRequest("Parent not found'") + } + + repo.Path = paths.Concatinate(spacePath.Value, in.UID) + + err = c.repoStore.Create(ctx, repo) + if err != nil { + return fmt.Errorf("failed to create repository in storage: %w", err) + } + + path := &types.Path{ + Version: 0, + Value: repo.Path, + IsPrimary: true, + TargetType: enum.PathTargetTypeRepo, + TargetID: repo.ID, + CreatedBy: repo.CreatedBy, + Created: repo.Created, + Updated: repo.Updated, + } + + err = c.pathStore.Create(ctx, path) + if err != nil { + return fmt.Errorf("failed to create path: %w", err) + } + + return nil + }) + if err != nil { + if err := c.DeleteRepositoryRPC(ctx, session, repo); err != nil { + log.Ctx(ctx).Warn().Err(err).Msg("gitrpc failed to delete repo for cleanup") + } + + return nil, err + } + + err = c.importer.Run(ctx, jobUID, importer.Input{ + RepoID: repo.ID, + GitUser: in.Username, + GitPass: in.Password, + CloneURL: repoInfo.CloneURL, + }) + if err != nil { + log.Ctx(ctx).Err(err).Msg("failed to start import repository job") + } + + repo.GitURL = c.urlProvider.GenerateRepoCloneURL(repo.Path) + + return repo, nil +} + +func (c *Controller) sanitizeImportInput(in *ImportInput) error { + if err := c.validateParentRef(in.ParentRef); err != nil { + return err + } + + if err := c.uidCheck(in.UID, false); err != nil { + return err + } + + if in.Provider == "" { + return usererror.BadRequest("provider must be provided") + } + + if in.RepoSlug == "" { + return usererror.BadRequest("repo slug must be provided") + } + + return nil +} + +func (c *Controller) createEmptyGitRepository( + ctx context.Context, + session *auth.Session, +) (*gitrpc.CreateRepositoryOutput, error) { + // generate envars (add everything githook CLI needs for execution) + envVars, err := githook.GenerateEnvironmentVariables( + ctx, + c.urlProvider.GetAPIBaseURLInternal(), + 0, + session.Principal.ID, + true, + ) + if err != nil { + return nil, fmt.Errorf("failed to generate git hook environment variables: %w", err) + } + + actor := rpcIdentityFromPrincipal(session.Principal) + committer := rpcIdentityFromPrincipal(bootstrap.NewSystemServiceSession().Principal) + now := time.Now() + + resp, err := c.gitRPCClient.CreateRepository(ctx, &gitrpc.CreateRepositoryParams{ + Actor: *actor, + EnvVars: envVars, + DefaultBranch: c.defaultBranch, + Files: nil, + Author: actor, + AuthorDate: &now, + Committer: committer, + CommitterDate: &now, + }) + if err != nil { + return nil, fmt.Errorf("failed to create repo on gitrpc: %w", err) + } + + return resp, nil +} diff --git a/internal/api/controller/repo/import_progress.go b/internal/api/controller/repo/import_progress.go new file mode 100644 index 000000000..72564f88c --- /dev/null +++ b/internal/api/controller/repo/import_progress.go @@ -0,0 +1,31 @@ +// Copyright 2022 Harness Inc. All rights reserved. +// Use of this source code is governed by the Polyform Free Trial License +// that can be found in the LICENSE.md file for this repository. + +package repo + +import ( + "context" + + apiauth "github.com/harness/gitness/internal/api/auth" + "github.com/harness/gitness/internal/auth" + "github.com/harness/gitness/types" + "github.com/harness/gitness/types/enum" +) + +// ImportProgress returns progress of the import job. +func (c *Controller) ImportProgress(ctx context.Context, + session *auth.Session, + repoRef string, +) (types.JobProgress, error) { + repo, err := c.repoStore.FindByRef(ctx, repoRef) + if err != nil { + return types.JobProgress{}, err + } + + if err = apiauth.CheckRepo(ctx, c.authorizer, session, repo, enum.PermissionRepoView, false); err != nil { + return types.JobProgress{}, err + } + + return c.importer.GetProgress(ctx, repo) +} diff --git a/internal/api/controller/repo/wire.go b/internal/api/controller/repo/wire.go index fba02dba7..ed4873239 100644 --- a/internal/api/controller/repo/wire.go +++ b/internal/api/controller/repo/wire.go @@ -7,6 +7,7 @@ package repo import ( "github.com/harness/gitness/gitrpc" "github.com/harness/gitness/internal/auth/authz" + "github.com/harness/gitness/internal/services/importer" "github.com/harness/gitness/internal/store" "github.com/harness/gitness/internal/url" "github.com/harness/gitness/types" @@ -24,7 +25,11 @@ var WireSet = wire.NewSet( func ProvideController(config *types.Config, db *sqlx.DB, urlProvider *url.Provider, uidCheck check.PathUID, authorizer authz.Authorizer, pathStore store.PathStore, repoStore store.RepoStore, spaceStore store.SpaceStore, pipelineStore store.PipelineStore, - principalStore store.PrincipalStore, rpcClient gitrpc.Interface) *Controller { - return NewController(config.Git.DefaultBranch, db, urlProvider, uidCheck, - authorizer, pathStore, repoStore, spaceStore, pipelineStore, principalStore, rpcClient) + principalStore store.PrincipalStore, rpcClient gitrpc.Interface, + importer *importer.Repository, +) *Controller { + return NewController(config.Git.DefaultBranch, db, urlProvider, + uidCheck, authorizer, pathStore, repoStore, + spaceStore, pipelineStore, principalStore, rpcClient, + importer) } diff --git a/internal/api/handler/repo/import.go b/internal/api/handler/repo/import.go new file mode 100644 index 000000000..d1b07afbd --- /dev/null +++ b/internal/api/handler/repo/import.go @@ -0,0 +1,36 @@ +// Copyright 2022 Harness Inc. All rights reserved. +// Use of this source code is governed by the Polyform Free Trial License +// that can be found in the LICENSE.md file for this repository. + +package repo + +import ( + "encoding/json" + "net/http" + + "github.com/harness/gitness/internal/api/controller/repo" + "github.com/harness/gitness/internal/api/render" + "github.com/harness/gitness/internal/api/request" +) + +func HandleImport(repoCtrl *repo.Controller) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + session, _ := request.AuthSessionFrom(ctx) + + in := new(repo.ImportInput) + err := json.NewDecoder(r.Body).Decode(in) + if err != nil { + render.BadRequestf(w, "Invalid Request Body: %s.", err) + return + } + + repo, err := repoCtrl.Import(ctx, session, in) + if err != nil { + render.TranslatedUserError(w, err) + return + } + + render.JSON(w, http.StatusCreated, repo) + } +} diff --git a/internal/api/handler/repo/import_progress.go b/internal/api/handler/repo/import_progress.go new file mode 100644 index 000000000..af98842e2 --- /dev/null +++ b/internal/api/handler/repo/import_progress.go @@ -0,0 +1,33 @@ +// Copyright 2022 Harness Inc. All rights reserved. +// Use of this source code is governed by the Polyform Free Trial License +// that can be found in the LICENSE.md file for this repository. + +package repo + +import ( + "net/http" + + "github.com/harness/gitness/internal/api/controller/repo" + "github.com/harness/gitness/internal/api/render" + "github.com/harness/gitness/internal/api/request" +) + +func HandleImportProgress(repoCtrl *repo.Controller) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + session, _ := request.AuthSessionFrom(ctx) + repoRef, err := request.GetRepoRefFromPath(r) + if err != nil { + render.TranslatedUserError(w, err) + return + } + + progress, err := repoCtrl.ImportProgress(ctx, session, repoRef) + if err != nil { + render.TranslatedUserError(w, err) + return + } + + render.JSON(w, http.StatusOK, progress) + } +} diff --git a/internal/router/api.go b/internal/router/api.go index 87f20dd62..7713d898c 100644 --- a/internal/router/api.go +++ b/internal/router/api.go @@ -232,6 +232,7 @@ func setupRepos(r chi.Router, r.Route("/repos", func(r chi.Router) { // Create takes path and parentId via body, not uri r.Post("/", handlerrepo.HandleCreate(repoCtrl)) + r.Post("/import", handlerrepo.HandleImport(repoCtrl)) r.Route(fmt.Sprintf("/{%s}", request.PathParamRepoRef), func(r chi.Router) { // repo level operations r.Get("/", handlerrepo.HandleFind(repoCtrl)) @@ -241,6 +242,8 @@ func setupRepos(r chi.Router, r.Post("/move", handlerrepo.HandleMove(repoCtrl)) r.Get("/service-accounts", handlerrepo.HandleListServiceAccounts(repoCtrl)) + r.Get("/import-progress", handlerrepo.HandleImportProgress(repoCtrl)) + // content operations // NOTE: this allows /content and /content/ to both be valid (without any other tricks.) // We don't expect there to be any other operations in that route (as that could overlap with file names) diff --git a/internal/services/importer/provider.go b/internal/services/importer/provider.go new file mode 100644 index 000000000..ecfce9668 --- /dev/null +++ b/internal/services/importer/provider.go @@ -0,0 +1,142 @@ +// Copyright 2022 Harness Inc. All rights reserved. +// Use of this source code is governed by the Polyform Free Trial License +// that can be found in the LICENSE.md file for this repository. + +package importer + +import ( + "context" + "errors" + "fmt" + "net/http" + + "github.com/harness/gitness/internal/api/usererror" + + "github.com/drone/go-scm/scm" + "github.com/drone/go-scm/scm/driver/github" + "github.com/drone/go-scm/scm/transport/oauth2" +) + +type ProviderType string + +const ( + ProviderTypeGitHub ProviderType = "github" +) + +type ProviderInfo struct { + Type ProviderType + Host string + User string + Pass string +} + +type RepositoryInfo struct { + Space string + UID string + CloneURL string + IsPublic bool + DefaultBranch string +} + +func getClient(provider ProviderInfo) (*scm.Client, error) { + var scmClient *scm.Client + + switch provider.Type { + case "": + return nil, usererror.BadRequest("provider can not be empty") + case ProviderTypeGitHub: + scmClient = github.NewDefault() + if provider.Pass != "" { + scmClient.Client = &http.Client{ + Transport: &oauth2.Transport{ + Source: oauth2.StaticTokenSource(&scm.Token{Token: provider.Pass}), + }, + } + } + default: + return nil, usererror.BadRequestf("unsupported provider: %s", provider) + } + + return scmClient, nil +} +func Repo(ctx context.Context, provider ProviderInfo, repoSlug string) (RepositoryInfo, error) { + scmClient, err := getClient(provider) + if err != nil { + return RepositoryInfo{}, err + } + + scmRepo, _, err := scmClient.Repositories.Find(ctx, repoSlug) + if errors.Is(err, scm.ErrNotFound) { + return RepositoryInfo{}, usererror.BadRequestf("repository %s not found at %s", repoSlug, provider) + } + if errors.Is(err, scm.ErrNotAuthorized) { + return RepositoryInfo{}, usererror.BadRequestf("bad credentials provided for %s at %s", repoSlug, provider) + } + if err != nil { + return RepositoryInfo{}, fmt.Errorf("failed to fetch repository %s from %s: %w", repoSlug, provider, err) + } + + return RepositoryInfo{ + Space: scmRepo.Namespace, + UID: scmRepo.Name, + CloneURL: scmRepo.Clone, + IsPublic: !scmRepo.Private, + DefaultBranch: scmRepo.Branch, + }, nil +} + +func Space(ctx context.Context, provider ProviderInfo, space string) (map[string]RepositoryInfo, error) { + scmClient, err := getClient(provider) + if err != nil { + return nil, err + } + + repoMap := make(map[string]RepositoryInfo) + + const pageSize = 50 + page := 1 + for { + scmRepos, scmResponse, err := scmClient.Repositories.ListV2(ctx, scm.RepoListOptions{ + ListOptions: scm.ListOptions{ + URL: "", + Page: page, + Size: pageSize, + }, + RepoSearchTerm: scm.RepoSearchTerm{ + RepoName: "", + User: space, + }, + }) + if errors.Is(err, scm.ErrNotFound) { + return nil, usererror.BadRequestf("space %s not found at %s", space, provider) + } + if errors.Is(err, scm.ErrNotAuthorized) { + return nil, usererror.BadRequestf("bad credentials provided for %s at %s", space, provider) + } + if err != nil { + return nil, fmt.Errorf("failed to fetch space %s from %s: %w", space, provider, err) + } + + for _, scmRepo := range scmRepos { + if !scmRepo.Perm.Pull { + continue + } + + repoMap[scmRepo.Name] = RepositoryInfo{ + Space: scmRepo.Namespace, + UID: scmRepo.Name, + CloneURL: scmRepo.Clone, + IsPublic: !scmRepo.Private, + DefaultBranch: scmRepo.Branch, + } + } + + if len(scmRepos) == 0 || page == scmResponse.Page.Last { + break + } + + page++ + } + + return repoMap, nil +} diff --git a/internal/services/importer/repository.go b/internal/services/importer/repository.go new file mode 100644 index 000000000..60c347bc4 --- /dev/null +++ b/internal/services/importer/repository.go @@ -0,0 +1,165 @@ +// Copyright 2022 Harness Inc. All rights reserved. +// Use of this source code is governed by the Polyform Free Trial License +// that can be found in the LICENSE.md file for this repository. + +package importer + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/url" + "strings" + "time" + + "github.com/harness/gitness/gitrpc" + "github.com/harness/gitness/internal/bootstrap" + "github.com/harness/gitness/internal/githook" + "github.com/harness/gitness/internal/services/job" + "github.com/harness/gitness/internal/store" + gitnessurl "github.com/harness/gitness/internal/url" + gitness_store "github.com/harness/gitness/store" + "github.com/harness/gitness/types" +) + +type Repository struct { + urlProvider *gitnessurl.Provider + git gitrpc.Interface + repoStore store.RepoStore + scheduler *job.Scheduler +} + +var _ job.Handler = (*Repository)(nil) + +type Input struct { + RepoID int64 `json:"repo_id"` + GitUser string `json:"git_user"` + GitPass string `json:"git_pass"` + CloneURL string `json:"clone_url"` +} + +const jobType = "repository_import" + +func (i *Repository) Register(executor *job.Executor) error { + return executor.Register(jobType, i) +} + +func (i *Repository) Run(ctx context.Context, jobUID string, input Input) error { + data, err := json.Marshal(input) + if err != nil { + return err + } + + strData := strings.TrimSpace(string(data)) + + return i.scheduler.RunJob(ctx, job.Definition{ + UID: jobUID, + Type: jobType, + MaxRetries: 1, + Timeout: 30 * time.Minute, + Data: strData, + }) +} + +// Handle is repository import background job handler. +func (i *Repository) Handle(ctx context.Context, data string, _ job.ProgressReporter) (string, error) { + var input Input + if err := json.NewDecoder(strings.NewReader(data)).Decode(&input); err != nil { + return "", fmt.Errorf("failed to unmarshal job input: %w", err) + } + + if input.CloneURL == "" { + return "", errors.New("missing git repository clone URL") + } + + if input.GitUser != "" || input.GitPass != "" { + repoURL, err := url.Parse(input.CloneURL) + if err != nil { + return "", fmt.Errorf("failed to parse git clone URL: %w", err) + } + + repoURL.User = url.UserPassword(input.GitUser, input.GitPass) + input.CloneURL = repoURL.String() + } + + repo, err := i.repoStore.Find(ctx, input.RepoID) + if err != nil { + return "", fmt.Errorf("failed to find repo by id: %w", err) + } + + if !repo.Importing { + return "", fmt.Errorf("repository %s is not being imported", repo.UID) + } + + writeParams, err := createRPCWriteParams(ctx, i.urlProvider, repo) + if err != nil { + return "", fmt.Errorf("failed to create write params: %w", err) + } + + syncOut, err := i.git.SyncRepository(ctx, &gitrpc.SyncRepositoryParams{ + WriteParams: writeParams, + Source: input.CloneURL, + CreateIfNotExists: false, + }) + if err != nil { + return "", fmt.Errorf("failed to sync repositories: %w", err) + } + + repo.Importing = false + repo.DefaultBranch = syncOut.DefaultBranch + + err = i.repoStore.Update(ctx, repo) + if err != nil { + return "", fmt.Errorf("failed to update repository after import: %w", err) + } + + return "", err +} + +func (i *Repository) GetProgress(ctx context.Context, repo *types.Repository) (types.JobProgress, error) { + if !repo.Importing || repo.ImportingJobUID == nil || *repo.ImportingJobUID == "" { + // if the repo is not being imported, or it's job ID has been cleared (or never existed) return state=finished + return job.DoneProgress(), nil + } + + progress, err := i.scheduler.GetJobProgress(ctx, *repo.ImportingJobUID) + if errors.Is(err, gitness_store.ErrResourceNotFound) { + // if the job is not found return state=failed + return job.FailProgress(), nil + } + if err != nil { + return types.JobProgress{}, fmt.Errorf("failed to get job progress: %w", err) + } + + return progress, nil +} + +// CreateRPCWriteParams creates base write parameters for gitrpc write operations. +func createRPCWriteParams(ctx context.Context, + urlProvider *gitnessurl.Provider, + repo *types.Repository, +) (gitrpc.WriteParams, error) { + gitnessSession := bootstrap.NewSystemServiceSession() + + // generate envars (add everything githook CLI needs for execution) + envVars, err := githook.GenerateEnvironmentVariables( + ctx, + urlProvider.GetAPIBaseURLInternal(), + repo.ID, + gitnessSession.Principal.ID, + false, + ) + if err != nil { + return gitrpc.WriteParams{}, fmt.Errorf("failed to generate git hook environment variables: %w", err) + } + + return gitrpc.WriteParams{ + Actor: gitrpc.Identity{ + Name: gitnessSession.Principal.DisplayName, + Email: gitnessSession.Principal.Email, + }, + RepoUID: repo.GitUID, + EnvVars: envVars, + }, nil +} diff --git a/internal/services/importer/wire.go b/internal/services/importer/wire.go new file mode 100644 index 000000000..da3e4aa1e --- /dev/null +++ b/internal/services/importer/wire.go @@ -0,0 +1,40 @@ +// Copyright 2022 Harness Inc. All rights reserved. +// Use of this source code is governed by the Polyform Free Trial License +// that can be found in the LICENSE.md file for this repository. + +package importer + +import ( + "github.com/harness/gitness/gitrpc" + "github.com/harness/gitness/internal/services/job" + "github.com/harness/gitness/internal/store" + "github.com/harness/gitness/internal/url" + + "github.com/google/wire" +) + +var WireSet = wire.NewSet( + ProvideRepoImporter, +) + +func ProvideRepoImporter( + urlProvider *url.Provider, + git gitrpc.Interface, + repoStore store.RepoStore, + scheduler *job.Scheduler, + executor *job.Executor, +) (*Repository, error) { + importer := &Repository{ + urlProvider: urlProvider, + git: git, + repoStore: repoStore, + scheduler: scheduler, + } + + err := executor.Register(jobType, importer) + if err != nil { + return nil, err + } + + return importer, nil +} diff --git a/internal/store/database/migrate/postgres/0024_alter_repo_add_importing.down.sql b/internal/store/database/migrate/postgres/0024_alter_repo_add_importing.down.sql new file mode 100644 index 000000000..313e77393 --- /dev/null +++ b/internal/store/database/migrate/postgres/0024_alter_repo_add_importing.down.sql @@ -0,0 +1,4 @@ +ALTER TABLE repositories + DROP CONSTRAINT fk_repo_importing_job_uid, + DROP COLUMN repo_importing_job_uid, + DROP COLUMN repo_importing; diff --git a/internal/store/database/migrate/postgres/0024_alter_repo_add_importing.up.sql b/internal/store/database/migrate/postgres/0024_alter_repo_add_importing.up.sql new file mode 100644 index 000000000..18efc0740 --- /dev/null +++ b/internal/store/database/migrate/postgres/0024_alter_repo_add_importing.up.sql @@ -0,0 +1,8 @@ +ALTER TABLE repositories + ADD COLUMN repo_importing BOOLEAN NOT NULL DEFAULT false, + ADD COLUMN repo_importing_job_uid TEXT, + ADD CONSTRAINT fk_repo_importing_job_uid + FOREIGN KEY (repo_importing_job_uid) + REFERENCES jobs(job_uid) + ON DELETE SET NULL + ON UPDATE NO ACTION; diff --git a/internal/store/database/migrate/sqlite/0024_alter_repo_add_importing.down.sql b/internal/store/database/migrate/sqlite/0024_alter_repo_add_importing.down.sql new file mode 100644 index 000000000..802f19b7a --- /dev/null +++ b/internal/store/database/migrate/sqlite/0024_alter_repo_add_importing.down.sql @@ -0,0 +1,2 @@ +ALTER TABLE repositories DROP COLUMN repo_importing_job_uid; +ALTER TABLE repositories DROP COLUMN repo_importing; diff --git a/internal/store/database/migrate/sqlite/0024_alter_repo_add_importing.up.sql b/internal/store/database/migrate/sqlite/0024_alter_repo_add_importing.up.sql new file mode 100644 index 000000000..38ad144dc --- /dev/null +++ b/internal/store/database/migrate/sqlite/0024_alter_repo_add_importing.up.sql @@ -0,0 +1,2 @@ +ALTER TABLE repositories ADD COLUMN repo_importing BOOLEAN NOT NULL DEFAULT false; +ALTER TABLE repositories ADD COLUMN repo_importing_job_uid TEXT; diff --git a/internal/store/database/repo.go b/internal/store/database/repo.go index 30b9b20d7..3091f9f85 100644 --- a/internal/store/database/repo.go +++ b/internal/store/database/repo.go @@ -58,7 +58,9 @@ const ( ,repo_num_pulls ,repo_num_closed_pulls ,repo_num_open_pulls - ,repo_num_merged_pulls` + ,repo_num_merged_pulls + ,repo_importing + ,repo_importing_job_uid` repoSelectBaseWithJoin = ` SELECT` + repoColumnsForJoin + ` @@ -124,6 +126,8 @@ func (s *RepoStore) Create(ctx context.Context, repo *types.Repository) error { ,repo_num_closed_pulls ,repo_num_open_pulls ,repo_num_merged_pulls + ,repo_importing + ,repo_importing_job_uid ) values ( :repo_version ,:repo_parent_id @@ -142,6 +146,8 @@ func (s *RepoStore) Create(ctx context.Context, repo *types.Repository) error { ,:repo_num_closed_pulls ,:repo_num_open_pulls ,:repo_num_merged_pulls + ,:repo_importing + ,:repo_importing_job_uid ) RETURNING repo_id` db := dbtx.GetAccessor(ctx, s.db) @@ -170,12 +176,15 @@ func (s *RepoStore) Update(ctx context.Context, repo *types.Repository) error { ,repo_uid = :repo_uid ,repo_description = :repo_description ,repo_is_public = :repo_is_public + ,repo_default_branch = :repo_default_branch ,repo_pullreq_seq = :repo_pullreq_seq ,repo_num_forks = :repo_num_forks ,repo_num_pulls = :repo_num_pulls ,repo_num_closed_pulls = :repo_num_closed_pulls ,repo_num_open_pulls = :repo_num_open_pulls ,repo_num_merged_pulls = :repo_num_merged_pulls + ,repo_importing = :repo_importing + ,repo_importing_job_uid = :repo_importing_job_uid WHERE repo_id = :repo_id AND repo_version = :repo_version - 1` updatedAt := time.Now() @@ -295,17 +304,13 @@ func (s *RepoStore) List(ctx context.Context, parentID int64, opts *types.RepoFi // NOTE: string concatenation is safe because the // order attribute is an enum and is not user-defined, // and is therefore not subject to injection attacks. - stmt = stmt.OrderBy("repo_uid " + opts.Order.String()) - //TODO: Postgres does not support COLLATE NOCASE for UTF8 - // stmt = stmt.OrderBy("repo_uid COLLATE NOCASE " + opts.Order.String()) + stmt = stmt.OrderBy("repo_importing desc, repo_uid " + opts.Order.String()) case enum.RepoAttrCreated: stmt = stmt.OrderBy("repo_created " + opts.Order.String()) case enum.RepoAttrUpdated: stmt = stmt.OrderBy("repo_updated " + opts.Order.String()) case enum.RepoAttrPath: - stmt = stmt.OrderBy("repo_path " + opts.Order.String()) - //TODO: Postgres does not support COLLATE NOCASE for UTF8 - // stmt = stmt.OrderBy("repo_path COLLATE NOCASE " + opts.Order.String()) + stmt = stmt.OrderBy("repo_importing desc, repo_path " + opts.Order.String()) } sql, args, err := stmt.ToSql() diff --git a/types/repo.go b/types/repo.go index c14635992..daf913e02 100644 --- a/types/repo.go +++ b/types/repo.go @@ -35,6 +35,9 @@ type Repository struct { NumOpenPulls int `db:"repo_num_open_pulls" json:"num_open_pulls"` NumMergedPulls int `db:"repo_num_merged_pulls" json:"num_merged_pulls"` + Importing bool `db:"repo_importing" json:"importing"` + ImportingJobUID *string `db:"repo_importing_job_uid" json:"-"` + // git urls GitURL string `db:"-" json:"git_url"` } From 414993a388c2b0b4fda671ff225cb8d37afb6666 Mon Sep 17 00:00:00 2001 From: Vistaar Juneja Date: Thu, 7 Sep 2023 23:58:04 +0100 Subject: [PATCH 02/17] add pubsub for events --- cmd/gitness/wire.go | 2 + cmd/gitness/wire_gen.go | 24 +++-- internal/api/controller/space/controller.go | 5 +- internal/api/controller/space/events.go | 33 ++++++ internal/api/controller/space/wire.go | 8 +- internal/api/handler/space/events_stream.go | 110 ++++++++++++++++++++ internal/api/openapi/account.go | 2 +- internal/pipeline/events/events.go | 72 +++++++++++++ internal/pipeline/events/wire.go | 23 ++++ internal/pipeline/manager/manager.go | 9 +- internal/pipeline/manager/setup.go | 29 +++++- internal/pipeline/manager/teardown.go | 11 +- internal/pipeline/manager/updater.go | 33 ++++++ internal/pipeline/manager/wire.go | 4 +- internal/router/api.go | 2 + types/enum/event_types.go | 15 +++ 16 files changed, 362 insertions(+), 20 deletions(-) create mode 100644 internal/api/controller/space/events.go create mode 100644 internal/api/handler/space/events_stream.go create mode 100644 internal/pipeline/events/events.go create mode 100644 internal/pipeline/events/wire.go create mode 100644 types/enum/event_types.go diff --git a/cmd/gitness/wire.go b/cmd/gitness/wire.go index cf8070c30..b94c730db 100644 --- a/cmd/gitness/wire.go +++ b/cmd/gitness/wire.go @@ -41,6 +41,7 @@ import ( gitevents "github.com/harness/gitness/internal/events/git" pullreqevents "github.com/harness/gitness/internal/events/pullreq" "github.com/harness/gitness/internal/pipeline/commit" + eventsstream "github.com/harness/gitness/internal/pipeline/events" "github.com/harness/gitness/internal/pipeline/file" "github.com/harness/gitness/internal/pipeline/manager" "github.com/harness/gitness/internal/pipeline/runner" @@ -125,6 +126,7 @@ func initSystem(ctx context.Context, config *types.Config) (*cliserver.System, e triggerer.WireSet, file.WireSet, runner.WireSet, + eventsstream.WireSet, scheduler.WireSet, commit.WireSet, trigger.WireSet, diff --git a/cmd/gitness/wire_gen.go b/cmd/gitness/wire_gen.go index a46fe2c07..d92d499c2 100644 --- a/cmd/gitness/wire_gen.go +++ b/cmd/gitness/wire_gen.go @@ -10,7 +10,7 @@ import ( "context" "github.com/harness/gitness/cli/server" "github.com/harness/gitness/encrypt" - "github.com/harness/gitness/events" + events2 "github.com/harness/gitness/events" "github.com/harness/gitness/gitrpc" server3 "github.com/harness/gitness/gitrpc/server" "github.com/harness/gitness/gitrpc/server/cron" @@ -36,9 +36,10 @@ import ( "github.com/harness/gitness/internal/auth/authn" "github.com/harness/gitness/internal/auth/authz" "github.com/harness/gitness/internal/bootstrap" - events3 "github.com/harness/gitness/internal/events/git" - events2 "github.com/harness/gitness/internal/events/pullreq" + events4 "github.com/harness/gitness/internal/events/git" + events3 "github.com/harness/gitness/internal/events/pullreq" "github.com/harness/gitness/internal/pipeline/commit" + "github.com/harness/gitness/internal/pipeline/events" "github.com/harness/gitness/internal/pipeline/file" "github.com/harness/gitness/internal/pipeline/manager" "github.com/harness/gitness/internal/pipeline/runner" @@ -137,10 +138,13 @@ func initSystem(ctx context.Context, config *types.Config) (*server.System, erro logStore := logs.ProvideLogStore(db, config) logStream := livelog.ProvideLogStream(config) logsController := logs2.ProvideController(db, authorizer, executionStore, repoStore, pipelineStore, stageStore, stepStore, logStore, logStream) + pubsubConfig := pubsub.ProvideConfig(config) + pubSub := pubsub.ProvidePubSub(pubsubConfig, universalClient) + eventsEvents := events.ProvideEventsStreaming(pubSub) secretStore := database.ProvideSecretStore(db) connectorStore := database.ProvideConnectorStore(db) templateStore := database.ProvideTemplateStore(db) - spaceController := space.ProvideController(db, provider, pathUID, authorizer, pathStore, pipelineStore, secretStore, connectorStore, templateStore, spaceStore, repoStore, principalStore, repoController, membershipStore) + spaceController := space.ProvideController(db, provider, eventsEvents, pathUID, authorizer, pathStore, pipelineStore, secretStore, connectorStore, templateStore, spaceStore, repoStore, principalStore, repoController, membershipStore) pipelineController := pipeline.ProvideController(db, pathUID, pathStore, repoStore, authorizer, pipelineStore) encrypter, err := encrypt.ProvideEncrypter(config) if err != nil { @@ -162,11 +166,11 @@ func initSystem(ctx context.Context, config *types.Config) (*server.System, erro if err != nil { return nil, err } - eventsSystem, err := events.ProvideSystem(eventsConfig, universalClient) + eventsSystem, err := events2.ProvideSystem(eventsConfig, universalClient) if err != nil { return nil, err } - reporter, err := events2.ProvideReporter(eventsSystem) + reporter, err := events3.ProvideReporter(eventsSystem) if err != nil { return nil, err } @@ -178,11 +182,11 @@ func initSystem(ctx context.Context, config *types.Config) (*server.System, erro } webhookStore := database.ProvideWebhookStore(db) webhookExecutionStore := database.ProvideWebhookExecutionStore(db) - readerFactory, err := events3.ProvideReaderFactory(eventsSystem) + readerFactory, err := events4.ProvideReaderFactory(eventsSystem) if err != nil { return nil, err } - eventsReaderFactory, err := events2.ProvideReaderFactory(eventsSystem) + eventsReaderFactory, err := events3.ProvideReaderFactory(eventsSystem) if err != nil { return nil, err } @@ -191,7 +195,7 @@ func initSystem(ctx context.Context, config *types.Config) (*server.System, erro return nil, err } webhookController := webhook2.ProvideController(webhookConfig, db, authorizer, webhookStore, webhookExecutionStore, repoStore, webhookService) - eventsReporter, err := events3.ProvideReporter(eventsSystem) + eventsReporter, err := events4.ProvideReporter(eventsSystem) if err != nil { return nil, err } @@ -206,7 +210,7 @@ func initSystem(ctx context.Context, config *types.Config) (*server.System, erro webHandler := router.ProvideWebHandler(config) routerRouter := router.ProvideRouter(config, apiHandler, gitHandler, webHandler) serverServer := server2.ProvideServer(config, routerRouter) - executionManager := manager.ProvideExecutionManager(config, executionStore, pipelineStore, provider, fileService, logStore, logStream, repoStore, schedulerScheduler, secretStore, stageStore, stepStore, principalStore) + executionManager := manager.ProvideExecutionManager(config, executionStore, pipelineStore, provider, eventsEvents, fileService, logStore, logStream, repoStore, schedulerScheduler, secretStore, stageStore, stepStore, principalStore) client := manager.ProvideExecutionClient(executionManager, config) runtimeRunner, err := runner.ProvideExecutionRunner(config, client, executionManager) if err != nil { diff --git a/internal/api/controller/space/controller.go b/internal/api/controller/space/controller.go index 28780f7dd..8d9dd449a 100644 --- a/internal/api/controller/space/controller.go +++ b/internal/api/controller/space/controller.go @@ -7,6 +7,7 @@ package space import ( "github.com/harness/gitness/internal/api/controller/repo" "github.com/harness/gitness/internal/auth/authz" + "github.com/harness/gitness/internal/pipeline/events" "github.com/harness/gitness/internal/store" "github.com/harness/gitness/internal/url" "github.com/harness/gitness/types/check" @@ -17,6 +18,7 @@ import ( type Controller struct { db *sqlx.DB urlProvider *url.Provider + eventsStream events.Events uidCheck check.PathUID authorizer authz.Authorizer pathStore store.PathStore @@ -31,7 +33,7 @@ type Controller struct { membershipStore store.MembershipStore } -func NewController(db *sqlx.DB, urlProvider *url.Provider, +func NewController(db *sqlx.DB, urlProvider *url.Provider, eventsStream events.Events, uidCheck check.PathUID, authorizer authz.Authorizer, pathStore store.PathStore, pipelineStore store.PipelineStore, secretStore store.SecretStore, connectorStore store.ConnectorStore, templateStore store.TemplateStore, spaceStore store.SpaceStore, @@ -41,6 +43,7 @@ func NewController(db *sqlx.DB, urlProvider *url.Provider, return &Controller{ db: db, urlProvider: urlProvider, + eventsStream: eventsStream, uidCheck: uidCheck, authorizer: authorizer, pathStore: pathStore, diff --git a/internal/api/controller/space/events.go b/internal/api/controller/space/events.go new file mode 100644 index 000000000..23df138a4 --- /dev/null +++ b/internal/api/controller/space/events.go @@ -0,0 +1,33 @@ +// Copyright 2022 Harness Inc. All rights reserved. +// Use of this source code is governed by the Polyform Free Trial License +// that can be found in the LICENSE.md file for this repository. + +package space + +import ( + "context" + "fmt" + + apiauth "github.com/harness/gitness/internal/api/auth" + "github.com/harness/gitness/internal/auth" + "github.com/harness/gitness/internal/pipeline/events" + "github.com/harness/gitness/types/enum" +) + +func (c *Controller) Events( + ctx context.Context, + session *auth.Session, + spaceRef string, +) (<-chan *events.Event, <-chan error, error) { + space, err := c.spaceStore.FindByRef(ctx, spaceRef) + if err != nil { + return nil, nil, fmt.Errorf("failed to find space ref: %w", err) + } + + if err = apiauth.CheckSpace(ctx, c.authorizer, session, space, enum.PermissionSpaceView, true); err != nil { + return nil, nil, fmt.Errorf("failed to authorize stream: %w", err) + } + + events, errc := c.eventsStream.Subscribe(ctx, space.ID) + return events, errc, nil +} diff --git a/internal/api/controller/space/wire.go b/internal/api/controller/space/wire.go index 5dcbd7dec..6fa3022d5 100644 --- a/internal/api/controller/space/wire.go +++ b/internal/api/controller/space/wire.go @@ -7,6 +7,7 @@ package space import ( "github.com/harness/gitness/internal/api/controller/repo" "github.com/harness/gitness/internal/auth/authz" + "github.com/harness/gitness/internal/pipeline/events" "github.com/harness/gitness/internal/store" "github.com/harness/gitness/internal/url" "github.com/harness/gitness/types/check" @@ -20,13 +21,14 @@ var WireSet = wire.NewSet( ProvideController, ) -func ProvideController(db *sqlx.DB, urlProvider *url.Provider, uidCheck check.PathUID, authorizer authz.Authorizer, - pathStore store.PathStore, pipelineStore store.PipelineStore, secretStore store.SecretStore, +func ProvideController(db *sqlx.DB, urlProvider *url.Provider, eventsStream events.Events, + uidCheck check.PathUID, authorizer authz.Authorizer, pathStore store.PathStore, + pipelineStore store.PipelineStore, secretStore store.SecretStore, connectorStore store.ConnectorStore, templateStore store.TemplateStore, spaceStore store.SpaceStore, repoStore store.RepoStore, principalStore store.PrincipalStore, repoCtrl *repo.Controller, membershipStore store.MembershipStore, ) *Controller { - return NewController(db, urlProvider, uidCheck, authorizer, + return NewController(db, urlProvider, eventsStream, uidCheck, authorizer, pathStore, pipelineStore, secretStore, connectorStore, templateStore, spaceStore, repoStore, principalStore, repoCtrl, membershipStore) } diff --git a/internal/api/handler/space/events_stream.go b/internal/api/handler/space/events_stream.go new file mode 100644 index 000000000..4c45a9945 --- /dev/null +++ b/internal/api/handler/space/events_stream.go @@ -0,0 +1,110 @@ +// Copyright 2022 Harness Inc. All rights reserved. +// Use of this source code is governed by the Polyform Free Trial License +// that can be found in the LICENSE.md file for this repository. + +package space + +import ( + "context" + "encoding/json" + "io" + "net/http" + "time" + + "github.com/harness/gitness/internal/api/controller/space" + "github.com/harness/gitness/internal/api/render" + "github.com/harness/gitness/internal/api/request" + + "github.com/rs/zerolog/log" +) + +var ( + pingInterval = 30 * time.Second + tailMaxTime = 2 * time.Hour +) + +// HandleEventsStream returns an http.HandlerFunc that watches for +// events on a space +func HandleEventsStream(spaceCtrl *space.Controller) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + session, _ := request.AuthSessionFrom(ctx) + + spaceRef, err := request.GetSpaceRefFromPath(r) + if err != nil { + render.TranslatedUserError(w, err) + return + } + + f, ok := w.(http.Flusher) + if !ok { + log.Error().Msg("http writer type assertion failed") + render.InternalError(w) + return + } + + io.WriteString(w, ": ping\n\n") + f.Flush() + + events, errc, err := spaceCtrl.Events(ctx, session, spaceRef) + if err != nil { + render.TranslatedUserError(w, err) + return + } + + // could not get error channel + if errc == nil { + io.WriteString(w, "event: error\ndata: eof\n\n") + return + } + + h := w.Header() + h.Set("Content-Type", "text/event-stream") + h.Set("Cache-Control", "no-cache") + h.Set("Connection", "keep-alive") + h.Set("X-Accel-Buffering", "no") + h.Set("Access-Control-Allow-Origin", "*") + + ctx, cancel := context.WithTimeout(r.Context(), tailMaxTime) + defer cancel() + + enc := json.NewEncoder(w) + + pingTimer := time.NewTimer(pingInterval) + defer pingTimer.Stop() + L: + for { + // ensure timer is stopped before resetting (see documentation) + if !pingTimer.Stop() { + // in this specific case the timer's channel could be both, empty or full + select { + case <-pingTimer.C: + default: + } + } + pingTimer.Reset(pingInterval) + select { + case <-ctx.Done(): + log.Debug().Msg("events: stream cancelled") + break L + case err := <-errc: + log.Err(err).Msg("events: received error in the tail channel") + break L + case <-pingTimer.C: + // if time b/w messages takes longer, send a ping + io.WriteString(w, ": ping\n\n") + f.Flush() + case event := <-events: + io.WriteString(w, "data: ") + enc.Encode(event) + io.WriteString(w, "\n\n") + f.Flush() + } + } + + io.WriteString(w, "event: error\ndata: eof\n\n") + f.Flush() + + log.Debug().Msg("events: stream closed") + } +} diff --git a/internal/api/openapi/account.go b/internal/api/openapi/account.go index 39bf47930..c97c1688c 100644 --- a/internal/api/openapi/account.go +++ b/internal/api/openapi/account.go @@ -7,12 +7,12 @@ package openapi import ( "net/http" - "github.com/gotidy/ptr" "github.com/harness/gitness/internal/api/controller/user" "github.com/harness/gitness/internal/api/request" "github.com/harness/gitness/internal/api/usererror" "github.com/harness/gitness/types" + "github.com/gotidy/ptr" "github.com/swaggest/openapi-go/openapi3" ) diff --git a/internal/pipeline/events/events.go b/internal/pipeline/events/events.go new file mode 100644 index 000000000..dbc26e2f7 --- /dev/null +++ b/internal/pipeline/events/events.go @@ -0,0 +1,72 @@ +// Copyright 2022 Harness Inc. All rights reserved. +// Use of this source code is governed by the Polyform Free Trial License +// that can be found in the LICENSE.md file for this repository. + +package events + +import ( + "context" + "encoding/json" + "strconv" + + "github.com/harness/gitness/pubsub" + "github.com/harness/gitness/types/enum" +) + +// Event is an event which is sent to the UI via server-sent events. +type Event struct { + Type enum.EventType `json:"type"` + Data json.RawMessage `json:"data"` +} + +type Events interface { + // Publish publishes an event to a given space ID. + Publish(ctx context.Context, spaceID int64, event *Event) error + + // Subscribe listens to events on a space ID. + Subscribe(ctx context.Context, spaceID int64) (<-chan *Event, <-chan error) +} + +type event struct { + pubsub pubsub.PubSub + topic string +} + +func New(pubsub pubsub.PubSub, topic string) Events { + return &event{ + pubsub: pubsub, + topic: topic, + } +} + +func (e *event) Publish(ctx context.Context, spaceID int64, event *Event) error { + bytes, err := json.Marshal(event) + if err != nil { + return err + } + option := pubsub.WithPublishNamespace(format(spaceID)) + return e.pubsub.Publish(ctx, e.topic, bytes, option) +} + +// format creates the namespace name which will be spaces- +func format(id int64) string { + return "spaces-" + strconv.Itoa(int(id)) +} + +func (e *event) Subscribe(ctx context.Context, spaceID int64) (<-chan *Event, <-chan error) { + chEvent := make(chan *Event, 100) // TODO: check best size here + chErr := make(chan error) + g := func(payload []byte) error { + event := &Event{} + err := json.Unmarshal(payload, event) + if err != nil { + // This should never happen + return err + } + chEvent <- event + return nil + } + option := pubsub.WithChannelNamespace(format(spaceID)) + e.pubsub.Subscribe(ctx, e.topic, g, option) + return chEvent, chErr +} diff --git a/internal/pipeline/events/wire.go b/internal/pipeline/events/wire.go new file mode 100644 index 000000000..6879e8782 --- /dev/null +++ b/internal/pipeline/events/wire.go @@ -0,0 +1,23 @@ +// Copyright 2022 Harness Inc. All rights reserved. +// Use of this source code is governed by the Polyform Free Trial License +// that can be found in the LICENSE.md file for this repository. + +package events + +import ( + "github.com/harness/gitness/pubsub" + + "github.com/google/wire" +) + +// WireSet provides a wire set for this package. +var WireSet = wire.NewSet( + ProvideEventsStreaming, +) + +func ProvideEventsStreaming(pubsub pubsub.PubSub) Events { + return &event{ + pubsub: pubsub, + topic: "events", + } +} diff --git a/internal/pipeline/manager/manager.go b/internal/pipeline/manager/manager.go index 9a9a82529..1500c0f5a 100644 --- a/internal/pipeline/manager/manager.go +++ b/internal/pipeline/manager/manager.go @@ -11,6 +11,7 @@ import ( "io" "time" + "github.com/harness/gitness/internal/pipeline/events" "github.com/harness/gitness/internal/pipeline/file" "github.com/harness/gitness/internal/pipeline/scheduler" "github.com/harness/gitness/internal/store" @@ -98,7 +99,7 @@ type Manager struct { Pipelines store.PipelineStore urlProvider *urlprovider.Provider // Converter store.ConvertService - // Events store.Pubsub + Events events.Events // Globals store.GlobalSecretStore Logs store.LogStore Logz livelog.LogStream @@ -119,6 +120,7 @@ func New( executionStore store.ExecutionStore, pipelineStore store.PipelineStore, urlProvider *urlprovider.Provider, + events events.Events, fileService file.FileService, logStore store.LogStore, logStream livelog.LogStream, @@ -134,6 +136,7 @@ func New( Executions: executionStore, Pipelines: pipelineStore, urlProvider: urlProvider, + Events: events, FileService: fileService, Logs: logStore, Logz: logStream, @@ -313,6 +316,7 @@ func (m *Manager) BeforeStep(ctx context.Context, step *types.Step) error { } updater := &updater{ Executions: m.Executions, + Events: m.Events, Repos: m.Repos, Steps: m.Steps, Stages: m.Stages, @@ -332,6 +336,7 @@ func (m *Manager) AfterStep(ctx context.Context, step *types.Step) error { var retErr error updater := &updater{ Executions: m.Executions, + Events: m.Events, Repos: m.Repos, Steps: m.Steps, Stages: m.Stages, @@ -352,6 +357,7 @@ func (m *Manager) AfterStep(ctx context.Context, step *types.Step) error { func (m *Manager) BeforeStage(ctx context.Context, stage *types.Stage) error { s := &setup{ Executions: m.Executions, + Events: m.Events, Repos: m.Repos, Steps: m.Steps, Stages: m.Stages, @@ -365,6 +371,7 @@ func (m *Manager) BeforeStage(ctx context.Context, stage *types.Stage) error { func (m *Manager) AfterStage(ctx context.Context, stage *types.Stage) error { t := &teardown{ Executions: m.Executions, + Events: m.Events, Logs: m.Logz, Repos: m.Repos, Scheduler: m.Scheduler, diff --git a/internal/pipeline/manager/setup.go b/internal/pipeline/manager/setup.go index c7bb9e40c..457878fc0 100644 --- a/internal/pipeline/manager/setup.go +++ b/internal/pipeline/manager/setup.go @@ -6,9 +6,11 @@ package manager import ( "context" + "encoding/json" "errors" "time" + "github.com/harness/gitness/internal/pipeline/events" "github.com/harness/gitness/internal/store" gitness_store "github.com/harness/gitness/store" "github.com/harness/gitness/types" @@ -19,6 +21,7 @@ import ( type setup struct { Executions store.ExecutionStore + Events events.Events Repos store.RepoStore Steps store.StepStore Stages store.StageStore @@ -39,7 +42,7 @@ func (s *setup) do(ctx context.Context, stage *types.Stage) error { Int64("repo.id", execution.RepoID). Logger() - _, err = s.Repos.Find(noContext, execution.RepoID) + repo, err := s.Repos.Find(noContext, execution.RepoID) if err != nil { log.Error().Err(err).Msg("manager: cannot find the repository") return err @@ -72,15 +75,37 @@ func (s *setup) do(ctx context.Context, stage *types.Stage) error { } } - _, err = s.updateExecution(ctx, execution) + _, err = s.updateExecution(noContext, execution) if err != nil { log.Error().Err(err).Msg("manager: cannot update the execution") return err } + stages, err := s.Stages.ListWithSteps(noContext, execution.ID) + if err != nil { + log.Error().Err(err).Msg("manager: could not list stages with steps") + return err + } + execution.Stages = stages + err = s.Events.Publish(noContext, repo.ParentID, executionEvent(enum.ExecutionRunning, execution)) + if err != nil { + log.Warn().Err(err).Msg("manager: could not publish execution event") + } return nil } +func executionEvent( + eventType enum.EventType, + execution *types.Execution, +) *events.Event { + // json.Marshal will not return an error here, it can be absorbed + bytes, _ := json.Marshal(execution) + return &events.Event{ + Type: eventType, + Data: bytes, + } +} + // helper function that updates the execution status from pending to running. // This accounts for the fact that another agent may have already updated // the execution status, which may happen if two stages execute concurrently. diff --git a/internal/pipeline/manager/teardown.go b/internal/pipeline/manager/teardown.go index 9365bc787..25cbb180c 100644 --- a/internal/pipeline/manager/teardown.go +++ b/internal/pipeline/manager/teardown.go @@ -8,6 +8,7 @@ import ( "context" "time" + "github.com/harness/gitness/internal/pipeline/events" "github.com/harness/gitness/internal/pipeline/scheduler" "github.com/harness/gitness/internal/store" "github.com/harness/gitness/livelog" @@ -20,6 +21,7 @@ import ( type teardown struct { Executions store.ExecutionStore + Events events.Events Logs livelog.LogStream Scheduler scheduler.Scheduler Repos store.RepoStore @@ -46,7 +48,7 @@ func (t *teardown) do(ctx context.Context, stage *types.Stage) error { Str("stage.status", stage.Status). Logger() - _, err = t.Repos.Find(noContext, execution.RepoID) + repo, err := t.Repos.Find(noContext, execution.RepoID) if err != nil { log.Error().Err(err).Msg("manager: cannot find the repository") return err @@ -131,6 +133,13 @@ func (t *teardown) do(ctx context.Context, stage *types.Stage) error { return err } + execution.Stages = stages + err = t.Events.Publish(noContext, repo.ParentID, executionEvent(enum.ExecutionCompleted, execution)) + if err != nil { + log.Warn().Err(err). + Msg("manager: could not publish execution completed event") + } + return nil } diff --git a/internal/pipeline/manager/updater.go b/internal/pipeline/manager/updater.go index 4a7787063..13d068486 100644 --- a/internal/pipeline/manager/updater.go +++ b/internal/pipeline/manager/updater.go @@ -7,8 +7,10 @@ package manager import ( "context" + "github.com/harness/gitness/internal/pipeline/events" "github.com/harness/gitness/internal/store" "github.com/harness/gitness/types" + "github.com/harness/gitness/types/enum" "github.com/rs/zerolog/log" ) @@ -16,6 +18,7 @@ import ( type updater struct { Executions store.ExecutionStore Repos store.RepoStore + Events events.Events Steps store.StepStore Stages store.StageStore } @@ -36,5 +39,35 @@ func (u *updater) do(ctx context.Context, step *types.Step) error { return err } + stage, err := u.Stages.Find(noContext, step.StageID) + if err != nil { + log.Error().Err(err).Msg("manager: cannot find stage") + return nil + } + + execution, err := u.Executions.Find(noContext, stage.ExecutionID) + if err != nil { + log.Error().Err(err).Msg("manager: cannot find execution") + return nil + } + + repo, err := u.Repos.Find(noContext, execution.RepoID) + if err != nil { + log.Error().Err(err).Msg("manager: cannot find repo") + return nil + } + + stages, err := u.Stages.ListWithSteps(noContext, stage.ExecutionID) + if err != nil { + log.Error().Err(err).Msg("manager: cannot find stages") + return nil + } + + execution.Stages = stages + err = u.Events.Publish(noContext, repo.ParentID, executionEvent(enum.ExecutionUpdated, execution)) + if err != nil { + log.Warn().Err(err).Msg("manager: cannot publish execution updated event") + } + return nil } diff --git a/internal/pipeline/manager/wire.go b/internal/pipeline/manager/wire.go index 7e56d5bec..4849ea89c 100644 --- a/internal/pipeline/manager/wire.go +++ b/internal/pipeline/manager/wire.go @@ -5,6 +5,7 @@ package manager import ( + "github.com/harness/gitness/internal/pipeline/events" "github.com/harness/gitness/internal/pipeline/file" "github.com/harness/gitness/internal/pipeline/scheduler" "github.com/harness/gitness/internal/store" @@ -28,6 +29,7 @@ func ProvideExecutionManager( executionStore store.ExecutionStore, pipelineStore store.PipelineStore, urlProvider *url.Provider, + events events.Events, fileService file.FileService, logStore store.LogStore, logStream livelog.LogStream, @@ -37,7 +39,7 @@ func ProvideExecutionManager( stageStore store.StageStore, stepStore store.StepStore, userStore store.PrincipalStore) ExecutionManager { - return New(config, executionStore, pipelineStore, urlProvider, fileService, logStore, + return New(config, executionStore, pipelineStore, urlProvider, events, fileService, logStore, logStream, repoStore, scheduler, secretStore, stageStore, stepStore, userStore) } diff --git a/internal/router/api.go b/internal/router/api.go index 7713d898c..c987315a3 100644 --- a/internal/router/api.go +++ b/internal/router/api.go @@ -188,6 +188,8 @@ func setupSpaces(r chi.Router, spaceCtrl *space.Controller) { r.Patch("/", handlerspace.HandleUpdate(spaceCtrl)) r.Delete("/", handlerspace.HandleDelete(spaceCtrl)) + r.Get("/stream", handlerspace.HandleEventsStream(spaceCtrl)) + r.Post("/move", handlerspace.HandleMove(spaceCtrl)) r.Get("/spaces", handlerspace.HandleListSpaces(spaceCtrl)) r.Get("/repos", handlerspace.HandleListRepos(spaceCtrl)) diff --git a/types/enum/event_types.go b/types/enum/event_types.go new file mode 100644 index 000000000..18d0d171c --- /dev/null +++ b/types/enum/event_types.go @@ -0,0 +1,15 @@ +// Copyright 2022 Harness Inc. All rights reserved. +// Use of this source code is governed by the Polyform Free Trial License +// that can be found in the LICENSE.md file for this repository. + +// Enums for event types delivered to the event stream for the UI +package enum + +// EventType defines the kind of event +type EventType string + +const ( + ExecutionUpdated = "execution_updated" + ExecutionRunning = "execution_running" + ExecutionCompleted = "execution_completed" +) From b81d3cbcceeface39627e67c0c001bfd3cff4ffd Mon Sep 17 00:00:00 2001 From: Vistaar Juneja Date: Fri, 8 Sep 2023 12:47:14 +0100 Subject: [PATCH 03/17] address comments --- internal/api/controller/space/events.go | 72 ++++++++++++++++-- internal/api/handler/space/events_stream.go | 82 +++------------------ internal/pipeline/events/events.go | 15 +++- internal/writer/writeflush.go | 33 +++++++++ 4 files changed, 122 insertions(+), 80 deletions(-) create mode 100644 internal/writer/writeflush.go diff --git a/internal/api/controller/space/events.go b/internal/api/controller/space/events.go index 23df138a4..d3b2ee000 100644 --- a/internal/api/controller/space/events.go +++ b/internal/api/controller/space/events.go @@ -6,28 +6,88 @@ package space import ( "context" + "encoding/json" "fmt" + "io" + "time" apiauth "github.com/harness/gitness/internal/api/auth" "github.com/harness/gitness/internal/auth" - "github.com/harness/gitness/internal/pipeline/events" + "github.com/harness/gitness/internal/writer" "github.com/harness/gitness/types/enum" + "github.com/rs/zerolog/log" +) + +var ( + pingInterval = 30 * time.Second + tailMaxTime = 2 * time.Hour ) func (c *Controller) Events( ctx context.Context, session *auth.Session, spaceRef string, -) (<-chan *events.Event, <-chan error, error) { + w writer.WriterFlusher, +) error { space, err := c.spaceStore.FindByRef(ctx, spaceRef) if err != nil { - return nil, nil, fmt.Errorf("failed to find space ref: %w", err) + return fmt.Errorf("failed to find space ref: %w", err) } if err = apiauth.CheckSpace(ctx, c.authorizer, session, space, enum.PermissionSpaceView, true); err != nil { - return nil, nil, fmt.Errorf("failed to authorize stream: %w", err) + return fmt.Errorf("failed to authorize stream: %w", err) } - events, errc := c.eventsStream.Subscribe(ctx, space.ID) - return events, errc, nil + ctx, cancel := context.WithTimeout(ctx, tailMaxTime) + defer cancel() + + io.WriteString(w, ": ping\n\n") + w.Flush() + + events, errc, consumer := c.eventsStream.Subscribe(ctx, space.ID) + defer c.eventsStream.Unsubscribe(ctx, consumer) + // could not get error channel + if errc == nil { + io.WriteString(w, "event: error\ndata: eof\n\n") + w.Flush() + return fmt.Errorf("could not get error channel") + } + pingTimer := time.NewTimer(pingInterval) + defer pingTimer.Stop() + + enc := json.NewEncoder(w) +L: + for { + // ensure timer is stopped before resetting (see documentation) + if !pingTimer.Stop() { + // in this specific case the timer's channel could be both, empty or full + select { + case <-pingTimer.C: + default: + } + } + pingTimer.Reset(pingInterval) + select { + case <-ctx.Done(): + log.Debug().Msg("events: stream cancelled") + break L + case err := <-errc: + log.Err(err).Msg("events: received error in the tail channel") + break L + case <-pingTimer.C: + // if time b/w messages takes longer, send a ping + io.WriteString(w, ": ping\n\n") + w.Flush() + case event := <-events: + io.WriteString(w, "data: ") + enc.Encode(event) + io.WriteString(w, "\n\n") + w.Flush() + } + } + + io.WriteString(w, "event: error\ndata: eof\n\n") + w.Flush() + log.Debug().Msg("events: stream closed") + return nil } diff --git a/internal/api/handler/space/events_stream.go b/internal/api/handler/space/events_stream.go index 4c45a9945..1af9e053d 100644 --- a/internal/api/handler/space/events_stream.go +++ b/internal/api/handler/space/events_stream.go @@ -5,24 +5,15 @@ package space import ( - "context" - "encoding/json" - "io" "net/http" - "time" "github.com/harness/gitness/internal/api/controller/space" "github.com/harness/gitness/internal/api/render" "github.com/harness/gitness/internal/api/request" - + "github.com/harness/gitness/internal/writer" "github.com/rs/zerolog/log" ) -var ( - pingInterval = 30 * time.Second - tailMaxTime = 2 * time.Hour -) - // HandleEventsStream returns an http.HandlerFunc that watches for // events on a space func HandleEventsStream(spaceCtrl *space.Controller) http.HandlerFunc { @@ -36,28 +27,6 @@ func HandleEventsStream(spaceCtrl *space.Controller) http.HandlerFunc { return } - f, ok := w.(http.Flusher) - if !ok { - log.Error().Msg("http writer type assertion failed") - render.InternalError(w) - return - } - - io.WriteString(w, ": ping\n\n") - f.Flush() - - events, errc, err := spaceCtrl.Events(ctx, session, spaceRef) - if err != nil { - render.TranslatedUserError(w, err) - return - } - - // could not get error channel - if errc == nil { - io.WriteString(w, "event: error\ndata: eof\n\n") - return - } - h := w.Header() h.Set("Content-Type", "text/event-stream") h.Set("Cache-Control", "no-cache") @@ -65,46 +34,19 @@ func HandleEventsStream(spaceCtrl *space.Controller) http.HandlerFunc { h.Set("X-Accel-Buffering", "no") h.Set("Access-Control-Allow-Origin", "*") - ctx, cancel := context.WithTimeout(r.Context(), tailMaxTime) - defer cancel() - - enc := json.NewEncoder(w) - - pingTimer := time.NewTimer(pingInterval) - defer pingTimer.Stop() - L: - for { - // ensure timer is stopped before resetting (see documentation) - if !pingTimer.Stop() { - // in this specific case the timer's channel could be both, empty or full - select { - case <-pingTimer.C: - default: - } - } - pingTimer.Reset(pingInterval) - select { - case <-ctx.Done(): - log.Debug().Msg("events: stream cancelled") - break L - case err := <-errc: - log.Err(err).Msg("events: received error in the tail channel") - break L - case <-pingTimer.C: - // if time b/w messages takes longer, send a ping - io.WriteString(w, ": ping\n\n") - f.Flush() - case event := <-events: - io.WriteString(w, "data: ") - enc.Encode(event) - io.WriteString(w, "\n\n") - f.Flush() - } + f, ok := w.(http.Flusher) + if !ok { + log.Error().Msg("http writer type assertion failed") + render.InternalError(w) + return } - io.WriteString(w, "event: error\ndata: eof\n\n") - f.Flush() + writer := writer.NewWriterFlusher(w, f) - log.Debug().Msg("events: stream closed") + err = spaceCtrl.Events(ctx, session, spaceRef, writer) + if err != nil { + render.TranslatedUserError(w, err) + return + } } } diff --git a/internal/pipeline/events/events.go b/internal/pipeline/events/events.go index dbc26e2f7..a3408e914 100644 --- a/internal/pipeline/events/events.go +++ b/internal/pipeline/events/events.go @@ -24,7 +24,10 @@ type Events interface { Publish(ctx context.Context, spaceID int64, event *Event) error // Subscribe listens to events on a space ID. - Subscribe(ctx context.Context, spaceID int64) (<-chan *Event, <-chan error) + Subscribe(ctx context.Context, spaceID int64) (<-chan *Event, <-chan error, pubsub.Consumer) + + // Unsubscribe unsubscribes the consumer. + Unsubscribe(ctx context.Context, consumer pubsub.Consumer) error } type event struct { @@ -53,7 +56,7 @@ func format(id int64) string { return "spaces-" + strconv.Itoa(int(id)) } -func (e *event) Subscribe(ctx context.Context, spaceID int64) (<-chan *Event, <-chan error) { +func (e *event) Subscribe(ctx context.Context, spaceID int64) (<-chan *Event, <-chan error, pubsub.Consumer) { chEvent := make(chan *Event, 100) // TODO: check best size here chErr := make(chan error) g := func(payload []byte) error { @@ -67,6 +70,10 @@ func (e *event) Subscribe(ctx context.Context, spaceID int64) (<-chan *Event, <- return nil } option := pubsub.WithChannelNamespace(format(spaceID)) - e.pubsub.Subscribe(ctx, e.topic, g, option) - return chEvent, chErr + consumer := e.pubsub.Subscribe(ctx, e.topic, g, option) + return chEvent, chErr, consumer +} + +func (e *event) Unsubscribe(ctx context.Context, consumer pubsub.Consumer) error { + return consumer.Unsubscribe(ctx, e.topic) } diff --git a/internal/writer/writeflush.go b/internal/writer/writeflush.go new file mode 100644 index 000000000..c2e37eea0 --- /dev/null +++ b/internal/writer/writeflush.go @@ -0,0 +1,33 @@ +package writer + +import "io" + +type Flusher interface { + Flush() +} + +type writeWithFlusher struct { + writer io.Writer + flusher Flusher +} + +type WriterFlusher interface { + io.Writer + Flusher +} + +func NewWriterFlusher(writer io.Writer, flusher Flusher) WriterFlusher { + return &writeWithFlusher{ + writer: writer, + flusher: flusher, + } +} + +func (w *writeWithFlusher) Write(p []byte) (int, error) { + n, err := w.writer.Write(p) + return n, err +} + +func (w *writeWithFlusher) Flush() { + w.flusher.Flush() +} From 5000e78b5eea89b261b230b518cf6067937d015c Mon Sep 17 00:00:00 2001 From: Vistaar Juneja Date: Fri, 8 Sep 2023 13:15:10 +0100 Subject: [PATCH 04/17] rename Events interface to EventsStreamer, add file header --- cmd/gitness/wire_gen.go | 8 +++----- internal/api/controller/space/controller.go | 4 ++-- internal/api/controller/space/wire.go | 2 +- internal/pipeline/events/events.go | 4 ++-- internal/pipeline/events/wire.go | 2 +- internal/pipeline/manager/manager.go | 4 ++-- internal/pipeline/manager/setup.go | 2 +- internal/pipeline/manager/teardown.go | 2 +- internal/pipeline/manager/updater.go | 2 +- internal/pipeline/manager/wire.go | 2 +- internal/writer/writeflush.go | 4 ++++ 11 files changed, 19 insertions(+), 17 deletions(-) diff --git a/cmd/gitness/wire_gen.go b/cmd/gitness/wire_gen.go index d92d499c2..7f21e8da1 100644 --- a/cmd/gitness/wire_gen.go +++ b/cmd/gitness/wire_gen.go @@ -138,13 +138,11 @@ func initSystem(ctx context.Context, config *types.Config) (*server.System, erro logStore := logs.ProvideLogStore(db, config) logStream := livelog.ProvideLogStream(config) logsController := logs2.ProvideController(db, authorizer, executionStore, repoStore, pipelineStore, stageStore, stepStore, logStore, logStream) - pubsubConfig := pubsub.ProvideConfig(config) - pubSub := pubsub.ProvidePubSub(pubsubConfig, universalClient) - eventsEvents := events.ProvideEventsStreaming(pubSub) + eventsStreamer := events.ProvideEventsStreaming(pubSub) secretStore := database.ProvideSecretStore(db) connectorStore := database.ProvideConnectorStore(db) templateStore := database.ProvideTemplateStore(db) - spaceController := space.ProvideController(db, provider, eventsEvents, pathUID, authorizer, pathStore, pipelineStore, secretStore, connectorStore, templateStore, spaceStore, repoStore, principalStore, repoController, membershipStore) + spaceController := space.ProvideController(db, provider, eventsStreamer, pathUID, authorizer, pathStore, pipelineStore, secretStore, connectorStore, templateStore, spaceStore, repoStore, principalStore, repoController, membershipStore) pipelineController := pipeline.ProvideController(db, pathUID, pathStore, repoStore, authorizer, pipelineStore) encrypter, err := encrypt.ProvideEncrypter(config) if err != nil { @@ -210,7 +208,7 @@ func initSystem(ctx context.Context, config *types.Config) (*server.System, erro webHandler := router.ProvideWebHandler(config) routerRouter := router.ProvideRouter(config, apiHandler, gitHandler, webHandler) serverServer := server2.ProvideServer(config, routerRouter) - executionManager := manager.ProvideExecutionManager(config, executionStore, pipelineStore, provider, eventsEvents, fileService, logStore, logStream, repoStore, schedulerScheduler, secretStore, stageStore, stepStore, principalStore) + executionManager := manager.ProvideExecutionManager(config, executionStore, pipelineStore, provider, eventsStreamer, fileService, logStore, logStream, repoStore, schedulerScheduler, secretStore, stageStore, stepStore, principalStore) client := manager.ProvideExecutionClient(executionManager, config) runtimeRunner, err := runner.ProvideExecutionRunner(config, client, executionManager) if err != nil { diff --git a/internal/api/controller/space/controller.go b/internal/api/controller/space/controller.go index 8d9dd449a..6b35a95fe 100644 --- a/internal/api/controller/space/controller.go +++ b/internal/api/controller/space/controller.go @@ -18,7 +18,7 @@ import ( type Controller struct { db *sqlx.DB urlProvider *url.Provider - eventsStream events.Events + eventsStream events.EventsStreamer uidCheck check.PathUID authorizer authz.Authorizer pathStore store.PathStore @@ -33,7 +33,7 @@ type Controller struct { membershipStore store.MembershipStore } -func NewController(db *sqlx.DB, urlProvider *url.Provider, eventsStream events.Events, +func NewController(db *sqlx.DB, urlProvider *url.Provider, eventsStream events.EventsStreamer, uidCheck check.PathUID, authorizer authz.Authorizer, pathStore store.PathStore, pipelineStore store.PipelineStore, secretStore store.SecretStore, connectorStore store.ConnectorStore, templateStore store.TemplateStore, spaceStore store.SpaceStore, diff --git a/internal/api/controller/space/wire.go b/internal/api/controller/space/wire.go index 6fa3022d5..ea39967a2 100644 --- a/internal/api/controller/space/wire.go +++ b/internal/api/controller/space/wire.go @@ -21,7 +21,7 @@ var WireSet = wire.NewSet( ProvideController, ) -func ProvideController(db *sqlx.DB, urlProvider *url.Provider, eventsStream events.Events, +func ProvideController(db *sqlx.DB, urlProvider *url.Provider, eventsStream events.EventsStreamer, uidCheck check.PathUID, authorizer authz.Authorizer, pathStore store.PathStore, pipelineStore store.PipelineStore, secretStore store.SecretStore, connectorStore store.ConnectorStore, templateStore store.TemplateStore, diff --git a/internal/pipeline/events/events.go b/internal/pipeline/events/events.go index a3408e914..0afa78d65 100644 --- a/internal/pipeline/events/events.go +++ b/internal/pipeline/events/events.go @@ -19,7 +19,7 @@ type Event struct { Data json.RawMessage `json:"data"` } -type Events interface { +type EventsStreamer interface { // Publish publishes an event to a given space ID. Publish(ctx context.Context, spaceID int64, event *Event) error @@ -35,7 +35,7 @@ type event struct { topic string } -func New(pubsub pubsub.PubSub, topic string) Events { +func New(pubsub pubsub.PubSub, topic string) EventsStreamer { return &event{ pubsub: pubsub, topic: topic, diff --git a/internal/pipeline/events/wire.go b/internal/pipeline/events/wire.go index 6879e8782..a848fc4bb 100644 --- a/internal/pipeline/events/wire.go +++ b/internal/pipeline/events/wire.go @@ -15,7 +15,7 @@ var WireSet = wire.NewSet( ProvideEventsStreaming, ) -func ProvideEventsStreaming(pubsub pubsub.PubSub) Events { +func ProvideEventsStreaming(pubsub pubsub.PubSub) EventsStreamer { return &event{ pubsub: pubsub, topic: "events", diff --git a/internal/pipeline/manager/manager.go b/internal/pipeline/manager/manager.go index 1500c0f5a..182529fdf 100644 --- a/internal/pipeline/manager/manager.go +++ b/internal/pipeline/manager/manager.go @@ -99,7 +99,7 @@ type Manager struct { Pipelines store.PipelineStore urlProvider *urlprovider.Provider // Converter store.ConvertService - Events events.Events + Events events.EventsStreamer // Globals store.GlobalSecretStore Logs store.LogStore Logz livelog.LogStream @@ -120,7 +120,7 @@ func New( executionStore store.ExecutionStore, pipelineStore store.PipelineStore, urlProvider *urlprovider.Provider, - events events.Events, + events events.EventsStreamer, fileService file.FileService, logStore store.LogStore, logStream livelog.LogStream, diff --git a/internal/pipeline/manager/setup.go b/internal/pipeline/manager/setup.go index 457878fc0..16126498c 100644 --- a/internal/pipeline/manager/setup.go +++ b/internal/pipeline/manager/setup.go @@ -21,7 +21,7 @@ import ( type setup struct { Executions store.ExecutionStore - Events events.Events + Events events.EventsStreamer Repos store.RepoStore Steps store.StepStore Stages store.StageStore diff --git a/internal/pipeline/manager/teardown.go b/internal/pipeline/manager/teardown.go index 25cbb180c..63d827686 100644 --- a/internal/pipeline/manager/teardown.go +++ b/internal/pipeline/manager/teardown.go @@ -21,7 +21,7 @@ import ( type teardown struct { Executions store.ExecutionStore - Events events.Events + Events events.EventsStreamer Logs livelog.LogStream Scheduler scheduler.Scheduler Repos store.RepoStore diff --git a/internal/pipeline/manager/updater.go b/internal/pipeline/manager/updater.go index 13d068486..0b9f801ba 100644 --- a/internal/pipeline/manager/updater.go +++ b/internal/pipeline/manager/updater.go @@ -18,7 +18,7 @@ import ( type updater struct { Executions store.ExecutionStore Repos store.RepoStore - Events events.Events + Events events.EventsStreamer Steps store.StepStore Stages store.StageStore } diff --git a/internal/pipeline/manager/wire.go b/internal/pipeline/manager/wire.go index 4849ea89c..f097fd710 100644 --- a/internal/pipeline/manager/wire.go +++ b/internal/pipeline/manager/wire.go @@ -29,7 +29,7 @@ func ProvideExecutionManager( executionStore store.ExecutionStore, pipelineStore store.PipelineStore, urlProvider *url.Provider, - events events.Events, + events events.EventsStreamer, fileService file.FileService, logStore store.LogStore, logStream livelog.LogStream, diff --git a/internal/writer/writeflush.go b/internal/writer/writeflush.go index c2e37eea0..141e10a9a 100644 --- a/internal/writer/writeflush.go +++ b/internal/writer/writeflush.go @@ -1,3 +1,7 @@ +// Copyright 2022 Harness Inc. All rights reserved. +// Use of this source code is governed by the Polyform Free Trial License +// that can be found in the LICENSE.md file for this repository. + package writer import "io" From 60ec1f9b4ee7a6cc5567c0043f353e688bfa1672 Mon Sep 17 00:00:00 2001 From: Vistaar Juneja Date: Fri, 8 Sep 2023 13:47:00 +0100 Subject: [PATCH 05/17] continue in case channel is full --- internal/pipeline/events/events.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/internal/pipeline/events/events.go b/internal/pipeline/events/events.go index 0afa78d65..632a799ec 100644 --- a/internal/pipeline/events/events.go +++ b/internal/pipeline/events/events.go @@ -66,7 +66,11 @@ func (e *event) Subscribe(ctx context.Context, spaceID int64) (<-chan *Event, <- // This should never happen return err } - chEvent <- event + select { + case chEvent <- event: + default: + } + return nil } option := pubsub.WithChannelNamespace(format(spaceID)) From 2d0fc6f3c9d5cf883e6298847f290a3e0e616a81 Mon Sep 17 00:00:00 2001 From: Dan Wilson Date: Fri, 8 Sep 2023 14:51:29 +0100 Subject: [PATCH 06/17] UI log streaming and some secrets WIP --- internal/api/handler/logs/tail.go | 14 ++-- web/config/webpack.dev.js | 3 +- .../components/Console/Console.module.scss | 4 +- web/src/components/Console/Console.tsx | 3 +- .../ConsoleLogs/ConsoleLogs.module.scss | 4 +- .../components/ConsoleLogs/ConsoleLogs.tsx | 25 ++---- .../ConsoleStep/ConsoleStep.module.scss | 13 +++ .../ConsoleStep/ConsoleStep.module.scss.d.ts | 3 +- .../components/ConsoleStep/ConsoleStep.tsx | 81 ++++++++++++++----- .../ExecutionPageHeader.module.scss | 12 +-- .../ExecutionPageHeader.tsx | 10 +-- .../ExecutionStageList/ExecutionStageList.tsx | 2 +- .../NewSecretModalButton.tsx | 13 +-- web/src/framework/strings/stringTypes.ts | 5 ++ web/src/i18n/strings.en.yaml | 5 ++ web/src/pages/ExecutionList/ExecutionList.tsx | 59 +++++++++----- web/src/pages/SecretList/SecretList.tsx | 78 ++++++++++++++---- web/src/services/code/index.tsx | 40 +++++---- web/src/services/code/swagger.yaml | 37 +++++---- web/src/utils/PipelineUtils.ts | 4 +- 20 files changed, 275 insertions(+), 140 deletions(-) diff --git a/internal/api/handler/logs/tail.go b/internal/api/handler/logs/tail.go index b611d0ad4..198b63e69 100644 --- a/internal/api/handler/logs/tail.go +++ b/internal/api/handler/logs/tail.go @@ -60,6 +60,13 @@ func HandleTail(logCtrl *logs.Controller) http.HandlerFunc { return } + h := w.Header() + h.Set("Content-Type", "text/event-stream") + h.Set("Cache-Control", "no-cache") + h.Set("Connection", "keep-alive") + h.Set("X-Accel-Buffering", "no") + h.Set("Access-Control-Allow-Origin", "*") + io.WriteString(w, ": ping\n\n") f.Flush() @@ -77,13 +84,6 @@ func HandleTail(logCtrl *logs.Controller) http.HandlerFunc { return } - h := w.Header() - h.Set("Content-Type", "text/event-stream") - h.Set("Cache-Control", "no-cache") - h.Set("Connection", "keep-alive") - h.Set("X-Accel-Buffering", "no") - h.Set("Access-Control-Allow-Origin", "*") - ctx, cancel := context.WithTimeout(r.Context(), tailMaxTime) defer cancel() diff --git a/web/config/webpack.dev.js b/web/config/webpack.dev.js index 7bec53907..d3436b7c0 100644 --- a/web/config/webpack.dev.js +++ b/web/config/webpack.dev.js @@ -57,7 +57,8 @@ const devConfig = { secure: false, changeOrigin: true } - } + }, + compress: false }, plugins: [ new MiniCssExtractPlugin({ diff --git a/web/src/components/Console/Console.module.scss b/web/src/components/Console/Console.module.scss index 6b2f0e1cd..0fb914b96 100644 --- a/web/src/components/Console/Console.module.scss +++ b/web/src/components/Console/Console.module.scss @@ -1,7 +1,7 @@ .container { display: flex; flex-direction: column; - background-color: black; + background-color: var(--black); height: 100%; overflow-y: auto; } @@ -14,7 +14,7 @@ .header { position: sticky; top: 0; - background-color: var(--black); + background-color: var(--black) !important; height: var(--log-content-header-height); .headerLayout { diff --git a/web/src/components/Console/Console.tsx b/web/src/components/Console/Console.tsx index 263f0276d..70815347a 100644 --- a/web/src/components/Console/Console.tsx +++ b/web/src/components/Console/Console.tsx @@ -26,8 +26,7 @@ const Console: FC = ({ stage, repoPath }) => { {stage?.started && stage?.stopped && ( - {/* this needs fixed */} - {timeDistance(stage?.started, stage?.stopped)} + {`completed ${timeDistance(stage?.stopped, Date.now())} ago`} )} diff --git a/web/src/components/ConsoleLogs/ConsoleLogs.module.scss b/web/src/components/ConsoleLogs/ConsoleLogs.module.scss index 2c19a31c4..62f38b756 100644 --- a/web/src/components/ConsoleLogs/ConsoleLogs.module.scss +++ b/web/src/components/ConsoleLogs/ConsoleLogs.module.scss @@ -1,9 +1,9 @@ .logLayout { - margin-left: var(--spacing-xxxlarge) !important; + margin-left: 30px !important; } .lineNumber { - width: var(--spacing-xlarge); + min-width: var(--spacing-xlarge); color: #999; margin-right: 16px; font-family: 'Roboto Mono' !important; diff --git a/web/src/components/ConsoleLogs/ConsoleLogs.tsx b/web/src/components/ConsoleLogs/ConsoleLogs.tsx index 589353d12..5e9c0ce24 100644 --- a/web/src/components/ConsoleLogs/ConsoleLogs.tsx +++ b/web/src/components/ConsoleLogs/ConsoleLogs.tsx @@ -1,35 +1,22 @@ -import { Layout, Text } from '@harnessio/uicore' +import { FlexExpander, Layout, Text } from '@harnessio/uicore' import React, { FC } from 'react' +import type { LivelogLine } from 'services/code' import css from './ConsoleLogs.module.scss' -// currently a string - should be an array of strings in future interface ConsoleLogsProps { - logs: string -} - -interface log { - pos: number - out: string - time: number -} - -const convertStringToLogArray = (logs: string): log[] => { - const logStrings = logs.split('\n').map(log => { - return JSON.parse(log) - }) - - return logStrings + logs: LivelogLine[] } const ConsoleLogs: FC = ({ logs }) => { - const logArray = convertStringToLogArray(logs) return ( <> - {logArray.map((log, index) => { + {logs.map((log, index) => { return ( {log.pos} {log.out} + + {log.time}s ) })} diff --git a/web/src/components/ConsoleStep/ConsoleStep.module.scss b/web/src/components/ConsoleStep/ConsoleStep.module.scss index 56f5deaae..465505cc1 100644 --- a/web/src/components/ConsoleStep/ConsoleStep.module.scss +++ b/web/src/components/ConsoleStep/ConsoleStep.module.scss @@ -7,3 +7,16 @@ .loading { margin-left: var(--spacing-xxxlarge) !important; } + +.spin { + animation: spin 1s linear infinite; +} + +@keyframes spin { + 0% { + transform: rotate(0deg); + } + 100% { + transform: rotate(360deg); + } +} diff --git a/web/src/components/ConsoleStep/ConsoleStep.module.scss.d.ts b/web/src/components/ConsoleStep/ConsoleStep.module.scss.d.ts index 7f68607e5..0d4fcb2ca 100644 --- a/web/src/components/ConsoleStep/ConsoleStep.module.scss.d.ts +++ b/web/src/components/ConsoleStep/ConsoleStep.module.scss.d.ts @@ -1,7 +1,8 @@ /* eslint-disable */ // this is an auto-generated file declare const styles: { - readonly stepLayout: string + readonly spin: string readonly loading: string + readonly stepLayout: string } export default styles diff --git a/web/src/components/ConsoleStep/ConsoleStep.tsx b/web/src/components/ConsoleStep/ConsoleStep.tsx index b09c5f5f5..900f29a33 100644 --- a/web/src/components/ConsoleStep/ConsoleStep.tsx +++ b/web/src/components/ConsoleStep/ConsoleStep.tsx @@ -1,9 +1,9 @@ import { Icon } from '@harnessio/icons' import { FlexExpander, Layout } from '@harnessio/uicore' -import React, { FC, useEffect } from 'react' +import React, { FC, useEffect, useRef, useState } from 'react' import { useGet } from 'restful-react' import { Text } from '@harnessio/uicore' -import type { TypesStep } from 'services/code' +import type { LivelogLine, TypesStep } from 'services/code' import { timeDistance } from 'utils/Utils' import ConsoleLogs from 'components/ConsoleLogs/ConsoleLogs' import { useStrings } from 'framework/strings' @@ -20,20 +20,68 @@ interface ConsoleStepProps { const ConsoleStep: FC = ({ step, stageNumber, repoPath, pipelineName, executionNumber }) => { const { getString } = useStrings() - const [isOpened, setIsOpened] = React.useState(false) + const [isOpened, setIsOpened] = useState(false) + const [streamingLogs, setStreamingLogs] = useState([]) + const eventSourceRef = useRef(null) - const { data, error, loading, refetch } = useGet({ + const shouldUseGet = step?.status !== 'running' && step?.status !== 'pending' + const isPending = step?.status === 'pending' + + const { data, error, loading, refetch } = useGet({ path: `/api/v1/repos/${repoPath}/+/pipelines/${pipelineName}/executions/${executionNumber}/logs/${String( stageNumber )}/${String(step?.number)}`, lazy: true }) - // this refetches any open steps when the stage number changes - really it shouldnt refetch until reopened... useEffect(() => { setIsOpened(false) - refetch() - }, [stageNumber, refetch]) + }, [stageNumber]) + + useEffect(() => { + if (step?.status === 'running') { + if (eventSourceRef.current) { + eventSourceRef.current.close() + setStreamingLogs([]) + } + eventSourceRef.current = new EventSource( + `/api/v1/repos/${repoPath}/+/pipelines/${pipelineName}/executions/${executionNumber}/logs/${String( + stageNumber + )}/${String(step?.number)}/stream` + ) + eventSourceRef.current.onmessage = event => { + const newLog = JSON.parse(event.data) + setStreamingLogs(existingLogs => { + return [...existingLogs, newLog] + }) + } + } + return () => { + if (eventSourceRef.current) eventSourceRef.current.close() + } + }, [executionNumber, pipelineName, repoPath, stageNumber, step?.number, step?.status]) + + let icon + if (step?.status === 'success') { + icon = + } else if (isPending) { + icon = + } else if (step?.status === 'running') { + icon = + } else { + icon = // Default icon in case of other statuses or unknown status + } + + let content + if (loading) { + content =
{getString('loading')}
+ } else if (error && step?.status !== 'running') { + content =
Error: {error.message}
+ } else if (streamingLogs.length) { + content = + } else if (data) { + content = + } return ( <> @@ -41,25 +89,20 @@ const ConsoleStep: FC = ({ step, stageNumber, repoPath, pipeli className={css.stepLayout} spacing="medium" onClick={() => { - setIsOpened(!isOpened) - if (!data && !loading) refetch() + if (!isPending) { + setIsOpened(!isOpened) + if (shouldUseGet && !isOpened) refetch() + } }}> - + {/* TODO - flesh icon logic out */} + {icon} {step?.name} {step?.started && step?.stopped &&
{timeDistance(step?.stopped, step?.started)}
} - {isOpened ? ( - loading ? ( -
{getString('loading')}
- ) : error ? ( -
Error: {error}
- ) : data ? ( - - ) : null - ) : null} + {isOpened && content} ) } diff --git a/web/src/components/ExecutionPageHeader/ExecutionPageHeader.module.scss b/web/src/components/ExecutionPageHeader/ExecutionPageHeader.module.scss index a890740b2..b0f95954d 100644 --- a/web/src/components/ExecutionPageHeader/ExecutionPageHeader.module.scss +++ b/web/src/components/ExecutionPageHeader/ExecutionPageHeader.module.scss @@ -5,16 +5,6 @@ gap: var(--spacing-small) !important; } -.header { - width: 100% !important; - gap: var(--spacing-small) !important; - - a { - font-size: var(--font-size-small); - color: var(--primary-7); - } -} - .breadcrumb { align-items: center; @@ -36,6 +26,8 @@ } .executionInfo { + display: flex !important; + gap: var(--spacing-small) !important; width: 100% !important; align-items: center !important; margin-left: 0 !important; diff --git a/web/src/components/ExecutionPageHeader/ExecutionPageHeader.tsx b/web/src/components/ExecutionPageHeader/ExecutionPageHeader.tsx index 4870c424b..44cd71ba4 100644 --- a/web/src/components/ExecutionPageHeader/ExecutionPageHeader.tsx +++ b/web/src/components/ExecutionPageHeader/ExecutionPageHeader.tsx @@ -1,5 +1,5 @@ import React, { Fragment } from 'react' -import { Layout, Text, PageHeader, Utils, Avatar, FlexExpander } from '@harnessio/uicore' +import { Layout, Text, PageHeader, Utils, Avatar, FlexExpander, Container } from '@harnessio/uicore' import { Icon } from '@harnessio/icons' import { Color } from '@harnessio/design-system' import { Link, useParams } from 'react-router-dom' @@ -78,8 +78,7 @@ export function ExecutionPageHeader({ } content={ executionInfo && ( - // TODO - margin left not playing ball... why? - + {executionInfo.message} @@ -106,14 +105,13 @@ export function ExecutionPageHeader({ {timeDistance(executionInfo.started, executionInfo.finished)} - - + {timeDistance(executionInfo.finished, Date.now())} ago - + ) } /> diff --git a/web/src/components/ExecutionStageList/ExecutionStageList.tsx b/web/src/components/ExecutionStageList/ExecutionStageList.tsx index eceeea265..2b5090586 100644 --- a/web/src/components/ExecutionStageList/ExecutionStageList.tsx +++ b/web/src/components/ExecutionStageList/ExecutionStageList.tsx @@ -32,7 +32,7 @@ const ExecutionStage: FC = ({ stage, isSelected = false, se status={getStatus(stage.status || ExecutionState.PENDING)} iconOnly noBackground - iconSize={16} + iconSize={18} className={css.statusIcon} /> diff --git a/web/src/components/NewSecretModalButton/NewSecretModalButton.tsx b/web/src/components/NewSecretModalButton/NewSecretModalButton.tsx index 279a4afd4..ff63e36c8 100644 --- a/web/src/components/NewSecretModalButton/NewSecretModalButton.tsx +++ b/web/src/components/NewSecretModalButton/NewSecretModalButton.tsx @@ -38,7 +38,7 @@ export interface NewSecretModalButtonProps extends Omit void + onSuccess: () => void } export const NewSecretModalButton: React.FC = ({ @@ -46,12 +46,12 @@ export const NewSecretModalButton: React.FC = ({ modalTitle, submitButtonTitle, cancelButtonTitle, - onSubmit, + onSuccess, ...props }) => { const ModalComponent: React.FC = () => { const { getString } = useStrings() - const { showError } = useToaster() + const { showError, showSuccess } = useToaster() const { mutate: createSecret, loading } = useMutate({ verb: 'POST', @@ -66,9 +66,10 @@ export const NewSecretModalButton: React.FC = ({ description: formData.description, uid: formData.name } - const response = await createSecret(payload) + await createSecret(payload) hideModal() - onSubmit(response) + showSuccess(getString('secrets.createSuccess')) + onSuccess() } catch (exception) { showError(getErrorMessage(exception), 0, getString('secrets.failedToCreate')) } @@ -152,7 +153,7 @@ export const NewSecretModalButton: React.FC = ({ ) } - const [openModal, hideModal] = useModalHook(ModalComponent, [onSubmit]) + const [openModal, hideModal] = useModalHook(ModalComponent, [onSuccess]) return + icon="play-outline" + disabled={mutateLoading} + onClick={handleClick}> ) const columns: Column[] = useMemo( @@ -75,14 +97,9 @@ const ExecutionList = () => { return ( - + {`#${record.number}.`} - {record.title} + {record.message} @@ -112,16 +129,18 @@ const ExecutionList = () => { const record = row.original return ( - - - - {timeDistance(record.started, record.finished)} - - + {record?.started && record?.finished && ( + + + + {timeDistance(record.started, record.finished)} + + + )} - {timeDistance(record.finished, Date.now())} ago + {timeDistance(record.created, Date.now())} ago diff --git a/web/src/pages/SecretList/SecretList.tsx b/web/src/pages/SecretList/SecretList.tsx index 856766be9..cb419fac8 100644 --- a/web/src/pages/SecretList/SecretList.tsx +++ b/web/src/pages/SecretList/SecretList.tsx @@ -6,16 +6,17 @@ import { Layout, PageBody, PageHeader, + StringSubstitute, TableV2 as Table, - Text + Text, + useToaster } from '@harnessio/uicore' -import { Color } from '@harnessio/design-system' +import { Color, Intent } from '@harnessio/design-system' import cx from 'classnames' import type { CellProps, Column } from 'react-table' import Keywords from 'react-keywords' -import { useHistory } from 'react-router-dom' -import { useGet } from 'restful-react' -import { useStrings } from 'framework/strings' +import { useGet, useMutate } from 'restful-react' +import { String, useStrings } from 'framework/strings' import { LoadingSpinner } from 'components/LoadingSpinner/LoadingSpinner' import { SearchInputWithSpinner } from 'components/SearchInputWithSpinner/SearchInputWithSpinner' import { NoResultCard } from 'components/NoResultCard/NoResultCard' @@ -26,14 +27,13 @@ import { useQueryParams } from 'hooks/useQueryParams' import { useGetSpaceParam } from 'hooks/useGetSpaceParam' import { ResourceListingPagination } from 'components/ResourceListingPagination/ResourceListingPagination' import { NewSecretModalButton } from 'components/NewSecretModalButton/NewSecretModalButton' -import { useAppContext } from 'AppContext' +import { useConfirmAct } from 'hooks/useConfirmAction' +import { OptionsMenuButton } from 'components/OptionsMenuButton/OptionsMenuButton' import noSecretsImage from '../RepositoriesListing/no-repo.svg' import css from './SecretList.module.scss' const SecretList = () => { - const { routes } = useAppContext() const space = useGetSpaceParam() - const history = useHistory() const { getString } = useStrings() const [searchTerm, setSearchTerm] = useState() const pageBrowser = useQueryParams() @@ -58,9 +58,7 @@ const SecretList = () => { text={getString('secrets.newSecretButton')} variation={ButtonVariation.PRIMARY} icon="plus" - onSubmit={secretInfo => - history.push(routes.toCODESecret({ space, secret: secretInfo.uid as string })) - }> + onSuccess={() => refetch()}> ) const columns: Column[] = useMemo( @@ -97,9 +95,62 @@ const SecretList = () => { ) }, disableSortBy: true + }, + { + id: 'action', + width: '30px', + Cell: ({ row }: CellProps) => { + const { mutate: deleteSecret } = useMutate({ + verb: 'DELETE', + path: `/api/v1/secrets/${space}/${row.original.uid}` + }) + const { showSuccess, showError } = useToaster() + const confirmDeleteSecret = useConfirmAct() + + // TODO - add edit option + return ( + + confirmDeleteSecret({ + title: getString('secrets.deleteSecret'), + confirmText: getString('delete'), + intent: Intent.DANGER, + message: ( + + ), + action: async () => { + deleteSecret({}) + .then(() => { + showSuccess( + , + 5000 + ) + refetch() + }) + .catch(secretDeleteError => { + showError(getErrorMessage(secretDeleteError), 0, 'secrets.failedToDeleteSecret') + }) + } + }) + } + ]} + /> + ) + } } ], - [getString, searchTerm] + [getString, refetch, searchTerm, space] ) return ( @@ -130,9 +181,6 @@ const SecretList = () => { className={css.table} columns={columns} data={secrets || []} - onRowClick={secretInfo => - history.push(routes.toCODESecret({ space: 'root', secret: secretInfo.uid as string })) - } getRowClassName={row => cx(css.row, !row.original.description && css.noDesc)} /> )} diff --git a/web/src/services/code/index.tsx b/web/src/services/code/index.tsx index 57d954a64..3779663e7 100644 --- a/web/src/services/code/index.tsx +++ b/web/src/services/code/index.tsx @@ -108,6 +108,12 @@ export interface GitrpcSignature { when?: string } +export interface LivelogLine { + out?: string + pos?: number + time?: number +} + export interface OpenapiAdminUsersCreateRequest { display_name?: string email?: string @@ -179,10 +185,6 @@ export interface OpenapiCreateConnectorRequest { uid?: string } -export interface OpenapiCreateExecutionRequest { - status?: string -} - export interface OpenapiCreatePathRequest { path?: string } @@ -738,6 +740,7 @@ export interface TypesRepository { fork_id?: number git_url?: string id?: number + importing?: boolean is_public?: boolean num_closed_pulls?: number num_forks?: number @@ -790,7 +793,7 @@ export interface TypesSpace { export interface TypesStage { arch?: string - depends_on?: string[] | null + depends_on?: string[] errignore?: boolean error?: string execution_id?: number @@ -807,6 +810,7 @@ export interface TypesStage { on_failure?: boolean on_success?: boolean os?: string + repo_id?: number started?: number status?: string steps?: TypesStep[] @@ -817,7 +821,7 @@ export interface TypesStage { } export interface TypesStep { - depends_on?: string[] | null + depends_on?: string[] detached?: boolean errignore?: boolean error?: string @@ -2459,19 +2463,26 @@ export const useListExecutions = ({ repo_ref, pipeline_uid, ...props }: UseListE { base: getConfig('code/api/v1'), pathParams: { repo_ref, pipeline_uid }, ...props } ) +export interface CreateExecutionQueryParams { + /** + * Branch to run the execution for. + */ + branch?: string +} + export interface CreateExecutionPathParams { repo_ref: string pipeline_uid: string } export type CreateExecutionProps = Omit< - MutateProps, + MutateProps, 'path' | 'verb' > & CreateExecutionPathParams export const CreateExecution = ({ repo_ref, pipeline_uid, ...props }: CreateExecutionProps) => ( - + verb="POST" path={`/repos/${repo_ref}/pipelines/${pipeline_uid}/executions`} base={getConfig('code/api/v1')} @@ -2480,13 +2491,13 @@ export const CreateExecution = ({ repo_ref, pipeline_uid, ...props }: CreateExec ) export type UseCreateExecutionProps = Omit< - UseMutateProps, + UseMutateProps, 'path' | 'verb' > & CreateExecutionPathParams export const useCreateExecution = ({ repo_ref, pipeline_uid, ...props }: UseCreateExecutionProps) => - useMutate( + useMutate( 'POST', (paramsInPath: CreateExecutionPathParams) => `/repos/${paramsInPath.repo_ref}/pipelines/${paramsInPath.pipeline_uid}/executions`, @@ -2600,7 +2611,8 @@ export interface ViewLogsPathParams { step_number: string } -export type ViewLogsProps = Omit, 'path'> & ViewLogsPathParams +export type ViewLogsProps = Omit, 'path'> & + ViewLogsPathParams export const ViewLogs = ({ repo_ref, @@ -2610,14 +2622,14 @@ export const ViewLogs = ({ step_number, ...props }: ViewLogsProps) => ( - + path={`/repos/${repo_ref}/pipelines/${pipeline_uid}/executions/${execution_number}/logs/${stage_number}/${step_number}`} base={getConfig('code/api/v1')} {...props} /> ) -export type UseViewLogsProps = Omit, 'path'> & +export type UseViewLogsProps = Omit, 'path'> & ViewLogsPathParams export const useViewLogs = ({ @@ -2628,7 +2640,7 @@ export const useViewLogs = ({ step_number, ...props }: UseViewLogsProps) => - useGet( + useGet( (paramsInPath: ViewLogsPathParams) => `/repos/${paramsInPath.repo_ref}/pipelines/${paramsInPath.pipeline_uid}/executions/${paramsInPath.execution_number}/logs/${paramsInPath.stage_number}/${paramsInPath.step_number}`, { diff --git a/web/src/services/code/swagger.yaml b/web/src/services/code/swagger.yaml index 84eb98c18..27a2a1659 100644 --- a/web/src/services/code/swagger.yaml +++ b/web/src/services/code/swagger.yaml @@ -2294,6 +2294,12 @@ paths: post: operationId: createExecution parameters: + - description: Branch to run the execution for. + in: query + name: branch + required: false + schema: + type: string - in: path name: repo_ref required: true @@ -2304,11 +2310,6 @@ paths: required: true schema: type: string - requestBody: - content: - application/json: - schema: - $ref: '#/components/schemas/OpenapiCreateExecutionRequest' responses: '201': content: @@ -2535,9 +2536,11 @@ paths: responses: '200': content: - text/plain: + application/json: schema: - type: string + items: + $ref: '#/components/schemas/LivelogLine' + type: array description: OK '401': content: @@ -6370,6 +6373,15 @@ components: format: date-time type: string type: object + LivelogLine: + properties: + out: + type: string + pos: + type: integer + time: + type: integer + type: object OpenapiAdminUsersCreateRequest: properties: display_name: @@ -6497,11 +6509,6 @@ components: uid: type: string type: object - OpenapiCreateExecutionRequest: - properties: - status: - type: string - type: object OpenapiCreatePathRequest: properties: path: @@ -7446,6 +7453,8 @@ components: type: string id: type: integer + importing: + type: boolean is_public: type: boolean num_closed_pulls: @@ -7539,7 +7548,6 @@ components: depends_on: items: type: string - nullable: true type: array errignore: type: boolean @@ -7571,6 +7579,8 @@ components: type: boolean os: type: string + repo_id: + type: integer started: type: integer status: @@ -7593,7 +7603,6 @@ components: depends_on: items: type: string - nullable: true type: array detached: type: boolean diff --git a/web/src/utils/PipelineUtils.ts b/web/src/utils/PipelineUtils.ts index 7ba46dd10..e7cfae3c3 100644 --- a/web/src/utils/PipelineUtils.ts +++ b/web/src/utils/PipelineUtils.ts @@ -1,6 +1,6 @@ import { ExecutionState } from 'components/ExecutionStatus/ExecutionStatus' -export const getStatus = (status: string): ExecutionState => { +export const getStatus = (status: string | undefined): ExecutionState => { switch (status) { case 'success': return ExecutionState.SUCCESS @@ -10,6 +10,8 @@ export const getStatus = (status: string): ExecutionState => { return ExecutionState.RUNNING case 'pending': return ExecutionState.PENDING + case 'error': + return ExecutionState.ERROR default: return ExecutionState.PENDING } From 1a477a23a0defda651b4d50ae7178aa404fad03f Mon Sep 17 00:00:00 2001 From: Tan Nhu Date: Fri, 8 Sep 2023 15:51:54 +0000 Subject: [PATCH 07/17] Enable View Raw file + optimize fetching current user (#413) --- web/src/AppContext.tsx | 10 +++++++-- web/src/RouteDestinations.tsx | 1 - .../FileContent/FileContent.tsx | 22 ++++++++++++------- 3 files changed, 22 insertions(+), 11 deletions(-) diff --git a/web/src/AppContext.tsx b/web/src/AppContext.tsx index 763b1ee5e..d11cbebce 100644 --- a/web/src/AppContext.tsx +++ b/web/src/AppContext.tsx @@ -1,4 +1,5 @@ -import React, { useState, useContext, useEffect } from 'react' +import React, { useState, useContext, useEffect, useMemo } from 'react' +import { matchPath } from 'react-router-dom' import { noop } from 'lodash-es' import { useGet } from 'restful-react' import type { AppProps } from 'AppProps' @@ -33,8 +34,13 @@ export const AppContextProvider: React.FC<{ value: AppProps }> = React.memo(func value: initialValue, children }) { + const lazy = useMemo( + () => initialValue.standalone && !!matchPath(location.pathname, { path: '/(signin|register)' }), + [initialValue.standalone] + ) const { data: currentUser = defaultCurrentUser } = useGet({ - path: '/api/v1/user' + path: '/api/v1/user', + lazy }) const [appStates, setAppStates] = useState(initialValue) diff --git a/web/src/RouteDestinations.tsx b/web/src/RouteDestinations.tsx index a30325981..d7cef09bb 100644 --- a/web/src/RouteDestinations.tsx +++ b/web/src/RouteDestinations.tsx @@ -36,7 +36,6 @@ import NewPipeline from 'pages/NewPipeline/NewPipeline' export const RouteDestinations: React.FC = React.memo(function RouteDestinations() { const { getString } = useStrings() const repoPath = `${pathProps.space}/${pathProps.repoName}` - const { OPEN_SOURCE_PIPELINES, OPEN_SOURCE_SECRETS } = useFeatureFlag() return ( diff --git a/web/src/pages/Repository/RepositoryContent/FileContent/FileContent.tsx b/web/src/pages/Repository/RepositoryContent/FileContent/FileContent.tsx index 08a065234..7eb3c4cdd 100644 --- a/web/src/pages/Repository/RepositoryContent/FileContent/FileContent.tsx +++ b/web/src/pages/Repository/RepositoryContent/FileContent/FileContent.tsx @@ -31,6 +31,7 @@ import { useAppContext } from 'AppContext' import { LatestCommitForFile } from 'components/LatestCommit/LatestCommit' import { useCommitModal } from 'components/CommitModalButton/CommitModalButton' import { useStrings } from 'framework/strings' +import { getConfig } from 'services/config' import { OptionsMenuButton } from 'components/OptionsMenuButton/OptionsMenuButton' import { PlainButton } from 'components/PlainButton/PlainButton' import { CommitsView } from 'components/CommitsView/CommitsView' @@ -57,7 +58,6 @@ export function FileContent({ useFileContentViewerDecision({ repoMetadata, gitRef, resourcePath, resourceContent }) const history = useHistory() const [activeTab, setActiveTab] = React.useState(FileSection.CONTENT) - const content = useMemo( () => decodeGitContent((resourceContent?.content as RepoFileContent)?.data), [resourceContent?.content] @@ -199,13 +199,18 @@ export function FileContent({ style={{ padding: '5px' }} width="145px" items={[ - // { - // hasIcon: true, - // iconName: 'arrow-right', - // text: getString('viewRaw'), - // onClick: () => window.open(rawURL, '_blank') // TODO: This is still not working due to token is not stored in cookies - // }, - // '-', + { + hasIcon: true, + iconName: 'arrow-right', + text: getString('viewRaw'), + onClick: () => { + const url = standalone + ? rawURL.replace(/^\/code/, '') + : getConfig(rawURL).replace('//', '/') + window.open(url, '_blank') + } + }, + '-', { hasIcon: true, iconName: 'cloud-download', @@ -243,6 +248,7 @@ export function FileContent({
+ rawURL: {rawURL} { From 742769bba189c1859edd28616271f34081b58b6e Mon Sep 17 00:00:00 2001 From: Dan Wilson Date: Fri, 8 Sep 2023 17:03:01 +0100 Subject: [PATCH 08/17] fix Tan PR comments --- web/src/components/Console/Console.tsx | 5 +++-- web/src/components/ConsoleStep/ConsoleStep.tsx | 13 +++++++------ 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/web/src/components/Console/Console.tsx b/web/src/components/Console/Console.tsx index 70815347a..2caf96d42 100644 --- a/web/src/components/Console/Console.tsx +++ b/web/src/components/Console/Console.tsx @@ -6,7 +6,7 @@ import type { CODEProps } from 'RouteDefinitions' import type { TypesStage } from 'services/code' import ConsoleStep from 'components/ConsoleStep/ConsoleStep' import { timeDistance } from 'utils/Utils' -// import { useGetRepositoryMetadata } from 'hooks/useGetRepositoryMetadata' +import { useStrings } from 'framework/strings' import css from './Console.module.scss' interface ConsoleProps { @@ -16,6 +16,7 @@ interface ConsoleProps { const Console: FC = ({ stage, repoPath }) => { const { pipeline, execution: executionNum } = useParams() + const { getString } = useStrings() return (
@@ -26,7 +27,7 @@ const Console: FC = ({ stage, repoPath }) => { {stage?.started && stage?.stopped && ( - {`completed ${timeDistance(stage?.stopped, Date.now())} ago`} + {getString('executions.completedTime', { timeString: timeDistance(stage?.stopped, Date.now()) })} )} diff --git a/web/src/components/ConsoleStep/ConsoleStep.tsx b/web/src/components/ConsoleStep/ConsoleStep.tsx index 900f29a33..71c8b9724 100644 --- a/web/src/components/ConsoleStep/ConsoleStep.tsx +++ b/web/src/components/ConsoleStep/ConsoleStep.tsx @@ -7,6 +7,7 @@ import type { LivelogLine, TypesStep } from 'services/code' import { timeDistance } from 'utils/Utils' import ConsoleLogs from 'components/ConsoleLogs/ConsoleLogs' import { useStrings } from 'framework/strings' +import { ExecutionState } from 'components/ExecutionStatus/ExecutionStatus' import css from './ConsoleStep.module.scss' interface ConsoleStepProps { @@ -24,8 +25,8 @@ const ConsoleStep: FC = ({ step, stageNumber, repoPath, pipeli const [streamingLogs, setStreamingLogs] = useState([]) const eventSourceRef = useRef(null) - const shouldUseGet = step?.status !== 'running' && step?.status !== 'pending' - const isPending = step?.status === 'pending' + const shouldUseGet = step?.status !== ExecutionState.RUNNING && step?.status !== ExecutionState.PENDING + const isPending = step?.status === ExecutionState.PENDING const { data, error, loading, refetch } = useGet({ path: `/api/v1/repos/${repoPath}/+/pipelines/${pipelineName}/executions/${executionNumber}/logs/${String( @@ -39,7 +40,7 @@ const ConsoleStep: FC = ({ step, stageNumber, repoPath, pipeli }, [stageNumber]) useEffect(() => { - if (step?.status === 'running') { + if (step?.status === ExecutionState.RUNNING) { if (eventSourceRef.current) { eventSourceRef.current.close() setStreamingLogs([]) @@ -62,11 +63,11 @@ const ConsoleStep: FC = ({ step, stageNumber, repoPath, pipeli }, [executionNumber, pipelineName, repoPath, stageNumber, step?.number, step?.status]) let icon - if (step?.status === 'success') { + if (step?.status === ExecutionState.SUCCESS) { icon = } else if (isPending) { icon = - } else if (step?.status === 'running') { + } else if (step?.status === ExecutionState.RUNNING) { icon = } else { icon = // Default icon in case of other statuses or unknown status @@ -75,7 +76,7 @@ const ConsoleStep: FC = ({ step, stageNumber, repoPath, pipeli let content if (loading) { content =
{getString('loading')}
- } else if (error && step?.status !== 'running') { + } else if (error && step?.status !== ExecutionState.RUNNING) { content =
Error: {error.message}
} else if (streamingLogs.length) { content = From 65f4ae8ab0a1daa90f2a4fc88af2987dfebd68a8 Mon Sep 17 00:00:00 2001 From: Calvin Lee Date: Fri, 8 Sep 2023 16:25:18 +0000 Subject: [PATCH 09/17] fix: [code-828]: remove second api call (#414) --- web/src/pages/Home/Home.tsx | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/web/src/pages/Home/Home.tsx b/web/src/pages/Home/Home.tsx index 2ce141d40..6b091a72d 100644 --- a/web/src/pages/Home/Home.tsx +++ b/web/src/pages/Home/Home.tsx @@ -9,6 +9,7 @@ import { useStrings } from 'framework/strings' import { LoadingSpinner } from 'components/LoadingSpinner/LoadingSpinner' import { useAppContext } from 'AppContext' import { useGetRepositoryMetadata } from 'hooks/useGetRepositoryMetadata' +import { voidFn } from 'utils/Utils' import { NewSpaceModalButton } from 'components/NewSpaceModalButton/NewSpaceModalButton' import css from './Home.module.scss' @@ -22,9 +23,7 @@ export default function Home() { const { space } = useGetRepositoryMetadata() const spaces = [] - const { refetch } = useGet({ - path: '/api/v1/user/memberships' - }) + const NewSpaceButton = ( {})} handleNavigation={spaceName => { history.push(routes.toCODERepositories({ space: spaceName })) }} From e61be1ec2d260cd90302547cdbe01e7805c9dee2 Mon Sep 17 00:00:00 2001 From: Tan Nhu Date: Fri, 8 Sep 2023 15:51:54 +0000 Subject: [PATCH 10/17] Enable View Raw file + optimize fetching current user (#413) --- web/src/AppContext.tsx | 10 +++++++-- web/src/RouteDestinations.tsx | 1 - .../FileContent/FileContent.tsx | 22 ++++++++++++------- 3 files changed, 22 insertions(+), 11 deletions(-) diff --git a/web/src/AppContext.tsx b/web/src/AppContext.tsx index 763b1ee5e..d11cbebce 100644 --- a/web/src/AppContext.tsx +++ b/web/src/AppContext.tsx @@ -1,4 +1,5 @@ -import React, { useState, useContext, useEffect } from 'react' +import React, { useState, useContext, useEffect, useMemo } from 'react' +import { matchPath } from 'react-router-dom' import { noop } from 'lodash-es' import { useGet } from 'restful-react' import type { AppProps } from 'AppProps' @@ -33,8 +34,13 @@ export const AppContextProvider: React.FC<{ value: AppProps }> = React.memo(func value: initialValue, children }) { + const lazy = useMemo( + () => initialValue.standalone && !!matchPath(location.pathname, { path: '/(signin|register)' }), + [initialValue.standalone] + ) const { data: currentUser = defaultCurrentUser } = useGet({ - path: '/api/v1/user' + path: '/api/v1/user', + lazy }) const [appStates, setAppStates] = useState(initialValue) diff --git a/web/src/RouteDestinations.tsx b/web/src/RouteDestinations.tsx index a30325981..d7cef09bb 100644 --- a/web/src/RouteDestinations.tsx +++ b/web/src/RouteDestinations.tsx @@ -36,7 +36,6 @@ import NewPipeline from 'pages/NewPipeline/NewPipeline' export const RouteDestinations: React.FC = React.memo(function RouteDestinations() { const { getString } = useStrings() const repoPath = `${pathProps.space}/${pathProps.repoName}` - const { OPEN_SOURCE_PIPELINES, OPEN_SOURCE_SECRETS } = useFeatureFlag() return ( diff --git a/web/src/pages/Repository/RepositoryContent/FileContent/FileContent.tsx b/web/src/pages/Repository/RepositoryContent/FileContent/FileContent.tsx index 08a065234..7eb3c4cdd 100644 --- a/web/src/pages/Repository/RepositoryContent/FileContent/FileContent.tsx +++ b/web/src/pages/Repository/RepositoryContent/FileContent/FileContent.tsx @@ -31,6 +31,7 @@ import { useAppContext } from 'AppContext' import { LatestCommitForFile } from 'components/LatestCommit/LatestCommit' import { useCommitModal } from 'components/CommitModalButton/CommitModalButton' import { useStrings } from 'framework/strings' +import { getConfig } from 'services/config' import { OptionsMenuButton } from 'components/OptionsMenuButton/OptionsMenuButton' import { PlainButton } from 'components/PlainButton/PlainButton' import { CommitsView } from 'components/CommitsView/CommitsView' @@ -57,7 +58,6 @@ export function FileContent({ useFileContentViewerDecision({ repoMetadata, gitRef, resourcePath, resourceContent }) const history = useHistory() const [activeTab, setActiveTab] = React.useState(FileSection.CONTENT) - const content = useMemo( () => decodeGitContent((resourceContent?.content as RepoFileContent)?.data), [resourceContent?.content] @@ -199,13 +199,18 @@ export function FileContent({ style={{ padding: '5px' }} width="145px" items={[ - // { - // hasIcon: true, - // iconName: 'arrow-right', - // text: getString('viewRaw'), - // onClick: () => window.open(rawURL, '_blank') // TODO: This is still not working due to token is not stored in cookies - // }, - // '-', + { + hasIcon: true, + iconName: 'arrow-right', + text: getString('viewRaw'), + onClick: () => { + const url = standalone + ? rawURL.replace(/^\/code/, '') + : getConfig(rawURL).replace('//', '/') + window.open(url, '_blank') + } + }, + '-', { hasIcon: true, iconName: 'cloud-download', @@ -243,6 +248,7 @@ export function FileContent({
+ rawURL: {rawURL} { From 4b3b79c9566520a38a489f5f6bab888ff2643cd4 Mon Sep 17 00:00:00 2001 From: Calvin Lee Date: Fri, 8 Sep 2023 16:25:18 +0000 Subject: [PATCH 11/17] fix: [code-828]: remove second api call (#414) --- web/src/pages/Home/Home.tsx | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/web/src/pages/Home/Home.tsx b/web/src/pages/Home/Home.tsx index 2ce141d40..6b091a72d 100644 --- a/web/src/pages/Home/Home.tsx +++ b/web/src/pages/Home/Home.tsx @@ -9,6 +9,7 @@ import { useStrings } from 'framework/strings' import { LoadingSpinner } from 'components/LoadingSpinner/LoadingSpinner' import { useAppContext } from 'AppContext' import { useGetRepositoryMetadata } from 'hooks/useGetRepositoryMetadata' +import { voidFn } from 'utils/Utils' import { NewSpaceModalButton } from 'components/NewSpaceModalButton/NewSpaceModalButton' import css from './Home.module.scss' @@ -22,9 +23,7 @@ export default function Home() { const { space } = useGetRepositoryMetadata() const spaces = [] - const { refetch } = useGet({ - path: '/api/v1/user/memberships' - }) + const NewSpaceButton = ( {})} handleNavigation={spaceName => { history.push(routes.toCODERepositories({ space: spaceName })) }} From 6c0b219176eee721260847fc0e88409bceec93c2 Mon Sep 17 00:00:00 2001 From: Dan Wilson Date: Fri, 8 Sep 2023 18:44:59 +0100 Subject: [PATCH 12/17] add string --- web/src/i18n/strings.en.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/web/src/i18n/strings.en.yaml b/web/src/i18n/strings.en.yaml index 71a6aba31..4f0c5d966 100644 --- a/web/src/i18n/strings.en.yaml +++ b/web/src/i18n/strings.en.yaml @@ -632,6 +632,7 @@ executions: name: Execution Name description: Description time: Time + completedTime: completed {{timeString}} ago selectRange: Shift-click to select a range allCommits: All Commits secrets: From 4b36713c0d99eaf3fa16793c37e120060c6d2e55 Mon Sep 17 00:00:00 2001 From: Tan Nhu Date: Fri, 8 Sep 2023 18:44:52 +0000 Subject: [PATCH 13/17] Update Monaco lib / Fix Git Blame font (#417) --- web/package.json | 34 +--- web/src/pages/Home/Home.tsx | 9 +- .../FileContent/GitBlame.module.scss | 57 +++--- .../FileContent/GitBlame.tsx | 4 +- web/src/utils/utils.scss | 4 +- web/yarn.lock | 192 +++--------------- 6 files changed, 63 insertions(+), 237 deletions(-) diff --git a/web/package.json b/web/package.json index aff42ed68..f9ecef4cf 100644 --- a/web/package.json +++ b/web/package.json @@ -61,7 +61,7 @@ "marked": "^4.0.12", "masonry-layout": "^4.2.2", "moment": "^2.25.3", - "monaco-editor": "^0.37.1", + "monaco-editor": "^0.40.0", "monaco-editor-webpack-plugin": "^7.0.1", "qs": "^6.9.4", "react": "^17.0.2", @@ -71,7 +71,7 @@ "react-intersection-observer": "^9.4.1", "react-jsx-match": "^1.1.5", "react-keywords": "^0.0.5", - "react-monaco-editor": "^0.52.0", + "react-monaco-editor": "^0.54.0", "react-pdf": "^7.1.2", "react-resize-detector": "^7.1.2", "react-router-dom": "^5.2.1", @@ -129,7 +129,6 @@ "identity-obj-proxy": "^3.0.0", "jest": "^26.2.0", "js-yaml": "^4.1.0", - "lint-staged": "^11.0.0", "lodash": "^4.17.21", "mini-css-extract-plugin": "^2.4.2", "mustache": "^4.0.1", @@ -156,34 +155,5 @@ }, "engines": { "node": ">=14.16.0" - }, - "lint-staged": { - "*.{ts,tsx}": [ - "eslint --rulesdir ./scripts/eslint-rules --resolve-plugins-relative-to", - "sh scripts/typecheck-staged.sh", - "prettier --check" - ], - "*.scss": [ - "prettier --check" - ], - "strings.*.yaml": [ - "npm strings:check" - ], - "webpack.devServerProxy.config.js": [ - "exit 1" - ] - }, - "i18nSettings": { - "extensionToLanguageMap": { - "es": [ - "es" - ], - "en": [ - "en", - "en-US", - "en-IN", - "en-UK" - ] - } } } diff --git a/web/src/pages/Home/Home.tsx b/web/src/pages/Home/Home.tsx index 6b091a72d..c9f577cc2 100644 --- a/web/src/pages/Home/Home.tsx +++ b/web/src/pages/Home/Home.tsx @@ -1,15 +1,12 @@ import React from 'react' import { ButtonVariation, Container, Layout, PageBody, Text } from '@harnessio/uicore' import { FontVariation } from '@harnessio/design-system' -import { useGet } from 'restful-react' +import { noop } from 'lodash-es' import { useHistory } from 'react-router-dom' -// import type { TypesSpace } from 'services/code' import { useStrings } from 'framework/strings' -// import { usePageIndex } from 'hooks/usePageIndex' import { LoadingSpinner } from 'components/LoadingSpinner/LoadingSpinner' import { useAppContext } from 'AppContext' import { useGetRepositoryMetadata } from 'hooks/useGetRepositoryMetadata' -import { voidFn } from 'utils/Utils' import { NewSpaceModalButton } from 'components/NewSpaceModalButton/NewSpaceModalButton' import css from './Home.module.scss' @@ -17,8 +14,6 @@ export default function Home() { const { getString } = useStrings() const history = useHistory() const { routes } = useAppContext() - // const [searchTerm, setSearchTerm] = useState('') - // const [page, setPage] = usePageIndex(1) const { currentUser } = useAppContext() const { space } = useGetRepositoryMetadata() @@ -33,7 +28,7 @@ export default function Home() { icon="plus" width={173} height={48} - onRefetch={voidFn(() => {})} + onRefetch={noop} handleNavigation={spaceName => { history.push(routes.toCODERepositories({ space: spaceName })) }} diff --git a/web/src/pages/Repository/RepositoryContent/FileContent/GitBlame.module.scss b/web/src/pages/Repository/RepositoryContent/FileContent/GitBlame.module.scss index 5c272e27d..d4153a958 100644 --- a/web/src/pages/Repository/RepositoryContent/FileContent/GitBlame.module.scss +++ b/web/src/pages/Repository/RepositoryContent/FileContent/GitBlame.module.scss @@ -1,45 +1,42 @@ @import 'src/utils/utils'; .main { - flex-grow: 1; + --code-editor-border-color: var(--grey-200); - :global { - .cm-editor { - border: none !important; + .codeViewer { + flex-grow: 1; - .cm-scroller { - padding: 0 !important; + :global { + .cm-gutter { + background-color: var(--grey-50); } - .cm-content { - padding: 0; - } + .cm-editor { + border: none !important; - .cm-line { - &, - * { - @include mono-font; + .cm-scroller { + padding: 0 !important; + + .cm-line { + &, + * { + @include mono-font; + } + } + } + + .cm-content { + padding: 0; } } } - } - .lineNo { - min-width: 70px !important; - text-align: right; - padding-right: 10px; - height: 100%; - color: var(--grey-400); - } -} - -.gitBlame { - --code-editor-border-color: var(--grey-200); - // padding: 0 var(--spacing-xlarge) 0 var(--spacing-small) !important; - - :global { - .cm-gutter { - background-color: var(--grey-50); + .lineNo { + min-width: 70px !important; + text-align: right; + padding-right: 10px; + height: 100%; + color: var(--grey-400); } } diff --git a/web/src/pages/Repository/RepositoryContent/FileContent/GitBlame.tsx b/web/src/pages/Repository/RepositoryContent/FileContent/GitBlame.tsx index e83d37673..f9439006e 100644 --- a/web/src/pages/Repository/RepositoryContent/FileContent/GitBlame.tsx +++ b/web/src/pages/Repository/RepositoryContent/FileContent/GitBlame.tsx @@ -148,7 +148,7 @@ export const GitBlame: React.FC + {Object.values(blameBlocks).map(blameInfo => ( @@ -246,7 +246,7 @@ const GitBlameRenderer = React.memo(function GitBlameSourceViewer({ filename={filename} content={source} readonly={true} - className={css.main} + className={css.codeViewer} onViewUpdate={onViewUpdate} extensions={extensions.of([])} maxHeight="auto" diff --git a/web/src/utils/utils.scss b/web/src/utils/utils.scss index dd1093631..07ce2d680 100644 --- a/web/src/utils/utils.scss +++ b/web/src/utils/utils.scss @@ -2,9 +2,11 @@ $code-editor-font-family: ui-monospace, 'Cascadia Code', 'Source Code Pro', Menl monospace; @mixin mono-font { - font-family: var(--font-family-mono) !important; + font-family: Menlo, Monaco, 'Courier New', monospace var(--font-family-mono) !important; + font-weight: normal !important; font-size: 13px !important; font-feature-settings: 'liga' 0, 'calt' 0; + font-variation-settings: normal; line-height: 18px; letter-spacing: 0px; } diff --git a/web/yarn.lock b/web/yarn.lock index ada8c1cd1..b28574b68 100644 --- a/web/yarn.lock +++ b/web/yarn.lock @@ -2214,14 +2214,6 @@ agent-base@6: dependencies: debug "4" -aggregate-error@^3.0.0: - version "3.1.0" - resolved "https://registry.yarnpkg.com/aggregate-error/-/aggregate-error-3.1.0.tgz#92670ff50f5359bdb7a3e0d40d0ec30c5737687a" - integrity sha512-4I7Td01quW/RpocfNayFdFVk1qSuoh0E7JrbRJ16nH01HhKFQ88INq9Sd+nd72zqRySlr9BmDA8xlEJ6vJMrYA== - dependencies: - clean-stack "^2.0.0" - indent-string "^4.0.0" - ajv-formats@^2.1.1: version "2.1.1" resolved "https://registry.yarnpkg.com/ajv-formats/-/ajv-formats-2.1.1.tgz#6e669400659eb74973bbf2e33327180a0996b520" @@ -2288,7 +2280,7 @@ ansi-colors@^4.1.1: resolved "https://registry.yarnpkg.com/ansi-colors/-/ansi-colors-4.1.3.tgz#37611340eb2243e70cc604cad35d63270d48781b" integrity sha512-/6w/C21Pm1A7aZitlI5Ni/2J6FFQN8i1Cvz3kHABAAbw93v/NlvKdVOqz7CCWz/3iv/JplRSEEZ83XION15ovw== -ansi-escapes@^4.2.1, ansi-escapes@^4.3.0: +ansi-escapes@^4.2.1: version "4.3.2" resolved "https://registry.yarnpkg.com/ansi-escapes/-/ansi-escapes-4.3.2.tgz#6b2291d1db7d98b6521d5f1efa42d0f3a9feb65e" integrity sha512-gKXj5ALrKWQLsYG9jlTRmR/xKluxHV+Z9QEwNIgCfM1/uwPMCuzVVnh5mwTd+OuBZcwSIMbqssNWRm1lE51QaQ== @@ -2989,11 +2981,6 @@ clean-css@^5.2.2: dependencies: source-map "~0.6.0" -clean-stack@^2.0.0: - version "2.2.0" - resolved "https://registry.yarnpkg.com/clean-stack/-/clean-stack-2.2.0.tgz#ee8472dbb129e727b31e8a10a427dee9dfe4008b" - integrity sha512-4diC9HaTE+KRAMWhDhrGOECgWZxoevMc5TlkObMqNSsVU62PYzXZ/SMTjzyGAFF1YusgxGcSWTEXBhp0CPwQ1A== - cli-boxes@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/cli-boxes/-/cli-boxes-1.0.0.tgz#4fa917c3e59c94a004cd61f8ee509da651687143" @@ -3006,14 +2993,6 @@ cli-cursor@^3.1.0: dependencies: restore-cursor "^3.1.0" -cli-truncate@2.1.0, cli-truncate@^2.1.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/cli-truncate/-/cli-truncate-2.1.0.tgz#c39e28bf05edcde5be3b98992a22deed5a2b93c7" - integrity sha512-n8fOixwDD6b/ObinzTrp1ZKFzbgvKZvuz/TvejnLn1aQfC6r52XEx85FmuC+3HI+JM7coBRXUvNqEU2PHVrHpg== - dependencies: - slice-ansi "^3.0.0" - string-width "^4.2.0" - cli-width@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/cli-width/-/cli-width-3.0.0.tgz#a2f48437a2caa9a22436e794bf071ec9e61cedf6" @@ -3122,12 +3101,7 @@ color-name@~1.1.4: resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.4.tgz#c2a09a87acbde69543de6f63fa3995c826c536a2" integrity sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA== -colorette@^1.4.0: - version "1.4.0" - resolved "https://registry.yarnpkg.com/colorette/-/colorette-1.4.0.tgz#5190fbb87276259a86ad700bff2c6d6faa3fca40" - integrity sha512-Y2oEozpomLn7Q3HFP7dpww7AtMJplbM9lGZP6RDfHqmbeRjiwRg4n6VM6j4KLmRke85uWEI7JqF17f3pqdRA0g== - -colorette@^2.0.10, colorette@^2.0.14, colorette@^2.0.16: +colorette@^2.0.10, colorette@^2.0.14: version "2.0.20" resolved "https://registry.yarnpkg.com/colorette/-/colorette-2.0.20.tgz#9eb793e6833067f7235902fcd3b09917a000a95a" integrity sha512-IfEDxwoWIjkeXL1eXcDiow4UbKjhLdq6/EuSVR9GMN7KVH3r9gQ83e73hsz1Nd1T3ijd5xv1wcWRYO+D6kCI2w== @@ -3169,7 +3143,7 @@ commander@^4.1.0: resolved "https://registry.yarnpkg.com/commander/-/commander-4.1.1.tgz#9fd602bd936294e9e9ef46a3f4d6964044b18068" integrity sha512-NOKm8xhkzAjzFx8B2v5OAHT+u5pRQc2UCa2Vq9jYL/31o2wi9mxBA7LIFs3sV5VSC49z6pEhfbMULvShKj26WA== -commander@^8.2.0, commander@^8.3.0: +commander@^8.3.0: version "8.3.0" resolved "https://registry.yarnpkg.com/commander/-/commander-8.3.0.tgz#4837ea1b2da67b9c616a67afbb0fafee567bca66" integrity sha512-OkTL9umf+He2DZkUq8f8J9of7yL6RJKI24dVITBmNfZBmri9zYZQrKkuXiKhyfPSu8tUhnVBB1iKXevvnlR4Ww== @@ -3284,17 +3258,6 @@ cosmiconfig@^6.0.0: path-type "^4.0.0" yaml "^1.7.2" -cosmiconfig@^7.0.1: - version "7.1.0" - resolved "https://registry.yarnpkg.com/cosmiconfig/-/cosmiconfig-7.1.0.tgz#1443b9afa596b670082ea46cbd8f6a62b84635f6" - integrity sha512-AdmX6xUzdNASswsFtmwSt7Vj8po9IuqXm0UXz7QKPuEUmPB4XyjGfaAr2PSuELMwkRMVH1EpIkX5bTZGRB3eCA== - dependencies: - "@types/parse-json" "^4.0.0" - import-fresh "^3.2.1" - parse-json "^5.0.0" - path-type "^4.0.0" - yaml "^1.10.0" - create-error-class@^3.0.0: version "3.0.2" resolved "https://registry.yarnpkg.com/create-error-class/-/create-error-class-3.0.2.tgz#06be7abef947a3f14a30fd610671d401bca8b7b6" @@ -3431,7 +3394,7 @@ debug@2.6.9, debug@^2.2.0, debug@^2.3.3: dependencies: ms "2.0.0" -debug@4, debug@^4.0.0, debug@^4.0.1, debug@^4.1.0, debug@^4.1.1, debug@^4.3.1, debug@^4.3.2, debug@^4.3.4: +debug@4, debug@^4.0.0, debug@^4.0.1, debug@^4.1.0, debug@^4.1.1, debug@^4.3.1, debug@^4.3.4: version "4.3.4" resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.4.tgz#1319f6579357f2338d3337d2cdd4914bb5dcc865" integrity sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ== @@ -3821,7 +3784,7 @@ enhanced-resolve@^5.0.0, enhanced-resolve@^5.13.0, enhanced-resolve@^5.7.0: graceful-fs "^4.2.4" tapable "^2.2.0" -enquirer@^2.3.5, enquirer@^2.3.6: +enquirer@^2.3.5: version "2.3.6" resolved "https://registry.yarnpkg.com/enquirer/-/enquirer-2.3.6.tgz#2a7fe5dd634a1e4125a975ec994ff5456dc3734d" integrity sha512-yjNnPr315/FjS4zIsUxYguYUPP2e1NK4d7E7ZOLiyYCcbFBiTMyID+2wvm2w6+pZ/odMA7cRkjhsPbltwBOrLg== @@ -4255,7 +4218,7 @@ execa@^4.0.0: signal-exit "^3.0.2" strip-final-newline "^2.0.0" -execa@^5.0.0, execa@^5.1.1: +execa@^5.0.0: version "5.1.1" resolved "https://registry.yarnpkg.com/execa/-/execa-5.1.1.tgz#f80ad9cbf4298f7bd1d4c9555c21e93741c411dd" integrity sha512-8uSpZZocAZRBAPIEINJj3Lo9HyGitllczc27Eh5YYojjMFMn8yHMDMaUHE2Jqfq05D/wucwI4JGURyXt1vchyg== @@ -4740,11 +4703,6 @@ get-intrinsic@^1.0.2, get-intrinsic@^1.1.1, get-intrinsic@^1.1.3, get-intrinsic@ has "^1.0.3" has-symbols "^1.0.3" -get-own-enumerable-property-symbols@^3.0.0: - version "3.0.2" - resolved "https://registry.yarnpkg.com/get-own-enumerable-property-symbols/-/get-own-enumerable-property-symbols-3.0.2.tgz#b5fde77f22cbe35f390b4e089922c50bce6ef664" - integrity sha512-I0UBV/XOz1XkIJHEUDMZAbzCThU/H8DxmSfmdGcKPnVhu2VfFqr34jr9777IyaTYvxjedWhqVIilEDsCdP5G6g== - get-package-type@^0.1.0: version "0.1.0" resolved "https://registry.yarnpkg.com/get-package-type/-/get-package-type-0.1.0.tgz#8de2d803cff44df3bc6c456e6668b36c3926e11a" @@ -5806,7 +5764,7 @@ is-number@^7.0.0: resolved "https://registry.yarnpkg.com/is-number/-/is-number-7.0.0.tgz#7535345b896734d5f80c4d06c50955527a14f12b" integrity sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng== -is-obj@^1.0.0, is-obj@^1.0.1: +is-obj@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/is-obj/-/is-obj-1.0.1.tgz#3e4729ac1f5fde025cd7d83a896dab9f4f67db0f" integrity sha512-l4RyHgRqGN4Y3+9JHVrNqO+tN0rV5My76uW5/nuO4K1b6vw5G8d/cmFjP9tRfEsdhZNt0IFdZuK/c2Vr4Nb+Qg== @@ -5853,11 +5811,6 @@ is-regex@^1.0.4, is-regex@^1.1.4: call-bind "^1.0.2" has-tostringtag "^1.0.0" -is-regexp@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/is-regexp/-/is-regexp-1.0.0.tgz#fd2d883545c46bac5a633e7b9a09e87fa2cb5069" - integrity sha512-7zjFAPO4/gwyQAAgRRmqeEeyIICSdmCqa3tsVHMdBzaXXRiqopZL4Cyghg/XulGWrtABTpbnYYzzIRffLkP4oA== - is-retry-allowed@^1.0.0: version "1.2.0" resolved "https://registry.yarnpkg.com/is-retry-allowed/-/is-retry-allowed-1.2.0.tgz#d778488bd0a4666a3be8a1482b9f2baafedea8b4" @@ -6746,40 +6699,6 @@ lines-and-columns@^1.1.6: resolved "https://registry.yarnpkg.com/lines-and-columns/-/lines-and-columns-1.2.4.tgz#eca284f75d2965079309dc0ad9255abb2ebc1632" integrity sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg== -lint-staged@^11.0.0: - version "11.2.6" - resolved "https://registry.yarnpkg.com/lint-staged/-/lint-staged-11.2.6.tgz#f477b1af0294db054e5937f171679df63baa4c43" - integrity sha512-Vti55pUnpvPE0J9936lKl0ngVeTdSZpEdTNhASbkaWX7J5R9OEifo1INBGQuGW4zmy6OG+TcWPJ3m5yuy5Q8Tg== - dependencies: - cli-truncate "2.1.0" - colorette "^1.4.0" - commander "^8.2.0" - cosmiconfig "^7.0.1" - debug "^4.3.2" - enquirer "^2.3.6" - execa "^5.1.1" - listr2 "^3.12.2" - micromatch "^4.0.4" - normalize-path "^3.0.0" - please-upgrade-node "^3.2.0" - string-argv "0.3.1" - stringify-object "3.3.0" - supports-color "8.1.1" - -listr2@^3.12.2: - version "3.14.0" - resolved "https://registry.yarnpkg.com/listr2/-/listr2-3.14.0.tgz#23101cc62e1375fd5836b248276d1d2b51fdbe9e" - integrity sha512-TyWI8G99GX9GjE54cJ+RrNMcIFBfwMPxc3XTFiAYGN4s10hWROGtOg7+O6u6LE3mNkyld7RSLE6nrKBvTfcs3g== - dependencies: - cli-truncate "^2.1.0" - colorette "^2.0.16" - log-update "^4.0.0" - p-map "^4.0.0" - rfdc "^1.3.0" - rxjs "^7.5.1" - through "^2.3.8" - wrap-ansi "^7.0.0" - load-json-file@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/load-json-file/-/load-json-file-4.0.0.tgz#2f5f45ab91e33216234fd53adab668eb4ec0993b" @@ -6848,16 +6767,6 @@ lodash@4.x, lodash@^4.17.15, lodash@^4.17.19, lodash@^4.17.20, lodash@^4.17.21, resolved "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c" integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg== -log-update@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/log-update/-/log-update-4.0.0.tgz#589ecd352471f2a1c0c570287543a64dfd20e0a1" - integrity sha512-9fkkDevMefjg0mmzWFBW8YkFP91OrizzkW3diF7CpG+S2EYdy4+TVfGwz1zeF8x7hCx1ovSPTOE9Ngib74qqUg== - dependencies: - ansi-escapes "^4.3.0" - cli-cursor "^3.1.0" - slice-ansi "^4.0.0" - wrap-ansi "^6.2.0" - longest-streak@^3.0.0: version "3.1.0" resolved "https://registry.yarnpkg.com/longest-streak/-/longest-streak-3.1.0.tgz#62fa67cd958742a1574af9f39866364102d90cd4" @@ -7563,10 +7472,10 @@ monaco-editor-webpack-plugin@^7.0.1: dependencies: loader-utils "^2.0.2" -monaco-editor@^0.37.1: - version "0.37.1" - resolved "https://registry.yarnpkg.com/monaco-editor/-/monaco-editor-0.37.1.tgz#d6f5ffb593e019e74e19bf8a2bdef5a691876f4e" - integrity sha512-jLXEEYSbqMkT/FuJLBZAVWGuhIb4JNwHE9kPTorAVmsdZ4UzHAfgWxLsVtD7pLRFaOwYPhNG9nUCpmFL1t/dIg== +monaco-editor@^0.40.0: + version "0.40.0" + resolved "https://registry.npmjs.org/monaco-editor/-/monaco-editor-0.40.0.tgz#d10834e15ad50a15ec61fd01892e508464ebe2fe" + integrity sha512-1wymccLEuFSMBvCk/jT1YDW/GuxMLYwnFwF9CDyYCxoTw2Pt379J3FUhwy9c43j51JdcxVPjwk0jm0EVDsBS2g== mri@^1.1.0: version "1.2.0" @@ -8071,13 +7980,6 @@ p-locate@^4.1.0: dependencies: p-limit "^2.2.0" -p-map@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/p-map/-/p-map-4.0.0.tgz#bb2f95a5eda2ec168ec9274e06a747c3e2904d2b" - integrity sha512-/bjOqmgETBYB5BoEeGVea8dmvHb2m9GLy1E9W43yeyfP6QQCZGFNa+XRceJEuDB6zqr+gKpIAmlLebMpykw/MQ== - dependencies: - aggregate-error "^3.0.0" - p-retry@^4.5.0: version "4.6.2" resolved "https://registry.yarnpkg.com/p-retry/-/p-retry-4.6.2.tgz#9baae7184057edd4e17231cee04264106e092a16" @@ -8304,13 +8206,6 @@ pkg-dir@^4.2.0: dependencies: find-up "^4.0.0" -please-upgrade-node@^3.2.0: - version "3.2.0" - resolved "https://registry.yarnpkg.com/please-upgrade-node/-/please-upgrade-node-3.2.0.tgz#aeddd3f994c933e4ad98b99d9a556efa0e2fe942" - integrity sha512-gQR3WpIgNIKwBMVLkpMUeR3e1/E1y42bqDQZfql+kDeXd8COYfM8PQA4X6y7a8u9Ua9FHmsrrmirW2vHs45hWg== - dependencies: - semver-compare "^1.0.0" - popper.js@^1.14.4, popper.js@^1.15.0: version "1.16.1" resolved "https://registry.yarnpkg.com/popper.js/-/popper.js-1.16.1.tgz#2a223cb3dc7b6213d740e40372be40de43e65b1b" @@ -8662,13 +8557,20 @@ react-markdown@~8.0.0: unist-util-visit "^4.0.0" vfile "^5.0.0" -react-monaco-editor@*, react-monaco-editor@^0.52.0: +react-monaco-editor@*: version "0.52.0" resolved "https://registry.yarnpkg.com/react-monaco-editor/-/react-monaco-editor-0.52.0.tgz#69a7c450a22830064d2fc1b446674b1b7da0f540" integrity sha512-jhQSekf2JABbcpN45gKZlWfTS0QcBOZAbAWKGyfqy/KEcMXTwJpCOYGcn2Ur11SUUxWJUzhKjE2fx9BGBc5S8g== dependencies: prop-types "^15.8.1" +react-monaco-editor@^0.54.0: + version "0.54.0" + resolved "https://registry.npmjs.org/react-monaco-editor/-/react-monaco-editor-0.54.0.tgz#ec9293249a991b08264be723c1ec0ca3a6d480d8" + integrity sha512-9JwO69851mfpuhYLHlKbae7omQWJ/2ICE2lbL0VHyNyZR8rCOH7440u+zAtDgiOMpLwmYdY1sEZCdRefywX6GQ== + dependencies: + prop-types "^15.8.1" + react-pdf@^7.1.2: version "7.1.2" resolved "https://registry.yarnpkg.com/react-pdf/-/react-pdf-7.1.2.tgz#c6979cff9ac09c3e5ab7ea9e0182f79a499768e5" @@ -9245,11 +9147,6 @@ reusify@^1.0.4: resolved "https://registry.yarnpkg.com/reusify/-/reusify-1.0.4.tgz#90da382b1e126efc02146e90845a88db12925d76" integrity sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw== -rfdc@^1.3.0: - version "1.3.0" - resolved "https://registry.yarnpkg.com/rfdc/-/rfdc-1.3.0.tgz#d0b7c441ab2720d05dc4cf26e01c89631d9da08b" - integrity sha512-V2hovdzFbOi77/WajaSMXk2OLm+xNIeQdMMuB7icj7bk6zi2F8GGAxigcnDFpJHbNyNcgyJDiP+8nOrY5cZGrA== - rimraf@^3.0.0, rimraf@^3.0.2: version "3.0.2" resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-3.0.2.tgz#f1a5402ba6220ad52cc1282bac1ae3aa49fd061a" @@ -9281,13 +9178,6 @@ rxjs@^6.6.0: dependencies: tslib "^1.9.0" -rxjs@^7.5.1: - version "7.8.0" - resolved "https://registry.yarnpkg.com/rxjs/-/rxjs-7.8.0.tgz#90a938862a82888ff4c7359811a595e14e1e09a4" - integrity sha512-F2+gxDshqmIub1KdvZkaEfGDwLNpPvk9Fs6LD/MyQxNgMds/WH9OdDDXOmxUZpME+iSK3rQCctkL0DYyytUqMg== - dependencies: - tslib "^2.1.0" - sade@^1.7.3: version "1.8.1" resolved "https://registry.yarnpkg.com/sade/-/sade-1.8.1.tgz#0a78e81d658d394887be57d2a409bf703a3b2701" @@ -9413,11 +9303,6 @@ selfsigned@^2.1.1: dependencies: node-forge "^1" -semver-compare@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/semver-compare/-/semver-compare-1.0.0.tgz#0dee216a1c941ab37e9efb1788f6afc5ff5537fc" - integrity sha512-YM3/ITh2MJ5MtzaM429anh+x2jiLVjqILF4m4oyQB18W7Ggea7BfqdH/wGMK7dDiMghv/6WG7znWMwUDzJiXow== - semver-diff@^2.0.0: version "2.1.0" resolved "https://registry.yarnpkg.com/semver-diff/-/semver-diff-2.1.0.tgz#4bbb8437c8d37e4b0cf1a68fd726ec6d645d6d36" @@ -9630,15 +9515,6 @@ slash@^3.0.0: resolved "https://registry.yarnpkg.com/slash/-/slash-3.0.0.tgz#6539be870c165adbd5240220dbe361f1bc4d4634" integrity sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q== -slice-ansi@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/slice-ansi/-/slice-ansi-3.0.0.tgz#31ddc10930a1b7e0b67b08c96c2f49b77a789787" - integrity sha512-pSyv7bSTC7ig9Dcgbw9AuRNUb5k5V6oDudjZoMBSr13qpLBG7tB+zgCkARjq7xIUgdz5P1Qe8u+rSGdouOOIyQ== - dependencies: - ansi-styles "^4.0.0" - astral-regex "^2.0.0" - is-fullwidth-code-point "^3.0.0" - slice-ansi@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/slice-ansi/-/slice-ansi-4.0.0.tgz#500e8dd0fd55b05815086255b3195adf2a45fe6b" @@ -9844,11 +9720,6 @@ stop-iteration-iterator@^1.0.0: dependencies: internal-slot "^1.0.4" -string-argv@0.3.1: - version "0.3.1" - resolved "https://registry.yarnpkg.com/string-argv/-/string-argv-0.3.1.tgz#95e2fbec0427ae19184935f816d74aaa4c5c19da" - integrity sha512-a1uQGz7IyVy9YwhqjZIZu1c8JO8dNIe20xBmSS6qu9kv++k3JGzCVmprbNN5Kn+BgzD5E7YYwg1CcjuJMRNsvg== - string-length@^4.0.1: version "4.0.2" resolved "https://registry.yarnpkg.com/string-length/-/string-length-4.0.2.tgz#a8a8dc7bd5c1a82b9b3c8b87e125f66871b6e57a" @@ -9947,15 +9818,6 @@ string_decoder@~1.1.1: dependencies: safe-buffer "~5.1.0" -stringify-object@3.3.0: - version "3.3.0" - resolved "https://registry.yarnpkg.com/stringify-object/-/stringify-object-3.3.0.tgz#703065aefca19300d3ce88af4f5b3956d7556629" - integrity sha512-rHqiFh1elqCQ9WPLIC8I0Q/g/wj5J1eMkyoiD6eoQApWHP0FtlK7rqnhmabL5VUY9JQCcqwwvlOaSuutekgyrw== - dependencies: - get-own-enumerable-property-symbols "^3.0.0" - is-obj "^1.0.1" - is-regexp "^1.0.0" - strip-ansi@^3.0.0, strip-ansi@^3.0.1: version "3.0.1" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-3.0.1.tgz#6a385fb8853d952d5ff05d0e8aaf94278dc63dcf" @@ -10031,13 +9893,6 @@ style-to-object@^0.4.0: dependencies: inline-style-parser "0.1.1" -supports-color@8.1.1, supports-color@^8.0.0: - version "8.1.1" - resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-8.1.1.tgz#cd6fc17e28500cff56c1b86c0a7fd4a54a73005c" - integrity sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q== - dependencies: - has-flag "^4.0.0" - supports-color@^5.3.0: version "5.5.0" resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-5.5.0.tgz#e2e69a44ac8772f78a1ec0b35b689df6530efc8f" @@ -10052,6 +9907,13 @@ supports-color@^7.0.0, supports-color@^7.1.0: dependencies: has-flag "^4.0.0" +supports-color@^8.0.0: + version "8.1.1" + resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-8.1.1.tgz#cd6fc17e28500cff56c1b86c0a7fd4a54a73005c" + integrity sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q== + dependencies: + has-flag "^4.0.0" + supports-hyperlinks@^2.0.0: version "2.3.0" resolved "https://registry.yarnpkg.com/supports-hyperlinks/-/supports-hyperlinks-2.3.0.tgz#3943544347c1ff90b15effb03fc14ae45ec10624" @@ -10168,7 +10030,7 @@ throat@^5.0.0: resolved "https://registry.yarnpkg.com/throat/-/throat-5.0.0.tgz#c5199235803aad18754a667d659b5e72ce16764b" integrity sha512-fcwX4mndzpLQKBS1DVYhGAcYaYt7vsHNIvQV+WXMvnow5cgjPphq5CaayLaGsjRdSCKZFNGt7/GYAuXaNOiYCA== -through@^2.3.6, through@^2.3.8: +through@^2.3.6: version "2.3.8" resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" integrity sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg== From 32888b07ce19831b7c831cd2ecb13a4897feb17a Mon Sep 17 00:00:00 2001 From: Calvin Lee Date: Fri, 8 Sep 2023 23:48:29 +0000 Subject: [PATCH 14/17] fix: [code-836]: fix issue where code comments cant be added (#420) --- web/src/components/DiffViewer/DiffViewerUtils.tsx | 3 ++- web/src/pages/Webhooks/Webhooks.tsx | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/web/src/components/DiffViewer/DiffViewerUtils.tsx b/web/src/components/DiffViewer/DiffViewerUtils.tsx index 03ac6d20f..ded5497dd 100644 --- a/web/src/components/DiffViewer/DiffViewerUtils.tsx +++ b/web/src/components/DiffViewer/DiffViewerUtils.tsx @@ -143,7 +143,8 @@ export function getCommentLineInfo( ) { const isSideBySideView = viewStyle === ViewStyle.SIDE_BY_SIDE const { left, lineNumber, filePath } = commentEntry - const filePathBody = contentDOM?.querySelector(`[data="${filePath}"`) + console.log(commentEntry) + const filePathBody = filePath ? contentDOM?.querySelector(`[data="${filePath}"`) : contentDOM const diffBody = filePathBody?.querySelector( `${isSideBySideView ? `.d2h-file-side-diff${left ? '.left' : '.right'} ` : ''}.d2h-diff-tbody` diff --git a/web/src/pages/Webhooks/Webhooks.tsx b/web/src/pages/Webhooks/Webhooks.tsx index ff2f725b0..fc360be03 100644 --- a/web/src/pages/Webhooks/Webhooks.tsx +++ b/web/src/pages/Webhooks/Webhooks.tsx @@ -286,7 +286,7 @@ export default function Webhooks() { showWhen={() => webhooks?.length === 0} forSearch={!!searchTerm} message={getString('webhookEmpty')} - buttonText={getString('createWebhook')} + buttonText={getString('newWebhook')} onButtonClick={() => history.push( routes.toCODEWebhookNew({ From 0d086a1a4de0cfc4c4a22463b60ccb376f095e81 Mon Sep 17 00:00:00 2001 From: Johannes Batzill Date: Fri, 8 Sep 2023 23:52:19 +0000 Subject: [PATCH 15/17] [Events] This PR introduces the Trigger Service for Event Consumption + Some Minor improvements (#419) --- cli/server/config.go | 32 +++--- cmd/gitness/wire.go | 7 +- cmd/gitness/wire_gen.go | 13 ++- events/reader.go | 10 +- .../pullreq/{pullreq.go => service.go} | 0 internal/services/trigger/handler_branch.go | 22 ++++ internal/services/trigger/handler_pullreq.go | 27 +++++ internal/services/trigger/handler_tag.go | 22 ++++ internal/services/trigger/service.go | 103 ++++++++++++++++++ internal/services/trigger/wire.go | 28 +++++ .../webhook/{branch.go => handler_branch.go} | 0 .../{pullreq.go => handler_pullreq.go} | 0 .../webhook/{tag.go => handler_tag.go} | 0 internal/services/webhook/service.go | 18 ++- internal/services/{services.go => wire.go} | 15 +-- types/config.go | 18 +++ 16 files changed, 272 insertions(+), 43 deletions(-) rename internal/services/pullreq/{pullreq.go => service.go} (100%) create mode 100644 internal/services/trigger/handler_branch.go create mode 100644 internal/services/trigger/handler_pullreq.go create mode 100644 internal/services/trigger/handler_tag.go create mode 100644 internal/services/trigger/service.go create mode 100644 internal/services/trigger/wire.go rename internal/services/webhook/{branch.go => handler_branch.go} (100%) rename internal/services/webhook/{pullreq.go => handler_pullreq.go} (100%) rename internal/services/webhook/{tag.go => handler_tag.go} (100%) rename internal/services/{services.go => wire.go} (72%) diff --git a/cli/server/config.go b/cli/server/config.go index b1dea701f..e79512606 100644 --- a/cli/server/config.go +++ b/cli/server/config.go @@ -13,6 +13,7 @@ import ( "github.com/harness/gitness/events" "github.com/harness/gitness/gitrpc" "github.com/harness/gitness/gitrpc/server" + "github.com/harness/gitness/internal/services/trigger" "github.com/harness/gitness/internal/services/webhook" "github.com/harness/gitness/lock" "github.com/harness/gitness/store/database" @@ -147,23 +148,26 @@ func ProvideEventsConfig() (events.Config, error) { return config, nil } -// ProvideWebhookConfig loads the webhook config from the environment. -// It backfills certain config elements if required. -func ProvideWebhookConfig() (webhook.Config, error) { - config := webhook.Config{} - err := envconfig.Process("", &config) - if err != nil { - return webhook.Config{}, fmt.Errorf("failed to load events config: %w", err) +// ProvideWebhookConfig loads the webhook service config from the main config. +func ProvideWebhookConfig(config *types.Config) webhook.Config { + return webhook.Config{ + UserAgentIdentity: config.Webhook.UserAgentIdentity, + HeaderIdentity: config.Webhook.HeaderIdentity, + EventReaderName: config.InstanceID, + Concurrency: config.Webhook.Concurrency, + MaxRetries: config.Webhook.MaxRetries, + AllowPrivateNetwork: config.Webhook.AllowPrivateNetwork, + AllowLoopback: config.Webhook.AllowLoopback, } +} - if config.EventReaderName == "" { - config.EventReaderName, err = getSanitizedMachineName() - if err != nil { - return webhook.Config{}, fmt.Errorf("failed to get sanitized machine name: %w", err) - } +// ProvideTriggerConfig loads the trigger service config from the main config. +func ProvideTriggerConfig(config *types.Config) trigger.Config { + return trigger.Config{ + EventReaderName: config.InstanceID, + Concurrency: config.Webhook.Concurrency, + MaxRetries: config.Webhook.MaxRetries, } - - return config, nil } // ProvideLockConfig generates the `lock` package config from the gitness config. diff --git a/cmd/gitness/wire.go b/cmd/gitness/wire.go index b94c730db..8675dbc8c 100644 --- a/cmd/gitness/wire.go +++ b/cmd/gitness/wire.go @@ -32,7 +32,7 @@ import ( "github.com/harness/gitness/internal/api/controller/space" "github.com/harness/gitness/internal/api/controller/system" "github.com/harness/gitness/internal/api/controller/template" - "github.com/harness/gitness/internal/api/controller/trigger" + controllertrigger "github.com/harness/gitness/internal/api/controller/trigger" "github.com/harness/gitness/internal/api/controller/user" controllerwebhook "github.com/harness/gitness/internal/api/controller/webhook" "github.com/harness/gitness/internal/auth/authn" @@ -54,6 +54,7 @@ import ( "github.com/harness/gitness/internal/services/importer" "github.com/harness/gitness/internal/services/job" pullreqservice "github.com/harness/gitness/internal/services/pullreq" + "github.com/harness/gitness/internal/services/trigger" "github.com/harness/gitness/internal/services/webhook" "github.com/harness/gitness/internal/store" "github.com/harness/gitness/internal/store/cache" @@ -106,6 +107,8 @@ func initSystem(ctx context.Context, config *types.Config) (*cliserver.System, e events.WireSet, cliserver.ProvideWebhookConfig, webhook.WireSet, + cliserver.ProvideTriggerConfig, + trigger.WireSet, githook.WireSet, cliserver.ProvideLockConfig, lock.WireSet, @@ -129,7 +132,7 @@ func initSystem(ctx context.Context, config *types.Config) (*cliserver.System, e eventsstream.WireSet, scheduler.WireSet, commit.WireSet, - trigger.WireSet, + controllertrigger.WireSet, plugin.WireSet, importer.WireSet, ) diff --git a/cmd/gitness/wire_gen.go b/cmd/gitness/wire_gen.go index 7f21e8da1..248028317 100644 --- a/cmd/gitness/wire_gen.go +++ b/cmd/gitness/wire_gen.go @@ -52,6 +52,7 @@ import ( "github.com/harness/gitness/internal/services/importer" "github.com/harness/gitness/internal/services/job" pullreq2 "github.com/harness/gitness/internal/services/pullreq" + trigger2 "github.com/harness/gitness/internal/services/trigger" "github.com/harness/gitness/internal/services/webhook" "github.com/harness/gitness/internal/store" "github.com/harness/gitness/internal/store/cache" @@ -174,10 +175,7 @@ func initSystem(ctx context.Context, config *types.Config) (*server.System, erro } migrator := codecomments.ProvideMigrator(gitrpcInterface) pullreqController := pullreq.ProvideController(db, provider, authorizer, pullReqStore, pullReqActivityStore, codeCommentView, pullReqReviewStore, pullReqReviewerStore, repoStore, principalStore, gitrpcInterface, reporter, mutexManager, migrator) - webhookConfig, err := server.ProvideWebhookConfig() - if err != nil { - return nil, err - } + webhookConfig := server.ProvideWebhookConfig(config) webhookStore := database.ProvideWebhookStore(db) webhookExecutionStore := database.ProvideWebhookExecutionStore(db) readerFactory, err := events4.ProvideReaderFactory(eventsSystem) @@ -236,7 +234,12 @@ func initSystem(ctx context.Context, config *types.Config) (*server.System, erro if err != nil { return nil, err } - servicesServices := services.ProvideServices(webhookService, pullreqService, executor, jobScheduler) + triggerConfig := server.ProvideTriggerConfig(config) + triggerService, err := trigger2.ProvideService(ctx, triggerConfig, readerFactory, eventsReaderFactory) + if err != nil { + return nil, err + } + servicesServices := services.ProvideServices(webhookService, pullreqService, triggerService, jobScheduler) serverSystem := server.NewSystem(bootstrapBootstrap, serverServer, poller, grpcServer, cronManager, servicesServices) return serverSystem, nil } diff --git a/events/reader.go b/events/reader.go index c625c82bb..19fc18ee8 100644 --- a/events/reader.go +++ b/events/reader.go @@ -43,9 +43,9 @@ func (f *ReaderFactory[R]) Launch(ctx context.Context, // setup ctx with copied logger that has extra fields set log := log.Ctx(ctx).With(). - Str("events_category", f.category). - Str("events_group_name", groupName). - Str("events_reader_name", readerName). + Str("events.category", f.category). + Str("events.group_name", groupName). + Str("events.reader_name", readerName). Logger() // create new stream consumer using factory method @@ -191,8 +191,8 @@ func ReaderRegisterEvent[T interface{}](reader *GenericReader, // update ctx with event type for proper logging log := log.Ctx(ctx).With(). - Str("events_type", string(eventType)). - Str("events_id", event.ID). + Str("events.type", string(eventType)). + Str("events.id", event.ID). Logger() ctx = log.WithContext(ctx) diff --git a/internal/services/pullreq/pullreq.go b/internal/services/pullreq/service.go similarity index 100% rename from internal/services/pullreq/pullreq.go rename to internal/services/pullreq/service.go diff --git a/internal/services/trigger/handler_branch.go b/internal/services/trigger/handler_branch.go new file mode 100644 index 000000000..ff9207ef5 --- /dev/null +++ b/internal/services/trigger/handler_branch.go @@ -0,0 +1,22 @@ +// Copyright 2022 Harness Inc. All rights reserved. +// Use of this source code is governed by the Polyform Free Trial License +// that can be found in the LICENSE.md file for this repository. + +package trigger + +import ( + "context" + + "github.com/harness/gitness/events" + gitevents "github.com/harness/gitness/internal/events/git" +) + +func (s *Service) handleEventBranchCreated(ctx context.Context, + event *events.Event[*gitevents.BranchCreatedPayload]) error { + return events.NewDiscardEventErrorf("not implemented") +} + +func (s *Service) handleEventBranchUpdated(ctx context.Context, + event *events.Event[*gitevents.BranchUpdatedPayload]) error { + return events.NewDiscardEventErrorf("not implemented") +} diff --git a/internal/services/trigger/handler_pullreq.go b/internal/services/trigger/handler_pullreq.go new file mode 100644 index 000000000..7f209d85a --- /dev/null +++ b/internal/services/trigger/handler_pullreq.go @@ -0,0 +1,27 @@ +// Copyright 2022 Harness Inc. All rights reserved. +// Use of this source code is governed by the Polyform Free Trial License +// that can be found in the LICENSE.md file for this repository. + +package trigger + +import ( + "context" + + "github.com/harness/gitness/events" + pullreqevents "github.com/harness/gitness/internal/events/pullreq" +) + +func (s *Service) handleEventPullReqCreated(ctx context.Context, + event *events.Event[*pullreqevents.CreatedPayload]) error { + return events.NewDiscardEventErrorf("not implemented") +} + +func (s *Service) handleEventPullReqReopened(ctx context.Context, + event *events.Event[*pullreqevents.ReopenedPayload]) error { + return events.NewDiscardEventErrorf("not implemented") +} + +func (s *Service) handleEventPullReqBranchUpdated(ctx context.Context, + event *events.Event[*pullreqevents.BranchUpdatedPayload]) error { + return events.NewDiscardEventErrorf("not implemented") +} diff --git a/internal/services/trigger/handler_tag.go b/internal/services/trigger/handler_tag.go new file mode 100644 index 000000000..c69c76809 --- /dev/null +++ b/internal/services/trigger/handler_tag.go @@ -0,0 +1,22 @@ +// Copyright 2022 Harness Inc. All rights reserved. +// Use of this source code is governed by the Polyform Free Trial License +// that can be found in the LICENSE.md file for this repository. + +package trigger + +import ( + "context" + + "github.com/harness/gitness/events" + gitevents "github.com/harness/gitness/internal/events/git" +) + +func (s *Service) handleEventTagCreated(ctx context.Context, + event *events.Event[*gitevents.TagCreatedPayload]) error { + return events.NewDiscardEventErrorf("not implemented") +} + +func (s *Service) handleEventTagUpdated(ctx context.Context, + event *events.Event[*gitevents.TagUpdatedPayload]) error { + return events.NewDiscardEventErrorf("not implemented") +} diff --git a/internal/services/trigger/service.go b/internal/services/trigger/service.go new file mode 100644 index 000000000..c2e58a5a6 --- /dev/null +++ b/internal/services/trigger/service.go @@ -0,0 +1,103 @@ +// Copyright 2022 Harness Inc. All rights reserved. +// Use of this source code is governed by the Polyform Free Trial License +// that can be found in the LICENSE.md file for this repository. + +package trigger + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/harness/gitness/events" + gitevents "github.com/harness/gitness/internal/events/git" + pullreqevents "github.com/harness/gitness/internal/events/pullreq" + "github.com/harness/gitness/stream" +) + +const ( + eventsReaderGroupName = "gitness:trigger" +) + +type Config struct { + EventReaderName string + Concurrency int + MaxRetries int +} + +func (c *Config) Prepare() error { + if c == nil { + return errors.New("config is required") + } + if c.EventReaderName == "" { + return errors.New("config.EventReaderName is required") + } + if c.Concurrency < 1 { + return errors.New("config.Concurrency has to be a positive number") + } + if c.MaxRetries < 0 { + return errors.New("config.MaxRetries can't be negative") + } + + return nil +} + +type Service struct{} + +func New( + ctx context.Context, + config Config, + gitReaderFactory *events.ReaderFactory[*gitevents.Reader], + pullreqEvReaderFactory *events.ReaderFactory[*pullreqevents.Reader], +) (*Service, error) { + if err := config.Prepare(); err != nil { + return nil, fmt.Errorf("provided trigger service config is invalid: %w", err) + } + + service := &Service{} + + _, err := gitReaderFactory.Launch(ctx, eventsReaderGroupName, config.EventReaderName, + func(r *gitevents.Reader) error { + const idleTimeout = 1 * time.Minute + r.Configure( + stream.WithConcurrency(config.Concurrency), + stream.WithHandlerOptions( + stream.WithIdleTimeout(idleTimeout), + stream.WithMaxRetries(config.MaxRetries), + )) + + _ = r.RegisterBranchCreated(service.handleEventBranchCreated) + _ = r.RegisterBranchUpdated(service.handleEventBranchUpdated) + + _ = r.RegisterTagCreated(service.handleEventTagCreated) + _ = r.RegisterTagUpdated(service.handleEventTagUpdated) + + return nil + }) + if err != nil { + return nil, fmt.Errorf("failed to launch git events reader: %w", err) + } + + _, err = pullreqEvReaderFactory.Launch(ctx, eventsReaderGroupName, config.EventReaderName, + func(r *pullreqevents.Reader) error { + const idleTimeout = 1 * time.Minute + r.Configure( + stream.WithConcurrency(config.Concurrency), + stream.WithHandlerOptions( + stream.WithIdleTimeout(idleTimeout), + stream.WithMaxRetries(config.MaxRetries), + )) + + _ = r.RegisterCreated(service.handleEventPullReqCreated) + _ = r.RegisterBranchUpdated(service.handleEventPullReqBranchUpdated) + _ = r.RegisterReopened(service.handleEventPullReqReopened) + + return nil + }) + if err != nil { + return nil, fmt.Errorf("failed to launch pr events reader: %w", err) + } + + return service, nil +} diff --git a/internal/services/trigger/wire.go b/internal/services/trigger/wire.go new file mode 100644 index 000000000..fab566b95 --- /dev/null +++ b/internal/services/trigger/wire.go @@ -0,0 +1,28 @@ +// Copyright 2022 Harness Inc. All rights reserved. +// Use of this source code is governed by the Polyform Free Trial License +// that can be found in the LICENSE.md file for this repository. + +package trigger + +import ( + "context" + + "github.com/harness/gitness/events" + gitevents "github.com/harness/gitness/internal/events/git" + pullreqevents "github.com/harness/gitness/internal/events/pullreq" + + "github.com/google/wire" +) + +var WireSet = wire.NewSet( + ProvideService, +) + +func ProvideService( + ctx context.Context, + config Config, + gitReaderFactory *events.ReaderFactory[*gitevents.Reader], + pullReqEvFactory *events.ReaderFactory[*pullreqevents.Reader], +) (*Service, error) { + return New(ctx, config, gitReaderFactory, pullReqEvFactory) +} diff --git a/internal/services/webhook/branch.go b/internal/services/webhook/handler_branch.go similarity index 100% rename from internal/services/webhook/branch.go rename to internal/services/webhook/handler_branch.go diff --git a/internal/services/webhook/pullreq.go b/internal/services/webhook/handler_pullreq.go similarity index 100% rename from internal/services/webhook/pullreq.go rename to internal/services/webhook/handler_pullreq.go diff --git a/internal/services/webhook/tag.go b/internal/services/webhook/handler_tag.go similarity index 100% rename from internal/services/webhook/tag.go rename to internal/services/webhook/handler_tag.go diff --git a/internal/services/webhook/service.go b/internal/services/webhook/service.go index 52cd0b7dc..ea8da2c2a 100644 --- a/internal/services/webhook/service.go +++ b/internal/services/webhook/service.go @@ -27,17 +27,15 @@ const ( type Config struct { // UserAgentIdentity specifies the identity used for the user agent header // IMPORTANT: do not include version. - UserAgentIdentity string `envconfig:"GITNESS_WEBHOOK_USER_AGENT_IDENTITY" default:"Gitness"` + UserAgentIdentity string // HeaderIdentity specifies the identity used for headers in webhook calls (e.g. X-Gitness-Trigger, ...). // NOTE: If no value is provided, the UserAgentIdentity will be used. - HeaderIdentity string `envconfig:"GITNESS_WEBHOOK_HEADER_IDENTITY"` - // EventReaderName is the name used to read events from stream. - // Note: this should be different for every running instance. - EventReaderName string `envconfig:"GITNESS_WEBHOOK_EVENT_READER_NAME"` - Concurrency int `envconfig:"GITNESS_WEBHOOK_CONCURRENCY" default:"4"` - MaxRetries int `envconfig:"GITNESS_WEBHOOK_MAX_RETRIES" default:"3"` - AllowPrivateNetwork bool `envconfig:"GITNESS_WEBHOOK_ALLOW_PRIVATE_NETWORK" default:"false"` - AllowLoopback bool `envconfig:"GITNESS_WEBHOOK_ALLOW_LOOPBACK" default:"false"` + HeaderIdentity string + EventReaderName string + Concurrency int + MaxRetries int + AllowPrivateNetwork bool + AllowLoopback bool } func (c *Config) Prepare() error { @@ -91,7 +89,7 @@ func NewService(ctx context.Context, config Config, repoStore store.RepoStore, pullreqStore store.PullReqStore, urlProvider *url.Provider, principalStore store.PrincipalStore, gitRPCClient gitrpc.Interface) (*Service, error) { if err := config.Prepare(); err != nil { - return nil, fmt.Errorf("provided config is invalid: %w", err) + return nil, fmt.Errorf("provided webhook service config is invalid: %w", err) } service := &Service{ webhookStore: webhookStore, diff --git a/internal/services/services.go b/internal/services/wire.go similarity index 72% rename from internal/services/services.go rename to internal/services/wire.go index 1cb5efedd..919fb7b75 100644 --- a/internal/services/services.go +++ b/internal/services/wire.go @@ -7,6 +7,7 @@ package services import ( "github.com/harness/gitness/internal/services/job" "github.com/harness/gitness/internal/services/pullreq" + "github.com/harness/gitness/internal/services/trigger" "github.com/harness/gitness/internal/services/webhook" "github.com/google/wire" @@ -19,20 +20,20 @@ var WireSet = wire.NewSet( type Services struct { Webhook *webhook.Service PullReq *pullreq.Service - JobExecutor *job.Executor + Trigger *trigger.Service JobScheduler *job.Scheduler } func ProvideServices( - webhooksSrv *webhook.Service, - pullReqSrv *pullreq.Service, - jobExecutor *job.Executor, + webhooksSvc *webhook.Service, + pullReqSvc *pullreq.Service, + triggerSvc *trigger.Service, jobScheduler *job.Scheduler, ) Services { return Services{ - Webhook: webhooksSrv, - PullReq: pullReqSrv, - JobExecutor: jobExecutor, + Webhook: webhooksSvc, + PullReq: pullReqSvc, + Trigger: triggerSvc, JobScheduler: jobScheduler, } } diff --git a/types/config.go b/types/config.go index 37f2b7335..85421310a 100644 --- a/types/config.go +++ b/types/config.go @@ -200,4 +200,22 @@ type Config struct { // finished and failed jobs will be purged from the DB. PurgeFinishedOlderThan time.Duration `envconfig:"GITNESS_JOBS_PURGE_FINISHED_OLDER_THAN" default:"120h"` } + + Webhook struct { + // UserAgentIdentity specifies the identity used for the user agent header + // IMPORTANT: do not include version. + UserAgentIdentity string `envconfig:"GITNESS_WEBHOOK_USER_AGENT_IDENTITY" default:"Gitness"` + // HeaderIdentity specifies the identity used for headers in webhook calls (e.g. X-Gitness-Trigger, ...). + // NOTE: If no value is provided, the UserAgentIdentity will be used. + HeaderIdentity string `envconfig:"GITNESS_WEBHOOK_HEADER_IDENTITY"` + Concurrency int `envconfig:"GITNESS_WEBHOOK_CONCURRENCY" default:"4"` + MaxRetries int `envconfig:"GITNESS_WEBHOOK_MAX_RETRIES" default:"3"` + AllowPrivateNetwork bool `envconfig:"GITNESS_WEBHOOK_ALLOW_PRIVATE_NETWORK" default:"false"` + AllowLoopback bool `envconfig:"GITNESS_WEBHOOK_ALLOW_LOOPBACK" default:"false"` + } + + Trigger struct { + Concurrency int `envconfig:"GITNESS_TRIGGER_CONCURRENCY" default:"4"` + MaxRetries int `envconfig:"GITNESS_TRIGGER_MAX_RETRIES" default:"3"` + } } From 9c46f066dcccb8cd62240907344cde7a54765eaa Mon Sep 17 00:00:00 2001 From: calvin Date: Fri, 8 Sep 2023 17:56:04 -0600 Subject: [PATCH 16/17] fix: [code-836]: remove comment --- web/src/components/DiffViewer/DiffViewerUtils.tsx | 1 - 1 file changed, 1 deletion(-) diff --git a/web/src/components/DiffViewer/DiffViewerUtils.tsx b/web/src/components/DiffViewer/DiffViewerUtils.tsx index ded5497dd..db20f6cdd 100644 --- a/web/src/components/DiffViewer/DiffViewerUtils.tsx +++ b/web/src/components/DiffViewer/DiffViewerUtils.tsx @@ -143,7 +143,6 @@ export function getCommentLineInfo( ) { const isSideBySideView = viewStyle === ViewStyle.SIDE_BY_SIDE const { left, lineNumber, filePath } = commentEntry - console.log(commentEntry) const filePathBody = filePath ? contentDOM?.querySelector(`[data="${filePath}"`) : contentDOM const diffBody = filePathBody?.querySelector( From 8570810dbcea5709dcceeb3182d1bf63d8aab168 Mon Sep 17 00:00:00 2001 From: Johannes Batzill Date: Sat, 9 Sep 2023 00:59:35 +0000 Subject: [PATCH 17/17] fix stringtypes (#422) --- web/src/framework/strings/stringTypes.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/web/src/framework/strings/stringTypes.ts b/web/src/framework/strings/stringTypes.ts index bcf1a47af..d16e6f61c 100644 --- a/web/src/framework/strings/stringTypes.ts +++ b/web/src/framework/strings/stringTypes.ts @@ -169,6 +169,7 @@ export interface StringsMap { enterUser: string error: string error404Text: string + 'executions.completedTime': string 'executions.description': string 'executions.name': string 'executions.newExecutionButton': string