feat: [CI-14316]: Connectors support in gitness (#2699)

* fix postgres migration
* address comments
* fix lint
* fix lint issues
* fix db schema, remove unnecessary files
* normalize schema to enforce secret integrity
* connectors fixes
* add created_by field
* rebase connectors implementation with latest
* fix lint
* change UID -> identifier
* refactor files around and lint
* Basic implementation for connectors

app/api/controller/connector/controller.go

@@ -16,23 +16,28 @@ package connector
 
 import (
 	"github.com/harness/gitness/app/auth/authz"
+	"github.com/harness/gitness/app/connector"
 	"github.com/harness/gitness/app/store"
 )
 
 type Controller struct {
-	connectorStore store.ConnectorStore
+	connectorStore   store.ConnectorStore
-	authorizer     authz.Authorizer
+	connectorService *connector.Service
-	spaceStore     store.SpaceStore
+
+	authorizer authz.Authorizer
+	spaceStore store.SpaceStore
 }
 
 func NewController(
 	authorizer authz.Authorizer,
 	connectorStore store.ConnectorStore,
+	connectorService *connector.Service,
 	spaceStore store.SpaceStore,
 ) *Controller {
 	return &Controller{
-		connectorStore: connectorStore,
+		connectorStore:   connectorStore,
-		authorizer:     authorizer,
+		connectorService: connectorService,
-		spaceStore:     spaceStore,
+		authorizer:       authorizer,
+		spaceStore:       spaceStore,
 	}
 }

app/api/controller/connector/create.go

@@ -36,13 +36,11 @@ var (
 )
 
 type CreateInput struct {
-	Description string `json:"description"`
+	Description string             `json:"description"`
-	SpaceRef    string `json:"space_ref"` // Ref of the parent space
+	SpaceRef    string             `json:"space_ref"` // Ref of the parent space
-	// TODO [CODE-1363]: remove after identifier migration.
+	Identifier  string             `json:"identifier"`
-	UID        string `json:"uid" deprecated:"true"`
+	Type        enum.ConnectorType `json:"type"`
-	Identifier string `json:"identifier"`
+	types.ConnectorConfig
-	Type       string `json:"type"`
-	Data       string `json:"data"`
 }
 
 func (c *Controller) Create(
@@ -50,7 +48,7 @@ func (c *Controller) Create(
 	session *auth.Session,
 	in *CreateInput,
 ) (*types.Connector, error) {
-	if err := c.sanitizeCreateInput(in); err != nil {
+	if err := in.validate(); err != nil {
 		return nil, fmt.Errorf("failed to sanitize input: %w", err)
 	}
 
@@ -72,16 +70,19 @@ func (c *Controller) Create(
 	}
 
 	now := time.Now().UnixMilli()
+
 	connector := &types.Connector{
-		Description: in.Description,
+		Description:     in.Description,
-		Data:        in.Data,
+		CreatedBy:       session.Principal.ID,
-		Type:        in.Type,
+		Type:            in.Type,
-		SpaceID:     parentSpace.ID,
+		SpaceID:         parentSpace.ID,
-		Identifier:  in.Identifier,
+		Identifier:      in.Identifier,
-		Created:     now,
+		Created:         now,
-		Updated:     now,
+		Updated:         now,
-		Version:     0,
+		Version:         0,
+		ConnectorConfig: in.ConnectorConfig,
 	}
+
 	err = c.connectorStore.Create(ctx, connector)
 	if err != nil {
 		return nil, fmt.Errorf("connector creation failed: %w", err)
@@ -90,16 +91,20 @@ func (c *Controller) Create(
 	return connector, nil
 }
 
-func (c *Controller) sanitizeCreateInput(in *CreateInput) error {
+func (in *CreateInput) validate() error {
-	// TODO [CODE-1363]: remove after identifier migration.
+	parentRefAsID, err := strconv.ParseInt(in.SpaceRef, 10, 64)
-	if in.Identifier == "" {
+	if (err == nil && parentRefAsID <= 0) || (len(strings.TrimSpace(in.SpaceRef)) == 0) {
-		in.Identifier = in.UID
+		return errConnectorRequiresParent
 	}
 
-	parentRefAsID, _ := strconv.ParseInt(in.SpaceRef, 10, 64)
+	// check that the connector type is valid
+	if _, ok := in.Type.Sanitize(); !ok {
+		return usererror.BadRequest("invalid connector type")
+	}
 
-	if parentRefAsID <= 0 || len(strings.TrimSpace(in.SpaceRef)) == 0 {
+	// if the connector type is valid, validate the connector config
-		return errConnectorRequiresParent
+	if err := in.ConnectorConfig.Validate(in.Type); err != nil {
+		return usererror.BadRequest(fmt.Sprintf("invalid connector config: %s", err.Error()))
 	}
 
 	if err := check.Identifier(in.Identifier); err != nil {

app/api/controller/connector/test.go

@@ -0,0 +1,65 @@
+// Copyright 2023 Harness, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package connector
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	apiauth "github.com/harness/gitness/app/api/auth"
+	"github.com/harness/gitness/app/auth"
+	"github.com/harness/gitness/types"
+	"github.com/harness/gitness/types/enum"
+
+	"github.com/rs/zerolog/log"
+)
+
+func (c *Controller) Test(
+	ctx context.Context,
+	session *auth.Session,
+	spaceRef string,
+	identifier string,
+) (types.ConnectorTestResponse, error) {
+	space, err := c.spaceStore.FindByRef(ctx, spaceRef)
+	if err != nil {
+		return types.ConnectorTestResponse{}, fmt.Errorf("failed to find space: %w", err)
+	}
+	err = apiauth.CheckConnector(ctx, c.authorizer, session, space.Path, identifier, enum.PermissionConnectorAccess)
+	if err != nil {
+		return types.ConnectorTestResponse{}, fmt.Errorf("failed to authorize: %w", err)
+	}
+	connector, err := c.connectorStore.FindByIdentifier(ctx, space.ID, identifier)
+	if err != nil {
+		return types.ConnectorTestResponse{}, fmt.Errorf("failed to find connector: %w", err)
+	}
+
+	resp, err := c.connectorService.Test(ctx, connector)
+	if err != nil {
+		return types.ConnectorTestResponse{}, err
+	}
+	// Try to update connector last test information in DB. Log but ignore errors
+	_, err = c.connectorStore.UpdateOptLock(ctx, connector, func(original *types.Connector) error {
+		original.LastTestErrorMsg = resp.ErrorMsg
+		original.LastTestStatus = resp.Status
+		original.LastTestAttempt = time.Now().UnixMilli()
+		return nil
+	})
+	if err != nil {
+		log.Ctx(ctx).Warn().Err(err).Msg("failed to update test connection information in connector")
+	}
+
+	return resp, nil
+}

app/api/controller/connector/update.go

@@ -20,6 +20,7 @@ import (
 	"strings"
 
 	apiauth "github.com/harness/gitness/app/api/auth"
+	"github.com/harness/gitness/app/api/usererror"
 	"github.com/harness/gitness/app/auth"
 	"github.com/harness/gitness/types"
 	"github.com/harness/gitness/types/check"
@@ -28,11 +29,9 @@ import (
 
 // UpdateInput is used for updating a connector.
 type UpdateInput struct {
-	// TODO [CODE-1363]: remove after identifier migration.
-	UID         *string `json:"uid" deprecated:"true"`
 	Identifier  *string `json:"identifier"`
 	Description *string `json:"description"`
-	Data        *string `json:"data"`
+	*types.ConnectorConfig
 }
 
 func (c *Controller) Update(
@@ -42,7 +41,7 @@ func (c *Controller) Update(
 	identifier string,
 	in *UpdateInput,
 ) (*types.Connector, error) {
-	if err := c.sanitizeUpdateInput(in); err != nil {
+	if err := in.validate(); err != nil {
 		return nil, fmt.Errorf("failed to sanitize input: %w", err)
 	}
 
@@ -68,20 +67,22 @@ func (c *Controller) Update(
 		if in.Description != nil {
 			original.Description = *in.Description
 		}
-		if in.Data != nil {
+		// TODO: See if this can be made better. The PATCH API supports partial updates so
-			original.Data = *in.Data
+		// currently we keep all the top level fields the same unless they are explicitly provided.
+		// The connector config is a nested field so we only check whether it's provided at the top level, and not
+		// all the fields inside the config. Maybe PUT/POST would be a better option here?
+		// We can revisit this once we start adding more connectors.
+		if in.ConnectorConfig != nil {
+			if err := in.ConnectorConfig.Validate(connector.Type); err != nil {
+				return usererror.BadRequestf("failed to validate connector config: %s", err.Error())
+			}
+			original.ConnectorConfig = *in.ConnectorConfig
 		}
-
 		return nil
 	})
 }
 
-func (c *Controller) sanitizeUpdateInput(in *UpdateInput) error {
+func (in *UpdateInput) validate() error {
-	// TODO [CODE-1363]: remove after identifier migration.
-	if in.Identifier == nil {
-		in.Identifier = in.UID
-	}
-
 	if in.Identifier != nil {
 		if err := check.Identifier(*in.Identifier); err != nil {
 			return err
@@ -95,7 +96,5 @@ func (c *Controller) sanitizeUpdateInput(in *UpdateInput) error {
 		}
 	}
 
-	// TODO: Validate Data
-
 	return nil
 }

app/api/controller/connector/wire.go

@@ -16,6 +16,7 @@ package connector
 
 import (
 	"github.com/harness/gitness/app/auth/authz"
+	"github.com/harness/gitness/app/connector"
 	"github.com/harness/gitness/app/store"
 
 	"github.com/google/wire"
@@ -28,8 +29,9 @@ var WireSet = wire.NewSet(
 
 func ProvideController(
 	connectorStore store.ConnectorStore,
+	connectorService *connector.Service,
 	authorizer authz.Authorizer,
 	spaceStore store.SpaceStore,
 ) *Controller {
-	return NewController(authorizer, connectorStore, spaceStore)
+	return NewController(authorizer, connectorStore, connectorService, spaceStore)
 }

app/api/handler/connector/test.go

@@ -0,0 +1,49 @@
+// Copyright 2023 Harness, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package connector
+
+import (
+	"net/http"
+
+	"github.com/harness/gitness/app/api/controller/connector"
+	"github.com/harness/gitness/app/api/render"
+	"github.com/harness/gitness/app/api/request"
+	"github.com/harness/gitness/app/paths"
+)
+
+func HandleTest(connectorCtrl *connector.Controller) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		ctx := r.Context()
+		session, _ := request.AuthSessionFrom(ctx)
+		connectorRef, err := request.GetConnectorRefFromPath(r)
+		if err != nil {
+			render.TranslatedUserError(ctx, w, err)
+			return
+		}
+		spaceRef, connectorIdentifier, err := paths.DisectLeaf(connectorRef)
+		if err != nil {
+			render.TranslatedUserError(ctx, w, err)
+			return
+		}
+
+		resp, err := connectorCtrl.Test(ctx, session, spaceRef, connectorIdentifier)
+		if err != nil {
+			render.TranslatedUserError(ctx, w, err)
+			return
+		}
+
+		render.JSON(w, http.StatusOK, resp)
+	}
+}

app/api/openapi/connector.go

@@ -86,4 +86,16 @@ func connectorOperations(reflector *openapi3.Reflector) {
 	_ = reflector.SetJSONResponse(&opUpdate, new(usererror.Error), http.StatusForbidden)
 	_ = reflector.SetJSONResponse(&opUpdate, new(usererror.Error), http.StatusNotFound)
 	_ = reflector.Spec.AddOperation(http.MethodPatch, "/connectors/{connector_ref}", opUpdate)
+
+	opTest := openapi3.Operation{}
+	opTest.WithTags("connector")
+	opTest.WithMapOfAnything(map[string]interface{}{"operationId": "testConnector"})
+	_ = reflector.SetRequest(&opTest, nil, http.MethodPost)
+	_ = reflector.SetJSONResponse(&opTest, new(types.ConnectorTestResponse), http.StatusOK)
+	_ = reflector.SetJSONResponse(&opTest, new(usererror.Error), http.StatusBadRequest)
+	_ = reflector.SetJSONResponse(&opTest, new(usererror.Error), http.StatusInternalServerError)
+	_ = reflector.SetJSONResponse(&opTest, new(usererror.Error), http.StatusUnauthorized)
+	_ = reflector.SetJSONResponse(&opTest, new(usererror.Error), http.StatusForbidden)
+	_ = reflector.SetJSONResponse(&opTest, new(usererror.Error), http.StatusNotFound)
+	_ = reflector.Spec.AddOperation(http.MethodPost, "/connectors/{connector_ref}/test", opTest)
 }

app/connector/connector.go

@@ -0,0 +1,58 @@
+// Copyright 2023 Harness, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package connector
+
+import (
+	"context"
+	"time"
+
+	"github.com/harness/gitness/app/connector/scm"
+	"github.com/harness/gitness/app/store"
+	"github.com/harness/gitness/types"
+)
+
+var (
+	testConnectionTimeout = 5 * time.Second
+)
+
+type Service struct {
+	secretStore store.SecretStore
+	// A separate SCM connector service is helpful here since the go-scm library abstracts out all the specific
+	// SCM interactions, making the interfacing common for all the SCM connectors.
+	// There might be connectors (eg docker, gcr, etc) in the future which require separate implementations.
+	// Nevertheless, there should be an attempt to abstract out common functionality for different connector
+	// types if possible - otherwise separate implementations can be written here.
+	scmService *scm.Service
+}
+
+func New(secretStore store.SecretStore, scmService *scm.Service) *Service {
+	return &Service{
+		secretStore: secretStore,
+		scmService:  scmService,
+	}
+}
+
+func (s *Service) Test(
+	ctx context.Context,
+	connector *types.Connector,
+) (types.ConnectorTestResponse, error) {
+	// Set a timeout while testing connection.
+	ctxWithTimeout, cancel := context.WithDeadline(ctx, time.Now().Add(testConnectionTimeout))
+	defer cancel()
+	if connector.Type.IsSCM() {
+		return s.scmService.Test(ctxWithTimeout, connector)
+	}
+	return types.ConnectorTestResponse{}, nil
+}

app/connector/scm/provider.go

@@ -0,0 +1,112 @@
+// Copyright 2023 Harness, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package scm
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+
+	"github.com/harness/gitness/app/store"
+	"github.com/harness/gitness/types"
+	"github.com/harness/gitness/types/enum"
+
+	"github.com/drone/go-scm/scm"
+	"github.com/drone/go-scm/scm/driver/github"
+	"github.com/drone/go-scm/scm/transport/oauth2"
+)
+
+// getSCMProvider returns an SCM client given a connector.
+// The SCM client can be used as a common layer for interfacing with any SCM.
+func getSCMProvider(
+	ctx context.Context,
+	connector *types.Connector,
+	secretStore store.SecretStore,
+) (*scm.Client, error) {
+	var client *scm.Client
+	var err error
+	var transport http.RoundTripper
+
+	switch x := connector.Type; x {
+	case enum.ConnectorTypeGithub:
+		if connector.Github == nil {
+			return nil, fmt.Errorf("github connector is nil")
+		}
+		if connector.Github.APIURL == "" {
+			client = github.NewDefault()
+		} else {
+			client, err = github.New(connector.Github.APIURL)
+			if err != nil {
+				return nil, err
+			}
+		}
+		if connector.Github.Auth == nil {
+			return nil, fmt.Errorf("github auth needs to be provided")
+		}
+		if connector.Github.Auth.AuthType == enum.ConnectorAuthTypeBearer {
+			creds := connector.Github.Auth.Bearer
+			pass, err := resolveSecret(ctx, connector.SpaceID, creds.Token, secretStore)
+			if err != nil {
+				return nil, err
+			}
+			transport = oauthTransport(pass, oauth2.SchemeBearer)
+		} else {
+			return nil, fmt.Errorf("unsupported auth type for github connector: %s", connector.Github.Auth.AuthType)
+		}
+	default:
+		return nil, fmt.Errorf("unsupported scm provider type: %s", x)
+	}
+
+	// override default transport if available
+	if transport != nil {
+		client.Client = &http.Client{Transport: transport}
+	}
+
+	return client, nil
+}
+
+func oauthTransport(token string, scheme string) http.RoundTripper {
+	if token == "" {
+		return nil
+	}
+	return &oauth2.Transport{
+		Base:   defaultTransport(),
+		Scheme: scheme,
+		Source: oauth2.StaticTokenSource(&scm.Token{Token: token}),
+	}
+}
+
+// defaultTransport provides a default http.Transport.
+// This can be extended when needed for things like more advanced TLS config, proxies, etc.
+func defaultTransport() http.RoundTripper {
+	return &http.Transport{
+		Proxy: http.ProxyFromEnvironment,
+	}
+}
+
+// resolveSecret looks into the secret store to find the value of a secret.
+func resolveSecret(
+	ctx context.Context,
+	spaceID int64,
+	ref types.SecretRef,
+	secretStore store.SecretStore,
+) (string, error) {
+	// the secret should be in the same space as the connector
+	s, err := secretStore.FindByIdentifier(ctx, spaceID, ref.Identifier)
+	if err != nil {
+		return "", fmt.Errorf("could not find secret from store: %w", err)
+	}
+	return s.Data, nil
+}

app/connector/scm/scm.go

@@ -0,0 +1,50 @@
+// Copyright 2023 Harness, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package scm
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/harness/gitness/app/store"
+	"github.com/harness/gitness/types"
+	"github.com/harness/gitness/types/enum"
+)
+
+type Service struct {
+	secretStore store.SecretStore
+}
+
+func NewService(secretStore store.SecretStore) *Service {
+	return &Service{
+		secretStore: secretStore,
+	}
+}
+
+func (s *Service) Test(ctx context.Context, c *types.Connector) (types.ConnectorTestResponse, error) {
+	if !c.Type.IsSCM() {
+		return types.ConnectorTestResponse{}, fmt.Errorf("connector type: %s is not an SCM connector", c.Type.String())
+	}
+	client, err := getSCMProvider(ctx, c, s.secretStore)
+	if err != nil {
+		return types.ConnectorTestResponse{}, err
+	}
+	// Check whether a valid user exists - if yes, the connection is successful
+	_, _, err = client.Users.Find(ctx)
+	if err != nil {
+		return types.ConnectorTestResponse{Status: enum.ConnectorStatusFailed, ErrorMsg: err.Error()}, nil //nolint:nilerr
+	}
+	return types.ConnectorTestResponse{Status: enum.ConnectorStatusSuccess}, nil
+}

app/connector/wire.go

@@ -0,0 +1,42 @@
+// Copyright 2023 Harness, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package connector
+
+import (
+	"github.com/harness/gitness/app/connector/scm"
+	"github.com/harness/gitness/app/store"
+
+	"github.com/google/wire"
+)
+
+// WireSet provides a wire set for this package.
+var WireSet = wire.NewSet(
+	ProvideConnectorHandler,
+	ProvideSCMConnectorHandler,
+)
+
+// ProvideConnectorHandler provides a connector handler for handling connector-related ops.
+func ProvideConnectorHandler(
+	secretStore store.SecretStore,
+	scmService *scm.Service,
+) *Service {
+	return New(secretStore, scmService)
+}
+
+// ProvideSCMConnectorHandler provides a SCM connector handler for specifically handling
+// SCM connector related ops.
+func ProvideSCMConnectorHandler(secretStore store.SecretStore) *scm.Service {
+	return scm.NewService(secretStore)
+}

app/router/api.go

@@ -503,6 +503,7 @@ func setupConnectors(
 			r.Get("/", handlerconnector.HandleFind(connectorCtrl))
 			r.Patch("/", handlerconnector.HandleUpdate(connectorCtrl))
 			r.Delete("/", handlerconnector.HandleDelete(connectorCtrl))
+			r.Post("/test", handlerconnector.HandleTest(connectorCtrl))
 		})
 	})
 }

app/store/database/connector.go

@@ -16,6 +16,7 @@ package database
 
 import (
 	"context"
+	"database/sql"
 	"fmt"
 	"strings"
 	"time"
@@ -25,6 +26,7 @@ import (
 	"github.com/harness/gitness/store/database"
 	"github.com/harness/gitness/store/database/dbtx"
 	"github.com/harness/gitness/types"
+	"github.com/harness/gitness/types/enum"
 
 	"github.com/jmoiron/sqlx"
 	"github.com/pkg/errors"
@@ -40,25 +42,265 @@ const (
 
 	connectorColumns = `
 	connector_id,
+	connector_identifier,
 	connector_description,
+	connector_type,
+	connector_auth_type,
+	connector_created_by,
 	connector_space_id,
-	connector_uid,
+	connector_last_test_attempt,
-	connector_data,
+	connector_last_test_error_msg,
+	connector_last_test_status,
 	connector_created,
 	connector_updated,
-	connector_version
+	connector_version,
+	connector_address,
+	connector_insecure,
+	connector_username,
+	connector_github_app_installation_id,
+	connector_github_app_application_id,
+	connector_region,
+	connector_password,
+	connector_token,
+	connector_aws_key,
+	connector_aws_secret,
+	connector_github_app_private_key,
+	connector_token_refresh
 	`
 )
 
+type connector struct {
+	ID               int64  `db:"connector_id"`
+	Identifier       string `db:"connector_identifier"`
+	Description      string `db:"connector_description"`
+	Type             string `db:"connector_type"`
+	AuthType         string `db:"connector_auth_type"`
+	CreatedBy        int64  `db:"connector_created_by"`
+	SpaceID          int64  `db:"connector_space_id"`
+	LastTestAttempt  int64  `db:"connector_last_test_attempt"`
+	LastTestErrorMsg string `db:"connector_last_test_error_msg"`
+	LastTestStatus   string `db:"connector_last_test_status"`
+	Created          int64  `db:"connector_created"`
+	Updated          int64  `db:"connector_updated"`
+	Version          int64  `db:"connector_version"`
+
+	Address                 sql.NullString `db:"connector_address"`
+	Insecure                sql.NullBool   `db:"connector_insecure"`
+	Username                sql.NullString `db:"connector_username"`
+	GithubAppInstallationID sql.NullString `db:"connector_github_app_installation_id"`
+	GithubAppApplicationID  sql.NullString `db:"connector_github_app_application_id"`
+	Region                  sql.NullString `db:"connector_region"`
+	// Password fields are stored as reference to secrets table
+	Password            sql.NullInt64 `db:"connector_password"`
+	Token               sql.NullInt64 `db:"connector_token"`
+	AWSKey              sql.NullInt64 `db:"connector_aws_key"`
+	AWSSecret           sql.NullInt64 `db:"connector_aws_secret"`
+	GithubAppPrivateKey sql.NullInt64 `db:"connector_github_app_private_key"`
+	TokenRefresh        sql.NullInt64 `db:"connector_token_refresh"`
+}
+
 // NewConnectorStore returns a new ConnectorStore.
-func NewConnectorStore(db *sqlx.DB) store.ConnectorStore {
+// The secret store is used to resolve the secret references.
+func NewConnectorStore(db *sqlx.DB, secretStore store.SecretStore) store.ConnectorStore {
 	return &connectorStore{
-		db: db,
+		db:          db,
+		secretStore: secretStore,
 	}
 }
 
+func (s *connectorStore) mapFromDBConnectors(ctx context.Context, src []*connector) ([]*types.Connector, error) {
+	dst := make([]*types.Connector, len(src))
+	for i, v := range src {
+		m, err := s.mapFromDBConnector(ctx, v)
+		if err != nil {
+			return nil, fmt.Errorf("could not map from db connector: %w", err)
+		}
+		dst[i] = m
+	}
+	return dst, nil
+}
+
+func (s *connectorStore) mapToDBConnector(ctx context.Context, v *types.Connector) (*connector, error) {
+	to := connector{
+		ID:               v.ID,
+		Identifier:       v.Identifier,
+		Description:      v.Description,
+		Type:             v.Type.String(),
+		SpaceID:          v.SpaceID,
+		CreatedBy:        v.CreatedBy,
+		Created:          v.Created,
+		Updated:          v.Updated,
+		Version:          v.Version,
+		LastTestAttempt:  v.LastTestAttempt,
+		LastTestErrorMsg: v.LastTestErrorMsg,
+		LastTestStatus:   v.LastTestStatus.String(),
+	}
+	// Parse connector specific configs
+	err := s.convertConfigToDB(ctx, v, &to)
+	if err != nil {
+		return nil, fmt.Errorf("could not convert config to db: %w", err)
+	}
+	return &to, nil
+}
+
+func (s *connectorStore) convertConfigToDB(
+	ctx context.Context,
+	source *types.Connector,
+	to *connector,
+) error {
+	switch {
+	case source.Github != nil:
+		to.Address = sql.NullString{String: source.Github.APIURL, Valid: true}
+		to.Insecure = sql.NullBool{Bool: source.Github.Insecure, Valid: true}
+		if source.Github.Auth == nil {
+			return fmt.Errorf("auth is required for github connectors")
+		}
+		if source.Github.Auth.AuthType != enum.ConnectorAuthTypeBearer {
+			return fmt.Errorf("only bearer token auth is supported for github connectors")
+		}
+		to.AuthType = source.Github.Auth.AuthType.String()
+		creds := source.Github.Auth.Bearer
+		// use the same space ID as the connector
+		tokenID, err := s.secretIdentiferToID(ctx, creds.Token.Identifier, source.SpaceID)
+		if err != nil {
+			return fmt.Errorf("could not find secret: %w", err)
+		}
+		to.Token = sql.NullInt64{Int64: tokenID, Valid: true}
+	default:
+		return fmt.Errorf("no connector config found for type: %s", source.Type)
+	}
+	return nil
+}
+
+// secretIdentiferToID finds the secret ID given the space ID and the identifier.
+func (s *connectorStore) secretIdentiferToID(
+	ctx context.Context,
+	identifier string,
+	spaceID int64,
+) (int64, error) {
+	secret, err := s.secretStore.FindByIdentifier(ctx, spaceID, identifier)
+	if err != nil {
+		return 0, err
+	}
+	return secret.ID, nil
+}
+
+func (s *connectorStore) mapFromDBConnector(
+	ctx context.Context,
+	dbConnector *connector,
+) (*types.Connector, error) {
+	connector := &types.Connector{
+		ID:               dbConnector.ID,
+		Identifier:       dbConnector.Identifier,
+		Description:      dbConnector.Description,
+		Type:             enum.ConnectorType(dbConnector.Type),
+		SpaceID:          dbConnector.SpaceID,
+		CreatedBy:        dbConnector.CreatedBy,
+		LastTestAttempt:  dbConnector.LastTestAttempt,
+		LastTestErrorMsg: dbConnector.LastTestErrorMsg,
+		LastTestStatus:   enum.ConnectorStatus(dbConnector.LastTestStatus),
+		Created:          dbConnector.Created,
+		Updated:          dbConnector.Updated,
+		Version:          dbConnector.Version,
+	}
+	err := s.populateConnectorData(ctx, dbConnector, connector)
+	if err != nil {
+		return nil, fmt.Errorf("could not populate connector data: %w", err)
+	}
+	return connector, nil
+}
+
+func (s *connectorStore) populateConnectorData(
+	ctx context.Context,
+	source *connector,
+	to *types.Connector,
+) error {
+	switch enum.ConnectorType(source.Type) {
+	case enum.ConnectorTypeGithub:
+		githubData, err := s.parseGithubConnectorData(ctx, source)
+		if err != nil {
+			return fmt.Errorf("could not parse github connector data: %w", err)
+		}
+		to.Github = githubData
+	// Cases for other connectors can be added here
+	default:
+		return fmt.Errorf("unsupported connector type: %s", source.Type)
+	}
+	return nil
+}
+
+func (s *connectorStore) parseGithubConnectorData(
+	ctx context.Context,
+	connector *connector,
+) (*types.GithubConnectorData, error) {
+	auth, err := s.parseAuthenticationData(ctx, connector)
+	if err != nil {
+		return nil, fmt.Errorf("could not parse authentication data: %w", err)
+	}
+	return &types.GithubConnectorData{
+		APIURL:   connector.Address.String,
+		Insecure: connector.Insecure.Bool,
+		Auth:     auth,
+	}, nil
+}
+
+func (s *connectorStore) parseAuthenticationData(
+	ctx context.Context,
+	connector *connector,
+) (*types.ConnectorAuth, error) {
+	authType, err := enum.ParseConnectorAuthType(connector.AuthType)
+	if err != nil {
+		return nil, err
+	}
+
+	switch authType {
+	case enum.ConnectorAuthTypeBasic:
+		if !connector.Username.Valid || !connector.Password.Valid {
+			return nil, fmt.Errorf("basic auth requires both username and password")
+		}
+		passwordRef, err := s.convertToRef(ctx, connector.Password.Int64)
+		if err != nil {
+			return nil, fmt.Errorf("could not convert basicauth password to ref: %w", err)
+		}
+		return &types.ConnectorAuth{
+			AuthType: enum.ConnectorAuthTypeBasic,
+			Basic: &types.BasicAuthCreds{
+				Username: connector.Username.String,
+				Password: passwordRef,
+			},
+		}, nil
+	case enum.ConnectorAuthTypeBearer:
+		if !connector.Token.Valid {
+			return nil, fmt.Errorf("bearer auth requires a token")
+		}
+		tokenRef, err := s.convertToRef(ctx, connector.Token.Int64)
+		if err != nil {
+			return nil, fmt.Errorf("could not convert bearer token to ref: %w", err)
+		}
+		return &types.ConnectorAuth{
+			AuthType: enum.ConnectorAuthTypeBearer,
+			Bearer: &types.BearerTokenCreds{
+				Token: tokenRef,
+			},
+		}, nil
+	default:
+		return nil, fmt.Errorf("unsupported auth type: %s", connector.AuthType)
+	}
+}
+
+func (s *connectorStore) convertToRef(ctx context.Context, id int64) (types.SecretRef, error) {
+	secret, err := s.secretStore.Find(ctx, id)
+	if err != nil {
+		return types.SecretRef{}, err
+	}
+	return types.SecretRef{
+		Identifier: secret.Identifier,
+	}, nil
+}
+
 type connectorStore struct {
-	db *sqlx.DB
+	db          *sqlx.DB
+	secretStore store.SecretStore
 }
 
 // Find returns a connector given a connector ID.
@@ -67,11 +309,11 @@ func (s *connectorStore) Find(ctx context.Context, id int64) (*types.Connector,
 		WHERE connector_id = $1`
 	db := dbtx.GetAccessor(ctx, s.db)
 
-	dst := new(types.Connector)
+	dst := new(connector)
 	if err := db.GetContext(ctx, dst, findQueryStmt, id); err != nil {
 		return nil, database.ProcessSQLErrorf(ctx, err, "Failed to find connector")
 	}
-	return dst, nil
+	return s.mapFromDBConnector(ctx, dst)
 }
 
 // FindByIdentifier returns a connector in a given space with a given identifier.
@@ -81,41 +323,77 @@ func (s *connectorStore) FindByIdentifier(
 	identifier string,
 ) (*types.Connector, error) {
 	const findQueryStmt = connectorQueryBase + `
-		WHERE connector_space_id = $1 AND connector_uid = $2`
+		WHERE connector_space_id = $1 AND connector_identifier = $2`
 	db := dbtx.GetAccessor(ctx, s.db)
 
-	dst := new(types.Connector)
+	dst := new(connector)
 	if err := db.GetContext(ctx, dst, findQueryStmt, spaceID, identifier); err != nil {
 		return nil, database.ProcessSQLErrorf(ctx, err, "Failed to find connector")
 	}
-	return dst, nil
+	return s.mapFromDBConnector(ctx, dst)
 }
 
 // Create creates a connector.
 func (s *connectorStore) Create(ctx context.Context, connector *types.Connector) error {
+	dbConnector, err := s.mapToDBConnector(ctx, connector)
+	if err != nil {
+		return err
+	}
 	const connectorInsertStmt = `
 	INSERT INTO connectors (
 		connector_description
 		,connector_type
+		,connector_created_by
 		,connector_space_id
-		,connector_uid
+		,connector_identifier
-		,connector_data
+		,connector_last_test_attempt
+		,connector_last_test_error_msg
+		,connector_last_test_status
 		,connector_created
 		,connector_updated
 		,connector_version
+		,connector_auth_type
+		,connector_address
+		,connector_insecure
+		,connector_username
+		,connector_github_app_installation_id
+		,connector_github_app_application_id
+		,connector_region
+		,connector_password
+		,connector_token
+		,connector_aws_key
+		,connector_aws_secret
+		,connector_github_app_private_key
+		,connector_token_refresh
 	) VALUES (
 		:connector_description
 		,:connector_type
+		,:connector_created_by
 		,:connector_space_id
-		,:connector_uid
+		,:connector_identifier
-		,:connector_data
+		,:connector_last_test_attempt
+		,:connector_last_test_error_msg
+		,:connector_last_test_status
 		,:connector_created
 		,:connector_updated
 		,:connector_version
+		,:connector_auth_type
+		,:connector_address
+		,:connector_insecure
+		,:connector_username
+		,:connector_github_app_installation_id
+		,:connector_github_app_application_id
+		,:connector_region
+		,:connector_password
+		,:connector_token
+		,:connector_aws_key
+		,:connector_aws_secret
+		,:connector_github_app_private_key
+		,:connector_token_refresh
 	) RETURNING connector_id`
 	db := dbtx.GetAccessor(ctx, s.db)
 
-	query, arg, err := db.BindNamed(connectorInsertStmt, connector)
+	query, arg, err := db.BindNamed(connectorInsertStmt, dbConnector)
 	if err != nil {
 		return database.ProcessSQLErrorf(ctx, err, "Failed to bind connector object")
 	}
@@ -128,24 +406,42 @@ func (s *connectorStore) Create(ctx context.Context, connector *types.Connector)
 }
 
 func (s *connectorStore) Update(ctx context.Context, p *types.Connector) error {
+	conn, err := s.mapToDBConnector(ctx, p)
+	if err != nil {
+		return err
+	}
 	const connectorUpdateStmt = `
 	UPDATE connectors
 	SET
 		connector_description = :connector_description
-		,connector_uid = :connector_uid
+		,connector_identifier = :connector_identifier
-		,connector_data = :connector_data
+		,connector_last_test_attempt = :connector_last_test_attempt
-		,connector_type = :connector_type
+		,connector_last_test_error_msg = :connector_last_test_error_msg
+		,connector_last_test_status = :connector_last_test_status
 		,connector_updated = :connector_updated
 		,connector_version = :connector_version
+		,connector_auth_type = :connector_auth_type
+		,connector_address = :connector_address
+		,connector_insecure = :connector_insecure
+		,connector_username = :connector_username
+		,connector_github_app_installation_id = :connector_github_app_installation_id
+		,connector_github_app_application_id = :connector_github_app_application_id
+		,connector_region = :connector_region
+		,connector_password = :connector_password
+		,connector_token = :connector_token
+		,connector_aws_key = :connector_aws_key
+		,connector_aws_secret = :connector_aws_secret
+		,connector_github_app_private_key = :connector_github_app_private_key
+		,connector_token_refresh = :connector_token_refresh
 	WHERE connector_id = :connector_id AND connector_version = :connector_version - 1`
-	connector := *p
+	o := *conn
 
-	connector.Version++
+	o.Version++
-	connector.Updated = time.Now().UnixMilli()
+	o.Updated = time.Now().UnixMilli()
 
 	db := dbtx.GetAccessor(ctx, s.db)
 
-	query, arg, err := db.BindNamed(connectorUpdateStmt, connector)
+	query, arg, err := db.BindNamed(connectorUpdateStmt, o)
 	if err != nil {
 		return database.ProcessSQLErrorf(ctx, err, "Failed to bind connector object")
 	}
@@ -164,8 +460,8 @@ func (s *connectorStore) Update(ctx context.Context, p *types.Connector) error {
 		return gitness_store.ErrVersionConflict
 	}
 
-	p.Version = connector.Version
+	p.Version = o.Version
-	p.Updated = connector.Updated
+	p.Updated = o.Updated
 	return nil
 }
 
@@ -209,7 +505,7 @@ func (s *connectorStore) List(
 		Where("connector_space_id = ?", fmt.Sprint(parentID))
 
 	if filter.Query != "" {
-		stmt = stmt.Where("LOWER(connector_uid) LIKE ?", fmt.Sprintf("%%%s%%", strings.ToLower(filter.Query)))
+		stmt = stmt.Where("LOWER(connector_identifier) LIKE ?", fmt.Sprintf("%%%s%%", strings.ToLower(filter.Query)))
 	}
 
 	stmt = stmt.Limit(database.Limit(filter.Size))
@@ -222,12 +518,12 @@ func (s *connectorStore) List(
 
 	db := dbtx.GetAccessor(ctx, s.db)
 
-	dst := []*types.Connector{}
+	dst := []*connector{}
 	if err = db.SelectContext(ctx, &dst, sql, args...); err != nil {
 		return nil, database.ProcessSQLErrorf(ctx, err, "Failed executing custom list query")
 	}
 
-	return dst, nil
+	return s.mapFromDBConnectors(ctx, dst)
 }
 
 // Delete deletes a connector given a connector ID.
@@ -249,7 +545,7 @@ func (s *connectorStore) Delete(ctx context.Context, id int64) error {
 func (s *connectorStore) DeleteByIdentifier(ctx context.Context, spaceID int64, identifier string) error {
 	const connectorDeleteStmt = `
 	DELETE FROM connectors
-	WHERE connector_space_id = $1 AND connector_uid = $2`
+	WHERE connector_space_id = $1 AND connector_identifier = $2`
 
 	db := dbtx.GetAccessor(ctx, s.db)
 
@@ -268,7 +564,7 @@ func (s *connectorStore) Count(ctx context.Context, parentID int64, filter types
 		Where("connector_space_id = ?", parentID)
 
 	if filter.Query != "" {
-		stmt = stmt.Where("LOWER(connector_uid) LIKE ?", fmt.Sprintf("%%%s%%", filter.Query))
+		stmt = stmt.Where("LOWER(connector_identifier) LIKE ?", fmt.Sprintf("%%%s%%", filter.Query))
 	}
 
 	sql, args, err := stmt.ToSql()

app/store/database/migrate/postgres/0072_create_ar_table_images_and_alter_table_artifacts.up.sql

@@ -23,7 +23,7 @@ ALTER TABLE artifacts DROP CONSTRAINT fk_registries_registry_id;
 
 ALTER TABLE artifacts DROP COLUMN artifact_name;
 ALTER TABLE artifacts DROP COLUMN artifact_registry_id;
-ALTER TABLE artifacts DROP COLUMN artifact_labels;..
+ALTER TABLE artifacts DROP COLUMN artifact_labels;
 ALTER TABLE artifacts DROP COLUMN artifact_enabled;
 
 ALTER TABLE artifacts ADD COLUMN artifact_version TEXT NOT NULL;

app/store/database/migrate/postgres/0074_create_table_connectors.down.sql

@@ -0,0 +1 @@
+DROP TABLE IF exists connectors;

app/store/database/migrate/postgres/0074_create_table_connectors.up.sql

@@ -0,0 +1,88 @@
+-- Connectors table is not being used so can be dropped and recreated without
+-- worrying about a migration
+DROP TABLE IF EXISTS connectors;
+
+CREATE TABLE connectors (
+    -- Fields valid for all connectors
+    connector_id SERIAL PRIMARY KEY,
+    connector_identifier TEXT NOT NULL,
+    connector_description TEXT NOT NULL,
+    connector_type TEXT NOT NULL,
+    connector_auth_type TEXT NOT NULL, -- basicauth, oidc, oauth, aws
+    connector_created_by INTEGER NOT NULL,
+    connector_space_id INTEGER NOT NULL,
+    connector_last_test_attempt INTEGER NOT NULL,
+    connector_last_test_error_msg TEXT NOT NULL,
+    connector_last_test_status TEXT NOT NULL,
+    connector_created BIGINT NOT NULL,
+    connector_updated BIGINT NOT NULL,
+    connector_version INTEGER NOT NULL,
+    connector_address TEXT,
+    connector_insecure BOOLEAN,
+
+    -- Fields used by different connectors based on the auth_type
+    connector_username TEXT,
+    connector_github_app_installation_id TEXT,
+    connector_github_app_application_id TEXT,
+    connector_region TEXT,
+
+    -- secrets (foreign keys to the secrets table and restricted on delete)
+    connector_password INTEGER,
+    connector_token INTEGER,
+    connector_aws_key INTEGER,
+    connector_aws_secret INTEGER,
+    connector_github_app_private_key INTEGER,
+    connector_token_refresh INTEGER,
+    
+    -- Foreign key to spaces table
+    CONSTRAINT fk_connectors_space_id FOREIGN KEY (connector_space_id)
+        REFERENCES spaces (space_id)
+        ON UPDATE NO ACTION
+        ON DELETE CASCADE,
+
+    -- Foreign key to principals table
+    CONSTRAINT fk_connectors_created_by FOREIGN KEY (connector_created_by)
+        REFERENCES principals (principal_id)
+        ON UPDATE NO ACTION
+        ON DELETE NO ACTION,
+
+    -- Foreign key to secrets table
+    CONSTRAINT fk_connectors_password FOREIGN KEY (connector_password)
+        REFERENCES secrets (secret_id)
+        ON UPDATE NO ACTION
+        ON DELETE RESTRICT,
+
+    -- Foreign key to secrets table
+    CONSTRAINT fk_connectors_token FOREIGN KEY (connector_token)
+        REFERENCES secrets (secret_id)
+        ON UPDATE NO ACTION
+        ON DELETE RESTRICT,
+
+    -- Foreign key to secrets table
+    CONSTRAINT fk_connectors_aws_key FOREIGN KEY (connector_aws_key)
+        REFERENCES secrets (secret_id)
+        ON UPDATE NO ACTION
+        ON DELETE RESTRICT,
+
+    -- Foreign key to secrets table
+    CONSTRAINT fk_connectors_aws_secret FOREIGN KEY (connector_aws_secret)
+        REFERENCES secrets (secret_id)
+        ON UPDATE NO ACTION
+        ON DELETE RESTRICT,
+
+    -- Foreign key to secrets table
+    CONSTRAINT fk_connectors_github_app_private_key FOREIGN KEY (connector_github_app_private_key)
+        REFERENCES secrets (secret_id)
+        ON UPDATE NO ACTION
+        ON DELETE RESTRICT,
+
+    -- Foreign key to secrets table
+    CONSTRAINT fk_connectors_token_refresh FOREIGN KEY (connector_token_refresh)
+        REFERENCES secrets (secret_id)
+        ON UPDATE NO ACTION
+        ON DELETE RESTRICT
+);
+
+-- Creating a unique index for case-insensitive connector identifiers
+CREATE UNIQUE INDEX unique_connector_lowercase_identifier 
+ON connectors(connector_space_id, LOWER(connector_identifier));

app/store/database/migrate/sqlite/0074_create_table_connectors.down.sql

@@ -0,0 +1 @@
+DROP TABLE IF EXISTS connectors;

app/store/database/migrate/sqlite/0074_create_table_connectors.up.sql

@@ -0,0 +1,88 @@
+-- Connectors table is not being used so can be dropped and recreated without
+-- worrying about a migration
+DROP TABLE IF EXISTS connectors;
+
+CREATE TABLE connectors (
+    -- Fields valid for all connectors
+    connector_id INTEGER PRIMARY KEY AUTOINCREMENT,
+    connector_identifier TEXT NOT NULL,
+    connector_description TEXT NOT NULL,
+    connector_type TEXT NOT NULL,
+    connector_auth_type TEXT NOT NULL, -- basicauth, oidc, oauth, aws
+    connector_created_by INTEGER NOT NULL,
+    connector_space_id INTEGER NOT NULL,
+    connector_last_test_attempt INTEGER NOT NULL,
+    connector_last_test_error_msg TEXT NOT NULL,
+    connector_last_test_status TEXT NOT NULL,
+    connector_created INTEGER NOT NULL,
+    connector_updated INTEGER NOT NULL,
+    connector_version INTEGER NOT NULL,
+    connector_address TEXT,
+    connector_insecure BOOLEAN,
+
+    -- Fields used by different connectors based on the auth_type
+    connector_username TEXT,
+    connector_github_app_installation_id TEXT,
+    connector_github_app_application_id TEXT,
+    connector_region TEXT,
+
+    -- secrets (foreign keys to the secrets table and restricted on delete)
+    connector_password INTEGER,
+    connector_token INTEGER,
+    connector_aws_key INTEGER,
+    connector_aws_secret INTEGER,
+    connector_github_app_private_key INTEGER,
+    connector_token_refresh INTEGER,
+    
+    -- Foreign key to spaces table
+    CONSTRAINT fk_connectors_space_id FOREIGN KEY (connector_space_id)
+        REFERENCES spaces (space_id) MATCH SIMPLE
+        ON UPDATE NO ACTION
+        ON DELETE CASCADE,
+
+    -- Foreign key to principals table
+    CONSTRAINT fk_connectors_created_by FOREIGN KEY (connector_created_by)
+        REFERENCES principals (principal_id) MATCH SIMPLE
+        ON UPDATE NO ACTION
+        ON DELETE NO ACTION,
+
+    -- Foreign key to secrets table
+    CONSTRAINT fk_connectors_password FOREIGN KEY (connector_password)
+        REFERENCES secrets (secret_id) MATCH SIMPLE
+        ON UPDATE NO ACTION
+        ON DELETE RESTRICT,
+
+    -- Foreign key to secrets table
+    CONSTRAINT fk_connectors_token FOREIGN KEY (connector_token)
+        REFERENCES secrets (secret_id) MATCH SIMPLE
+        ON UPDATE NO ACTION
+        ON DELETE RESTRICT,
+
+    -- Foreign key to secrets table
+    CONSTRAINT fk_connectors_aws_key FOREIGN KEY (connector_aws_key)
+        REFERENCES secrets (secret_id) MATCH SIMPLE
+        ON UPDATE NO ACTION
+        ON DELETE RESTRICT,
+
+    -- Foreign key to secrets table
+    CONSTRAINT fk_connectors_aws_secret FOREIGN KEY (connector_aws_secret)
+        REFERENCES secrets (secret_id) MATCH SIMPLE
+        ON UPDATE NO ACTION
+        ON DELETE RESTRICT
+
+    -- Foreign key to secrets table
+    CONSTRAINT fk_connectors_github_app_private_key FOREIGN KEY (connector_github_app_private_key)
+        REFERENCES secrets (secret_id) MATCH SIMPLE
+        ON UPDATE NO ACTION
+        ON DELETE RESTRICT
+
+ -- Foreign key to secrets table
+    CONSTRAINT fk_connectors_token_refresh FOREIGN KEY (connector_token_refresh)
+        REFERENCES secrets (secret_id) MATCH SIMPLE
+        ON UPDATE NO ACTION
+        ON DELETE RESTRICT
+);
+
+-- Creating a unique index for case-insensitive connector identifiers
+CREATE UNIQUE INDEX unique_connector_lowercase_identifier 
+ON connectors(connector_space_id, LOWER(connector_identifier));

app/store/database/wire.go

@@ -205,8 +205,8 @@ func ProvideSecretStore(db *sqlx.DB) store.SecretStore {
 }
 
 // ProvideConnectorStore provides a connector store.
-func ProvideConnectorStore(db *sqlx.DB) store.ConnectorStore {
+func ProvideConnectorStore(db *sqlx.DB, secretStore store.SecretStore) store.ConnectorStore {
-	return NewConnectorStore(db)
+	return NewConnectorStore(db, secretStore)
 }
 
 // ProvideTemplateStore provides a template store.

cmd/gitness/wire.go

@@ -53,6 +53,7 @@ import (
 	"github.com/harness/gitness/app/auth/authn"
 	"github.com/harness/gitness/app/auth/authz"
 	"github.com/harness/gitness/app/bootstrap"
+	connectorservice "github.com/harness/gitness/app/connector"
 	gitevents "github.com/harness/gitness/app/events/git"
 	gitspaceevents "github.com/harness/gitness/app/events/gitspace"
 	gitspaceinfraevents "github.com/harness/gitness/app/events/gitspaceinfra"
@@ -215,6 +216,7 @@ func initSystem(ctx context.Context, config *types.Config) (*cliserver.System, e
 		controllerlogs.WireSet,
 		secret.WireSet,
 		connector.WireSet,
+		connectorservice.WireSet,
 		template.WireSet,
 		manager.WireSet,
 		triggerer.WireSet,

cmd/gitness/wire_gen.go

@@ -12,7 +12,7 @@ import (
 	aiagent2 "github.com/harness/gitness/app/api/controller/aiagent"
 	capabilities2 "github.com/harness/gitness/app/api/controller/capabilities"
 	check2 "github.com/harness/gitness/app/api/controller/check"
-	"github.com/harness/gitness/app/api/controller/connector"
+	connector2 "github.com/harness/gitness/app/api/controller/connector"
 	"github.com/harness/gitness/app/api/controller/execution"
 	"github.com/harness/gitness/app/api/controller/githook"
 	gitspace2 "github.com/harness/gitness/app/api/controller/gitspace"
@@ -42,6 +42,7 @@ import (
 	"github.com/harness/gitness/app/auth/authn"
 	"github.com/harness/gitness/app/auth/authz"
 	"github.com/harness/gitness/app/bootstrap"
+	"github.com/harness/gitness/app/connector"
 	events6 "github.com/harness/gitness/app/events/git"
 	events7 "github.com/harness/gitness/app/events/gitspace"
 	events3 "github.com/harness/gitness/app/events/gitspaceinfra"
@@ -269,7 +270,7 @@ func initSystem(ctx context.Context, config *types.Config) (*server.System, erro
 	logsController := logs2.ProvideController(authorizer, executionStore, repoStore, pipelineStore, stageStore, stepStore, logStore, logStream)
 	spaceIdentifier := check.ProvideSpaceIdentifierCheck()
 	secretStore := database.ProvideSecretStore(db)
-	connectorStore := database.ProvideConnectorStore(db)
+	connectorStore := database.ProvideConnectorStore(db, secretStore)
 	repoGitInfoView := database.ProvideRepoGitInfoView(db)
 	repoGitInfoCache := cache.ProvideRepoGitInfoCache(repoGitInfoView)
 	pullReqStore := database.ProvidePullReqStore(db, principalInfoCache)
@@ -306,7 +307,9 @@ func initSystem(ctx context.Context, config *types.Config) (*server.System, erro
 	pipelineController := pipeline.ProvideController(repoStore, triggerStore, authorizer, pipelineStore, reporter2)
 	secretController := secret.ProvideController(encrypter, secretStore, authorizer, spaceStore)
 	triggerController := trigger.ProvideController(authorizer, triggerStore, pipelineStore, repoStore)
-	connectorController := connector.ProvideController(connectorStore, authorizer, spaceStore)
+	scmService := connector.ProvideSCMConnectorHandler(secretStore)
+	connectorService := connector.ProvideConnectorHandler(secretStore, scmService)
+	connectorController := connector2.ProvideController(connectorStore, connectorService, authorizer, spaceStore)
 	templateController := template.ProvideController(templateStore, authorizer, spaceStore)
 	pluginController := plugin.ProvideController(pluginStore)
 	pullReqActivityStore := database.ProvidePullReqActivityStore(db, principalInfoCache)

types/connector.go

@@ -14,29 +14,24 @@
 
 package types
 
-import "encoding/json"
+import (
+	"github.com/harness/gitness/types/enum"
+)
 
 type Connector struct {
-	ID          int64  `db:"connector_id"              json:"-"`
+	ID               int64                `json:"-"`
-	Description string `db:"connector_description"     json:"description"`
+	Description      string               `json:"description"`
-	SpaceID     int64  `db:"connector_space_id"        json:"space_id"`
+	SpaceID          int64                `json:"space_id"`
-	Identifier  string `db:"connector_uid"             json:"identifier"`
+	Identifier       string               `json:"identifier"`
-	Type        string `db:"connector_type"            json:"type"`
+	CreatedBy        int64                `json:"created_by"`
-	Data        string `db:"connector_data"            json:"data"`
+	Type             enum.ConnectorType   `json:"type"`
-	Created     int64  `db:"connector_created"         json:"created"`
+	LastTestAttempt  int64                `json:"last_test_attempt"`
-	Updated     int64  `db:"connector_updated"         json:"updated"`
+	LastTestErrorMsg string               `json:"last_test_error_msg"`
-	Version     int64  `db:"connector_version"         json:"-"`
+	LastTestStatus   enum.ConnectorStatus `json:"last_test_status"`
-}
+	Created          int64                `json:"created"`
+	Updated          int64                `json:"updated"`
+	Version          int64                `json:"-"`
 
-// TODO [CODE-1363]: remove after identifier migration.
+	// Pointers to connector specific data
-func (s Connector) MarshalJSON() ([]byte, error) {
+	ConnectorConfig
-	// alias allows us to embed the original object while avoiding an infinite loop of marshaling.
-	type alias Connector
-	return json.Marshal(&struct {
-		alias
-		UID string `json:"uid"`
-	}{
-		alias: (alias)(s),
-		UID:   s.Identifier,
-	})
 }

types/connector_auth.go

@@ -0,0 +1,60 @@
+// Copyright 2023 Harness, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package types
+
+import (
+	"fmt"
+
+	"github.com/harness/gitness/types/enum"
+)
+
+// ConnectorAuth represents the authentication configuration for a connector.
+type ConnectorAuth struct {
+	AuthType enum.ConnectorAuthType `json:"type"`
+	Basic    *BasicAuthCreds        `json:"basic,omitempty"`
+	Bearer   *BearerTokenCreds      `json:"bearer,omitempty"`
+}
+
+// BasicAuthCreds represents credentials for basic authentication.
+type BasicAuthCreds struct {
+	Username string    `json:"username"`
+	Password SecretRef `json:"password"`
+}
+
+type BearerTokenCreds struct {
+	Token SecretRef `json:"token"`
+}
+
+func (c *ConnectorAuth) Validate() error {
+	switch c.AuthType {
+	case enum.ConnectorAuthTypeBasic:
+		if c.Basic == nil {
+			return fmt.Errorf("basic auth credentials are required")
+		}
+		if c.Basic.Username == "" || c.Basic.Password.Identifier == "" {
+			return fmt.Errorf("basic auth credentials are required")
+		}
+	case enum.ConnectorAuthTypeBearer:
+		if c.Bearer == nil {
+			return fmt.Errorf("bearer token credentials are required")
+		}
+		if c.Bearer.Token.Identifier == "" {
+			return fmt.Errorf("bearer token is required")
+		}
+	default:
+		return fmt.Errorf("unsupported auth type: %s", c.AuthType)
+	}
+	return nil
+}

types/connector_config.go

@@ -0,0 +1,38 @@
+// Copyright 2023 Harness, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package types
+
+import (
+	"fmt"
+
+	"github.com/harness/gitness/types/enum"
+)
+
+// ConnectorConfig is a list of all the connector and their associated config.
+type ConnectorConfig struct {
+	Github *GithubConnectorData `json:"github,omitempty"`
+}
+
+func (c ConnectorConfig) Validate(typ enum.ConnectorType) error {
+	switch typ {
+	case enum.ConnectorTypeGithub:
+		if c.Github != nil {
+			return c.Github.Validate()
+		}
+		return fmt.Errorf("github connector config is required")
+	default:
+		return fmt.Errorf("connector type %s is not supported", typ)
+	}
+}

types/connector_test_response.go

@@ -0,0 +1,22 @@
+// Copyright 2023 Harness, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package types
+
+import "github.com/harness/gitness/types/enum"
+
+type ConnectorTestResponse struct {
+	Status   enum.ConnectorStatus `json:"status"`
+	ErrorMsg string               `json:"error_msg,omitempty"`
+}

types/enum/connector_auth_type.go

@@ -0,0 +1,61 @@
+// Copyright 2023 Harness, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package enum
+
+import "fmt"
+
+// ConnectorAuthType represents the type of connector authentication.
+type ConnectorAuthType string
+
+const (
+	ConnectorAuthTypeBasic  ConnectorAuthType = "basic"
+	ConnectorAuthTypeBearer ConnectorAuthType = "bearer"
+)
+
+func ParseConnectorAuthType(s string) (ConnectorAuthType, error) {
+	switch s {
+	case "basic":
+		return ConnectorAuthTypeBasic, nil
+	case "bearer":
+		return ConnectorAuthTypeBearer, nil
+	default:
+		return "", fmt.Errorf("unknown connector auth type provided: %s", s)
+	}
+}
+
+func (t ConnectorAuthType) String() string {
+	switch t {
+	case ConnectorAuthTypeBasic:
+		return "basic"
+	case ConnectorAuthTypeBearer:
+		return "bearer"
+	default:
+		return "undefined"
+	}
+}
+
+func GetAllConnectorAuthTypes() []ConnectorAuthType {
+	return []ConnectorAuthType{
+		ConnectorAuthTypeBasic,
+		ConnectorAuthTypeBearer,
+	}
+}
+
+func (ConnectorAuthType) Enum() []interface{} { return toInterfaceSlice(GetAllConnectorAuthTypes()) }
+func (t ConnectorAuthType) Sanitize() (ConnectorAuthType, bool) {
+	return Sanitize(t, func() ([]ConnectorAuthType, ConnectorAuthType) {
+		return GetAllConnectorAuthTypes(), ""
+	})
+}

types/enum/connector_status.go

@@ -0,0 +1,49 @@
+// Copyright 2023 Harness, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package enum
+
+// ConnectorStatus represents the status of the connector after testing connection.
+type ConnectorStatus string
+
+const (
+	ConnectorStatusSuccess ConnectorStatus = "success"
+
+	ConnectorStatusFailed ConnectorStatus = "failed"
+)
+
+func (s ConnectorStatus) String() string {
+	switch s {
+	case ConnectorStatusSuccess:
+		return "success"
+	case ConnectorStatusFailed:
+		return "failed"
+	default:
+		return undefined
+	}
+}
+
+func GetAllConnectorStatus() ([]ConnectorStatus, ConnectorStatus) {
+	return connectorStatus, "" // No default value
+}
+
+var connectorStatus = sortEnum([]ConnectorStatus{
+	ConnectorStatusSuccess,
+	ConnectorStatusFailed,
+})
+
+func (ConnectorStatus) Enum() []interface{} { return toInterfaceSlice(connectorStatus) }
+func (s ConnectorStatus) Sanitize() (ConnectorStatus, bool) {
+	return Sanitize(s, GetAllConnectorStatus)
+}

types/enum/connector_type.go

@@ -0,0 +1,63 @@
+// Copyright 2023 Harness, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package enum
+
+import "fmt"
+
+// ConnectorType represents the type of connector.
+type ConnectorType string
+
+const (
+	// ConnectorTypeGithub is a github connector.
+	ConnectorTypeGithub ConnectorType = "github"
+)
+
+func ParseConnectorType(s string) (ConnectorType, error) {
+	switch s {
+	case "github":
+		return ConnectorTypeGithub, nil
+	default:
+		return "", fmt.Errorf("unknown connector type provided: %s", s)
+	}
+}
+
+func (t ConnectorType) String() string {
+	switch t {
+	case ConnectorTypeGithub:
+		return "github"
+	default:
+		return undefined
+	}
+}
+
+func (t ConnectorType) IsSCM() bool {
+	switch t {
+	case ConnectorTypeGithub:
+		return true
+	default:
+		return false
+	}
+}
+
+func GetAllConnectorTypes() ([]ConnectorType, ConnectorType) {
+	return connectorTypes, "" // No default value
+}
+
+var connectorTypes = sortEnum([]ConnectorType{
+	ConnectorTypeGithub,
+})
+
+func (ConnectorType) Enum() []interface{}               { return toInterfaceSlice(connectorTypes) }
+func (t ConnectorType) Sanitize() (ConnectorType, bool) { return Sanitize(t, GetAllConnectorTypes) }

types/github_connector_data.go

@@ -0,0 +1,44 @@
+// Copyright 2023 Harness, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package types
+
+import (
+	"fmt"
+
+	"github.com/harness/gitness/types/enum"
+)
+
+type GithubConnectorData struct {
+	APIURL   string         `json:"api_url"`
+	Insecure bool           `json:"insecure"`
+	Auth     *ConnectorAuth `json:"auth"`
+}
+
+func (g *GithubConnectorData) Validate() error {
+	if g.Auth == nil {
+		return fmt.Errorf("auth is required for github connectors")
+	}
+	if g.Auth.AuthType != enum.ConnectorAuthTypeBearer {
+		return fmt.Errorf("only bearer token auth is supported for github connectors")
+	}
+	if err := g.Auth.Validate(); err != nil {
+		return fmt.Errorf("invalid auth credentials: %w", err)
+	}
+	return nil
+}
+
+func (g *GithubConnectorData) Type() enum.ConnectorType {
+	return enum.ConnectorTypeGithub
+}

types/secret_ref.go

@@ -0,0 +1,19 @@
+// Copyright 2023 Harness, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package types
+
+type SecretRef struct {
+	Identifier string `json:"identifier"`
+}