diff --git a/internal/handlers/descriptors/adapters.yaml b/internal/handlers/descriptors/adapters.yaml index 123a0f18..0a730289 100644 --- a/internal/handlers/descriptors/adapters.yaml +++ b/internal/handlers/descriptors/adapters.yaml @@ -60,6 +60,18 @@ export: description: | Export an adapter configuration +load: + use: adapters + group: admin-essentials + description: | + Load all adapters + +dump: + use: adapters + group: admin-essentials + description: | + Dump all adapters + ############################################################################# # Platform commands ############################################################################# diff --git a/internal/handlers/descriptors/workflows.yaml b/internal/handlers/descriptors/workflows.yaml index 440187e3..05fe2e2f 100644 --- a/internal/handlers/descriptors/workflows.yaml +++ b/internal/handlers/descriptors/workflows.yaml @@ -111,18 +111,17 @@ export: ############################################################################## -# Gitter Commands +# Dataset Commands ############################################################################## -push: - use: workflow - group: automation-studio +load: + use: workflows + group: automation-studio description: | - Push a workflow directly to a repository + Load workflows -pull: - use: workflow +dump: + use: workflows group: automation-studio - description: | - Pull a workflow directly from a repository + Dump all workflows diff --git a/internal/handlers/handlers.go b/internal/handlers/handlers.go index a4299498..44173d34 100644 --- a/internal/handlers/handlers.go +++ b/internal/handlers/handlers.go @@ -69,13 +69,10 @@ func NewHandler(iapClient client.Client, cfg *config.Config) Handler { NewServerHandler(iapClient, cfg, descriptors), + NewLocalAAAHandler(iapClient, cfg, descriptors), NewLocalClientHandler(iapClient, cfg, descriptors), ) - if cfg.MongoUri != "" { - NewLocalAAAHandler(iapClient, cfg, descriptors) - } - return Handler{ Runtime: &Runtime{ Config: cfg, diff --git a/internal/runners/adapters.go b/internal/runners/adapters.go index c4831a2f..ea0a3754 100644 --- a/internal/runners/adapters.go +++ b/internal/runners/adapters.go @@ -412,6 +412,81 @@ func (r *AdapterRunner) Restart(in Request) (*Response, error) { ), nil } +////////////////////////////////////////////////////////////////////////////// +// Dumper Interface +// + +// Dump implements the `dump adapters...` command +func (r *AdapterRunner) Dump(in Request) (*Response, error) { + logger.Trace() + + res, err := r.service.GetAll() + if err != nil { + return nil, err + } + + var assets = map[string]interface{}{} + for _, ele := range res { + key := fmt.Sprintf("%s.adapter.json", ele.Name) + assets[key] = ele + } + + if err := dumpAssets(in, assets); err != nil { + return nil, err + } + + return NewResponse( + fmt.Sprintf("Dumped %v adapter(s)", len(assets)), + ), nil +} + +////////////////////////////////////////////////////////////////////////////// +// Loader Interface +// + +// Load implements the `load adapters ...` command +func (r *AdapterRunner) Load(in Request) (*Response, error) { + logger.Trace() + + elements, err := loadAssets(in) + if err != nil { + return nil, err + } + + var loaded int + var skipped int + + var output []string + + for fn, ele := range elements { + var adapter services.Adapter + + if err := loadUnmarshalAsset(ele, &adapter); err != nil { + output = append(output, fmt.Sprintf("Failed to load adapter from `%s`, skipping", fn)) + skipped++ + } else { + if _, err := r.service.Import(adapter); err != nil { + if !strings.HasSuffix(err.Error(), "already exists!\"") { + return nil, err + } + output = append(output, fmt.Sprintf("Skipping `%s`, adapter `%s` already exists", fn, adapter.Name)) + skipped++ + } else { + output = append(output, fmt.Sprintf("Loaded adapter `%s` successfully from `%s`", adapter.Name, fn)) + loaded++ + } + } + } + + output = append(output, fmt.Sprintf( + "\nSuccessfully loaded %v and skipped %v files from `%s`", loaded, skipped, in.Args[0], + )) + + return NewResponse( + strings.Join(output, "\n"), + ), nil +} + ////////////////////////////////////////////////////////////////////////////// // Private functions // diff --git a/internal/runners/dumper.go b/internal/runners/dumper.go index 70736ddd..771ad1dc 100644 --- a/internal/runners/dumper.go +++ b/internal/runners/dumper.go @@ -15,12 +15,5 @@ import ( // value must be the object instance. func dumpAssets(in Request, objects map[string]interface{}) error { logger.Trace() - - for key, value := range objects { - if err := exportAssetFromRequest(in, value, key); err != nil { - return err - } - } - - return nil + return exportAssets(in, objects) } diff --git a/internal/runners/export.go b/internal/runners/export.go deleted file mode 100644 index 09625fe0..00000000 --- a/internal/runners/export.go +++ /dev/null @@ -1,87 +0,0 @@ -// Copyright 2024 Itential Inc. All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited -// Proprietary and confidential - -package runners - -import ( - "errors" - "fmt" - "os" - "path/filepath" - - "github.com/itential/ipctl/internal/utils" - "github.com/itential/ipctl/pkg/logger" -) - -const defaultEncoding = "json" - -type ExportObject struct { - Name string `json:"name"` - Type string `json:"type"` - Encoding string `json:"encoding"` - Path string `json:"path"` - AbsPath string `json:"abspath"` -} - -type ExportOption func(*ExportObject) - -func Export(in any, opts ...ExportOption) (*ExportObject, error) { - logger.Trace() - - obj := new(ExportObject) - - for _, opt := range opts { - opt(obj) - } - - if obj.Type == "" { - return nil, errors.New("export type not specified") - } - - if obj.Path == "" { - path, err := os.Getwd() - if err != nil { - return nil, err - } - obj.Path = path - } - - if obj.Encoding == "" { - obj.Encoding = defaultEncoding - } - - fn := fmt.Sprintf("%s.%s.json", obj.Name, obj.Type) - - obj.AbsPath = filepath.Join(obj.Path, fn) - - if err := utils.WriteJsonToDisk(in, fn, obj.Path); err != nil { - return nil, err - } - - return obj, nil -} - -func WithExportName(v string) ExportOption { - return func(obj *ExportObject) { - obj.Name = v - } -} - -func WithExportType(v string) ExportOption { - return func(obj *ExportObject) { - obj.Type = v - } -} - -func WithExportEncoding(v string) ExportOption { - return func(obj *ExportObject) { - obj.Encoding = v - } -} - -func WithExportPath(v string) ExportOption { - return func(obj *ExportObject) { - obj.Path = v - } -} diff --git a/internal/runners/exporter.go b/internal/runners/exporter.go index c93720e9..80de258c 100644 --- a/internal/runners/exporter.go +++ b/internal/runners/exporter.go @@ -7,23 +7,54 @@ package runners import ( "os" "path/filepath" + "strings" "github.com/itential/ipctl/internal/flags" "github.com/itential/ipctl/internal/utils" "github.com/itential/ipctl/pkg/logger" + giturls "github.com/whilp/git-urls" ) -// exportNewRepository will create a new Repository object from an export -// request. -func exportNewRepositoryFromRequest(in Request) *Repository { - common := in.Common.(*flags.AssetExportCommon) +// exportNewRepository will create a new Repository object from an the incoming +// Request object. +func exportNewRepositoryFromRequest(in Request) (*Repository, error) { + logger.Trace() + + common := in.Common.(flags.Gitter) + + url := common.GetRepository() + privateKeyFile := common.GetPrivateKeyFile() + reference := common.GetReference() + + u, err := giturls.Parse(common.GetRepository()) + if err != nil { + panic(err) + } + + if u.Scheme == "file" && strings.HasPrefix(u.Path, "@") { + r, err := in.Config.GetRepository(u.Path[1:]) + if err != nil { + return nil, err + } + + url = r.Url + + if privateKeyFile == "" { + privateKeyFile = r.PrivateKeyFile + } + + if reference == "" { + reference = r.Reference + } + } + return NewRepository( - common.Repository, - WithReference(common.Reference), - WithPrivateKeyFile(common.PrivateKeyFile), + url, + WithReference(reference), + WithPrivateKeyFile(privateKeyFile), WithName(in.Config.GitName), WithEmail(in.Config.GitEmail), - ) + ), nil } // exportAssetFromRequest will take a request object and instance of an asset @@ -31,6 +62,15 @@ func exportNewRepositoryFromRequest(in Request) *Repository { // will write the asset to the repository and commit it. If not, this function // will simply write the asset to the local disk. func exportAssetFromRequest(in Request, o any, fn string) error { + return exportAssets(in, map[string]interface{}{fn: o}) +} + +// exportAssets accepts the Request object and a map of the assets and will +// write them to disk. If the request object includes repository settings, +// this function will push the assets into the repository. The assets argument +// must be a map where the key is the filename and the value is the asset to +// write to disk. +func exportAssets(in Request, assets map[string]interface{}) error { logger.Trace() path := in.Common.(flags.Committer).GetPath() @@ -39,16 +79,13 @@ func exportAssetFromRequest(in Request, o any, fn string) error { var repoPath string if in.Common.(flags.Gitter).GetRepository() != "" { - repo = NewRepository( - in.Common.(flags.Gitter).GetRepository(), - WithReference(in.Common.(flags.Gitter).GetReference()), - WithPrivateKeyFile(in.Common.(flags.Gitter).GetPrivateKeyFile()), - WithName(in.Config.GitName), - WithEmail(in.Config.GitEmail), - ) - var e error + repo, e = exportNewRepositoryFromRequest(in) + if e != nil { + return e + } + repoPath, e = repo.Clone() if e != nil { return e @@ -58,11 +95,13 @@ func exportAssetFromRequest(in Request, o any, fn string) error { path = filepath.Join(repoPath, in.Common.(flags.Committer).GetPath()) } - if err := utils.WriteJsonToDisk(o, fn, path); err != nil { - return err + for key, value := range assets { + if err := utils.WriteJsonToDisk(value, key, path); err != nil { + return err + } } - if in.Common.(flags.Gitter).GetRepository() != "" { + if repo != nil { msg := in.Common.(flags.Committer).GetMessage() if err := repo.CommitAndPush(repoPath, msg); err != nil { return err diff --git a/internal/runners/models.go b/internal/runners/models.go index 665ad712..8d9800a9 100644 --- a/internal/runners/models.go +++ b/internal/runners/models.go @@ -340,7 +340,10 @@ func (r *ModelRunner) Export(in Request) (*Response, error) { var repoPath string if common.Repository != "" { - repo = exportNewRepositoryFromRequest(in) + repo, err = exportNewRepositoryFromRequest(in) + if err != nil { + return nil, err + } var e error diff --git a/internal/runners/prebuilts.go b/internal/runners/prebuilts.go index d9aa479f..8d0d407d 100644 --- a/internal/runners/prebuilts.go +++ b/internal/runners/prebuilts.go @@ -338,7 +338,10 @@ func (r *PrebuiltRunner) Export(in Request) (*Response, error) { var repoPath string if common.Repository != "" { - repo = exportNewRepositoryFromRequest(in) + repo, err = exportNewRepositoryFromRequest(in) + if err != nil { + return nil, err + } var e error diff --git a/internal/runners/projects.go b/internal/runners/projects.go index db091ff1..0044a87a 100644 --- a/internal/runners/projects.go +++ b/internal/runners/projects.go @@ -358,7 +358,10 @@ func (r *ProjectRunner) Export(in Request) (*Response, error) { var repoPath string if common.Repository != "" { - repo = exportNewRepositoryFromRequest(in) + repo, err = exportNewRepositoryFromRequest(in) + if err != nil { + return nil, err + } var e error diff --git a/internal/runners/workflows.go b/internal/runners/workflows.go index 3f198675..b37e0555 100644 --- a/internal/runners/workflows.go +++ b/internal/runners/workflows.go @@ -284,6 +284,84 @@ func (r *WorkflowRunner) Export(in Request) (*Response, error) { ), nil } +////////////////////////////////////////////////////////////////////////////// +// Dumper Interface +// + +// Dump implements the `dump workflows...` command +func (r *WorkflowRunner) Dump(in Request) (*Response, error) { + logger.Trace() + + res, err := r.service.GetAll() + if err != nil { + return nil, err + } + + var assets = map[string]interface{}{} + + for _, ele := range res { + if !strings.HasPrefix(ele.Name, "@") { + key := fmt.Sprintf("%s.workflow.json", ele.Name) + assets[key] = ele + } + } + + if err := dumpAssets(in, assets); err != nil { + return nil, err + } + + return NewResponse( + fmt.Sprintf("Dumped %v workflow(s)", len(assets)), + ), nil +} + +////////////////////////////////////////////////////////////////////////////// +// Loader Interface +// + +// Load implements the `load workflows ...` command +func (r *WorkflowRunner) Load(in Request) (*Response, error) { + logger.Trace() + + elements, err := loadAssets(in) + if err != nil { + return nil, err + } + + var loaded int + var skipped int + + var output []string + + for fn, ele := range elements { + var workflow services.Workflow + + if err := loadUnmarshalAsset(ele, &workflow); err != nil { + output = append(output, fmt.Sprintf("Failed to load workflow from `%s`, skipping", fn)) + skipped++ + } else { + if err := r.importWorkflow(workflow, false); err != nil { + if !strings.HasPrefix(err.Error(), "worklow with name") { + return nil, err + } + output = append(output, fmt.Sprintf("Skipping `%s`, workflow `%s` already exists", fn, workflow.Name)) + skipped++ + } else { + output = append(output, fmt.Sprintf("Loaded workflow `%s` successfully from `%s`", workflow.Name, fn)) + loaded++ + } + } + } + + output = append(output, fmt.Sprintf( + "\nSuccessfully loaded %v and skipped %v files from `%s`", loaded, skipped, in.Args[0], + )) + + return NewResponse( + strings.Join(output, "\n"), + ), nil +} + ////////////////////////////////////////////////////////////////////////////// // Private functions // diff --git a/internal/utils/path.go b/internal/utils/path.go index 74de3497..60f3d5fa 100644 --- a/internal/utils/path.go +++ b/internal/utils/path.go @@ -93,12 +93,12 @@ func WriteBytesToDisk(b []byte, dst string, overwrite bool) error { // current working directory is used. func WriteJsonToDisk(o any, fn, fp string) error { logger.Trace() - logger.Debug("Writing file `%s` to path `%s`", fn, fp) dst, err := NormalizeFilename(fn, fp) if err != nil { return err } + logger.Debug("Writing file `%s` to path `%s`", fn, fp) b, err := json.MarshalIndent(o, "", " ") if err != nil { diff --git a/pkg/config/config.go b/pkg/config/config.go index 5599e30b..54c46c2e 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -19,8 +19,9 @@ import ( const ( defaultFileName = "config" - defaultAppWorkingDir = "~/.platform.d" - defaultAppDefaultProfile = "" + defaultAppWorkingDir = "~/.platform.d" + defaultAppDefaultProfile = "" + defaultAppDefaultRepository = "" defaultLogLevel = "INFO" defaultLogFileJson = false @@ -38,13 +39,17 @@ const ( type Config struct { // Application settings - WorkingDir string `json:"working_dir"` - DefaultProfile string `json:"default_profile"` + WorkingDir string `json:"working_dir"` + DefaultProfile string `json:"default_profile"` + DefaultRepository string `json:"default_repository"` // Profiles profileName string profiles map[string]*Profile + // Repositories + repositories map[string]*Repository + // Log settings LogLevel string `json:"log_level"` LogFileJSON bool `json:"log_file_json"` @@ -60,9 +65,6 @@ type Config struct { // Git settings GitName string `json:"git_name"` GitEmail string `json:"git_email"` - - // Mongo settings - MongoUri string `json:"mongo_uri"` } func NewConfig(defaults map[string]interface{}, envBinding map[string]string, appWorkingDir, sysConfigPath, fileName string) *Config { @@ -120,6 +122,7 @@ func (ac *Config) initConfig(defaultsVariables map[string]interface{}, environme } ac.profiles = map[string]*Profile{} + ac.repositories = map[string]*Repository{} var defaults map[string]interface{} @@ -130,7 +133,24 @@ func (ac *Config) initConfig(defaultsVariables map[string]interface{}, environme ac.profiles["default"] = loadProfile(defaults, defaults, map[string]interface{}{}) for key, value := range viper.AllSettings() { - if strings.HasPrefix(key, "profile ") { + if strings.HasPrefix(key, "repository ") { + parts := strings.Split(key, " ") + + if len(parts) > 2 { + handleError("repository names cannot contain spaces", nil) + } + + var overrides = map[string]interface{}{} + + for _, ele := range getRepositoryFields() { + if val, exists := os.LookupEnv(fmt.Sprintf("IPCTL_REPOSITORY_%s_%s", strings.ToUpper(parts[1]), strings.ToUpper(ele))); exists { + overrides[ele] = val + } + } + + ac.repositories[parts[1]] = loadRepository(value.(map[string]any), overrides) + + } else if strings.HasPrefix(key, "profile ") { parts := strings.Split(key, " ") if len(parts) > 2 { @@ -160,6 +180,7 @@ func (ac *Config) initConfig(defaultsVariables map[string]interface{}, environme func (ac *Config) populateFields() { ac.WorkingDir = GetAndExpandDirectory("application.working_dir") ac.DefaultProfile = viper.GetString("application.default_profile") + ac.DefaultRepository = viper.GetString("application.default_repository") ac.LogLevel = viper.GetString("log.level") ac.LogFileJSON = viper.GetBool("log.file_json") @@ -173,13 +194,12 @@ func (ac *Config) populateFields() { ac.GitName = viper.GetString("git.name") ac.GitEmail = viper.GetString("git.email") - - ac.MongoUri = viper.GetString("mongo.uri") } var defaultValues = map[string]interface{}{ - "application.working_dir": defaultAppWorkingDir, - "application.default_profile": defaultAppDefaultProfile, + "application.working_dir": defaultAppWorkingDir, + "application.default_profile": defaultAppDefaultProfile, + "application.default_repository": defaultAppDefaultRepository, "log.level": defaultLogLevel, "log.file_json": defaultLogFileJson, @@ -196,8 +216,9 @@ var defaultValues = map[string]interface{}{ } var defaultEnvVarBindings = map[string]string{ - "application.working_dir": "IPCTL_APPLICATION_WORKING_DIR", - "application.default_profile": "IPCTL_APPLICATION_DEFAULT_PROFILE", + "application.working_dir": "IPCTL_APPLICATION_WORKING_DIR", + "application.default_profile": "IPCTL_APPLICATION_DEFAULT_PROFILE", + "application.default_repository": "IPCTL_APPLICATION_DEFAULT_REPOSITORY", "log.level": "IPCTL_LOG_LEVEL", "log.file_json": "IPCTL_LOG_FILE_JSON", @@ -211,8 +232,6 @@ var defaultEnvVarBindings = map[string]string{ "git.name": "IPCTL_GIT_NAME", "git.email": "IPCTL_GIT_EMAIL", - - "mongo.uri": "IPCTL_MONGO_URI", } // getConfigFileFromFlag reads in the file passed in using the --config flag on the cli diff --git a/pkg/config/repository.go b/pkg/config/repository.go new file mode 100644 index 00000000..7b5c6a13 --- /dev/null +++ b/pkg/config/repository.go @@ -0,0 +1,76 @@ +// Copyright 2024 Itential Inc. All Rights Reserved +// Unauthorized copying of this file, via any medium is strictly prohibited +// Proprietary and confidential + +package config + +import ( + "errors" + "fmt" + "reflect" + + "github.com/mitchellh/go-homedir" +) + +type Repository struct { + Url string `json:"url"` + PrivateKey string `json:"private_key"` + PrivateKeyFile string `json:"private_key_file"` + Reference string `json:"reference"` +} + +func getRepositoryFields() []string { + fields := reflect.TypeOf((*Repository)(nil)).Elem() + var properties = []string{} + + for i := 0; i < fields.NumField(); i++ { + f := fields.Field(i) + properties = append(properties, f.Tag.Get("json")) + } + + return properties +} + +func loadRepository(values, overrides map[string]interface{}) *Repository { + r := &Repository{} + + for _, ele := range getRepositoryFields() { + var value any + + // First check if the value for the key exists in overrides; if it + // does, use it. If it doesn't, fallback to checking if it wwas + // defined in the profile (values) + if val, ok := overrides[ele]; ok { + value = val + } else { + value = values[ele] + } + + var v string + + if value != nil { + v = value.(string) + } + + switch ele { + case "url": + r.Url = v + case "private_key": + r.PrivateKey = v + case "private_key_file": + privateKeyFile, _ := homedir.Expand(v) + r.PrivateKeyFile = privateKeyFile + case "reference": + r.Reference = v + } + } + + return r +} + +func (ac *Config) GetRepository(name string) (*Repository, error) { + if v, exists := ac.repositories[name]; exists { + return v, nil + } + return nil, errors.New(fmt.Sprintf("repository `%s` does not exist", name)) +}