From 073ebe9bdc14c78ed60d9d97cf309a2a6039724c Mon Sep 17 00:00:00 2001 From: Monica Casanova Date: Wed, 10 May 2023 17:02:10 +0100 Subject: [PATCH 01/12] feat: Adds SARIF output --- cmd/main.go | 9 +++ reporting/report.go | 12 +++ reporting/sarif.go | 186 ++++++++++++++++++++++++++++++++++++++++++++ secrets/secrets.go | 14 ++-- 4 files changed, 215 insertions(+), 6 deletions(-) create mode 100644 reporting/sarif.go diff --git a/cmd/main.go b/cmd/main.go index 6f9aa7df..80bf3464 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -15,6 +15,7 @@ import ( ) const timeSleepInterval = 50 +const reportPath = "report-path" var rootCmd = &cobra.Command{ Use: "2ms", @@ -57,6 +58,7 @@ func Execute() { cobra.OnInitialize(initLog) rootCmd.Flags().BoolP("all", "", true, "scan all plugins") rootCmd.Flags().StringSlice("tags", []string{"all"}, "select rules to be applied") + rootCmd.Flags().StringP(reportPath, "r", "", "path to generate report file") for _, plugin := range allPlugins { err := plugin.DefineCommandLineArgs(rootCmd) @@ -88,6 +90,7 @@ func validateTags(tags []string) { func execute(cmd *cobra.Command, args []string) { tags, err := cmd.Flags().GetStringSlice("tags") + reportPath, _ := cmd.Flags().GetString("report-path") if err != nil { log.Fatal().Msg(err.Error()) } @@ -156,6 +159,12 @@ func execute(cmd *cobra.Command, args []string) { // Show Report if report.TotalItemsScanned > 0 { report.ShowReport() + if reportPath != "" { + err := report.Write(reportPath, secrets.OrderedRules) + if err != nil { + log.Error().Msgf("Failed to create sarif file report with error: %s", err) + } + } } else { log.Error().Msg("Scan completed with empty content") os.Exit(0) diff --git a/reporting/report.go b/reporting/report.go index 5ea7fe7f..f0d6bc3c 100644 --- a/reporting/report.go +++ b/reporting/report.go @@ -2,6 +2,8 @@ package reporting import ( "fmt" + "github.com/zricethezav/gitleaks/v8/config" + "os" "strings" ) @@ -57,3 +59,13 @@ func getItemId(fullPath string) string { itemLink := itemLinkStrings[len(itemLinkStrings)-1] return itemLink } + +func (r *Report) Write(reportPath string, orderedRules []config.Rule) error { + file, err := os.Create(reportPath) + if err != nil { + return err + } + writeSarif(*r, file, orderedRules) + + return nil +} diff --git a/reporting/sarif.go b/reporting/sarif.go new file mode 100644 index 00000000..fd815d77 --- /dev/null +++ b/reporting/sarif.go @@ -0,0 +1,186 @@ +package reporting + +import ( + "encoding/json" + "fmt" + "github.com/zricethezav/gitleaks/v8/config" + "io" +) + +func writeSarif(report Report, w io.WriteCloser, orderedRules []config.Rule) error { + sarif := Sarif{ + Schema: "https://schemastore.azurewebsites.net/schemas/json/sarif-2.1.0-rtm.5.json", + Version: "2.1.0", + Runs: getRuns(orderedRules, report), + } + + encoder := json.NewEncoder(w) + encoder.SetIndent("", " ") + return encoder.Encode(sarif) +} + +func getRuns(orderedRules []config.Rule, report Report) []Runs { + return []Runs{ + { + Tool: getTool(orderedRules), + Results: getResults(report), + }, + } +} + +func getTool(orderedRules []config.Rule) Tool { + tool := Tool{ + Driver: Driver{ + Name: "2ms", + SemanticVersion: "v1.2.3", //cmd.Version, + Rules: getRules(orderedRules), + }, + } + + // if this tool has no rules, ensure that it is represented as [] instead of null/nil + if hasEmptyRules(tool) { + tool.Driver.Rules = make([]Rules, 0) + } + + return tool +} + +func hasEmptyRules(tool Tool) bool { + return len(tool.Driver.Rules) == 0 +} + +func getRules(orderedRules []config.Rule) []Rules { + var rules []Rules + for _, rule := range orderedRules { + shortDescription := ShortDescription{ + Text: rule.Description, + } + if rule.Regex != nil { + shortDescription = ShortDescription{ + Text: rule.Regex.String(), + } + } else if rule.Path != nil { + shortDescription = ShortDescription{ + Text: rule.Path.String(), + } + } + rules = append(rules, Rules{ + ID: rule.RuleID, + Name: rule.Description, + Description: shortDescription, + }) + } + return rules +} + +func messageText(secret Secret) string { + return fmt.Sprintf("%s has detected secret for file %s.", secret.Description, secret.ID) +} + +func getResults(report Report) []Results { + var results []Results + for _, secrets := range report.Results { + for _, secret := range secrets { + r := Results{ + Message: Message{ + Text: messageText(secret), + }, + RuleId: secret.Description, + Locations: getLocation(secret), + } + results = append(results, r) + } + } + return results +} + +func getLocation(secret Secret) []Locations { + uri := secret.ID + return []Locations{ + { + PhysicalLocation: PhysicalLocation{ + ArtifactLocation: ArtifactLocation{ + URI: uri, + }, + Region: Region{ + StartLine: secret.StartLine, + EndLine: secret.EndLine, + StartColumn: secret.StartColumn, + EndColumn: secret.EndColumn, + Snippet: Snippet{ + Text: secret.Value, + }, + }, + }, + }, + } +} + +type Sarif struct { + Schema string `json:"$schema"` + Version string `json:"version"` + Runs []Runs `json:"runs"` +} +type ShortDescription struct { + Text string `json:"text"` +} + +type FullDescription struct { + Text string `json:"text"` +} + +type Rules struct { + ID string `json:"id"` + Name string `json:"name"` + Description ShortDescription `json:"shortDescription"` +} + +type Driver struct { + Name string `json:"name"` + SemanticVersion string `json:"semanticVersion"` + Rules []Rules `json:"rules"` +} + +type Tool struct { + Driver Driver `json:"driver"` +} + +type Message struct { + Text string `json:"text"` +} + +type ArtifactLocation struct { + URI string `json:"uri"` +} + +type Region struct { + StartLine int `json:"startLine"` + StartColumn int `json:"startColumn"` + EndLine int `json:"endLine"` + EndColumn int `json:"endColumn"` + Snippet Snippet `json:"snippet"` +} + +type Snippet struct { + Text string `json:"text"` +} + +type PhysicalLocation struct { + ArtifactLocation ArtifactLocation `json:"artifactLocation"` + Region Region `json:"region"` +} + +type Locations struct { + PhysicalLocation PhysicalLocation `json:"physicalLocation"` +} + +type Results struct { + Message Message `json:"message"` + RuleId string `json:"ruleId"` + Locations []Locations `json:"locations"` +} + +type Runs struct { + Tool Tool `json:"tool"` + Results []Results `json:"results"` +} diff --git a/secrets/secrets.go b/secrets/secrets.go index 6d109a54..82318945 100644 --- a/secrets/secrets.go +++ b/secrets/secrets.go @@ -11,8 +11,9 @@ import ( ) type Secrets struct { - rules map[string]config.Rule - detector detect.Detector + rules map[string]config.Rule + detector detect.Detector + OrderedRules []config.Rule } type Rule struct { @@ -45,15 +46,16 @@ func Init(tags []string) *Secrets { allRules, _ := loadAllRules() rulesToBeApplied := getRules(allRules, tags) - cfg := config.Config{ + config := config.Config{ Rules: rulesToBeApplied, } - detector := detect.NewDetector(cfg) + detector := detect.NewDetector(config) return &Secrets{ - rules: rulesToBeApplied, - detector: *detector, + rules: rulesToBeApplied, + detector: *detector, + OrderedRules: config.OrderedRules(), } } From 670da396ced6c577a5b667f342a211e04b6b1418 Mon Sep 17 00:00:00 2001 From: Monica Casanova Date: Wed, 10 May 2023 17:06:23 +0100 Subject: [PATCH 02/12] refactor: merge with master --- reporting/report.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/reporting/report.go b/reporting/report.go index a521245a..79626042 100644 --- a/reporting/report.go +++ b/reporting/report.go @@ -2,6 +2,8 @@ package reporting import ( "fmt" + "github.com/zricethezav/gitleaks/v8/config" + "os" "path/filepath" "strings" ) @@ -64,3 +66,13 @@ func getItemId(fullPath string) string { } return itemId } + +func (r *Report) Write(reportPath string, orderedRules []config.Rule) error { + file, err := os.Create(reportPath) + if err != nil { + return err + } + writeSarif(*r, file, orderedRules) + + return nil +} From 143ac4b2457d1148a053ea70c9966d3515609ff0 Mon Sep 17 00:00:00 2001 From: Monica Casanova Date: Fri, 12 May 2023 14:59:12 +0100 Subject: [PATCH 03/12] refactor: improvements on sarif report --- cmd/main.go | 7 ++- config/config.go | 10 ++++ reporting/report.go | 38 ++++++------ reporting/sarif.go | 143 +++++++++++++++----------------------------- secrets/secrets.go | 26 +++++--- 5 files changed, 98 insertions(+), 126 deletions(-) create mode 100644 config/config.go diff --git a/cmd/main.go b/cmd/main.go index 3fdc197a..1d13373a 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -1,6 +1,7 @@ package cmd import ( + "github.com/checkmarx/2ms/config" "github.com/checkmarx/2ms/plugins" "github.com/checkmarx/2ms/reporting" "github.com/checkmarx/2ms/secrets" @@ -107,6 +108,8 @@ func execute(cmd *cobra.Command, args []string) { var wg sync.WaitGroup + cfg := config.LoadConfig("2ms", Version) + // ------------------------------------- // Get content from plugins pluginsInitialized := 0 @@ -142,7 +145,7 @@ func execute(cmd *cobra.Command, args []string) { go secrets.Detect(secretsChannel, item, &wg) case secret := <-secretsChannel: report.TotalSecretsFound++ - report.Results[secret.ID] = append(report.Results[secret.ID], secret) + report.Results[secret.Source] = append(report.Results[secret.Source], secret) case err, ok := <-errorsChannel: if !ok { return @@ -161,7 +164,7 @@ func execute(cmd *cobra.Command, args []string) { if report.TotalItemsScanned > 0 { report.ShowReport() if reportPath != "" { - err := report.Write(reportPath, secrets.OrderedRules) + err := report.Write(reportPath, cfg) if err != nil { log.Error().Msgf("Failed to create sarif file report with error: %s", err) } diff --git a/config/config.go b/config/config.go new file mode 100644 index 00000000..bebe909b --- /dev/null +++ b/config/config.go @@ -0,0 +1,10 @@ +package config + +type Config struct { + Name string + Version string +} + +func LoadConfig(name string, version string) *Config { + return &Config{Name: name, Version: version} +} diff --git a/reporting/report.go b/reporting/report.go index 79626042..37e8c844 100644 --- a/reporting/report.go +++ b/reporting/report.go @@ -2,10 +2,9 @@ package reporting import ( "fmt" - "github.com/zricethezav/gitleaks/v8/config" + "github.com/checkmarx/2ms/config" "os" "path/filepath" - "strings" ) type Report struct { @@ -16,6 +15,7 @@ type Report struct { type Secret struct { ID string + Source string Description string StartLine int EndLine int @@ -44,35 +44,31 @@ func (r *Report) ShowReport() { func (r *Report) generateResultsReport() { for source, secrets := range r.Results { - itemId := getItemId(source) - fmt.Printf("- Item ID: %s\n", itemId) - fmt.Printf(" - Item Full Path: %s\n", source) + fmt.Printf(" - Item Source: %s\n", source) fmt.Println(" - Secrets:") for _, secret := range secrets { + fmt.Printf(" - Item ID: %s\n", secret.ID) fmt.Printf(" - Type: %s\n", secret.Description) - fmt.Printf(" - Value: %.40s\n", secret.Value) + fmt.Printf(" - Value: %.40s\n", secret.Value) } } } -func getItemId(fullPath string) string { - var itemId string - if strings.Contains(fullPath, "/") { - itemLinkStrings := strings.Split(fullPath, "/") - itemId = itemLinkStrings[len(itemLinkStrings)-1] - } - if strings.Contains(fullPath, "\\") { - itemId = filepath.Base(fullPath) - } - return itemId -} - -func (r *Report) Write(reportPath string, orderedRules []config.Rule) error { +func (r *Report) Write(reportPath string, cfg *config.Config) error { file, err := os.Create(reportPath) if err != nil { return err } - writeSarif(*r, file, orderedRules) - return nil + fileExtension := filepath.Ext(reportPath) + switch fileExtension { + //case ".json": + // err = writeJson(*r, file, cfg) + //case ".csv": + // err = writeCsv(*r, file, cfg) + case ".sarif": + err = writeSarif(*r, file, cfg) + } + + return err } diff --git a/reporting/sarif.go b/reporting/sarif.go index fd815d77..8a626901 100644 --- a/reporting/sarif.go +++ b/reporting/sarif.go @@ -2,16 +2,15 @@ package reporting import ( "encoding/json" - "fmt" - "github.com/zricethezav/gitleaks/v8/config" + "github.com/checkmarx/2ms/config" "io" ) -func writeSarif(report Report, w io.WriteCloser, orderedRules []config.Rule) error { +func writeSarif(report Report, w io.WriteCloser, cfg *config.Config) error { sarif := Sarif{ Schema: "https://schemastore.azurewebsites.net/schemas/json/sarif-2.1.0-rtm.5.json", Version: "2.1.0", - Runs: getRuns(orderedRules, report), + Runs: getRuns(report, cfg), } encoder := json.NewEncoder(w) @@ -19,62 +18,39 @@ func writeSarif(report Report, w io.WriteCloser, orderedRules []config.Rule) err return encoder.Encode(sarif) } -func getRuns(orderedRules []config.Rule, report Report) []Runs { +func getRuns(report Report, cfg *config.Config) []Runs { return []Runs{ { - Tool: getTool(orderedRules), + Tool: getTool(cfg), + Summary: getSummary(report), Results: getResults(report), }, } } -func getTool(orderedRules []config.Rule) Tool { +func getTool(cfg *config.Config) Tool { tool := Tool{ - Driver: Driver{ - Name: "2ms", - SemanticVersion: "v1.2.3", //cmd.Version, - Rules: getRules(orderedRules), - }, + Name: cfg.Name, + SemanticVersion: cfg.Version, } // if this tool has no rules, ensure that it is represented as [] instead of null/nil if hasEmptyRules(tool) { - tool.Driver.Rules = make([]Rules, 0) + tool.Rules = make([]Rules, 0) } return tool } func hasEmptyRules(tool Tool) bool { - return len(tool.Driver.Rules) == 0 + return len(tool.Rules) == 0 } -func getRules(orderedRules []config.Rule) []Rules { - var rules []Rules - for _, rule := range orderedRules { - shortDescription := ShortDescription{ - Text: rule.Description, - } - if rule.Regex != nil { - shortDescription = ShortDescription{ - Text: rule.Regex.String(), - } - } else if rule.Path != nil { - shortDescription = ShortDescription{ - Text: rule.Path.String(), - } - } - rules = append(rules, Rules{ - ID: rule.RuleID, - Name: rule.Description, - Description: shortDescription, - }) +func getSummary(report Report) Summary { + return Summary{TotalItemsScanned: report.TotalItemsScanned, + TotalItemsWithSecrets: len(report.Results), + TotalSecretsFound: report.TotalSecretsFound, } - return rules -} - -func messageText(secret Secret) string { - return fmt.Sprintf("%s has detected secret for file %s.", secret.Description, secret.ID) } func getResults(report Report) []Results { @@ -82,11 +58,9 @@ func getResults(report Report) []Results { for _, secrets := range report.Results { for _, secret := range secrets { r := Results{ - Message: Message{ - Text: messageText(secret), - }, - RuleId: secret.Description, - Locations: getLocation(secret), + ItemSource: secret.Source, + RuleId: secret.Description, + Locations: getLocations(secret), } results = append(results, r) } @@ -94,22 +68,17 @@ func getResults(report Report) []Results { return results } -func getLocation(secret Secret) []Locations { - uri := secret.ID +func getLocations(secret Secret) []Locations { return []Locations{ { - PhysicalLocation: PhysicalLocation{ - ArtifactLocation: ArtifactLocation{ - URI: uri, - }, - Region: Region{ - StartLine: secret.StartLine, - EndLine: secret.EndLine, - StartColumn: secret.StartColumn, - EndColumn: secret.EndColumn, - Snippet: Snippet{ - Text: secret.Value, - }, + ItemId: secret.ID, + Region: Region{ + StartLine: secret.StartLine, + EndLine: secret.EndLine, + StartColumn: secret.StartColumn, + EndColumn: secret.EndColumn, + Value: Value{ + Text: secret.Value, }, }, }, @@ -121,66 +90,48 @@ type Sarif struct { Version string `json:"version"` Runs []Runs `json:"runs"` } -type ShortDescription struct { - Text string `json:"text"` -} - -type FullDescription struct { - Text string `json:"text"` -} type Rules struct { - ID string `json:"id"` - Name string `json:"name"` - Description ShortDescription `json:"shortDescription"` + Name string `json:"name"` } -type Driver struct { +type Tool struct { Name string `json:"name"` SemanticVersion string `json:"semanticVersion"` Rules []Rules `json:"rules"` } -type Tool struct { - Driver Driver `json:"driver"` -} - -type Message struct { - Text string `json:"text"` -} - -type ArtifactLocation struct { - URI string `json:"uri"` -} - type Region struct { - StartLine int `json:"startLine"` - StartColumn int `json:"startColumn"` - EndLine int `json:"endLine"` - EndColumn int `json:"endColumn"` - Snippet Snippet `json:"snippet"` + StartLine int `json:"startLine"` + StartColumn int `json:"startColumn"` + EndLine int `json:"endLine"` + EndColumn int `json:"endColumn"` + Value Value `json:"value"` } -type Snippet struct { +type Value struct { Text string `json:"text"` } -type PhysicalLocation struct { - ArtifactLocation ArtifactLocation `json:"artifactLocation"` - Region Region `json:"region"` -} - type Locations struct { - PhysicalLocation PhysicalLocation `json:"physicalLocation"` + ItemId string `json:"itemId"` + Region Region `json:"region"` } type Results struct { - Message Message `json:"message"` - RuleId string `json:"ruleId"` - Locations []Locations `json:"locations"` + ItemSource string `json:"itemSource"` + RuleId string `json:"ruleId"` + Locations []Locations `json:"location"` +} + +type Summary struct { + TotalItemsScanned int `json:"totalItemsScanned"` + TotalItemsWithSecrets int `json:"totalItemsWithSecrets"` + TotalSecretsFound int `json:"totalSecretsFound"` } type Runs struct { Tool Tool `json:"tool"` + Summary Summary `json:"summary"` Results []Results `json:"results"` } diff --git a/secrets/secrets.go b/secrets/secrets.go index 82318945..b62fcb33 100644 --- a/secrets/secrets.go +++ b/secrets/secrets.go @@ -6,14 +6,14 @@ import ( "github.com/zricethezav/gitleaks/v8/cmd/generate/config/rules" "github.com/zricethezav/gitleaks/v8/config" "github.com/zricethezav/gitleaks/v8/detect" + "path/filepath" "strings" "sync" ) type Secrets struct { - rules map[string]config.Rule - detector detect.Detector - OrderedRules []config.Rule + rules map[string]config.Rule + detector detect.Detector } type Rule struct { @@ -53,9 +53,8 @@ func Init(tags []string) *Secrets { detector := detect.NewDetector(config) return &Secrets{ - rules: rulesToBeApplied, - detector: *detector, - OrderedRules: config.OrderedRules(), + rules: rulesToBeApplied, + detector: *detector, } } @@ -66,10 +65,23 @@ func (s *Secrets) Detect(secretsChannel chan reporting.Secret, item plugins.Item Raw: item.Content, } for _, value := range s.detector.Detect(fragment) { - secretsChannel <- reporting.Secret{ID: item.ID, Description: value.Description, StartLine: value.StartLine, StartColumn: value.StartColumn, EndLine: value.EndLine, EndColumn: value.EndColumn, Value: value.Secret} + itemId := getItemId(item.ID) + secretsChannel <- reporting.Secret{ID: itemId, Source: item.ID, Description: value.Description, StartLine: value.StartLine, StartColumn: value.StartColumn, EndLine: value.EndLine, EndColumn: value.EndColumn, Value: value.Secret} } } +func getItemId(fullPath string) string { + var itemId string + if strings.Contains(fullPath, "/") { + itemLinkStrings := strings.Split(fullPath, "/") + itemId = itemLinkStrings[len(itemLinkStrings)-1] + } + if strings.Contains(fullPath, "\\") { + itemId = filepath.Base(fullPath) + } + return itemId +} + func getRules(allRules []Rule, tags []string) map[string]config.Rule { rulesToBeApplied := make(map[string]config.Rule) From 7b4ff83238ebec1c1d524d24f06417b3c66d0120 Mon Sep 17 00:00:00 2001 From: Monica Casanova Date: Fri, 12 May 2023 15:04:49 +0100 Subject: [PATCH 04/12] refactor: adds import after merge with master --- cmd/main.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/main.go b/cmd/main.go index b5af3de7..a9a056ff 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -6,6 +6,7 @@ import ( "sync" "time" + "github.com/checkmarx/2ms/config" "github.com/checkmarx/2ms/plugins" "github.com/checkmarx/2ms/reporting" "github.com/checkmarx/2ms/secrets" From 97fc1e21533ac6b5e6b0d6f790ec2b85eaa148c5 Mon Sep 17 00:00:00 2001 From: Monica Casanova Date: Mon, 15 May 2023 12:35:49 +0100 Subject: [PATCH 05/12] feat: adds both JSON and Yaml as output file formats --- cmd/main.go | 2 +- go.mod | 1 + go.sum | 2 + reporting/json.go | 15 +++++++ reporting/report.go | 30 ++++++------- reporting/sarif.go | 107 +++++++++++++++++++++++--------------------- reporting/yaml.go | 14 ++++++ 7 files changed, 103 insertions(+), 68 deletions(-) create mode 100644 reporting/json.go create mode 100644 reporting/yaml.go diff --git a/cmd/main.go b/cmd/main.go index a9a056ff..b3b7226a 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -168,7 +168,7 @@ func execute(cmd *cobra.Command, args []string) { if reportPath != "" { err := report.Write(reportPath, cfg) if err != nil { - log.Error().Msgf("Failed to create sarif file report with error: %s", err) + log.Error().Msgf("Failed to create report file with error: %s", err) } } } else { diff --git a/go.mod b/go.mod index 3fa40718..5b18c6bb 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/spf13/cobra v1.6.1 github.com/stretchr/testify v1.8.1 github.com/zricethezav/gitleaks/v8 v8.16.1 + gopkg.in/yaml.v2 v2.4.0 ) require ( diff --git a/go.sum b/go.sum index b1ea402f..7503ea95 100644 --- a/go.sum +++ b/go.sum @@ -524,6 +524,8 @@ gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/reporting/json.go b/reporting/json.go new file mode 100644 index 00000000..f6631933 --- /dev/null +++ b/reporting/json.go @@ -0,0 +1,15 @@ +package reporting + +import ( + "encoding/json" + "io" +) + +func writeJson(report Report, w io.WriteCloser) error { + if len(report.Results) == 0 { + report.Results = map[string][]Secret{} + } + encoder := json.NewEncoder(w) + encoder.SetIndent("", " ") + return encoder.Encode(report) +} diff --git a/reporting/report.go b/reporting/report.go index 37e8c844..22df543c 100644 --- a/reporting/report.go +++ b/reporting/report.go @@ -8,20 +8,20 @@ import ( ) type Report struct { - Results map[string][]Secret - TotalItemsScanned int - TotalSecretsFound int + TotalItemsScanned int `json:"totalItemsScanned"` + TotalSecretsFound int `json:"totalSecretsFound"` + Results map[string][]Secret `json:"results"` } type Secret struct { - ID string - Source string - Description string - StartLine int - EndLine int - StartColumn int - EndColumn int - Value string + ID string `json:"id"` + Source string `json:"source"` + Description string `json:"description"` + StartLine int `json:"startLine"` + EndLine int `json:"endLine"` + StartColumn int `json:"startColumn"` + EndColumn int `json:"endColumn"` + Value string `json:"value"` } func Init() *Report { @@ -62,10 +62,10 @@ func (r *Report) Write(reportPath string, cfg *config.Config) error { fileExtension := filepath.Ext(reportPath) switch fileExtension { - //case ".json": - // err = writeJson(*r, file, cfg) - //case ".csv": - // err = writeCsv(*r, file, cfg) + case ".json": + err = writeJson(*r, file) + case ".yaml": + err = writeYaml(*r, file) case ".sarif": err = writeSarif(*r, file, cfg) } diff --git a/reporting/sarif.go b/reporting/sarif.go index 8a626901..0fe6c377 100644 --- a/reporting/sarif.go +++ b/reporting/sarif.go @@ -2,6 +2,7 @@ package reporting import ( "encoding/json" + "fmt" "github.com/checkmarx/2ms/config" "io" ) @@ -22,7 +23,6 @@ func getRuns(report Report, cfg *config.Config) []Runs { return []Runs{ { Tool: getTool(cfg), - Summary: getSummary(report), Results: getResults(report), }, } @@ -30,27 +30,17 @@ func getRuns(report Report, cfg *config.Config) []Runs { func getTool(cfg *config.Config) Tool { tool := Tool{ - Name: cfg.Name, - SemanticVersion: cfg.Version, - } - - // if this tool has no rules, ensure that it is represented as [] instead of null/nil - if hasEmptyRules(tool) { - tool.Rules = make([]Rules, 0) + Driver: Driver{ + Name: cfg.Name, + SemanticVersion: cfg.Version, + }, } return tool } -func hasEmptyRules(tool Tool) bool { - return len(tool.Rules) == 0 -} - -func getSummary(report Report) Summary { - return Summary{TotalItemsScanned: report.TotalItemsScanned, - TotalItemsWithSecrets: len(report.Results), - TotalSecretsFound: report.TotalSecretsFound, - } +func messageText(secret Secret) string { + return fmt.Sprintf("%s has detected secret for file %s.", secret.Description, secret.ID) } func getResults(report Report) []Results { @@ -58,9 +48,11 @@ func getResults(report Report) []Results { for _, secrets := range report.Results { for _, secret := range secrets { r := Results{ - ItemSource: secret.Source, - RuleId: secret.Description, - Locations: getLocations(secret), + Message: Message{ + Text: messageText(secret), + }, + RuleId: secret.Description, + Locations: getLocation(secret), } results = append(results, r) } @@ -68,17 +60,21 @@ func getResults(report Report) []Results { return results } -func getLocations(secret Secret) []Locations { +func getLocation(secret Secret) []Locations { return []Locations{ { - ItemId: secret.ID, - Region: Region{ - StartLine: secret.StartLine, - EndLine: secret.EndLine, - StartColumn: secret.StartColumn, - EndColumn: secret.EndColumn, - Value: Value{ - Text: secret.Value, + PhysicalLocation: PhysicalLocation{ + ArtifactLocation: ArtifactLocation{ + URI: secret.ID, + }, + Region: Region{ + StartLine: secret.StartLine, + EndLine: secret.EndLine, + StartColumn: secret.StartColumn, + EndColumn: secret.EndColumn, + Snippet: Snippet{ + Text: secret.Value, + }, }, }, }, @@ -90,48 +86,55 @@ type Sarif struct { Version string `json:"version"` Runs []Runs `json:"runs"` } +type ShortDescription struct { + Text string `json:"text"` +} -type Rules struct { - Name string `json:"name"` +type Driver struct { + Name string `json:"name"` + SemanticVersion string `json:"semanticVersion"` } type Tool struct { - Name string `json:"name"` - SemanticVersion string `json:"semanticVersion"` - Rules []Rules `json:"rules"` + Driver Driver `json:"driver"` +} + +type Message struct { + Text string `json:"text"` +} + +type ArtifactLocation struct { + URI string `json:"uri"` } type Region struct { - StartLine int `json:"startLine"` - StartColumn int `json:"startColumn"` - EndLine int `json:"endLine"` - EndColumn int `json:"endColumn"` - Value Value `json:"value"` + StartLine int `json:"startLine"` + StartColumn int `json:"startColumn"` + EndLine int `json:"endLine"` + EndColumn int `json:"endColumn"` + Snippet Snippet `json:"snippet"` } -type Value struct { +type Snippet struct { Text string `json:"text"` } -type Locations struct { - ItemId string `json:"itemId"` - Region Region `json:"region"` +type PhysicalLocation struct { + ArtifactLocation ArtifactLocation `json:"artifactLocation"` + Region Region `json:"region"` } -type Results struct { - ItemSource string `json:"itemSource"` - RuleId string `json:"ruleId"` - Locations []Locations `json:"location"` +type Locations struct { + PhysicalLocation PhysicalLocation `json:"physicalLocation"` } -type Summary struct { - TotalItemsScanned int `json:"totalItemsScanned"` - TotalItemsWithSecrets int `json:"totalItemsWithSecrets"` - TotalSecretsFound int `json:"totalSecretsFound"` +type Results struct { + Message Message `json:"message"` + RuleId string `json:"ruleId"` + Locations []Locations `json:"locations"` } type Runs struct { Tool Tool `json:"tool"` - Summary Summary `json:"summary"` Results []Results `json:"results"` } diff --git a/reporting/yaml.go b/reporting/yaml.go new file mode 100644 index 00000000..5ea79127 --- /dev/null +++ b/reporting/yaml.go @@ -0,0 +1,14 @@ +package reporting + +import ( + "gopkg.in/yaml.v2" + "io" +) + +func writeYaml(report Report, w io.WriteCloser) error { + if len(report.Results) == 0 { + report.Results = map[string][]Secret{} + } + enc := yaml.NewEncoder(w) + return enc.Encode(report) +} From 3ae9c830b296b5a427004a074d70522b2474782f Mon Sep 17 00:00:00 2001 From: Monica Casanova Date: Mon, 15 May 2023 14:57:35 +0100 Subject: [PATCH 06/12] feat: adds list of files option to report path --- cmd/main.go | 6 +++--- reporting/report.go | 31 ++++++++++++++++--------------- 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index b3b7226a..eb4d4da1 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -62,7 +62,7 @@ func Execute() { cobra.OnInitialize(initLog) rootCmd.Flags().BoolP("all", "", true, "scan all plugins") rootCmd.Flags().StringSlice("tags", []string{"all"}, "select rules to be applied") - rootCmd.Flags().StringP(reportPath, "r", "", "path to generate report file") + rootCmd.Flags().StringSlice(reportPath, []string{"all"}, "path to generate report file") for _, plugin := range allPlugins { err := plugin.DefineCommandLineArgs(rootCmd) @@ -94,7 +94,7 @@ func validateTags(tags []string) { func execute(cmd *cobra.Command, args []string) { tags, err := cmd.Flags().GetStringSlice("tags") - reportPath, _ := cmd.Flags().GetString("report-path") + reportPath, _ := cmd.Flags().GetStringSlice("report-path") if err != nil { log.Fatal().Msg(err.Error()) } @@ -165,7 +165,7 @@ func execute(cmd *cobra.Command, args []string) { // Show Report if report.TotalItemsScanned > 0 { report.ShowReport() - if reportPath != "" { + if len(reportPath) > 0 { err := report.Write(reportPath, cfg) if err != nil { log.Error().Msgf("Failed to create report file with error: %s", err) diff --git a/reporting/report.go b/reporting/report.go index 22df543c..b5cd3543 100644 --- a/reporting/report.go +++ b/reporting/report.go @@ -54,21 +54,22 @@ func (r *Report) generateResultsReport() { } } -func (r *Report) Write(reportPath string, cfg *config.Config) error { - file, err := os.Create(reportPath) - if err != nil { - return err - } +func (r *Report) Write(reportPath []string, cfg *config.Config) error { + for _, path := range reportPath { + file, err := os.Create(path) + if err != nil { + return err + } - fileExtension := filepath.Ext(reportPath) - switch fileExtension { - case ".json": - err = writeJson(*r, file) - case ".yaml": - err = writeYaml(*r, file) - case ".sarif": - err = writeSarif(*r, file, cfg) + fileExtension := filepath.Ext(path) + switch fileExtension { + case ".json": + err = writeJson(*r, file) + case ".yaml": + err = writeYaml(*r, file) + case ".sarif": + err = writeSarif(*r, file, cfg) + } } - - return err + return nil } From 47a6254e1066a437d0235ba8ab09852d1911f72b Mon Sep 17 00:00:00 2001 From: Monica Casanova Date: Tue, 16 May 2023 12:05:41 +0100 Subject: [PATCH 07/12] feat: adds format output option --- cmd/main.go | 21 ++++++++++++++------- reporting/json.go | 12 +++++++++++- reporting/report.go | 41 ++++++++++++++++------------------------- reporting/sarif.go | 18 +++++++++++++++++- reporting/yaml.go | 12 +++++++++++- 5 files changed, 69 insertions(+), 35 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index eb4d4da1..deb09276 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -16,8 +16,12 @@ import ( "github.com/spf13/cobra" ) -const timeSleepInterval = 50 -const reportPath = "report-path" +const ( + timeSleepInterval = 50 + reportPath = "report-path" + tags = "tags" + stdoutFormat = "stdout-format" +) var rootCmd = &cobra.Command{ Use: "2ms", @@ -61,8 +65,9 @@ func initLog() { func Execute() { cobra.OnInitialize(initLog) rootCmd.Flags().BoolP("all", "", true, "scan all plugins") - rootCmd.Flags().StringSlice("tags", []string{"all"}, "select rules to be applied") + rootCmd.Flags().StringSlice(tags, []string{"all"}, "select rules to be applied") rootCmd.Flags().StringSlice(reportPath, []string{"all"}, "path to generate report file") + rootCmd.Flags().StringP(stdoutFormat, "", "yaml", "scan all plugins") for _, plugin := range allPlugins { err := plugin.DefineCommandLineArgs(rootCmd) @@ -93,8 +98,10 @@ func validateTags(tags []string) { } func execute(cmd *cobra.Command, args []string) { - tags, err := cmd.Flags().GetStringSlice("tags") - reportPath, _ := cmd.Flags().GetStringSlice("report-path") + tags, err := cmd.Flags().GetStringSlice(tags) + reportPath, _ := cmd.Flags().GetStringSlice(reportPath) + stdoutFormat, _ := cmd.Flags().GetString(stdoutFormat) + if err != nil { log.Fatal().Msg(err.Error()) } @@ -164,9 +171,9 @@ func execute(cmd *cobra.Command, args []string) { // ------------------------------------- // Show Report if report.TotalItemsScanned > 0 { - report.ShowReport() + report.ShowReport(stdoutFormat, cfg) if len(reportPath) > 0 { - err := report.Write(reportPath, cfg) + err := report.WriteFile(reportPath, cfg) if err != nil { log.Error().Msgf("Failed to create report file with error: %s", err) } diff --git a/reporting/json.go b/reporting/json.go index f6631933..212abffd 100644 --- a/reporting/json.go +++ b/reporting/json.go @@ -3,9 +3,10 @@ package reporting import ( "encoding/json" "io" + "log" ) -func writeJson(report Report, w io.WriteCloser) error { +func writeJsonFile(report Report, w io.WriteCloser) error { if len(report.Results) == 0 { report.Results = map[string][]Secret{} } @@ -13,3 +14,12 @@ func writeJson(report Report, w io.WriteCloser) error { encoder.SetIndent("", " ") return encoder.Encode(report) } + +func writeJsonStdOut(report Report) string { + jsonReport, err := json.MarshalIndent(report, "", " ") + if err != nil { + log.Fatalf("failed to create Json report with error: %v", err) + } + + return string(jsonReport) +} diff --git a/reporting/report.go b/reporting/report.go index b5cd3543..b7c65925 100644 --- a/reporting/report.go +++ b/reporting/report.go @@ -30,31 +30,22 @@ func Init() *Report { } } -func (r *Report) ShowReport() { - fmt.Println("Summary:") - fmt.Printf("- Total items scanned: %d\n", r.TotalItemsScanned) - fmt.Printf("- Total items with secrets: %d\n", len(r.Results)) - if len(r.Results) > 0 { - fmt.Printf("- Total secrets found: %d\n", r.TotalSecretsFound) - fmt.Println("Detailed Report:") - r.generateResultsReport() - } - -} - -func (r *Report) generateResultsReport() { - for source, secrets := range r.Results { - fmt.Printf(" - Item Source: %s\n", source) - fmt.Println(" - Secrets:") - for _, secret := range secrets { - fmt.Printf(" - Item ID: %s\n", secret.ID) - fmt.Printf(" - Type: %s\n", secret.Description) - fmt.Printf(" - Value: %.40s\n", secret.Value) - } +func (r *Report) ShowReport(format string, cfg *config.Config) { + fileExtension := format + var output string + switch fileExtension { + case "json": + output = writeJsonStdOut(*r) + case "yaml": + output = writeYamlStdOut(*r) + case "sarif": + output = writeSarifStdOut(*r, cfg) } + fmt.Println("Summary:") + fmt.Printf("%s", output) } -func (r *Report) Write(reportPath []string, cfg *config.Config) error { +func (r *Report) WriteFile(reportPath []string, cfg *config.Config) error { for _, path := range reportPath { file, err := os.Create(path) if err != nil { @@ -64,11 +55,11 @@ func (r *Report) Write(reportPath []string, cfg *config.Config) error { fileExtension := filepath.Ext(path) switch fileExtension { case ".json": - err = writeJson(*r, file) + err = writeJsonFile(*r, file) case ".yaml": - err = writeYaml(*r, file) + err = writeYamlFile(*r, file) case ".sarif": - err = writeSarif(*r, file, cfg) + err = writeSarifFile(*r, file, cfg) } } return nil diff --git a/reporting/sarif.go b/reporting/sarif.go index 0fe6c377..d5665a3e 100644 --- a/reporting/sarif.go +++ b/reporting/sarif.go @@ -5,9 +5,10 @@ import ( "fmt" "github.com/checkmarx/2ms/config" "io" + "log" ) -func writeSarif(report Report, w io.WriteCloser, cfg *config.Config) error { +func writeSarifFile(report Report, w io.WriteCloser, cfg *config.Config) error { sarif := Sarif{ Schema: "https://schemastore.azurewebsites.net/schemas/json/sarif-2.1.0-rtm.5.json", Version: "2.1.0", @@ -19,6 +20,21 @@ func writeSarif(report Report, w io.WriteCloser, cfg *config.Config) error { return encoder.Encode(sarif) } +func writeSarifStdOut(report Report, cfg *config.Config) string { + sarif := Sarif{ + Schema: "https://schemastore.azurewebsites.net/schemas/json/sarif-2.1.0-rtm.5.json", + Version: "2.1.0", + Runs: getRuns(report, cfg), + } + + sarifReport, err := json.MarshalIndent(sarif, "", " ") + if err != nil { + log.Fatalf("failed to create Sarif report with error: %v", err) + } + + return string(sarifReport) +} + func getRuns(report Report, cfg *config.Config) []Runs { return []Runs{ { diff --git a/reporting/yaml.go b/reporting/yaml.go index 5ea79127..677e6d6f 100644 --- a/reporting/yaml.go +++ b/reporting/yaml.go @@ -3,12 +3,22 @@ package reporting import ( "gopkg.in/yaml.v2" "io" + "log" ) -func writeYaml(report Report, w io.WriteCloser) error { +func writeYamlFile(report Report, w io.WriteCloser) error { if len(report.Results) == 0 { report.Results = map[string][]Secret{} } enc := yaml.NewEncoder(w) return enc.Encode(report) } + +func writeYamlStdOut(report Report) string { + yamlReport, err := yaml.Marshal(&report) + if err != nil { + log.Fatalf("failed to create Yaml report with error: %v", err) + } + + return string(yamlReport) +} From a815001b97ae1308aeb861071f7d3770b6d5717d Mon Sep 17 00:00:00 2001 From: Monica Casanova Date: Tue, 16 May 2023 14:23:30 +0100 Subject: [PATCH 08/12] refactor: merge with master and small code improvements --- cmd/main.go | 76 +++++++++++++++++++++-------------------------------- 1 file changed, 30 insertions(+), 46 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index 4e4232bb..0396656f 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -2,13 +2,14 @@ package cmd import ( "fmt" + "github.com/checkmarx/2ms/config" "os" + "path/filepath" "strings" "sync" "time" - "github.com/checkmarx/2ms/config" "github.com/checkmarx/2ms/plugins" "github.com/checkmarx/2ms/reporting" "github.com/checkmarx/2ms/secrets" @@ -18,20 +19,20 @@ import ( "github.com/spf13/cobra" ) +var Version = "0.0.0" + const ( timeSleepInterval = 50 + tagsFlagName = "tags" + logLevelFlagName = "log-level" reportPath = "report-path" stdoutFormat = "stdout-format" - tagsFlagName = "tags" - logLevelFlagName = "log-level" ) -var Version = "0.0.0" - var rootCmd = &cobra.Command{ Use: "2ms", Short: "2ms Secrets Detection", - Run: execute, + Long: "2ms Secrets Detection: A tool to detect secrets in public websites and communication services.", Version: Version, } @@ -78,6 +79,8 @@ func Execute() { cobra.OnInitialize(initLog) rootCmd.PersistentFlags().StringSlice(tagsFlagName, []string{"all"}, "select rules to be applied") rootCmd.PersistentFlags().String(logLevelFlagName, "info", "log level (trace, debug, info, warn, error, fatal)") + rootCmd.PersistentFlags().StringSlice(reportPath, []string{""}, "path to generate report file. Available formats are: json, yaml and sarif") + rootCmd.PersistentFlags().String(stdoutFormat, "yaml", "stdout output format, available formats are: json, yaml and sarif") rootCmd.PersistentPreRun = preRun rootCmd.PersistentPostRun = postRun @@ -94,8 +97,6 @@ func Execute() { rootCmd.AddCommand(subCommand) } - rootCmd.PersistentFlags().StringP("log-level", "", "info", "log level (trace, debug, info, warn, error, fatal)") - if err := rootCmd.Execute(); err != nil { log.Fatal().Msg(err.Error()) } @@ -115,10 +116,21 @@ func validateTags(tags []string) { } } +func validateFormat(stdout string, reportPath []string) { + if !(strings.EqualFold(stdout, "yaml") || strings.EqualFold(stdout, "json") || strings.EqualFold(stdout, "sarif")) { + log.Fatal().Msgf(`invalid output format: %s, available formats are: json, yaml and sarif`, stdout) + } + for _, path := range reportPath { + + fileExtension := filepath.Ext(path) + if !(strings.EqualFold(fileExtension, ".yaml") || strings.EqualFold(fileExtension, ".json") || strings.EqualFold(fileExtension, ".sarif")) { + log.Fatal().Msgf(`invalid report extension: %s, available extensions are: json, yaml and sarif`, fileExtension) + } + } +} + func preRun(cmd *cobra.Command, args []string) { tags, err := cmd.Flags().GetStringSlice(tagsFlagName) - reportPath, _ := cmd.Flags().GetStringSlice(reportPath) - stdoutFormat, _ := cmd.Flags().GetString(stdoutFormat) if err != nil { log.Fatal().Msg(err.Error()) } @@ -126,41 +138,6 @@ func preRun(cmd *cobra.Command, args []string) { validateTags(tags) secrets := secrets.Init(tags) - report := reporting.Init() - - cfg := config.LoadConfig("2ms", Version) - - var itemsChannel = make(chan plugins.Item) - var secretsChannel = make(chan reporting.Secret) - var errorsChannel = make(chan error) - - var wg sync.WaitGroup - - // ------------------------------------- - // Get content from plugins - pluginsInitialized := 0 - for _, plugin := range allPlugins { - err := plugin.Initialize(cmd) - if err != nil { - log.Error().Msg(err.Error()) - continue - } - pluginsInitialized += 1 - } - - if pluginsInitialized == 0 { - log.Fatal().Msg("no scan plugin initialized. At least one plugin must be initialized to proceed. Stopping") - os.Exit(1) - } - - for _, plugin := range allPlugins { - if !plugin.IsEnabled() { - continue - } - - wg.Add(1) - go plugin.GetItems(itemsChannel, errorsChannel, &wg) - } go func() { for { @@ -171,7 +148,7 @@ func preRun(cmd *cobra.Command, args []string) { go secrets.Detect(secretsChan, item, channels.WaitGroup) case secret := <-secretsChan: report.TotalSecretsFound++ - report.Results[secret.Source] = append(report.Results[secret.ID], secret) + report.Results[secret.ID] = append(report.Results[secret.ID], secret) case err, ok := <-channels.Errors: if !ok { return @@ -185,6 +162,13 @@ func preRun(cmd *cobra.Command, args []string) { func postRun(cmd *cobra.Command, args []string) { channels.WaitGroup.Wait() + reportPath, _ := cmd.Flags().GetStringSlice(reportPath) + stdoutFormat, _ := cmd.Flags().GetString(stdoutFormat) + + validateFormat(stdoutFormat, reportPath) + + cfg := config.LoadConfig("2ms", Version) + // Wait for last secret to be added to report time.Sleep(time.Millisecond * timeSleepInterval) From 99fb360584f153aabff3bf8e91599b27171fb272 Mon Sep 17 00:00:00 2001 From: Monica Casanova Date: Tue, 16 May 2023 14:34:07 +0100 Subject: [PATCH 09/12] refactor: fixes linter --- reporting/report_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reporting/report_test.go b/reporting/report_test.go index 6ed251c6..c05cdbe4 100644 --- a/reporting/report_test.go +++ b/reporting/report_test.go @@ -24,7 +24,7 @@ JPcHeO7M6FohKgcEHX84koQDN98J/L7pFlSoU7WOl6f8BKavIdeSTPS9qQYWdQuT -----END RSA PRIVATE KEY-----`) results := map[string][]Secret{} - report := Report{results, 1, 1} + report := Report{len(results), 1, results} secret := Secret{Description: "bla", StartLine: 0, StartColumn: 0, EndLine: 0, EndColumn: 0, Value: secretValue} source := "directory\\rawStringAsFile.txt" From 24528c27e217b8f6354fd4b4e43e9c85cbe6a0ea Mon Sep 17 00:00:00 2001 From: Monica Casanova Date: Tue, 16 May 2023 14:36:40 +0100 Subject: [PATCH 10/12] refactor: fixes linter --- reporting/report.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/reporting/report.go b/reporting/report.go index b7c65925..9ba3811f 100644 --- a/reporting/report.go +++ b/reporting/report.go @@ -61,6 +61,9 @@ func (r *Report) WriteFile(reportPath []string, cfg *config.Config) error { case ".sarif": err = writeSarifFile(*r, file, cfg) } + if err != nil { + return err + } } return nil } From a2af449d1164750f2abd7ea95dcb862f2205f735 Mon Sep 17 00:00:00 2001 From: Monica Casanova Date: Tue, 16 May 2023 21:52:43 +0100 Subject: [PATCH 11/12] refactor: code reviewers suggestions --- cmd/main.go | 14 ++++++++----- reporting/json.go | 12 +---------- reporting/report.go | 49 +++++++++++++++++++++++++-------------------- reporting/sarif.go | 26 +++++++++++------------- reporting/yaml.go | 11 +--------- 5 files changed, 50 insertions(+), 62 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index 0396656f..4aba7cd3 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -27,6 +27,9 @@ const ( logLevelFlagName = "log-level" reportPath = "report-path" stdoutFormat = "stdout-format" + jsonFormat = "json" + yamlFormat = "yaml" + sarifFormat = "sarif" ) var rootCmd = &cobra.Command{ @@ -79,8 +82,8 @@ func Execute() { cobra.OnInitialize(initLog) rootCmd.PersistentFlags().StringSlice(tagsFlagName, []string{"all"}, "select rules to be applied") rootCmd.PersistentFlags().String(logLevelFlagName, "info", "log level (trace, debug, info, warn, error, fatal)") - rootCmd.PersistentFlags().StringSlice(reportPath, []string{""}, "path to generate report file. Available formats are: json, yaml and sarif") - rootCmd.PersistentFlags().String(stdoutFormat, "yaml", "stdout output format, available formats are: json, yaml and sarif") + rootCmd.PersistentFlags().StringSlice(reportPath, []string{""}, "path to generate report files. The output format will be determined by the file extension (.json, .yaml, .sarif)") + rootCmd.PersistentFlags().String(stdoutFormat, "yaml", "stdout output format, available formats are: json, yaml, sarif") rootCmd.PersistentPreRun = preRun rootCmd.PersistentPostRun = postRun @@ -117,14 +120,15 @@ func validateTags(tags []string) { } func validateFormat(stdout string, reportPath []string) { - if !(strings.EqualFold(stdout, "yaml") || strings.EqualFold(stdout, "json") || strings.EqualFold(stdout, "sarif")) { + if !(strings.EqualFold(stdout, yamlFormat) || strings.EqualFold(stdout, jsonFormat) || strings.EqualFold(stdout, sarifFormat)) { log.Fatal().Msgf(`invalid output format: %s, available formats are: json, yaml and sarif`, stdout) } for _, path := range reportPath { fileExtension := filepath.Ext(path) - if !(strings.EqualFold(fileExtension, ".yaml") || strings.EqualFold(fileExtension, ".json") || strings.EqualFold(fileExtension, ".sarif")) { - log.Fatal().Msgf(`invalid report extension: %s, available extensions are: json, yaml and sarif`, fileExtension) + format := strings.TrimPrefix(fileExtension, ".") + if !(strings.EqualFold(format, yamlFormat) || strings.EqualFold(format, jsonFormat) || strings.EqualFold(format, sarifFormat)) { + log.Fatal().Msgf(`invalid report extension: %s, available extensions are: json, yaml and sarif`, format) } } } diff --git a/reporting/json.go b/reporting/json.go index 212abffd..14a8df1a 100644 --- a/reporting/json.go +++ b/reporting/json.go @@ -2,20 +2,10 @@ package reporting import ( "encoding/json" - "io" "log" ) -func writeJsonFile(report Report, w io.WriteCloser) error { - if len(report.Results) == 0 { - report.Results = map[string][]Secret{} - } - encoder := json.NewEncoder(w) - encoder.SetIndent("", " ") - return encoder.Encode(report) -} - -func writeJsonStdOut(report Report) string { +func writeJson(report Report) string { jsonReport, err := json.MarshalIndent(report, "", " ") if err != nil { log.Fatalf("failed to create Json report with error: %v", err) diff --git a/reporting/report.go b/reporting/report.go index 9ba3811f..7c665144 100644 --- a/reporting/report.go +++ b/reporting/report.go @@ -5,6 +5,13 @@ import ( "github.com/checkmarx/2ms/config" "os" "path/filepath" + "strings" +) + +const ( + jsonFormat = "json" + yamlFormat = "yaml" + sarifFormat = "sarif" ) type Report struct { @@ -31,18 +38,10 @@ func Init() *Report { } func (r *Report) ShowReport(format string, cfg *config.Config) { - fileExtension := format - var output string - switch fileExtension { - case "json": - output = writeJsonStdOut(*r) - case "yaml": - output = writeYamlStdOut(*r) - case "sarif": - output = writeSarifStdOut(*r, cfg) - } + output := r.getOutput(format, cfg) + fmt.Println("Summary:") - fmt.Printf("%s", output) + fmt.Print(output) } func (r *Report) WriteFile(reportPath []string, cfg *config.Config) error { @@ -53,17 +52,23 @@ func (r *Report) WriteFile(reportPath []string, cfg *config.Config) error { } fileExtension := filepath.Ext(path) - switch fileExtension { - case ".json": - err = writeJsonFile(*r, file) - case ".yaml": - err = writeYamlFile(*r, file) - case ".sarif": - err = writeSarifFile(*r, file, cfg) - } - if err != nil { - return err - } + format := strings.TrimPrefix(fileExtension, ".") + output := r.getOutput(format, cfg) + + file.WriteString(output) } return nil } + +func (r *Report) getOutput(format string, cfg *config.Config) string { + var output string + switch format { + case jsonFormat: + output = writeJson(*r) + case yamlFormat: + output = writeYaml(*r) + case sarifFormat: + output = writeSarif(*r, cfg) + } + return output +} diff --git a/reporting/sarif.go b/reporting/sarif.go index d5665a3e..e2357a91 100644 --- a/reporting/sarif.go +++ b/reporting/sarif.go @@ -4,23 +4,10 @@ import ( "encoding/json" "fmt" "github.com/checkmarx/2ms/config" - "io" "log" ) -func writeSarifFile(report Report, w io.WriteCloser, cfg *config.Config) error { - sarif := Sarif{ - Schema: "https://schemastore.azurewebsites.net/schemas/json/sarif-2.1.0-rtm.5.json", - Version: "2.1.0", - Runs: getRuns(report, cfg), - } - - encoder := json.NewEncoder(w) - encoder.SetIndent("", " ") - return encoder.Encode(sarif) -} - -func writeSarifStdOut(report Report, cfg *config.Config) string { +func writeSarif(report Report, cfg *config.Config) string { sarif := Sarif{ Schema: "https://schemastore.azurewebsites.net/schemas/json/sarif-2.1.0-rtm.5.json", Version: "2.1.0", @@ -55,12 +42,23 @@ func getTool(cfg *config.Config) Tool { return tool } +func hasNoResults(report Report) bool { + return len(report.Results) == 0 +} + func messageText(secret Secret) string { return fmt.Sprintf("%s has detected secret for file %s.", secret.Description, secret.ID) } func getResults(report Report) []Results { var results []Results + + // if this report has no results, ensure that it is represented as [] instead of null/nil + if hasNoResults(report) { + results = make([]Results, 0) + return results + } + for _, secrets := range report.Results { for _, secret := range secrets { r := Results{ diff --git a/reporting/yaml.go b/reporting/yaml.go index 677e6d6f..f91bf7d1 100644 --- a/reporting/yaml.go +++ b/reporting/yaml.go @@ -2,19 +2,10 @@ package reporting import ( "gopkg.in/yaml.v2" - "io" "log" ) -func writeYamlFile(report Report, w io.WriteCloser) error { - if len(report.Results) == 0 { - report.Results = map[string][]Secret{} - } - enc := yaml.NewEncoder(w) - return enc.Encode(report) -} - -func writeYamlStdOut(report Report) string { +func writeYaml(report Report) string { yamlReport, err := yaml.Marshal(&report) if err != nil { log.Fatalf("failed to create Yaml report with error: %v", err) From 086dcd2e0086a5b3cdc648d71c706772df80641c Mon Sep 17 00:00:00 2001 From: Monica Casanova Date: Tue, 16 May 2023 21:56:32 +0100 Subject: [PATCH 12/12] refactor: fixes linter --- reporting/report.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/reporting/report.go b/reporting/report.go index 7c665144..1d4771c2 100644 --- a/reporting/report.go +++ b/reporting/report.go @@ -55,7 +55,10 @@ func (r *Report) WriteFile(reportPath []string, cfg *config.Config) error { format := strings.TrimPrefix(fileExtension, ".") output := r.getOutput(format, cfg) - file.WriteString(output) + _, err = file.WriteString(output) + if err != nil { + return err + } } return nil }