From 048c92fb33348c3e121e19c0f88d2bb1fc7ec686 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Wed, 16 Aug 2023 11:56:52 +0000 Subject: [PATCH 1/4] cleanup migration code --- .gitignore | 1 + v2/pkg/runner/config.go | 12 ++++++-- v2/pkg/runner/options.go | 61 +++------------------------------------- 3 files changed, 15 insertions(+), 59 deletions(-) diff --git a/.gitignore b/.gitignore index 5e2c8c922..b0dd47ba3 100644 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,5 @@ v2/subfinder vendor/ .idea .devcontainer +.vscode dist \ No newline at end of file diff --git a/v2/pkg/runner/config.go b/v2/pkg/runner/config.go index 79022ea69..1f4a10da1 100644 --- a/v2/pkg/runner/config.go +++ b/v2/pkg/runner/config.go @@ -30,14 +30,22 @@ func GetConfigDirectory() (string, error) { return config, nil } -// CreateProviderConfigYAML marshals the input map to the given location on the disk -func CreateProviderConfigYAML(configFilePath string, sourcesRequiringApiKeysMap map[string][]string) error { +// createProviderConfigYAML marshals the input map to the given location on the disk +func createProviderConfigYAML(configFilePath string) error { configFile, err := os.Create(configFilePath) if err != nil { return err } defer configFile.Close() + sourcesRequiringApiKeysMap := make(map[string][]string) + for _, source := range passive.AllSources { + if source.NeedsKey() { + sourceName := strings.ToLower(source.Name()) + sourcesRequiringApiKeysMap[sourceName] = []string{} + } + } + return yaml.NewEncoder(configFile).Encode(sourcesRequiringApiKeysMap) } diff --git a/v2/pkg/runner/options.go b/v2/pkg/runner/options.go index 570b218b5..34a349b55 100644 --- a/v2/pkg/runner/options.go +++ b/v2/pkg/runner/options.go @@ -11,8 +11,6 @@ import ( "regexp" "strings" - "gopkg.in/yaml.v3" - "github.com/projectdiscovery/goflags" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/subfinder/v2/pkg/passive" @@ -76,18 +74,6 @@ type OnResultCallback func(result *resolve.HostEntry) func ParseOptions() *Options { logutil.DisableDefaultLogger() - // Migrate config to provider config - if fileutil.FileExists(defaultConfigLocation) && !fileutil.FileExists(defaultProviderConfigLocation) { - gologger.Info().Msgf("Detected old %s config file, trying to migrate providers to %s\n", defaultConfigLocation, defaultProviderConfigLocation) - if err := migrateToProviderConfig(defaultConfigLocation, defaultProviderConfigLocation); err != nil { - gologger.Warning().Msgf("Could not migrate providers from existing config %s to provider config %s: %s\n", defaultConfigLocation, defaultProviderConfigLocation, err) - } else { - // cleanup the existing config file post migration - _ = os.Remove(defaultConfigLocation) - gologger.Info().Msgf("Migration successful from %s to %s.\n", defaultConfigLocation, defaultProviderConfigLocation) - } - } - options := &Options{} var err error @@ -159,6 +145,10 @@ func ParseOptions() *Options { os.Exit(1) } + if err := createProviderConfigYAML(defaultProviderConfigLocation); err != nil { + gologger.Error().Msgf("Could not create provider config file: %s\n", err) + } + if options.Config != defaultConfigLocation { // An empty source file is not a fatal error if err := flagSet.MergeConfigFile(options.Config); err != nil && !errors.Is(err, io.EOF) { @@ -226,49 +216,6 @@ func (options *Options) loadProvidersFrom(location string) { } } -func migrateToProviderConfig(defaultConfigLocation, defaultProviderLocation string) error { - configs, err := unMarshalToLowerCaseMap(defaultConfigLocation) - if err != nil { - return err - } - - sourcesRequiringApiKeysMap := make(map[string][]string) - for _, source := range passive.AllSources { - if source.NeedsKey() { - sourceName := strings.ToLower(source.Name()) - if sourceKeys, ok := configs[sourceName]; ok { - sourcesRequiringApiKeysMap[sourceName] = sourceKeys - } else { - sourcesRequiringApiKeysMap[sourceName] = []string{} - } - } - } - - return CreateProviderConfigYAML(defaultProviderLocation, sourcesRequiringApiKeysMap) -} - -func unMarshalToLowerCaseMap(defaultConfigLocation string) (map[string][]string, error) { - defaultConfigFile, err := os.Open(defaultConfigLocation) - if err != nil { - return nil, err - } - defer defaultConfigFile.Close() - - configs := map[string][]string{} - if err := yaml.NewDecoder(defaultConfigFile).Decode(configs); isFatalErr(err) { - return nil, err - } - - for k, v := range configs { - configs[strings.ToLower(k)] = v - } - return configs, nil -} - -func isFatalErr(err error) bool { - return err != nil && !errors.Is(err, io.EOF) -} - func listSources(options *Options) { gologger.Info().Msgf("Current list of available sources. [%d]\n", len(passive.AllSources)) gologger.Info().Msgf("Sources marked with an * need key(s) or token(s) to work.\n") From 3c907cfee368ea739e2509d810eef8709d897729 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Thu, 17 Aug 2023 13:35:28 +0000 Subject: [PATCH 2/4] create if not exists --- v2/pkg/runner/options.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/v2/pkg/runner/options.go b/v2/pkg/runner/options.go index 34a349b55..7867923da 100644 --- a/v2/pkg/runner/options.go +++ b/v2/pkg/runner/options.go @@ -145,8 +145,9 @@ func ParseOptions() *Options { os.Exit(1) } - if err := createProviderConfigYAML(defaultProviderConfigLocation); err != nil { - gologger.Error().Msgf("Could not create provider config file: %s\n", err) + if exists := fileutil.FileExists(defaultConfigLocation); !exists { + if err := createProviderConfigYAML(defaultProviderConfigLocation); err != nil { + gologger.Error().Msgf("Could not create provider config file: %s\n", err) } if options.Config != defaultConfigLocation { From f565351f74ec7ca179e6d525bfd72729589888bf Mon Sep 17 00:00:00 2001 From: mzack Date: Fri, 18 Aug 2023 10:10:04 +0200 Subject: [PATCH 3/4] fixing syntax error --- v2/pkg/runner/options.go | 1 + 1 file changed, 1 insertion(+) diff --git a/v2/pkg/runner/options.go b/v2/pkg/runner/options.go index 7867923da..c18ffc6c3 100644 --- a/v2/pkg/runner/options.go +++ b/v2/pkg/runner/options.go @@ -148,6 +148,7 @@ func ParseOptions() *Options { if exists := fileutil.FileExists(defaultConfigLocation); !exists { if err := createProviderConfigYAML(defaultProviderConfigLocation); err != nil { gologger.Error().Msgf("Could not create provider config file: %s\n", err) + } } if options.Config != defaultConfigLocation { From 2916dff2ea848fd6aae513cbb9c116262f813797 Mon Sep 17 00:00:00 2001 From: mzack Date: Fri, 18 Aug 2023 10:21:17 +0200 Subject: [PATCH 4/4] fixing variable name --- v2/pkg/runner/options.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/pkg/runner/options.go b/v2/pkg/runner/options.go index c18ffc6c3..9f1219011 100644 --- a/v2/pkg/runner/options.go +++ b/v2/pkg/runner/options.go @@ -145,7 +145,7 @@ func ParseOptions() *Options { os.Exit(1) } - if exists := fileutil.FileExists(defaultConfigLocation); !exists { + if exists := fileutil.FileExists(defaultProviderConfigLocation); !exists { if err := createProviderConfigYAML(defaultProviderConfigLocation); err != nil { gologger.Error().Msgf("Could not create provider config file: %s\n", err) }