Script 'mail_helper' called by obssrc Hello community, here is the log from the commit of package subfinder for openSUSE:Factory checked in at 2025-02-03 21:43:58 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/subfinder (Old) and /work/SRC/openSUSE:Factory/.subfinder.new.2316 (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "subfinder" Mon Feb 3 21:43:58 2025 rev:2 rq:1242173 version:2.6.8 Changes: -------- --- /work/SRC/openSUSE:Factory/subfinder/subfinder.changes 2025-01-31 16:04:27.384083669 +0100 +++ /work/SRC/openSUSE:Factory/.subfinder.new.2316/subfinder.changes 2025-02-03 21:44:54.741946279 +0100 @@ -1,0 +2,20 @@ +Sat Feb 1 23:29:38 UTC 2025 - Dominik Heidler <[email protected]> + +- Disable version check + * Add disable-version-check.patch + +------------------------------------------------------------------- +Sat Feb 1 10:40:15 UTC 2025 - Dominik Heidler <[email protected]> + +- Version 2.6.8 + * Added hudsonrock source by @dogancanbakir in #1481 + * Fixed negative statistics for sitedossier source by @dogancanbakir in #1432 + * Fixed GitHub source' performance issue by @dogancanbakir in #1421 + * Fixed the issue with source initialization to prevent the program from exiting when encountering invalid sources during initialization by @dogancanbakir in #1468 + * Removed the columbus source due to consistently encountering 502 errors by @dogancanbakir in #1430 + * Make failing sources non-default and exclude from the tests by @dogancanbakir in #1475 + * Removed subdomaincenter as its output quality was unsatisfactory by @dogancanbakir in #1500 + * Updated dnsdumpster source to use its new API by @dogancanbakir in #1484 + * Improved the usability of the Subfinder SDK by enabling the return of the source map by @mielverkerken in #1466 + +------------------------------------------------------------------- Old: ---- subfinder-2.6.7.tar.gz New: ---- disable-version-check.patch subfinder-2.6.8.tar.gz BETA DEBUG BEGIN: New:- Disable version check * Add disable-version-check.patch BETA DEBUG END: ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ subfinder.spec ++++++ --- /var/tmp/diff_new_pack.jSY6hi/_old 2025-02-03 21:44:55.849992014 +0100 +++ /var/tmp/diff_new_pack.jSY6hi/_new 2025-02-03 21:44:55.849992014 +0100 @@ -1,7 +1,7 @@ # # spec file for package subfinder # -# Copyright (c) 2024 SUSE LLC +# Copyright (c) 2025 SUSE LLC # # All modifications and additions to the file contributed by third parties # remain the property of their copyright owners, unless otherwise agreed @@ -17,17 +17,18 @@ Name: subfinder -Version: 2.6.7 +Version: 2.6.8 Release: 0 Summary: Fast passive subdomain enumeration tool License: MIT URL: https://github.com/projectdiscovery/subfinder Source0: https://github.com/projectdiscovery/%{name}/archive/v%{version}.tar.gz#/%{name}-%{version}.tar.gz Source1: vendor.tar.zstd +Patch1: disable-version-check.patch +BuildRequires: binutils BuildRequires: help2man BuildRequires: udev BuildRequires: zstd -BuildRequires: binutils BuildRequires: golang(API) >= 1.16 %description ++++++ disable-version-check.patch ++++++ diff --git a/v2/pkg/runner/banners.go b/v2/pkg/runner/banners.go index 1e9af43..181013c 100644 --- a/v2/pkg/runner/banners.go +++ b/v2/pkg/runner/banners.go @@ -2,7 +2,6 @@ package runner import ( "github.com/projectdiscovery/gologger" - updateutils "github.com/projectdiscovery/utils/update" ) const banner = ` @@ -29,6 +28,5 @@ func showBanner() { func GetUpdateCallback() func() { return func() { showBanner() - updateutils.GetUpdateToolCallback("subfinder", version)() } } diff --git a/v2/pkg/runner/options.go b/v2/pkg/runner/options.go index 5e3d4b1..6068386 100644 --- a/v2/pkg/runner/options.go +++ b/v2/pkg/runner/options.go @@ -18,7 +18,6 @@ import ( fileutil "github.com/projectdiscovery/utils/file" folderutil "github.com/projectdiscovery/utils/folder" logutil "github.com/projectdiscovery/utils/log" - updateutils "github.com/projectdiscovery/utils/update" ) var ( @@ -180,17 +179,6 @@ func ParseOptions() *Options { options.ConfigureOutput() showBanner() - if !options.DisableUpdateCheck { - latestVersion, err := updateutils.GetToolVersionCallback("subfinder", version)() - if err != nil { - if options.Verbose { - gologger.Error().Msgf("subfinder version check failed: %v", err.Error()) - } - } else { - gologger.Info().Msgf("Current subfinder version %v %v", version, updateutils.GetVersionDescription(version, latestVersion)) - } - } - if options.ListSources { listSources(options) os.Exit(0) ++++++ subfinder-2.6.7.tar.gz -> subfinder-2.6.8.tar.gz ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/subfinder-2.6.7/.github/workflows/release-binary.yml new/subfinder-2.6.8/.github/workflows/release-binary.yml --- old/subfinder-2.6.7/.github/workflows/release-binary.yml 2024-10-25 17:46:39.000000000 +0200 +++ new/subfinder-2.6.8/.github/workflows/release-binary.yml 2025-01-14 10:37:59.000000000 +0100 @@ -23,7 +23,7 @@ - name: "Create release on GitHub" uses: goreleaser/goreleaser-action@v3 with: - args: "release --rm-dist" + args: "release --clean" version: latest workdir: v2/ env: diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/subfinder-2.6.7/v2/examples/main.go new/subfinder-2.6.8/v2/examples/main.go --- old/subfinder-2.6.7/v2/examples/main.go 2024-10-25 17:46:39.000000000 +0200 +++ new/subfinder-2.6.8/v2/examples/main.go 2025-01-14 10:37:59.000000000 +0100 @@ -30,8 +30,9 @@ } output := &bytes.Buffer{} + var sourceMap map[string]map[string]struct{} // To run subdomain enumeration on a single domain - if err = subfinder.EnumerateSingleDomainWithCtx(context.Background(), "hackerone.com", []io.Writer{output}); err != nil { + if sourceMap, err = subfinder.EnumerateSingleDomainWithCtx(context.Background(), "hackerone.com", []io.Writer{output}); err != nil { log.Fatalf("failed to enumerate single domain: %v", err) } @@ -47,4 +48,13 @@ // print the output log.Println(output.String()) + + // Or use sourceMap to access the results in your application + for subdomain, sources := range sourceMap { + sourcesList := make([]string, 0, len(sources)) + for source := range sources { + sourcesList = append(sourcesList, source) + } + log.Printf("%s %s (%d)\n", subdomain, sourcesList, len(sources)) + } } diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/subfinder-2.6.7/v2/go.mod new/subfinder-2.6.8/v2/go.mod --- old/subfinder-2.6.7/v2/go.mod 2024-10-25 17:46:39.000000000 +0200 +++ new/subfinder-2.6.8/v2/go.mod 2025-01-14 10:37:59.000000000 +0100 @@ -10,10 +10,10 @@ github.com/projectdiscovery/chaos-client v0.5.2 github.com/projectdiscovery/dnsx v1.2.1 github.com/projectdiscovery/fdmax v0.0.4 - github.com/projectdiscovery/gologger v1.1.27 - github.com/projectdiscovery/ratelimit v0.0.55 - github.com/projectdiscovery/retryablehttp-go v1.0.82 - github.com/projectdiscovery/utils v0.2.15 + github.com/projectdiscovery/gologger v1.1.40 + github.com/projectdiscovery/ratelimit v0.0.68 + github.com/projectdiscovery/retryablehttp-go v1.0.95 + github.com/projectdiscovery/utils v0.4.6 github.com/rs/xid v1.5.0 github.com/stretchr/testify v1.9.0 github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 @@ -71,8 +71,8 @@ github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/projectdiscovery/blackrock v0.0.1 // indirect github.com/projectdiscovery/cdncheck v1.1.0 // indirect - github.com/projectdiscovery/fastdialer v0.2.9 // indirect - github.com/projectdiscovery/hmap v0.0.62 // indirect + github.com/projectdiscovery/fastdialer v0.2.14 // indirect + github.com/projectdiscovery/hmap v0.0.74 // indirect github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 // indirect github.com/projectdiscovery/networkpolicy v0.0.9 // indirect github.com/refraction-networking/utls v1.6.7 // indirect @@ -102,12 +102,12 @@ github.com/zmap/zcrypto v0.0.0-20230422215203-9a665e1e9968 // indirect go.etcd.io/bbolt v1.3.7 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.27.0 // indirect + golang.org/x/crypto v0.31.0 // indirect golang.org/x/mod v0.17.0 // indirect golang.org/x/oauth2 v0.11.0 // indirect - golang.org/x/sync v0.8.0 // indirect - golang.org/x/term v0.24.0 // indirect - golang.org/x/text v0.18.0 // indirect + golang.org/x/sync v0.10.0 // indirect + golang.org/x/term v0.27.0 // indirect + golang.org/x/text v0.21.0 // indirect golang.org/x/time v0.5.0 // indirect golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect google.golang.org/appengine v1.6.7 // indirect @@ -125,7 +125,7 @@ github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/projectdiscovery/goflags v0.1.64 - github.com/projectdiscovery/retryabledns v1.0.80 // indirect - golang.org/x/net v0.29.0 // indirect - golang.org/x/sys v0.25.0 // indirect + github.com/projectdiscovery/retryabledns v1.0.93 // indirect + golang.org/x/net v0.33.0 // indirect + golang.org/x/sys v0.28.0 // indirect ) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/subfinder-2.6.7/v2/go.sum new/subfinder-2.6.8/v2/go.sum --- old/subfinder-2.6.7/v2/go.sum 2024-10-25 17:46:39.000000000 +0200 +++ new/subfinder-2.6.8/v2/go.sum 2025-01-14 10:37:59.000000000 +0100 @@ -196,28 +196,28 @@ github.com/projectdiscovery/chaos-client v0.5.2/go.mod h1:KnoJ/NJPhll42uaqlDga6oafFfNw5l2XI2ajRijtDuU= github.com/projectdiscovery/dnsx v1.2.1 h1:TxslYvp1Z/YZ4CP/J0gx5RYpvXREnVmyoacmTcGu5yg= github.com/projectdiscovery/dnsx v1.2.1/go.mod h1:6dAsMCEDu7FArZy2qjyTeUQrqpZ4ITLU11fcmUvFqt0= -github.com/projectdiscovery/fastdialer v0.2.9 h1:vDCqxVMCyUu3oVEizEK1K8K+CCcLkVDW3X2HfiWaVFA= -github.com/projectdiscovery/fastdialer v0.2.9/go.mod h1:mYv5QaNBDDSHlZO9DI0niRMw+G5hUzwIhs8QixSElUI= +github.com/projectdiscovery/fastdialer v0.2.14 h1:/cndy+5celjoYzbk4LksHYOCTpFGIJY8RF/EK31Opjs= +github.com/projectdiscovery/fastdialer v0.2.14/go.mod h1:z5yKQ/YWaVrBMfdL6f5J7VytUx9wxc5vs/Lf51QelCw= github.com/projectdiscovery/fdmax v0.0.4 h1:K9tIl5MUZrEMzjvwn/G4drsHms2aufTn1xUdeVcmhmc= github.com/projectdiscovery/fdmax v0.0.4/go.mod h1:oZLqbhMuJ5FmcoaalOm31B1P4Vka/CqP50nWjgtSz+I= github.com/projectdiscovery/goflags v0.1.64 h1:FDfwdt9N97Hi8OuhbkDlKtVttpc/CRMIWQVa08VsHsI= github.com/projectdiscovery/goflags v0.1.64/go.mod h1:3FyHIVQtnycNOc1LE3O1jj/XR5XuMdF9QfHd0ujhnX4= -github.com/projectdiscovery/gologger v1.1.27 h1:et/adsKS0jAkPZNUuZTJ+J4U/Ofadxu6Bj3NlUYs1e8= -github.com/projectdiscovery/gologger v1.1.27/go.mod h1:TXLvCbofuDyQlweDkSBanN083w51QKT4EmyKAVMQ+Ts= -github.com/projectdiscovery/hmap v0.0.62 h1:Pb3omgGQkRHB2EBDhgXniUXVC8Jz2CBzpSMKZYRUKFs= -github.com/projectdiscovery/hmap v0.0.62/go.mod h1:pNheW3ukrLOcRUDFWrq8OAZiizhosDonIKRVg/4PGnM= +github.com/projectdiscovery/gologger v1.1.40 h1:FSIhKnYKzuIEIz3RTg6JX9JtDKgkEzIEf2v5RYckoQ4= +github.com/projectdiscovery/gologger v1.1.40/go.mod h1:8AUxYXmClqOWJgZ5wknNn5rRK3UlrXQ/r9JjX+gp5Gg= +github.com/projectdiscovery/hmap v0.0.74 h1:j0TpS9fJxisfdGcIRjaZ3qgyjP3pBkRJf12ZWv64rOI= +github.com/projectdiscovery/hmap v0.0.74/go.mod h1:qEPAdq/gWQU/IEI+QMzSyL+HYdqayR64V9vGTI/W38c= github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 h1:ZScLodGSezQVwsQDtBSMFp72WDq0nNN+KE/5DHKY5QE= github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983/go.mod h1:3G3BRKui7nMuDFAZKR/M2hiOLtaOmyukT20g88qRQjI= github.com/projectdiscovery/networkpolicy v0.0.9 h1:IrlDoYZagNNO8y+7iZeHT8k5izE+nek7TdtvEBwCxqk= github.com/projectdiscovery/networkpolicy v0.0.9/go.mod h1:XFJ2Lnv8BE/ziQCFjBHMsH1w6VmkPiQtk+NlBpdMU7M= -github.com/projectdiscovery/ratelimit v0.0.55 h1:K72IbJX/Lm4vbCtTcZ6Z8C5lWKL4vEhPYeiopFOWdqg= -github.com/projectdiscovery/ratelimit v0.0.55/go.mod h1:IpuZAnf3OIoUkXuO8CTAC/l0Fv50/ZfRrbRi6gufTwE= -github.com/projectdiscovery/retryabledns v1.0.80 h1:P1oqWHZdF/IMtxXV4mrtENTNsghYojVlrNcv8dZAMu8= -github.com/projectdiscovery/retryabledns v1.0.80/go.mod h1:BdCE4+Lph6hkNx+f+EjOnG+K8Z6shFWs9ZiTj6YcNvU= -github.com/projectdiscovery/retryablehttp-go v1.0.82 h1:XS+ZEKO291KGZVhHMO+RhvNvVE+IK8VHeM5i6CQ4Ixc= -github.com/projectdiscovery/retryablehttp-go v1.0.82/go.mod h1:JvlWId6aXT8en1ly22///wHoRQwSsMNAttdJ/YkhHHY= -github.com/projectdiscovery/utils v0.2.15 h1:FO3n7uhLazHtat3qBxWIq0JCfLoR/IvP5JXAOSKaQ5w= -github.com/projectdiscovery/utils v0.2.15/go.mod h1:2NAFRu8j/82bkVqx2TcsZFdgtUOnHEUi7u6s3lv79Lo= +github.com/projectdiscovery/ratelimit v0.0.68 h1:gMLD1aB4R8w7BIpKvtQf6TNb6+5zsJO9WSRWZ9pxwe4= +github.com/projectdiscovery/ratelimit v0.0.68/go.mod h1:ieU9nNu9Ie8nVMKdj3bsX3JA3kfNI8qn4pkNXsyRxsw= +github.com/projectdiscovery/retryabledns v1.0.93 h1:iKcEEEH77WwUf5EGimhHxCDdqBF2kOl7WhQi3VQXB8Q= +github.com/projectdiscovery/retryabledns v1.0.93/go.mod h1:f5HmPdVr3CUm4tHHiB0UyiZVQTYYAKTqfoj8M2gCvqo= +github.com/projectdiscovery/retryablehttp-go v1.0.95 h1:5CHhWLMovX1MD9W3HzlsMBY3xA+dyeqta2gSWo3j92E= +github.com/projectdiscovery/retryablehttp-go v1.0.95/go.mod h1:/7CHaD7vqnqBD++AI0JsJdcYyq1Wbf4vMhddjy7sZjI= +github.com/projectdiscovery/utils v0.4.6 h1:lwbS5d/f70wyDwuwF6lAVkn390hEI/0LOtqyqJEI+qE= +github.com/projectdiscovery/utils v0.4.6/go.mod h1:eevtW7+x7ydrBdmOenmHdqqJKRv3VqY2QUR7vs4qRfU= github.com/refraction-networking/utls v1.6.7 h1:zVJ7sP1dJx/WtVuITug3qYUq034cDq9B2MR1K67ULZM= github.com/refraction-networking/utls v1.6.7/go.mod h1:BC3O4vQzye5hqpmDTWUqi4P5DDhzJfkV1tdqtawQIH0= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= @@ -333,8 +333,8 @@ golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= -golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A= -golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70= +golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/exp v0.0.0-20230420155640-133eef4313cb h1:rhjz/8Mbfa8xROFiH+MQphmAmgqRM0bOMnytznhWEXk= golang.org/x/exp v0.0.0-20230420155640-133eef4313cb/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= @@ -355,8 +355,8 @@ golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= -golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= -golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I= golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw= @@ -366,8 +366,8 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= -golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -392,8 +392,8 @@ golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= -golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -401,8 +401,8 @@ golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o= -golang.org/x/term v0.24.0 h1:Mh5cbb+Zk2hqqXNO7S1iTjEphVL+jb8ZWaqh/g+JWkM= -golang.org/x/term v0.24.0/go.mod h1:lOBK/LVxemqiMij05LGJ0tzNr8xlmwBRJ81PX6wVLH8= +golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q= +golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -413,8 +413,8 @@ golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= -golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/subfinder-2.6.7/v2/pkg/passive/sources.go new/subfinder-2.6.8/v2/pkg/passive/sources.go --- old/subfinder-2.6.7/v2/pkg/passive/sources.go 2024-10-25 17:46:39.000000000 +0200 +++ new/subfinder-2.6.8/v2/pkg/passive/sources.go 2025-01-14 10:37:59.000000000 +0100 @@ -18,7 +18,6 @@ "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/certspotter" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/chaos" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/chinaz" - "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/columbus" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/commoncrawl" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/crtsh" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/digitorus" @@ -30,6 +29,7 @@ "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/fullhunt" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/github" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/hackertarget" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/hudsonrock" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/hunter" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/intelx" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/leakix" @@ -42,7 +42,6 @@ "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/securitytrails" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/shodan" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/sitedossier" - "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/subdomaincenter" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/threatbook" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/virustotal" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/waybackarchive" @@ -62,7 +61,6 @@ &certspotter.Source{}, &chaos.Source{}, &chinaz.Source{}, - &columbus.Source{}, &commoncrawl.Source{}, &crtsh.Source{}, &digitorus.Source{}, @@ -95,7 +93,7 @@ // &threatminer.Source{}, // failing api // &reconcloud.Source{}, // failing due to cloudflare bot protection &builtwith.Source{}, - &subdomaincenter.Source{}, + &hudsonrock.Source{}, } var sourceWarnings = mapsutil.NewSyncLockMap[string, string]( @@ -128,7 +126,7 @@ if len(sourceNames) > 0 { for _, source := range sourceNames { if NameSourceMap[source] == nil { - gologger.Fatal().Msgf("There is no source with the name: %s", source) + gologger.Warning().Msgf("There is no source with the name: %s", source) } else { sources[source] = NameSourceMap[source] } diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/subfinder-2.6.7/v2/pkg/passive/sources_test.go new/subfinder-2.6.8/v2/pkg/passive/sources_test.go --- old/subfinder-2.6.7/v2/pkg/passive/sources_test.go 2024-10-25 17:46:39.000000000 +0200 +++ new/subfinder-2.6.8/v2/pkg/passive/sources_test.go 2025-01-14 10:37:59.000000000 +0100 @@ -21,7 +21,6 @@ "certspotter", "chaos", "chinaz", - "columbus", "commoncrawl", "crtsh", "digitorus", @@ -54,7 +53,7 @@ // "threatminer", // "reconcloud", "builtwith", - "subdomaincenter", + "hudsonrock", } expectedDefaultSources = []string{ @@ -67,7 +66,6 @@ "censys", "chaos", "chinaz", - "columbus", "crtsh", "digitorus", "dnsdumpster", @@ -91,7 +89,6 @@ // "threatminer", // "reconcloud", "builtwith", - "subdomaincenter", } expectedDefaultRecursiveSources = []string{ @@ -100,7 +97,6 @@ "bufferover", "certspotter", "crtsh", - "dnsdumpster", "dnsdb", "digitorus", "hackertarget", diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/subfinder-2.6.7/v2/pkg/passive/sources_wo_auth_test.go new/subfinder-2.6.8/v2/pkg/passive/sources_wo_auth_test.go --- old/subfinder-2.6.7/v2/pkg/passive/sources_wo_auth_test.go 2024-10-25 17:46:39.000000000 +0200 +++ new/subfinder-2.6.8/v2/pkg/passive/sources_wo_auth_test.go 2025-01-14 10:37:59.000000000 +0100 @@ -24,13 +24,14 @@ } ignoredSources := []string{ - "commoncrawl", // commoncrawl is under resourced and will likely time-out so step over it for this test https://groups.google.com/u/2/g/common-crawl/c/3QmQjFA_3y4/m/vTbhGqIBBQAJ - "riddler", // failing due to cloudfront protection - "crtsh", // Fails in GH Action (possibly IP-based ban) causing a timeout. - "hackertarget", // Fails in GH Action (possibly IP-based ban) but works locally - "waybackarchive", // Fails randomly - "alienvault", // 503 Service Temporarily Unavailable - "digitorus", // failing with "Failed to retrieve certificate" + "commoncrawl", // commoncrawl is under resourced and will likely time-out so step over it for this test https://groups.google.com/u/2/g/common-crawl/c/3QmQjFA_3y4/m/vTbhGqIBBQAJ + "riddler", // failing due to cloudfront protection + "crtsh", // Fails in GH Action (possibly IP-based ban) causing a timeout. + "hackertarget", // Fails in GH Action (possibly IP-based ban) but works locally + "waybackarchive", // Fails randomly + "alienvault", // 503 Service Temporarily Unavailable + "digitorus", // failing with "Failed to retrieve certificate" + "dnsdumpster", // failing with "unexpected status code 403 received" } domain := "hackerone.com" diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/subfinder-2.6.7/v2/pkg/runner/banners.go new/subfinder-2.6.8/v2/pkg/runner/banners.go --- old/subfinder-2.6.7/v2/pkg/runner/banners.go 2024-10-25 17:46:39.000000000 +0200 +++ new/subfinder-2.6.8/v2/pkg/runner/banners.go 2025-01-14 10:37:59.000000000 +0100 @@ -17,7 +17,7 @@ const ToolName = `subfinder` // Version is the current version of subfinder -const version = `v2.6.7` +const version = `v2.6.8` // showBanner is used to show the banner to the user func showBanner() { diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/subfinder-2.6.7/v2/pkg/runner/enumerate.go new/subfinder-2.6.8/v2/pkg/runner/enumerate.go --- old/subfinder-2.6.7/v2/pkg/runner/enumerate.go 2024-10-25 17:46:39.000000000 +0200 +++ new/subfinder-2.6.8/v2/pkg/runner/enumerate.go 2025-01-14 10:37:59.000000000 +0100 @@ -19,18 +19,20 @@ const maxNumCount = 2 var replacer = strings.NewReplacer( + "â¢.", "", + "â¢", "", "*.", "", "http://";, "", "https://";, "", ) // EnumerateSingleDomain wraps EnumerateSingleDomainWithCtx with an empty context -func (r *Runner) EnumerateSingleDomain(domain string, writers []io.Writer) error { +func (r *Runner) EnumerateSingleDomain(domain string, writers []io.Writer) (map[string]map[string]struct{}, error) { return r.EnumerateSingleDomainWithCtx(context.Background(), domain, writers) } // EnumerateSingleDomainWithCtx performs subdomain enumeration against a single domain -func (r *Runner) EnumerateSingleDomainWithCtx(ctx context.Context, domain string, writers []io.Writer) error { +func (r *Runner) EnumerateSingleDomainWithCtx(ctx context.Context, domain string, writers []io.Writer) (map[string]map[string]struct{}, error) { gologger.Info().Msgf("Enumerating subdomains for %s\n", domain) // Check if the user has asked to remove wildcards explicitly. @@ -61,14 +63,15 @@ for result := range passiveResults { switch result.Type { case subscraping.Error: - gologger.Warning().Msgf("Could not run source %s: %s\n", result.Source, result.Error) + gologger.Warning().Msgf("Encountered an error with source %s: %s\n", result.Source, result.Error) case subscraping.Subdomain: + subdomain := replacer.Replace(result.Value) + // Validate the subdomain found and remove wildcards from - if !strings.HasSuffix(result.Value, "."+domain) { + if !strings.HasSuffix(subdomain, "."+domain) { skippedCounts[result.Source]++ continue } - subdomain := replacer.Replace(result.Value) if matchSubdomain := r.filterAndMatchSubdomain(subdomain); matchSubdomain { if _, ok := uniqueMap[subdomain]; !ok { @@ -145,7 +148,7 @@ } if err != nil { gologger.Error().Msgf("Could not write results for %s: %s\n", domain, err) - return err + return nil, err } } @@ -186,7 +189,7 @@ printStatistics(statistics) } - return nil + return sourceMap, nil } func (r *Runner) filterAndMatchSubdomain(subdomain string) bool { diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/subfinder-2.6.7/v2/pkg/runner/options.go new/subfinder-2.6.8/v2/pkg/runner/options.go --- old/subfinder-2.6.7/v2/pkg/runner/options.go 2024-10-25 17:46:39.000000000 +0200 +++ new/subfinder-2.6.8/v2/pkg/runner/options.go 2025-01-14 10:37:59.000000000 +0100 @@ -257,5 +257,5 @@ "netlas=1/s", // "gitlab=2/s", "github=83/m", - "subdomaincenter=2/m", + "hudsonrock=5/s", } diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/subfinder-2.6.7/v2/pkg/runner/runner.go new/subfinder-2.6.8/v2/pkg/runner/runner.go --- old/subfinder-2.6.7/v2/pkg/runner/runner.go 2024-10-25 17:46:39.000000000 +0200 +++ new/subfinder-2.6.8/v2/pkg/runner/runner.go 2025-01-14 10:37:59.000000000 +0100 @@ -136,7 +136,7 @@ return err } - err = r.EnumerateSingleDomainWithCtx(ctx, domain, append(writers, file)) + _, err = r.EnumerateSingleDomainWithCtx(ctx, domain, append(writers, file)) file.Close() } else if r.options.OutputDirectory != "" { @@ -154,11 +154,11 @@ return err } - err = r.EnumerateSingleDomainWithCtx(ctx, domain, append(writers, file)) + _, err = r.EnumerateSingleDomainWithCtx(ctx, domain, append(writers, file)) file.Close() } else { - err = r.EnumerateSingleDomainWithCtx(ctx, domain, writers) + _, err = r.EnumerateSingleDomainWithCtx(ctx, domain, writers) } if err != nil { return err diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/subfinder-2.6.7/v2/pkg/subscraping/sources/columbus/columbus.go new/subfinder-2.6.8/v2/pkg/subscraping/sources/columbus/columbus.go --- old/subfinder-2.6.7/v2/pkg/subscraping/sources/columbus/columbus.go 2024-10-25 17:46:39.000000000 +0200 +++ new/subfinder-2.6.8/v2/pkg/subscraping/sources/columbus/columbus.go 1970-01-01 01:00:00.000000000 +0100 @@ -1,92 +0,0 @@ -// Package columbus logic -package columbus - -import ( - "context" - "fmt" - "time" - - jsoniter "github.com/json-iterator/go" - - "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" -) - -// Source is the passive scraping agent -type Source struct { - timeTaken time.Duration - errors int - results int -} - -// Run function returns all subdomains found with the service -func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { - results := make(chan subscraping.Result) - s.errors = 0 - s.results = 0 - - go func() { - defer func(startTime time.Time) { - s.timeTaken = time.Since(startTime) - close(results) - }(time.Now()) - - resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://columbus.elmasy.com/api/lookup/%s";, domain)) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - s.errors++ - session.DiscardHTTPResponse(resp) - return - } - - var subdomains []string - err = jsoniter.NewDecoder(resp.Body).Decode(&subdomains) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - s.errors++ - resp.Body.Close() - return - } - - resp.Body.Close() - - for _, record := range subdomains { - if record == "" { - continue - } - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record + "." + domain} - s.results++ - } - - }() - - return results -} - -// Name returns the name of the source -func (s *Source) Name() string { - return "columbus" -} - -func (s *Source) IsDefault() bool { - return true -} - -func (s *Source) HasRecursiveSupport() bool { - return false -} - -func (s *Source) NeedsKey() bool { - return false -} - -func (s *Source) AddApiKeys(_ []string) { - // no key needed -} - -func (s *Source) Statistics() subscraping.Statistics { - return subscraping.Statistics{ - Errors: s.errors, - Results: s.results, - TimeTaken: s.timeTaken, - } -} diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/subfinder-2.6.7/v2/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go new/subfinder-2.6.8/v2/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go --- old/subfinder-2.6.7/v2/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go 2024-10-25 17:46:39.000000000 +0200 +++ new/subfinder-2.6.8/v2/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go 2025-01-14 10:37:59.000000000 +0100 @@ -3,67 +3,29 @@ import ( "context" + "encoding/json" "fmt" - "io" - "net/url" - "regexp" - "strings" "time" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" ) -// CSRFSubMatchLength CSRF regex submatch length -const CSRFSubMatchLength = 2 - -var re = regexp.MustCompile("<input type=\"hidden\" name=\"csrfmiddlewaretoken\" value=\"(.*)\">") - -// getCSRFToken gets the CSRF Token from the page -func getCSRFToken(page string) string { - if subs := re.FindStringSubmatch(page); len(subs) == CSRFSubMatchLength { - return strings.TrimSpace(subs[1]) - } - return "" -} - -// postForm posts a form for a domain and returns the response -func postForm(ctx context.Context, session *subscraping.Session, token, domain string) (string, error) { - params := url.Values{ - "csrfmiddlewaretoken": {token}, - "targetip": {domain}, - "user": {"free"}, - } - - resp, err := session.HTTPRequest( - ctx, - "POST", - "https://dnsdumpster.com/";, - fmt.Sprintf("csrftoken=%s; Domain=dnsdumpster.com", token), - map[string]string{ - "Content-Type": "application/x-www-form-urlencoded", - "Referer": "https://dnsdumpster.com";, - "X-CSRF-Token": token, - }, - strings.NewReader(params.Encode()), - subscraping.BasicAuth{}, - ) - - if err != nil { - session.DiscardHTTPResponse(resp) - return "", err - } - - // Now, grab the entire page - in, err := io.ReadAll(resp.Body) - resp.Body.Close() - return string(in), err +type response struct { + A []struct { + Host string `json:"host"` + } `json:"a"` + Ns []struct { + Host string `json:"host"` + } `json:"ns"` } // Source is the passive scraping agent type Source struct { + apiKeys []string timeTaken time.Duration errors int results int + skipped bool } // Run function returns all subdomains found with the service @@ -78,35 +40,35 @@ close(results) }(time.Now()) - resp, err := session.SimpleGet(ctx, "https://dnsdumpster.com/";) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - s.errors++ - session.DiscardHTTPResponse(resp) + randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) + if randomApiKey == "" { + s.skipped = true return } - body, err := io.ReadAll(resp.Body) + resp, err := session.Get(ctx, fmt.Sprintf("https://api.dnsdumpster.com/domain/%s";, domain), "", map[string]string{"X-API-Key": randomApiKey}) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ - resp.Body.Close() + session.DiscardHTTPResponse(resp) return } - resp.Body.Close() + defer resp.Body.Close() - csrfToken := getCSRFToken(string(body)) - data, err := postForm(ctx, session, csrfToken, domain) + var response response + err = json.NewDecoder(resp.Body).Decode(&response) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ + resp.Body.Close() return } - for _, subdomain := range session.Extractor.Extract(data) { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} + for _, record := range append(response.A, response.Ns...) { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Host} s.results++ } + }() return results @@ -122,15 +84,15 @@ } func (s *Source) HasRecursiveSupport() bool { - return true + return false } func (s *Source) NeedsKey() bool { - return false + return true } -func (s *Source) AddApiKeys(_ []string) { - // no key needed +func (s *Source) AddApiKeys(keys []string) { + s.apiKeys = keys } func (s *Source) Statistics() subscraping.Statistics { @@ -138,5 +100,6 @@ Errors: s.errors, Results: s.results, TimeTaken: s.timeTaken, + Skipped: s.skipped, } } diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/subfinder-2.6.7/v2/pkg/subscraping/sources/github/github.go new/subfinder-2.6.8/v2/pkg/subscraping/sources/github/github.go --- old/subfinder-2.6.7/v2/pkg/subscraping/sources/github/github.go 2024-10-25 17:46:39.000000000 +0200 +++ new/subfinder-2.6.8/v2/pkg/subscraping/sources/github/github.go 2025-01-14 10:37:59.000000000 +0100 @@ -11,6 +11,7 @@ "regexp" "strconv" "strings" + "sync" "time" jsoniter "github.com/json-iterator/go" @@ -142,40 +143,58 @@ // proccesItems process github response items func (s *Source) proccesItems(ctx context.Context, items []item, domainRegexp *regexp.Regexp, name string, session *subscraping.Session, results chan subscraping.Result) error { - for _, item := range items { - // find subdomains in code - resp, err := session.SimpleGet(ctx, rawURL(item.HTMLURL)) - if err != nil { - if resp != nil && resp.StatusCode != http.StatusNotFound { - session.DiscardHTTPResponse(resp) + var wg sync.WaitGroup + errChan := make(chan error, len(items)) + + for _, responseItem := range items { + wg.Add(1) + go func(responseItem item) { + defer wg.Done() + + // find subdomains in code + resp, err := session.SimpleGet(ctx, rawURL(responseItem.HTMLURL)) + if err != nil { + if resp != nil && resp.StatusCode != http.StatusNotFound { + session.DiscardHTTPResponse(resp) + } + errChan <- err + return } - return err - } - if resp.StatusCode == http.StatusOK { - scanner := bufio.NewScanner(resp.Body) - for scanner.Scan() { - line := scanner.Text() - if line == "" { - continue + if resp.StatusCode == http.StatusOK { + scanner := bufio.NewScanner(resp.Body) + for scanner.Scan() { + line := scanner.Text() + if line == "" { + continue + } + for _, subdomain := range domainRegexp.FindAllString(normalizeContent(line), -1) { + results <- subscraping.Result{Source: name, Type: subscraping.Subdomain, Value: subdomain} + s.results++ + } } - for _, subdomain := range domainRegexp.FindAllString(normalizeContent(line), -1) { + resp.Body.Close() + } + + // find subdomains in text matches + for _, textMatch := range responseItem.TextMatches { + for _, subdomain := range domainRegexp.FindAllString(normalizeContent(textMatch.Fragment), -1) { results <- subscraping.Result{Source: name, Type: subscraping.Subdomain, Value: subdomain} s.results++ - } } - resp.Body.Close() - } + }(responseItem) + } - // find subdomains in text matches - for _, textMatch := range item.TextMatches { - for _, subdomain := range domainRegexp.FindAllString(normalizeContent(textMatch.Fragment), -1) { - results <- subscraping.Result{Source: name, Type: subscraping.Subdomain, Value: subdomain} - s.results++ - } + wg.Wait() + close(errChan) + + for err := range errChan { + if err != nil { + return err } } + return nil } diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/subfinder-2.6.7/v2/pkg/subscraping/sources/hudsonrock/hudsonrock.go new/subfinder-2.6.8/v2/pkg/subscraping/sources/hudsonrock/hudsonrock.go --- old/subfinder-2.6.7/v2/pkg/subscraping/sources/hudsonrock/hudsonrock.go 1970-01-01 01:00:00.000000000 +0100 +++ new/subfinder-2.6.8/v2/pkg/subscraping/sources/hudsonrock/hudsonrock.go 2025-01-14 10:37:59.000000000 +0100 @@ -0,0 +1,100 @@ +// Package hudsonrock logic +package hudsonrock + +import ( + "context" + "encoding/json" + "fmt" + "time" + + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +type hudsonrockResponse struct { + Data struct { + EmployeesUrls []struct { + URL string `json:"url"` + } `json:"employees_urls"` + ClientsUrls []struct { + URL string `json:"url"` + } `json:"clients_urls"` + } `json:"data"` +} + +// Source is the passive scraping agent +type Source struct { + timeTaken time.Duration + errors int + results int +} + +// Run function returns all subdomains found with the service +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + s.errors = 0 + s.results = 0 + + go func() { + defer func(startTime time.Time) { + s.timeTaken = time.Since(startTime) + close(results) + }(time.Now()) + + resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://cavalier.hudsonrock.com/api/json/v2/osint-tools/urls-by-domain?domain=%s";, domain)) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + session.DiscardHTTPResponse(resp) + return + } + defer resp.Body.Close() + + var response hudsonrockResponse + err = json.NewDecoder(resp.Body).Decode(&response) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + resp.Body.Close() + return + } + + for _, record := range append(response.Data.EmployeesUrls, response.Data.ClientsUrls...) { + for _, subdomain := range session.Extractor.Extract(record.URL) { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} + s.results++ + } + } + + }() + + return results +} + +// Name returns the name of the source +func (s *Source) Name() string { + return "hudsonrock" +} + +func (s *Source) IsDefault() bool { + return false +} + +func (s *Source) HasRecursiveSupport() bool { + return false +} + +func (s *Source) NeedsKey() bool { + return false +} + +func (s *Source) AddApiKeys(_ []string) { + // no key needed +} + +func (s *Source) Statistics() subscraping.Statistics { + return subscraping.Statistics{ + Errors: s.errors, + Results: s.results, + TimeTaken: s.timeTaken, + } +} diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/subfinder-2.6.7/v2/pkg/subscraping/sources/sitedossier/sitedossier.go new/subfinder-2.6.8/v2/pkg/subscraping/sources/sitedossier/sitedossier.go --- old/subfinder-2.6.7/v2/pkg/subscraping/sources/sitedossier/sitedossier.go 2024-10-25 17:46:39.000000000 +0200 +++ new/subfinder-2.6.8/v2/pkg/subscraping/sources/sitedossier/sitedossier.go 2025-01-14 10:37:59.000000000 +0100 @@ -18,12 +18,6 @@ var reNext = regexp.MustCompile(`<a href="([A-Za-z0-9/.]+)"><b>`) -type agent struct { - results chan subscraping.Result - errors int - session *subscraping.Session -} - // Source is the passive scraping agent type Source struct { timeTaken time.Duration @@ -37,58 +31,52 @@ s.errors = 0 s.results = 0 - a := agent{ - session: session, - results: results, - } - go func() { defer func(startTime time.Time) { s.timeTaken = time.Since(startTime) - close(a.results) + close(results) }(time.Now()) - a.enumerate(ctx, fmt.Sprintf("http://www.sitedossier.com/parentdomain/%s";, domain)) - s.errors = a.errors - s.results = len(a.results) + s.enumerate(ctx, session, fmt.Sprintf("http://www.sitedossier.com/parentdomain/%s";, domain), results) }() - return a.results + return results } -func (a *agent) enumerate(ctx context.Context, baseURL string) { +func (s *Source) enumerate(ctx context.Context, session *subscraping.Session, baseURL string, results chan subscraping.Result) { select { case <-ctx.Done(): return default: } - resp, err := a.session.SimpleGet(ctx, baseURL) + resp, err := session.SimpleGet(ctx, baseURL) isnotfound := resp != nil && resp.StatusCode == http.StatusNotFound if err != nil && !isnotfound { - a.results <- subscraping.Result{Source: "sitedossier", Type: subscraping.Error, Error: err} - a.errors++ - a.session.DiscardHTTPResponse(resp) + results <- subscraping.Result{Source: "sitedossier", Type: subscraping.Error, Error: err} + s.errors++ + session.DiscardHTTPResponse(resp) return } body, err := io.ReadAll(resp.Body) if err != nil { - a.results <- subscraping.Result{Source: "sitedossier", Type: subscraping.Error, Error: err} - a.errors++ + results <- subscraping.Result{Source: "sitedossier", Type: subscraping.Error, Error: err} + s.errors++ resp.Body.Close() return } resp.Body.Close() src := string(body) - for _, subdomain := range a.session.Extractor.Extract(src) { - a.results <- subscraping.Result{Source: "sitedossier", Type: subscraping.Subdomain, Value: subdomain} + for _, subdomain := range session.Extractor.Extract(src) { + results <- subscraping.Result{Source: "sitedossier", Type: subscraping.Subdomain, Value: subdomain} + s.results++ } match := reNext.FindStringSubmatch(src) if len(match) > 0 { - a.enumerate(ctx, fmt.Sprintf("http://www.sitedossier.com%s";, match[1])) + s.enumerate(ctx, session, fmt.Sprintf("http://www.sitedossier.com%s";, match[1]), results) } } diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/subfinder-2.6.7/v2/pkg/subscraping/sources/subdomaincenter/subdomaincenter.go new/subfinder-2.6.8/v2/pkg/subscraping/sources/subdomaincenter/subdomaincenter.go --- old/subfinder-2.6.7/v2/pkg/subscraping/sources/subdomaincenter/subdomaincenter.go 2024-10-25 17:46:39.000000000 +0200 +++ new/subfinder-2.6.8/v2/pkg/subscraping/sources/subdomaincenter/subdomaincenter.go 1970-01-01 01:00:00.000000000 +0100 @@ -1,87 +0,0 @@ -// Package subdomaincenter logic -package subdomaincenter - -import ( - "context" - "fmt" - "time" - - jsoniter "github.com/json-iterator/go" - - "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" -) - -// Source is the passive scraping agent -type Source struct { - timeTaken time.Duration - errors int - results int -} - -// Run function returns all subdomains found with the service -func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { - results := make(chan subscraping.Result) - s.errors = 0 - s.results = 0 - - go func() { - defer func(startTime time.Time) { - s.timeTaken = time.Since(startTime) - close(results) - }(time.Now()) - - resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://api.subdomain.center/?domain=%s";, domain)) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - s.errors++ - session.DiscardHTTPResponse(resp) - return - } - - var subdomains []string - err = jsoniter.NewDecoder(resp.Body).Decode(&subdomains) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - s.errors++ - resp.Body.Close() - return - } - resp.Body.Close() - - for _, subdomain := range subdomains { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} - s.results++ - } - }() - - return results -} - -// Name returns the name of the source -func (s *Source) Name() string { - return "subdomaincenter" -} - -func (s *Source) IsDefault() bool { - return true -} - -func (s *Source) HasRecursiveSupport() bool { - return false -} - -func (s *Source) NeedsKey() bool { - return false -} - -func (s *Source) AddApiKeys(_ []string) { - // no key needed -} - -func (s *Source) Statistics() subscraping.Statistics { - return subscraping.Statistics{ - Errors: s.errors, - Results: s.results, - TimeTaken: s.timeTaken, - } -} ++++++ vendor.tar.zstd ++++++ Binary files /var/tmp/diff_new_pack.jSY6hi/_old and /var/tmp/diff_new_pack.jSY6hi/_new differ
