Merge pull request #42 from slicingmelon/proper-raw-urls

added raw urls support - on parsing and sending sending requests

Author: devploit

cmd/api.go

@@ -12,6 +12,8 @@ import (
 	"os"
 	"strings"
 	"time"
+
+	"github.com/slicingmelon/go-rawurlparser"
 )
 
 // parseFile reads a file given its filename and returns a list containing each of its lines.
@@ -46,7 +48,6 @@ type header struct {
 }
 
 // request makes an HTTP request using headers `headers` and proxy `proxy`.
-//
 // If `method` is empty, it defaults to "GET".
 func request(method, uri string, headers []header, proxy *url.URL, rateLimit bool, timeout int, redirect bool) (int, []byte, error) {
 	if method == "" {
@@ -57,21 +58,23 @@ func request(method, uri string, headers []header, proxy *url.URL, rateLimit boo
 		proxy = nil
 	}
 
-	client := &http.Client{
-		Transport: &http.Transport{
-			Proxy: http.ProxyURL(proxy),
-			TLSClientConfig: &tls.Config{
-				InsecureSkipVerify: true,
-			},
-			DialContext: (&net.Dialer{
-				Timeout:   time.Duration(timeout) / 1000 * time.Second,
-				KeepAlive: 30 * time.Second,
-			}).DialContext,
-			MaxIdleConns:          100,
-			IdleConnTimeout:       90 * time.Second,
-			TLSHandshakeTimeout:   10 * time.Second,
-			ExpectContinueTimeout: 1 * time.Second,
+	customTransport := &http.Transport{
+		Proxy: http.ProxyURL(proxy),
+		TLSClientConfig: &tls.Config{
+			InsecureSkipVerify: true,
 		},
+		DialContext: (&net.Dialer{
+			Timeout:   time.Duration(timeout) / 1000 * time.Second,
+			KeepAlive: 30 * time.Second,
+		}).DialContext,
+		MaxIdleConns:          100,
+		IdleConnTimeout:       90 * time.Second,
+		TLSHandshakeTimeout:   10 * time.Second,
+		ExpectContinueTimeout: 1 * time.Second,
+	}
+
+	client := &http.Client{
+		Transport: customTransport,
 	}
 
 	if !redirect {
@@ -80,11 +83,30 @@ func request(method, uri string, headers []header, proxy *url.URL, rateLimit boo
 		}
 	}
 
-	req, err := http.NewRequest(method, uri, nil)
-	if err != nil {
-		return 0, nil, nil
-	}
-	req.Close = true
+	// Use  raw URL parser instead
+	parsedURL := rawurlparser.RawURLParse(uri)
+
+	// Create new request
+	req := &http.Request{
+		Method: method,
+		URL: &url.URL{
+			Scheme: parsedURL.Scheme,
+			Host:   parsedURL.Host,
+			Opaque: parsedURL.Path, // Use Opaque to prevent path normalization
+		},
+		Header: make(http.Header),
+		Close:  true,
+	}
+
+	//log.Printf("Debug - Raw URL parsed: %s", uri)
+	// Don't use URL.String() for debugging, as it will perform encodings and normalization
+	// log.Printf("Debug - Request Components - Scheme: %s, Host: %s, Path: %s, RawPath: %s, Opaque: %s",
+	// 	req.URL.Scheme,
+	// 	req.URL.Host,
+	// 	req.URL.Path,
+	// 	req.URL.RawPath,
+	// 	req.URL.Opaque,
+	// )
 
 	for _, header := range headers {
 		req.Header.Add(header.key, header.value)

cmd/requester.go

@@ -15,6 +15,7 @@ import (
 	"unicode"
 
 	"github.com/fatih/color"
+	"github.com/slicingmelon/go-rawurlparser"
 	"github.com/zenthangplus/goccm"
 )
 
@@ -377,7 +378,7 @@ func requestMidPaths(options RequestOptions) {
 	x := strings.Split(options.uri, "/")
 	var uripath string
 
-	parsedURL, err := url.Parse(options.uri)
+	parsedURL := rawurlparser.RawURLParse(options.uri)
 	if err != nil {
 		log.Println(err)
 	}
@@ -481,11 +482,11 @@ func parseCurlOutput(output string, httpVersion string) Result {
 func requestPathCaseSwitching(options RequestOptions) {
 	color.Cyan("\n━━━━━━━━━━━━ PATH CASE SWITCHING ━━━━━━━━━━━━━")
 
-	parsedURL, err := url.Parse(options.uri)
-	if err != nil {
-		log.Println(err)
-		return
-	}
+	parsedURL := rawurlparser.RawURLParse(options.uri)
+	// if err != nil {
+	// 	log.Println(err)
+	// 	return
+	// }
 
 	baseuri := parsedURL.Scheme + "://" + parsedURL.Host
 	uripath := strings.Trim(parsedURL.Path, "/")

go.mod

@@ -1,6 +1,6 @@
 module nomore403
 
-go 1.19
+go 1.23.1
 
 require (
 	github.com/fatih/color v1.13.0
@@ -19,6 +19,7 @@ require (
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/pelletier/go-toml v1.9.5 // indirect
 	github.com/pelletier/go-toml/v2 v2.0.5 // indirect
+	github.com/slicingmelon/go-rawurlparser v0.0.0-20241101212355-a74dbff109f7 // indirect
 	github.com/spf13/afero v1.8.2 // indirect
 	github.com/spf13/cast v1.5.0 // indirect
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect

go.sum

@@ -159,6 +159,8 @@ github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/slicingmelon/go-rawurlparser v0.0.0-20241101212355-a74dbff109f7 h1:6xhzr51FYZpfVeGHp8M15WuIF+MbyN5ZcnoQRBEAX2A=
+github.com/slicingmelon/go-rawurlparser v0.0.0-20241101212355-a74dbff109f7/go.mod h1:xUf2JAg8Wkk4L26ByKhkAcqBeHzDdQ+m2QPZAY+g7y4=
 github.com/spf13/afero v1.8.2 h1:xehSyVa0YnHWsJ49JFljMpg1HX19V6NDZ1fkm1Xznbo=
 github.com/spf13/afero v1.8.2/go.mod h1:CtAatgMJh6bJEIs48Ay/FOnkljP3WeGUG0MC1RfAqwo=
 github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w=