feat(tartarus): implement support

go.mod

@@ -1,14 +1,16 @@
 module github.com/lizthgrey/tor-fetcher
 
-go 1.21.0
+go 1.25.0
 
 require (
-	github.com/cretz/bine v0.2.0
-	github.com/ipsn/go-libtor v1.0.380
-	golang.org/x/crypto v0.25.0
+	github.com/refraction-networking/utls v1.8.2
+	golang.org/x/crypto v0.47.0
+	golang.org/x/net v0.49.0
 )
 
 require (
-	golang.org/x/net v0.21.0 // indirect
-	golang.org/x/sys v0.22.0 // indirect
+	github.com/andybalholm/brotli v1.0.6 // indirect
+	github.com/klauspost/compress v1.17.4 // indirect
+	golang.org/x/sys v0.40.0 // indirect
+	golang.org/x/text v0.33.0 // indirect
 )

go.sum

@@ -1,33 +1,14 @@
-github.com/cretz/bine v0.1.0/go.mod h1:6PF6fWAvYtwjRGkAuDEJeWNOv3a2hUouSP/yRYXmvHw=
-github.com/cretz/bine v0.2.0 h1:8GiDRGlTgz+o8H9DSnsl+5MeBK4HsExxgl6WgzOCuZo=
-github.com/cretz/bine v0.2.0/go.mod h1:WU4o9QR9wWp8AVKtTM1XD5vUHkEqnf2vVSo6dBqbetI=
-github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
-github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/ipsn/go-libtor v1.0.380 h1:hCmALDBe3bPpgwMunonMLArrG41MxzpE91Bk8KQYnYM=
-github.com/ipsn/go-libtor v1.0.380/go.mod h1:6rIeHU7irp8ZH8E/JqaEOKlD6s4vSSUh4ngHelhlSMw=
-github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
-github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
-github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-golang.org/x/crypto v0.0.0-20190404164418-38d8ce5564a5/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE=
-golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8=
-golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30=
-golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M=
-golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
-golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
-golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
-golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
-golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
-gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+github.com/andybalholm/brotli v1.0.6 h1:Yf9fFpf49Zrxb9NlQaluyE92/+X7UVHlhMNJN2sxfOI=
+github.com/andybalholm/brotli v1.0.6/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
+github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4=
+github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM=
+github.com/refraction-networking/utls v1.8.2 h1:j4Q1gJj0xngdeH+Ox/qND11aEfhpgoEvV+S9iJ2IdQo=
+github.com/refraction-networking/utls v1.8.2/go.mod h1:jkSOEkLqn+S/jtpEHPOsVv/4V4EVnelwbMQl4vCWXAM=
+golang.org/x/crypto v0.47.0 h1:V6e3FRj+n4dbpw86FJ8Fv7XVOql7TEwpHapKoMJ/GO8=
+golang.org/x/crypto v0.47.0/go.mod h1:ff3Y9VzzKbwSSEzWqJsJVBnWmRwRSHt/6Op5n9bQc4A=
+golang.org/x/net v0.49.0 h1:eeHFmOGUTtaaPSGNmjBKpbng9MulQsJURQUAfUwY++o=
+golang.org/x/net v0.49.0/go.mod h1:/ysNB2EvaqvesRkuLAyjI1ycPZlQHM3q01F02UY/MV8=
+golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ=
+golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
+golang.org/x/text v0.33.0 h1:B3njUFyqtHDUI5jMn1YIr5B0IE2U0qck04r6d4KPAxE=
+golang.org/x/text v0.33.0/go.mod h1:LuMebE6+rBincTi9+xWTY8TztLzKHc/9C1uBCG27+q8=

main.go

@@ -4,41 +4,40 @@ import (
 	"bufio"
 	"compress/gzip"
 	"context"
-	"crypto/tls"
+	"crypto/sha256"
+	"encoding/binary"
 	"flag"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"log"
+	"net"
 	"net/http"
 	"net/http/cookiejar"
 	"net/url"
 	"os"
 	"strconv"
 	"strings"
-	"time"
+	"sync"
 
+	utls "github.com/refraction-networking/utls"
 	"golang.org/x/crypto/argon2"
-
-	"github.com/cretz/bine/tor"
-	"github.com/ipsn/go-libtor"
+	"golang.org/x/net/http2"
+	"golang.org/x/net/proxy"
 )
 
 var parallelism = flag.Int("p", 1, "Parallelism")
 var length = flag.Int("l", 32, "Length")
 var target = flag.String("target", "", "The URL to retrieve (required)")
-var ua = flag.String("ua", "Mozilla/5.0 (Windows NT 10.0; rv:102.0) Gecko/20100101 Firefox/102.0", "Tor user agent by default")
+var ua = flag.String("ua", "Mozilla/5.0 (Windows NT 10.0; rv:140.0) Gecko/20100101 Firefox/140.0", "Tor user agent by default")
+var socksAddr = flag.String("proxy", "socks5://127.0.0.1:9050", "SOCKS5 proxy address for Tor")
 
 func main() {
 	flag.Parse()
 	if *target == "" {
 		flag.Usage()
 		os.Exit(1)
 	}
-	http.DefaultTransport.(*http.Transport).TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
-
 	tc := NewTorClient()
-	defer tc.Close()
 	resp, err := tc.Fetch(*target, "")
 	if err != nil {
 		log.Fatal(err)
@@ -57,7 +56,7 @@ func main() {
 	default:
 		reader = resp.Body
 	}
-	body, err := ioutil.ReadAll(reader)
+	body, err := io.ReadAll(reader)
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -75,6 +74,9 @@ type ArgonParams struct {
 }
 
 func (p ArgonParams) Check(n int) bool {
+	if p.difficulty == 0 {
+		return true
+	}
 	password := fmt.Sprintf("%s%d", p.prefix, n)
 	hash := argon2.IDKey([]byte(password), []byte(p.salt), p.iterations, p.memory, p.parallelism, p.keyLength)
 	for i, v := range hash[:(p.difficulty+1)/2] {
@@ -91,105 +93,309 @@ func (p ArgonParams) Check(n int) bool {
 	return false
 }
 
-type TorClient struct {
-	c      http.Client
-	torCtx *tor.Tor
+type TartarusParams struct {
+	salt       string
+	difficulty uint
 }
 
-func (tc *TorClient) Get(target, referer string) (*http.Response, error) {
-	req, err := http.NewRequest("GET", target, nil)
-	if err != nil {
-		return nil, err
+func (p TartarusParams) Check(n int) bool {
+	input := p.salt + strconv.Itoa(n)
+	hash := sha256.Sum256([]byte(input))
+	val := binary.BigEndian.Uint32(hash[:4])
+	return val < (1 << (32 - p.difficulty))
+}
+
+// extractAttr extracts the value of an HTML attribute from a string.
+// e.g. extractAttr(`<html data-foo="bar">`, "data-foo") returns "bar".
+func extractAttr(s, attr string) string {
+	key := attr + `="`
+	idx := strings.Index(s, key)
+	if idx == -1 {
+		return ""
 	}
+	start := idx + len(key)
+	end := strings.Index(s[start:], `"`)
+	if end == -1 {
+		return ""
+	}
+	return s[start : start+end]
+}
+
+type TorClient struct {
+	c http.Client
+}
+
+func setHeaders(req *http.Request, referer string) {
 	if referer != "" {
 		req.Header.Set("Referer", referer)
 	}
-
 	req.Header.Set("User-Agent", *ua)
 	req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8")
 	req.Header.Set("Accept-Encoding", "gzip, deflate, br")
 	req.Header.Set("Accept-Language", "en-US,en;q=0.5")
 	req.Header.Set("Upgrade-Insecure-Requests", "1")
+}
+
+func (tc *TorClient) Get(target, referer string) (*http.Response, error) {
+	req, err := http.NewRequest("GET", target, nil)
+	if err != nil {
+		return nil, err
+	}
+	setHeaders(req, referer)
 	return tc.c.Do(req)
 }
 
-func (tc *TorClient) PostForm(target string, data url.Values) (*http.Response, error) {
+func (tc *TorClient) PostForm(target, referer string, data url.Values) (*http.Response, error) {
 	req, err := http.NewRequest("POST", target, strings.NewReader(data.Encode()))
 	if err != nil {
 		return nil, err
 	}
-	req.Header.Set("Referer", target)
-
-	req.Header.Set("User-Agent", *ua)
+	setHeaders(req, referer)
 	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
-	req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8")
-	req.Header.Set("Accept-Encoding", "gzip, deflate, br")
-	req.Header.Set("Accept-Language", "en-US,en;q=0.5")
-	req.Header.Set("Upgrade-Insecure-Requests", "1")
 	return tc.c.Do(req)
 }
 
-func (tc *TorClient) Close() {
-	tc.torCtx.Close()
+// utlsTransport is an http.RoundTripper that dials TLS with utls
+// (for browser-like fingerprints) and dispatches to HTTP/2 or HTTP/1.1
+// based on the ALPN-negotiated protocol.
+type utlsTransport struct {
+	dialTLS func(ctx context.Context, network, addr string) (net.Conn, error)
+
+	mu      sync.Mutex
+	h2Conns map[string]*http2.ClientConn
 }
 
-func NewTorClient() *TorClient {
-	ctx := context.Background()
-	torCtx, err := tor.Start(
-		ctx,
-		&tor.StartConf{ProcessCreator: libtor.Creator},
-	)
-	if err != nil {
-		log.Fatalf("Failed to create Tor Context = %v\n", err)
+func (t *utlsTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+	addr := req.URL.Hostname()
+	port := req.URL.Port()
+	if port == "" {
+		port = "443"
 	}
-	var dialer *tor.Dialer
-	for retry := 0; retry < 3; retry++ {
-		timeoutCtx, done := context.WithTimeout(ctx, 15*time.Second)
-		dialer, err = torCtx.Dialer(timeoutCtx, nil)
-		done()
+	hostPort := net.JoinHostPort(addr, port)
+
+	// Try reusing a cached HTTP/2 connection.
+	t.mu.Lock()
+	cc := t.h2Conns[hostPort]
+	t.mu.Unlock()
+	if cc != nil {
+		//log.Printf("transport: reusing h2 conn for %s %s", req.Method, req.URL)
+		resp, err := cc.RoundTrip(req)
 		if err == nil {
-			break
+			return resp, nil
+		}
+		//log.Printf("transport: cached h2 conn failed: %v, dialing new", err)
+		t.mu.Lock()
+		delete(t.h2Conns, hostPort)
+		t.mu.Unlock()
+	} else {
+		//log.Printf("transport: no cached conn for %s %s, dialing new", req.Method, req.URL)
+	}
+
+	conn, err := t.dialTLS(req.Context(), "tcp", hostPort)
+	if err != nil {
+		return nil, err
+	}
+
+	// Check ALPN negotiated protocol.
+	alpn := ""
+	if uconn, ok := conn.(*utls.UConn); ok {
+		alpn = uconn.ConnectionState().NegotiatedProtocol
+	}
+
+	if alpn == "h2" {
+		cc, err := (&http2.Transport{}).NewClientConn(conn)
+		if err != nil {
+			conn.Close()
+			return nil, err
+		}
+		t.mu.Lock()
+		if t.h2Conns == nil {
+			t.h2Conns = make(map[string]*http2.ClientConn)
 		}
+		t.h2Conns[hostPort] = cc
+		t.mu.Unlock()
+		return cc.RoundTrip(req)
+	}
+
+	// HTTP/1.1 fallback.
+	if err := req.Write(conn); err != nil {
+		conn.Close()
+		return nil, err
+	}
+	resp, err := http.ReadResponse(bufio.NewReader(conn), req)
+	if err != nil {
+		conn.Close()
+		return nil, err
+	}
+	return resp, nil
+}
+
+func NewTorClient() *TorClient {
+	proxyURL, err := url.Parse(*socksAddr)
+	if err != nil {
+		log.Fatalf("Failed to parse proxy URL %q: %v\n", *socksAddr, err)
 	}
+	socksDialer, err := proxy.FromURL(proxyURL, proxy.Direct)
 	if err != nil {
-		log.Fatalf("Failed to create dialer for Tor Context - %v\n", err)
+		log.Fatalf("Failed to create SOCKS dialer: %v\n", err)
 	}
+
+	dialTLS := func(ctx context.Context, network, addr string) (net.Conn, error) {
+		host, _, err := net.SplitHostPort(addr)
+		if err != nil {
+			host = addr
+		}
+		// TCP dial through the SOCKS5 proxy.
+		rawConn, err := socksDialer.(proxy.ContextDialer).DialContext(ctx, network, addr)
+		if err != nil {
+			return nil, err
+		}
+		// TLS handshake with Firefox fingerprint.
+		cfg := &utls.Config{ServerName: host}
+		uConn := utls.UClient(rawConn, cfg, utls.HelloFirefox_Auto)
+		if err := uConn.HandshakeContext(ctx); err != nil {
+			rawConn.Close()
+			return nil, err
+		}
+		return uConn, nil
+	}
+
 	jar, _ := cookiejar.New(nil)
 	httpClient := http.Client{
-		Transport: &http.Transport{
-			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
-			Proxy:           http.ProxyFromEnvironment,
-			DialContext:     dialer.DialContext,
+		Transport: &utlsTransport{dialTLS: dialTLS},
+		Jar:       jar,
+		CheckRedirect: func(req *http.Request, via []*http.Request) error {
+			// Don't follow redirects automatically; Fetch() handles them.
+			return http.ErrUseLastResponse
 		},
-		Jar: jar,
-	}
-	return &TorClient{
-		c:      httpClient,
-		torCtx: torCtx,
 	}
+	return &TorClient{c: httpClient}
 }
 
 func (tc *TorClient) Fetch(target, referer string) (*http.Response, error) {
-	resp, err := tc.Get(target, referer)
+	currentURL := target
+	currentReferer := referer
+
+	for range 10 { // max redirect/challenge hops
+		resp, err := tc.Get(currentURL, currentReferer)
+		if err != nil {
+			return nil, err
+		}
+
+		// Follow redirects manually (we disabled auto-follow).
+		if loc := resp.Header.Get("Location"); loc != "" &&
+			(resp.StatusCode >= 300 && resp.StatusCode < 400) {
+			resp.Body.Close()
+			resolved, err := resp.Request.URL.Parse(loc)
+			if err != nil {
+				return nil, fmt.Errorf("bad redirect Location %q: %w", loc, err)
+			}
+			//log.Printf("Following redirect: %s -> %s", currentURL, resolved)
+			currentReferer = currentURL
+			currentURL = resolved.String()
+			continue
+		}
+
+		// Not a challenge — return directly.
+		if resp.StatusCode != http.StatusForbidden && resp.StatusCode != http.StatusNonAuthoritativeInfo {
+			return resp, nil
+		}
+
+		// Read the challenge body.
+		bodyBytes, err := io.ReadAll(resp.Body)
+		resp.Body.Close()
+		if err != nil {
+			return nil, err
+		}
+		body := string(bodyBytes)
+		requestURL := resp.Request.URL
+
+		if strings.Contains(body, "data-ttrs-challenge") {
+			challengeResp, err := tc.solveTartarus(requestURL, body)
+			if err != nil {
+				return nil, err
+			}
+			// solveTartarus returns the re-GET response; loop to
+			// handle further redirects or challenges on the new domain.
+			if loc := challengeResp.Header.Get("Location"); loc != "" &&
+				(challengeResp.StatusCode >= 300 && challengeResp.StatusCode < 400) {
+				challengeResp.Body.Close()
+				resolved, err := requestURL.Parse(loc)
+				if err != nil {
+					return nil, fmt.Errorf("bad redirect Location %q: %w", loc, err)
+				}
+				//log.Printf("Following redirect after challenge: %s -> %s", requestURL, resolved)
+				currentReferer = requestURL.String()
+				currentURL = resolved.String()
+				continue
+			}
+			return challengeResp, nil
+		}
+		return tc.solveBasedFlare(requestURL, body)
+	}
+	return nil, fmt.Errorf("too many redirects/challenges")
+}
+
+func (tc *TorClient) solveTartarus(requestURL *url.URL, body string) (*http.Response, error) {
+	salt := extractAttr(body, "data-ttrs-challenge")
+	diffStr := extractAttr(body, "data-ttrs-difficulty")
+	difficulty, err := strconv.Atoi(diffStr)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("parsing tartarus difficulty: %w", err)
 	}
 
-	// Check whether we were allowed direct access.
-	if resp.StatusCode != http.StatusForbidden {
-		// If so (eg due to passing captcha earlier), then return
-		// the http.Response for caller to do what it will.
-		return resp, nil
+	p := TartarusParams{salt: salt, difficulty: uint(difficulty)}
+
+	// Brute-force SHA256 PoW from nonce=0.
+	var nonce int
+	for n := 0; ; n++ {
+		if p.Check(n) {
+			nonce = n
+			break
+		}
 	}
 
-	// Otherwise, do the captcha dance.
-	defer resp.Body.Close()
+	// POST the solution to /.ttrs/challenge as an XHR.
+	challengeURL := fmt.Sprintf("%s://%s/.ttrs/challenge", requestURL.Scheme, requestURL.Host)
+	values := url.Values{}
+	values.Set("salt", salt)
+	values.Set("nonce", strconv.Itoa(nonce))
+	//log.Printf("Tartarus: salt=%s difficulty=%d nonce=%d", salt, difficulty, nonce)
+	req, err := http.NewRequest("POST", challengeURL, strings.NewReader(values.Encode()))
+	if err != nil {
+		return nil, fmt.Errorf("building tartarus POST: %w", err)
+	}
+	req.Header.Set("Referer", requestURL.String())
+	req.Header.Set("User-Agent", *ua)
+	req.Header.Set("Accept", "application/json")
+	req.Header.Set("Accept-Language", "en-US,en;q=0.5")
+	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+	postResp, err := tc.c.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("posting tartarus solution: %w", err)
+	}
+	postBody, _ := io.ReadAll(postResp.Body)
+	postResp.Body.Close()
+	//log.Printf("Tartarus POST: status=%d body=%s", postResp.StatusCode, postBody)
+	//log.Printf("Tartarus POST: Set-Cookie=%v", postResp.Header["Set-Cookie"])
+	if postResp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("tartarus challenge POST returned %d", postResp.StatusCode)
+	}
 
+	// Check what cookies the jar has for this URL.
+	if tc.c.Jar != nil {
+		cookies := tc.c.Jar.Cookies(requestURL)
+		//log.Printf("Tartarus: cookies for %s: %v", requestURL, cookies)
+	}
+
+	// Re-GET the original target (cookie jar preserves ttrs_clearance).
+	return tc.Get(requestURL.String(), requestURL.String())
+}
+
+func (tc *TorClient) solveBasedFlare(requestURL *url.URL, body string) (*http.Response, error) {
 	var p ArgonParams
 	var pow string
-	scanner := bufio.NewScanner(resp.Body)
-	for scanner.Scan() {
-		l := scanner.Text()
+	for _, l := range strings.Split(body, "\n") {
 		if !strings.HasPrefix(l, "\t<body data") {
 			continue
 		}
@@ -202,43 +408,36 @@ func (tc *TorClient) Fetch(target, referer string) (*http.Response, error) {
 			value := split[1][1 : len(split[1])-1]
 			switch key {
 			case "data-pow":
-				// data-pow="234a8b1a036dd6aee9c2745b31ffb1b8#2b8e80f38873205a65c14f9055b6ad0567b7690d8cd0fc73ac55882f32457045#fa725558ce6c1a9343265dd2abaddde7acfdd8af56c6e7269b3fddc4b6c29884"
 				pow = value
 				params := strings.Split(pow, "#")
 				p.salt = params[0]
 				p.prefix = params[1]
 			case "data-time":
-				// data-time="1"
 				iters, err := strconv.Atoi(value)
 				if err != nil {
-					log.Fatal(err)
+					return nil, fmt.Errorf("parsing basedflare time: %w", err)
 				}
 				p.iterations = uint32(iters)
 			case "data-diff":
-				// data-diff="24"
 				bits, err := strconv.Atoi(value)
 				if err != nil {
-					log.Fatal(err)
+					return nil, fmt.Errorf("parsing basedflare diff: %w", err)
 				}
 				p.difficulty = bits / 8
 			case "data-kb":
-				// data-kb="512"
 				mem, err := strconv.Atoi(value)
 				if err != nil {
-					log.Fatal(err)
+					return nil, fmt.Errorf("parsing basedflare kb: %w", err)
 				}
 				p.memory = uint32(mem)
 			default:
-				log.Fatalf("Unexpected key: %s", key)
+				return nil, fmt.Errorf("unexpected basedflare key: %s", key)
 			}
 		}
 		p.parallelism = uint8(*parallelism)
 		p.keyLength = uint32(*length)
 		break
 	}
-	if err := scanner.Err(); err != nil {
-		return nil, err
-	}
 
 	// Run the POW, single-threaded in case another circuit is running.
 	var result int
@@ -249,10 +448,9 @@ func (tc *TorClient) Fetch(target, referer string) (*http.Response, error) {
 		}
 	}
 
-	// Post the result back to the checker. This will yield a redirect to
-	// our true target.
+	// Post the result back to the checker.
 	values := url.Values{}
 	values.Set("pow_response", fmt.Sprintf("%s#%d", pow, result))
 	values.Set("submit", "submit")
-	return tc.PostForm(resp.Request.URL.String(), values)
+	return tc.PostForm(requestURL.String(), requestURL.String(), values)
 }

main_test.go

@@ -0,0 +1,158 @@
+package main
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/cookiejar"
+	"net/http/httptest"
+	"net/url"
+	"strconv"
+	"testing"
+)
+
+func TestTartarusCheck(t *testing.T) {
+	tests := []struct {
+		name       string
+		salt       string
+		difficulty uint
+		nonce      int
+		want       bool
+	}{
+		{"difficulty 1, nonce 0 fails", "testsalt", 1, 0, false},
+		{"difficulty 1, nonce 1 passes", "testsalt", 1, 1, true},
+		{"difficulty 8, nonce 0 fails", "testsalt", 8, 0, false},
+		{"difficulty 8, nonce 13 passes", "testsalt", 8, 13, true},
+		{"real urlscan vector, fails nonce 0", "a92a106fa4e8c2398ebcabecefebf28c_69853ed8", 16, 0, false},
+		{"real urlscan vector, passes known nonce", "a92a106fa4e8c2398ebcabecefebf28c_69853ed8", 16, 3026359506902472, true},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			p := TartarusParams{salt: tt.salt, difficulty: tt.difficulty}
+			if got := p.Check(tt.nonce); got != tt.want {
+				t.Errorf("TartarusParams{%q, %d}.Check(%d) = %v, want %v",
+					tt.salt, tt.difficulty, tt.nonce, got, tt.want)
+			}
+		})
+	}
+}
+
+func TestExtractAttr(t *testing.T) {
+	tests := []struct {
+		name string
+		html string
+		attr string
+		want string
+	}{
+		{"finds attribute", `<html data-ttrs-challenge="abc123" data-ttrs-difficulty="16">`, "data-ttrs-challenge", "abc123"},
+		{"finds second attribute", `<html data-ttrs-challenge="abc123" data-ttrs-difficulty="16">`, "data-ttrs-difficulty", "16"},
+		{"missing attribute", `<html data-foo="bar">`, "data-ttrs-challenge", ""},
+		{"empty value", `<html data-ttrs-challenge="">`, "data-ttrs-challenge", ""},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := extractAttr(tt.html, tt.attr); got != tt.want {
+				t.Errorf("extractAttr(%q, %q) = %q, want %q",
+					tt.html, tt.attr, got, tt.want)
+			}
+		})
+	}
+}
+
+func TestSolveTartarusFlow(t *testing.T) {
+	// Reproduce the real urlscan flow from
+	// https://urlscan.io/api/v1/result/019c307d-9f9d-72ac-a600-a6319d5708d7/
+	const (
+		wantSalt = "a92a106fa4e8c2398ebcabecefebf28c_69853ed8"
+		wantDiff = "16"
+	)
+
+	challengeHTML := fmt.Sprintf(
+		`<html data-ttrs-challenge="%s" data-ttrs-difficulty="%s"></html>`,
+		wantSalt, wantDiff)
+
+	var gotPost url.Values
+	var gotAccept, gotReferer, gotContentType string
+
+	ts := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch {
+		case r.Method == "GET" && r.URL.Path == "/":
+			// First GET returns 203 with challenge page.
+			w.WriteHeader(http.StatusNonAuthoritativeInfo)
+			fmt.Fprint(w, challengeHTML)
+		case r.Method == "POST" && r.URL.Path == "/.ttrs/challenge":
+			// Capture the POST for assertions.
+			body, _ := io.ReadAll(r.Body)
+			gotPost, _ = url.ParseQuery(string(body))
+			gotAccept = r.Header.Get("Accept")
+			gotReferer = r.Header.Get("Referer")
+			gotContentType = r.Header.Get("Content-Type")
+			// Set a cookie like the real server does.
+			http.SetCookie(w, &http.Cookie{Name: "ttrs_clearance", Value: "test"})
+			w.Header().Set("Content-Type", "application/json")
+			fmt.Fprint(w, `{"success":true}`)
+		case r.Method == "GET" && r.URL.Path == "/" && r.Header.Get("Cookie") != "":
+			// Re-GET after challenge solved.
+			w.WriteHeader(http.StatusOK)
+			fmt.Fprint(w, "<html>real page</html>")
+		default:
+			w.WriteHeader(http.StatusBadRequest)
+		}
+	}))
+	defer ts.Close()
+
+	// Use the test server's client (trusts its TLS cert) with a cookie jar.
+	jar, _ := cookiejar.New(nil)
+	testClient := ts.Client()
+	testClient.Jar = jar
+	tc := &TorClient{c: *testClient}
+	resp, err := tc.Fetch(ts.URL+"/", "")
+	if err != nil {
+		t.Fatalf("Fetch: %v", err)
+	}
+	resp.Body.Close()
+
+	// Verify POST fields match the real urlscan capture.
+	if got := gotPost.Get("salt"); got != wantSalt {
+		t.Errorf("POST salt = %q, want %q", got, wantSalt)
+	}
+	if gotNonce := gotPost.Get("nonce"); gotNonce == "" {
+		t.Error("POST nonce is empty")
+	} else {
+		n, err := strconv.Atoi(gotNonce)
+		if err != nil {
+			t.Errorf("POST nonce %q is not an integer: %v", gotNonce, err)
+		} else {
+			p := TartarusParams{salt: wantSalt, difficulty: 16}
+			if !p.Check(n) {
+				t.Errorf("POST nonce %d does not satisfy difficulty 16", n)
+			}
+		}
+	}
+	if gotAccept != "application/json" {
+		t.Errorf("POST Accept = %q, want %q", gotAccept, "application/json")
+	}
+	if gotContentType != "application/x-www-form-urlencoded" {
+		t.Errorf("POST Content-Type = %q, want %q", gotContentType, "application/x-www-form-urlencoded")
+	}
+	if gotReferer == "" {
+		t.Error("POST Referer is empty, want original page URL")
+	}
+}
+
+func TestArgonCheck(t *testing.T) {
+	// Use minimal parameters so the test runs quickly.
+	p := ArgonParams{
+		memory:      64,
+		iterations:  1,
+		parallelism: 1,
+		keyLength:   32,
+		difficulty:  0,
+		prefix:      "test",
+		salt:        "salt",
+	}
+	// difficulty=0 means 0 leading hex nibbles required, so any hash passes.
+	if !p.Check(0) {
+		t.Error("ArgonParams with difficulty=0 should accept any nonce")
+	}
+}