Skip to content
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 8 additions & 6 deletions go.mod
Original file line number Diff line number Diff line change
@@ -1,14 +1,16 @@
module github.com/lizthgrey/tor-fetcher

go 1.21.0
go 1.25.0

require (
github.com/cretz/bine v0.2.0
github.com/ipsn/go-libtor v1.0.380
golang.org/x/crypto v0.25.0
github.com/refraction-networking/utls v1.8.2
golang.org/x/crypto v0.47.0
golang.org/x/net v0.49.0
)

require (
golang.org/x/net v0.21.0 // indirect
golang.org/x/sys v0.22.0 // indirect
github.com/andybalholm/brotli v1.0.6 // indirect
github.com/klauspost/compress v1.17.4 // indirect
golang.org/x/sys v0.40.0 // indirect
golang.org/x/text v0.33.0 // indirect
)
47 changes: 14 additions & 33 deletions go.sum
Original file line number Diff line number Diff line change
@@ -1,33 +1,14 @@
github.com/cretz/bine v0.1.0/go.mod h1:6PF6fWAvYtwjRGkAuDEJeWNOv3a2hUouSP/yRYXmvHw=
github.com/cretz/bine v0.2.0 h1:8GiDRGlTgz+o8H9DSnsl+5MeBK4HsExxgl6WgzOCuZo=
github.com/cretz/bine v0.2.0/go.mod h1:WU4o9QR9wWp8AVKtTM1XD5vUHkEqnf2vVSo6dBqbetI=
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/ipsn/go-libtor v1.0.380 h1:hCmALDBe3bPpgwMunonMLArrG41MxzpE91Bk8KQYnYM=
github.com/ipsn/go-libtor v1.0.380/go.mod h1:6rIeHU7irp8ZH8E/JqaEOKlD6s4vSSUh4ngHelhlSMw=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
golang.org/x/crypto v0.0.0-20190404164418-38d8ce5564a5/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE=
golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8=
golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30=
golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
github.com/andybalholm/brotli v1.0.6 h1:Yf9fFpf49Zrxb9NlQaluyE92/+X7UVHlhMNJN2sxfOI=
github.com/andybalholm/brotli v1.0.6/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4=
github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM=
github.com/refraction-networking/utls v1.8.2 h1:j4Q1gJj0xngdeH+Ox/qND11aEfhpgoEvV+S9iJ2IdQo=
github.com/refraction-networking/utls v1.8.2/go.mod h1:jkSOEkLqn+S/jtpEHPOsVv/4V4EVnelwbMQl4vCWXAM=
golang.org/x/crypto v0.47.0 h1:V6e3FRj+n4dbpw86FJ8Fv7XVOql7TEwpHapKoMJ/GO8=
golang.org/x/crypto v0.47.0/go.mod h1:ff3Y9VzzKbwSSEzWqJsJVBnWmRwRSHt/6Op5n9bQc4A=
golang.org/x/net v0.49.0 h1:eeHFmOGUTtaaPSGNmjBKpbng9MulQsJURQUAfUwY++o=
golang.org/x/net v0.49.0/go.mod h1:/ysNB2EvaqvesRkuLAyjI1ycPZlQHM3q01F02UY/MV8=
golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ=
golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
golang.org/x/text v0.33.0 h1:B3njUFyqtHDUI5jMn1YIr5B0IE2U0qck04r6d4KPAxE=
golang.org/x/text v0.33.0/go.mod h1:LuMebE6+rBincTi9+xWTY8TztLzKHc/9C1uBCG27+q8=
356 changes: 277 additions & 79 deletions main.go
Original file line number Diff line number Diff line change
@@ -4,41 +4,40 @@ import (
"bufio"
"compress/gzip"
"context"
"crypto/tls"
"crypto/sha256"
"encoding/binary"
"flag"
"fmt"
"io"
"io/ioutil"
"log"
"net"
"net/http"
"net/http/cookiejar"
"net/url"
"os"
"strconv"
"strings"
"time"
"sync"

utls "github.com/refraction-networking/utls"
"golang.org/x/crypto/argon2"

"github.com/cretz/bine/tor"
"github.com/ipsn/go-libtor"
"golang.org/x/net/http2"
"golang.org/x/net/proxy"
)

var parallelism = flag.Int("p", 1, "Parallelism")
var length = flag.Int("l", 32, "Length")
var target = flag.String("target", "", "The URL to retrieve (required)")
var ua = flag.String("ua", "Mozilla/5.0 (Windows NT 10.0; rv:102.0) Gecko/20100101 Firefox/102.0", "Tor user agent by default")
var ua = flag.String("ua", "Mozilla/5.0 (Windows NT 10.0; rv:140.0) Gecko/20100101 Firefox/140.0", "Tor user agent by default")
var socksAddr = flag.String("proxy", "socks5://127.0.0.1:9050", "SOCKS5 proxy address for Tor")

func main() {
flag.Parse()
if *target == "" {
flag.Usage()
os.Exit(1)
}
http.DefaultTransport.(*http.Transport).TLSClientConfig = &tls.Config{InsecureSkipVerify: true}

tc := NewTorClient()
defer tc.Close()
resp, err := tc.Fetch(*target, "")
if err != nil {
log.Fatal(err)
@@ -57,7 +56,7 @@ func main() {
default:
reader = resp.Body
}
body, err := ioutil.ReadAll(reader)
body, err := io.ReadAll(reader)
if err != nil {
log.Fatal(err)
}
@@ -75,6 +74,9 @@ type ArgonParams struct {
}

func (p ArgonParams) Check(n int) bool {
if p.difficulty == 0 {
return true
}
password := fmt.Sprintf("%s%d", p.prefix, n)
hash := argon2.IDKey([]byte(password), []byte(p.salt), p.iterations, p.memory, p.parallelism, p.keyLength)
for i, v := range hash[:(p.difficulty+1)/2] {
@@ -91,105 +93,309 @@ func (p ArgonParams) Check(n int) bool {
return false
}

type TorClient struct {
c http.Client
torCtx *tor.Tor
type TartarusParams struct {
salt string
difficulty uint
}

func (tc *TorClient) Get(target, referer string) (*http.Response, error) {
req, err := http.NewRequest("GET", target, nil)
if err != nil {
return nil, err
func (p TartarusParams) Check(n int) bool {
input := p.salt + strconv.Itoa(n)
hash := sha256.Sum256([]byte(input))
val := binary.BigEndian.Uint32(hash[:4])
return val < (1 << (32 - p.difficulty))
}

// extractAttr extracts the value of an HTML attribute from a string.
// e.g. extractAttr(`<html data-foo="bar">`, "data-foo") returns "bar".
func extractAttr(s, attr string) string {
key := attr + `="`
idx := strings.Index(s, key)
if idx == -1 {
return ""
}
start := idx + len(key)
end := strings.Index(s[start:], `"`)
if end == -1 {
return ""
}
return s[start : start+end]
}

type TorClient struct {
c http.Client
}

func setHeaders(req *http.Request, referer string) {
if referer != "" {
req.Header.Set("Referer", referer)
}

req.Header.Set("User-Agent", *ua)
req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8")
req.Header.Set("Accept-Encoding", "gzip, deflate, br")
req.Header.Set("Accept-Language", "en-US,en;q=0.5")
req.Header.Set("Upgrade-Insecure-Requests", "1")
}

func (tc *TorClient) Get(target, referer string) (*http.Response, error) {
req, err := http.NewRequest("GET", target, nil)
if err != nil {
return nil, err
}
setHeaders(req, referer)
return tc.c.Do(req)
}

func (tc *TorClient) PostForm(target string, data url.Values) (*http.Response, error) {
func (tc *TorClient) PostForm(target, referer string, data url.Values) (*http.Response, error) {
req, err := http.NewRequest("POST", target, strings.NewReader(data.Encode()))
if err != nil {
return nil, err
}
req.Header.Set("Referer", target)

req.Header.Set("User-Agent", *ua)
setHeaders(req, referer)
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8")
req.Header.Set("Accept-Encoding", "gzip, deflate, br")
req.Header.Set("Accept-Language", "en-US,en;q=0.5")
req.Header.Set("Upgrade-Insecure-Requests", "1")
return tc.c.Do(req)
}

func (tc *TorClient) Close() {
tc.torCtx.Close()
// utlsTransport is an http.RoundTripper that dials TLS with utls
// (for browser-like fingerprints) and dispatches to HTTP/2 or HTTP/1.1
// based on the ALPN-negotiated protocol.
type utlsTransport struct {
dialTLS func(ctx context.Context, network, addr string) (net.Conn, error)

mu sync.Mutex
h2Conns map[string]*http2.ClientConn
}

func NewTorClient() *TorClient {
ctx := context.Background()
torCtx, err := tor.Start(
ctx,
&tor.StartConf{ProcessCreator: libtor.Creator},
)
if err != nil {
log.Fatalf("Failed to create Tor Context = %v\n", err)
func (t *utlsTransport) RoundTrip(req *http.Request) (*http.Response, error) {
addr := req.URL.Hostname()
port := req.URL.Port()
if port == "" {
port = "443"
}
var dialer *tor.Dialer
for retry := 0; retry < 3; retry++ {
timeoutCtx, done := context.WithTimeout(ctx, 15*time.Second)
dialer, err = torCtx.Dialer(timeoutCtx, nil)
done()
hostPort := net.JoinHostPort(addr, port)

// Try reusing a cached HTTP/2 connection.
t.mu.Lock()
cc := t.h2Conns[hostPort]
t.mu.Unlock()
if cc != nil {
//log.Printf("transport: reusing h2 conn for %s %s", req.Method, req.URL)
resp, err := cc.RoundTrip(req)
if err == nil {
break
return resp, nil
}
//log.Printf("transport: cached h2 conn failed: %v, dialing new", err)
t.mu.Lock()
delete(t.h2Conns, hostPort)
t.mu.Unlock()
} else {
//log.Printf("transport: no cached conn for %s %s, dialing new", req.Method, req.URL)
}

conn, err := t.dialTLS(req.Context(), "tcp", hostPort)
if err != nil {
return nil, err
}

// Check ALPN negotiated protocol.
alpn := ""
if uconn, ok := conn.(*utls.UConn); ok {
alpn = uconn.ConnectionState().NegotiatedProtocol
}

if alpn == "h2" {
cc, err := (&http2.Transport{}).NewClientConn(conn)
if err != nil {
conn.Close()
return nil, err
}
t.mu.Lock()
if t.h2Conns == nil {
t.h2Conns = make(map[string]*http2.ClientConn)
}
t.h2Conns[hostPort] = cc
t.mu.Unlock()
return cc.RoundTrip(req)
}

// HTTP/1.1 fallback.
if err := req.Write(conn); err != nil {
conn.Close()
return nil, err
}
resp, err := http.ReadResponse(bufio.NewReader(conn), req)
if err != nil {
conn.Close()
return nil, err
}
return resp, nil
}

func NewTorClient() *TorClient {
proxyURL, err := url.Parse(*socksAddr)
if err != nil {
log.Fatalf("Failed to parse proxy URL %q: %v\n", *socksAddr, err)
}
socksDialer, err := proxy.FromURL(proxyURL, proxy.Direct)
if err != nil {
log.Fatalf("Failed to create dialer for Tor Context - %v\n", err)
log.Fatalf("Failed to create SOCKS dialer: %v\n", err)
}

dialTLS := func(ctx context.Context, network, addr string) (net.Conn, error) {
host, _, err := net.SplitHostPort(addr)
if err != nil {
host = addr
}
// TCP dial through the SOCKS5 proxy.
rawConn, err := socksDialer.(proxy.ContextDialer).DialContext(ctx, network, addr)
if err != nil {
return nil, err
}
// TLS handshake with Firefox fingerprint.
cfg := &utls.Config{ServerName: host}
uConn := utls.UClient(rawConn, cfg, utls.HelloFirefox_Auto)
if err := uConn.HandshakeContext(ctx); err != nil {
rawConn.Close()
return nil, err
}
return uConn, nil
}

jar, _ := cookiejar.New(nil)
httpClient := http.Client{
Transport: &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
Proxy: http.ProxyFromEnvironment,
DialContext: dialer.DialContext,
Transport: &utlsTransport{dialTLS: dialTLS},
Jar: jar,
CheckRedirect: func(req *http.Request, via []*http.Request) error {
// Don't follow redirects automatically; Fetch() handles them.
return http.ErrUseLastResponse
},
Jar: jar,
}
return &TorClient{
c: httpClient,
torCtx: torCtx,
}
return &TorClient{c: httpClient}
}

func (tc *TorClient) Fetch(target, referer string) (*http.Response, error) {
resp, err := tc.Get(target, referer)
currentURL := target
currentReferer := referer

for range 10 { // max redirect/challenge hops
resp, err := tc.Get(currentURL, currentReferer)
if err != nil {
return nil, err
}

// Follow redirects manually (we disabled auto-follow).
if loc := resp.Header.Get("Location"); loc != "" &&
(resp.StatusCode >= 300 && resp.StatusCode < 400) {
resp.Body.Close()
resolved, err := resp.Request.URL.Parse(loc)
if err != nil {
return nil, fmt.Errorf("bad redirect Location %q: %w", loc, err)
}
//log.Printf("Following redirect: %s -> %s", currentURL, resolved)
currentReferer = currentURL
currentURL = resolved.String()
continue
}

// Not a challenge — return directly.
if resp.StatusCode != http.StatusForbidden && resp.StatusCode != http.StatusNonAuthoritativeInfo {
return resp, nil
}

// Read the challenge body.
bodyBytes, err := io.ReadAll(resp.Body)
resp.Body.Close()
if err != nil {
return nil, err
}
body := string(bodyBytes)
requestURL := resp.Request.URL

if strings.Contains(body, "data-ttrs-challenge") {
challengeResp, err := tc.solveTartarus(requestURL, body)
if err != nil {
return nil, err
}
// solveTartarus returns the re-GET response; loop to
// handle further redirects or challenges on the new domain.
if loc := challengeResp.Header.Get("Location"); loc != "" &&
(challengeResp.StatusCode >= 300 && challengeResp.StatusCode < 400) {
challengeResp.Body.Close()
resolved, err := requestURL.Parse(loc)
if err != nil {
return nil, fmt.Errorf("bad redirect Location %q: %w", loc, err)
}
//log.Printf("Following redirect after challenge: %s -> %s", requestURL, resolved)
currentReferer = requestURL.String()
currentURL = resolved.String()
continue
}
return challengeResp, nil
}
return tc.solveBasedFlare(requestURL, body)
}
return nil, fmt.Errorf("too many redirects/challenges")
}

func (tc *TorClient) solveTartarus(requestURL *url.URL, body string) (*http.Response, error) {
salt := extractAttr(body, "data-ttrs-challenge")
diffStr := extractAttr(body, "data-ttrs-difficulty")
difficulty, err := strconv.Atoi(diffStr)
if err != nil {
return nil, err
return nil, fmt.Errorf("parsing tartarus difficulty: %w", err)
}

// Check whether we were allowed direct access.
if resp.StatusCode != http.StatusForbidden {
// If so (eg due to passing captcha earlier), then return
// the http.Response for caller to do what it will.
return resp, nil
p := TartarusParams{salt: salt, difficulty: uint(difficulty)}

// Brute-force SHA256 PoW from nonce=0.
var nonce int
for n := 0; ; n++ {
if p.Check(n) {
nonce = n
break
}
}

// Otherwise, do the captcha dance.
defer resp.Body.Close()
// POST the solution to /.ttrs/challenge as an XHR.
challengeURL := fmt.Sprintf("%s://%s/.ttrs/challenge", requestURL.Scheme, requestURL.Host)
values := url.Values{}
values.Set("salt", salt)
values.Set("nonce", strconv.Itoa(nonce))
//log.Printf("Tartarus: salt=%s difficulty=%d nonce=%d", salt, difficulty, nonce)
req, err := http.NewRequest("POST", challengeURL, strings.NewReader(values.Encode()))
if err != nil {
return nil, fmt.Errorf("building tartarus POST: %w", err)
}
req.Header.Set("Referer", requestURL.String())
req.Header.Set("User-Agent", *ua)
req.Header.Set("Accept", "application/json")
req.Header.Set("Accept-Language", "en-US,en;q=0.5")
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
postResp, err := tc.c.Do(req)
if err != nil {
return nil, fmt.Errorf("posting tartarus solution: %w", err)
}
postBody, _ := io.ReadAll(postResp.Body)
postResp.Body.Close()
//log.Printf("Tartarus POST: status=%d body=%s", postResp.StatusCode, postBody)
//log.Printf("Tartarus POST: Set-Cookie=%v", postResp.Header["Set-Cookie"])
if postResp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("tartarus challenge POST returned %d", postResp.StatusCode)
}

// Check what cookies the jar has for this URL.
if tc.c.Jar != nil {
cookies := tc.c.Jar.Cookies(requestURL)
//log.Printf("Tartarus: cookies for %s: %v", requestURL, cookies)
}

// Re-GET the original target (cookie jar preserves ttrs_clearance).
return tc.Get(requestURL.String(), requestURL.String())
}

func (tc *TorClient) solveBasedFlare(requestURL *url.URL, body string) (*http.Response, error) {
var p ArgonParams
var pow string
scanner := bufio.NewScanner(resp.Body)
for scanner.Scan() {
l := scanner.Text()
for _, l := range strings.Split(body, "\n") {
if !strings.HasPrefix(l, "\t<body data") {
continue
}
@@ -202,43 +408,36 @@ func (tc *TorClient) Fetch(target, referer string) (*http.Response, error) {
value := split[1][1 : len(split[1])-1]
switch key {
case "data-pow":
// data-pow="234a8b1a036dd6aee9c2745b31ffb1b8#2b8e80f38873205a65c14f9055b6ad0567b7690d8cd0fc73ac55882f32457045#fa725558ce6c1a9343265dd2abaddde7acfdd8af56c6e7269b3fddc4b6c29884"
pow = value
params := strings.Split(pow, "#")
p.salt = params[0]
p.prefix = params[1]
case "data-time":
// data-time="1"
iters, err := strconv.Atoi(value)
if err != nil {
log.Fatal(err)
return nil, fmt.Errorf("parsing basedflare time: %w", err)
}
p.iterations = uint32(iters)
case "data-diff":
// data-diff="24"
bits, err := strconv.Atoi(value)
if err != nil {
log.Fatal(err)
return nil, fmt.Errorf("parsing basedflare diff: %w", err)
}
p.difficulty = bits / 8
case "data-kb":
// data-kb="512"
mem, err := strconv.Atoi(value)
if err != nil {
log.Fatal(err)
return nil, fmt.Errorf("parsing basedflare kb: %w", err)
}
p.memory = uint32(mem)
default:
log.Fatalf("Unexpected key: %s", key)
return nil, fmt.Errorf("unexpected basedflare key: %s", key)
}
}
p.parallelism = uint8(*parallelism)
p.keyLength = uint32(*length)
break
}
if err := scanner.Err(); err != nil {
return nil, err
}

// Run the POW, single-threaded in case another circuit is running.
var result int
@@ -249,10 +448,9 @@ func (tc *TorClient) Fetch(target, referer string) (*http.Response, error) {
}
}

// Post the result back to the checker. This will yield a redirect to
// our true target.
// Post the result back to the checker.
values := url.Values{}
values.Set("pow_response", fmt.Sprintf("%s#%d", pow, result))
values.Set("submit", "submit")
return tc.PostForm(resp.Request.URL.String(), values)
return tc.PostForm(requestURL.String(), requestURL.String(), values)
}
158 changes: 158 additions & 0 deletions main_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,158 @@
package main

import (
"fmt"
"io"
"net/http"
"net/http/cookiejar"
"net/http/httptest"
"net/url"
"strconv"
"testing"
)

func TestTartarusCheck(t *testing.T) {
tests := []struct {
name string
salt string
difficulty uint
nonce int
want bool
}{
{"difficulty 1, nonce 0 fails", "testsalt", 1, 0, false},
{"difficulty 1, nonce 1 passes", "testsalt", 1, 1, true},
{"difficulty 8, nonce 0 fails", "testsalt", 8, 0, false},
{"difficulty 8, nonce 13 passes", "testsalt", 8, 13, true},
{"real urlscan vector, fails nonce 0", "a92a106fa4e8c2398ebcabecefebf28c_69853ed8", 16, 0, false},
{"real urlscan vector, passes known nonce", "a92a106fa4e8c2398ebcabecefebf28c_69853ed8", 16, 3026359506902472, true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
p := TartarusParams{salt: tt.salt, difficulty: tt.difficulty}
if got := p.Check(tt.nonce); got != tt.want {
t.Errorf("TartarusParams{%q, %d}.Check(%d) = %v, want %v",
tt.salt, tt.difficulty, tt.nonce, got, tt.want)
}
})
}
}

func TestExtractAttr(t *testing.T) {
tests := []struct {
name string
html string
attr string
want string
}{
{"finds attribute", `<html data-ttrs-challenge="abc123" data-ttrs-difficulty="16">`, "data-ttrs-challenge", "abc123"},
{"finds second attribute", `<html data-ttrs-challenge="abc123" data-ttrs-difficulty="16">`, "data-ttrs-difficulty", "16"},
{"missing attribute", `<html data-foo="bar">`, "data-ttrs-challenge", ""},
{"empty value", `<html data-ttrs-challenge="">`, "data-ttrs-challenge", ""},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := extractAttr(tt.html, tt.attr); got != tt.want {
t.Errorf("extractAttr(%q, %q) = %q, want %q",
tt.html, tt.attr, got, tt.want)
}
})
}
}

func TestSolveTartarusFlow(t *testing.T) {
// Reproduce the real urlscan flow from
// https://urlscan.io/api/v1/result/019c307d-9f9d-72ac-a600-a6319d5708d7/
const (
wantSalt = "a92a106fa4e8c2398ebcabecefebf28c_69853ed8"
wantDiff = "16"
)

challengeHTML := fmt.Sprintf(
`<html data-ttrs-challenge="%s" data-ttrs-difficulty="%s"></html>`,
wantSalt, wantDiff)

var gotPost url.Values
var gotAccept, gotReferer, gotContentType string

ts := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch {
case r.Method == "GET" && r.URL.Path == "/":
// First GET returns 203 with challenge page.
w.WriteHeader(http.StatusNonAuthoritativeInfo)
fmt.Fprint(w, challengeHTML)
case r.Method == "POST" && r.URL.Path == "/.ttrs/challenge":
// Capture the POST for assertions.
body, _ := io.ReadAll(r.Body)
gotPost, _ = url.ParseQuery(string(body))
gotAccept = r.Header.Get("Accept")
gotReferer = r.Header.Get("Referer")
gotContentType = r.Header.Get("Content-Type")
// Set a cookie like the real server does.
http.SetCookie(w, &http.Cookie{Name: "ttrs_clearance", Value: "test"})
w.Header().Set("Content-Type", "application/json")
fmt.Fprint(w, `{"success":true}`)
case r.Method == "GET" && r.URL.Path == "/" && r.Header.Get("Cookie") != "":
// Re-GET after challenge solved.
w.WriteHeader(http.StatusOK)
fmt.Fprint(w, "<html>real page</html>")
default:
w.WriteHeader(http.StatusBadRequest)
}
}))
defer ts.Close()

// Use the test server's client (trusts its TLS cert) with a cookie jar.
jar, _ := cookiejar.New(nil)
testClient := ts.Client()
testClient.Jar = jar
tc := &TorClient{c: *testClient}
resp, err := tc.Fetch(ts.URL+"/", "")
if err != nil {
t.Fatalf("Fetch: %v", err)
}
resp.Body.Close()

// Verify POST fields match the real urlscan capture.
if got := gotPost.Get("salt"); got != wantSalt {
t.Errorf("POST salt = %q, want %q", got, wantSalt)
}
if gotNonce := gotPost.Get("nonce"); gotNonce == "" {
t.Error("POST nonce is empty")
} else {
n, err := strconv.Atoi(gotNonce)
if err != nil {
t.Errorf("POST nonce %q is not an integer: %v", gotNonce, err)
} else {
p := TartarusParams{salt: wantSalt, difficulty: 16}
if !p.Check(n) {
t.Errorf("POST nonce %d does not satisfy difficulty 16", n)
}
}
}
if gotAccept != "application/json" {
t.Errorf("POST Accept = %q, want %q", gotAccept, "application/json")
}
if gotContentType != "application/x-www-form-urlencoded" {
t.Errorf("POST Content-Type = %q, want %q", gotContentType, "application/x-www-form-urlencoded")
}
if gotReferer == "" {
t.Error("POST Referer is empty, want original page URL")
}
}

func TestArgonCheck(t *testing.T) {
// Use minimal parameters so the test runs quickly.
p := ArgonParams{
memory: 64,
iterations: 1,
parallelism: 1,
keyLength: 32,
difficulty: 0,
prefix: "test",
salt: "salt",
}
// difficulty=0 means 0 leading hex nibbles required, so any hash passes.
if !p.Check(0) {
t.Error("ArgonParams with difficulty=0 should accept any nonce")
}
}