(cache)extensions/LeanSERP-Studio-package/LeanSERP-Studio-URL-Mode/LeanSERP-URL-Normalizer.ps1 at main · Fengsemul/extensions · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
[CmdletBinding()]
param(
    [Parameter(Mandatory = $true)]
    [string[]]$InputFile,

    [Parameter(Mandatory = $false)]
    [string]$PublicSuffixList = "",

    [Parameter(Mandatory = $false)]
    [string]$ApprovedRulesDirectory = "",

    [Parameter(Mandatory = $false)]
    [string]$OutputDirectory =
        "$env:USERPROFILE\Downloads\LeanSERP-URL-Output",

    [Parameter(Mandatory = $false)]
    [ValidateRange(1000, 1000000)]
    [int]$ChunkSize = 250000,

    [Parameter(Mandatory = $false)]
    [switch]$KeepUnderscores,

    [Parameter(Mandatory = $false)]
    [switch]$RemoveCommonWwwLabels,

    [Parameter(Mandatory = $false)]
    [switch]$KeepTemporaryFiles,

    [Parameter(Mandatory = $false)]
    [ValidateSet(
        "None",
        "Normal",
        "Extended",
        "Regex"
    )]
    [string]$ReplacementMode = "None",

    [Parameter(Mandatory = $false)]
    [string]$FindText = "",

    [Parameter(Mandatory = $false)]
    [string]$ReplaceText = "",

    [Parameter(Mandatory = $false)]
    [switch]$CaseSensitive,

    [Parameter(Mandatory = $false)]
    [switch]$PreviewOnly,

    [Parameter(Mandatory = $false)]
    [ValidateRange(1, 1000)]
    [int]$PreviewLimit = 100,

    [Parameter(Mandatory = $false)]
    [string]$ProgressLogPath = ""
)

Set-StrictMode -Version Latest
$ErrorActionPreference = "Stop"

$scriptDirectory =
    Split-Path -Parent $MyInvocation.MyCommand.Path

if (
    [string]::IsNullOrWhiteSpace(
        $PublicSuffixList
    )
) {
    $PublicSuffixList =
        Join-Path `
            $scriptDirectory `
            "public_suffix_list.dat"
}

if (
    [string]::IsNullOrWhiteSpace(
        $ApprovedRulesDirectory
    )
) {
    $ApprovedRulesDirectory =
        Join-Path `
            $scriptDirectory `
            "Approved-Rules"
}

$MaximumHostnameLength = 253
$MaximumLabelLength = 63
$MaximumRejectedSamples = 100
$FormatName = "leanserp-url-package"
$FormatVersion = 1
$script:ProgressWriter = $null

function New-Utf8Writer {
    param(
        [Parameter(Mandatory = $true)]
        [string]$Path,

        [Parameter(Mandatory = $false)]
        [bool]$Append = $false
    )

    return [System.IO.StreamWriter]::new(
        $Path,
        $Append,
        [System.Text.UTF8Encoding]::new(
            $false
        ),
        1048576
    )
}

function Open-ProgressWriter {
    if (
        [string]::IsNullOrWhiteSpace(
            $ProgressLogPath
        )
    ) {
        return
    }

    $progressDirectory =
        Split-Path -Parent $ProgressLogPath

    if (
        -not [string]::IsNullOrWhiteSpace(
            $progressDirectory
        )
    ) {
        [void](
            New-Item `
                -ItemType Directory `
                -Path $progressDirectory `
                -Force
        )
    }

    $script:ProgressWriter =
        [System.IO.StreamWriter]::new(
            $ProgressLogPath,
            $true,
            [System.Text.UTF8Encoding]::new(
                $false
            ),
            4096
        )
            $rejectionReasons
        rejectionSamples =
            $rejectionSamples
        files =
            $filesMetadata
    }
    [System.IO.File]::WriteAllText(
        $metadataOutputPath,
        (
            $metadata |
            ConvertTo-Json -Depth 15
        ) +
        [Environment]::NewLine,
        [System.Text.UTF8Encoding]::new(
            $false
        )
    )
    if (-not $KeepTemporaryFiles) {
        Remove-Item `
            -LiteralPath $temporaryDirectory `
            -Recurse `
            -Force
    }
    Write-StudioProgress
    Write-StudioProgress `
        -Text "LeanSERP URL build completed."
    Write-StudioProgress `
        -Text (
            "Input lines: {0:N0}" -f
            $totalLines
        )
    Write-StudioProgress `
        -Text (
            "Accepted lines: {0:N0}" -f
            $acceptedLines
        )
    Write-StudioProgress `
        -Text (
            "Rejected lines: {0:N0}" -f
            $rejectedLines
        )
    Write-StudioProgress `
        -Text (
            "Unique labels: {0:N0}" -f
            $labelMerge.OutputCount
        )
    Write-StudioProgress `
        -Text (
            "Exact-host blocks: {0:N0}" -f
            $exactHostMerge.OutputCount
        )
    Write-StudioProgress `
        -Text (
            "Host-subdomain blocks: {0:N0}" -f
            $hostSubdomainMerge.OutputCount
        )
    Write-StudioProgress `
        -Text (
            "PSL label overrides: {0:N0}" -f
            $overrideMerge.OutputCount
        )
    Write-StudioProgress `
        -Text "Package:"
    Write-StudioProgress `
        -Text $buildDirectory
    Write-StudioProgress `
        -Text "Metadata:"
    Write-StudioProgress `
        -Text $metadataOutputPath
}
finally {
    Close-ProgressWriter
}
exit 0