(cache)extensions/LeanSERP-Studio-package/LeanSERP-Studio-URL-Mode/LeanSerpUrlNormalizer.cs at main · Fengsemul/extensions · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Security.Cryptography;
using System.Text;
using System.Text.Json;
using System.Text.RegularExpressions;

internal static class Program
{
    private const int MaxHostnameLength = 253;
    private const int MaxLabelLength = 63;
    private const int DefaultChunkSize = 500000;

    private sealed class Options
    {
        public readonly List<string> InputFiles = new List<string>();
        public string OutputDirectory = "";
        public string PublicSuffixList = "";
        public int ChunkSize = DefaultChunkSize;
        public bool KeepUnderscores;
        public bool RemoveCommonWwwLabels;
    }

    private sealed class PslRules
    {
        public readonly HashSet<string> Exact =
            new HashSet<string>(StringComparer.OrdinalIgnoreCase);

        public readonly HashSet<string> Wildcards =
            new HashSet<string>(StringComparer.OrdinalIgnoreCase);

        public readonly HashSet<string> Exceptions =
            new HashSet<string>(StringComparer.OrdinalIgnoreCase);

        public int Count =>
            Exact.Count + Wildcards.Count + Exceptions.Count;
    }

    private sealed class BuildMetadata
    {
        public string Format { get; set; } = "leanserp-compiled-label-package";
        public int Version { get; set; } = 1;
        public string CreatedAt { get; set; } = "";
        public double DurationSeconds { get; set; }
        public long InputLines { get; set; }
        public long AcceptedLines { get; set; }
        public long RejectedLines { get; set; }
        public long PublicSuffixOnlyLines { get; set; }
        public long UniqueLabels { get; set; }
        public long DuplicatesRemoved { get; set; }
        public bool KeepUnderscores { get; set; }
        public bool RemoveCommonWwwLabels { get; set; }
        public int PublicSuffixRuleCount { get; set; }
        public string LabelsSha256 { get; set; } = "";
        public string RejectionsSha256 { get; set; } = "";
        public List<SourceMetadata> Sources { get; set; } =
            new List<SourceMetadata>();
    }

    private sealed class SourceMetadata
    {
        public string Path { get; set; } = "";
        public long ByteLength { get; set; }
        public long Lines { get; set; }
        public long AcceptedLines { get; set; }
        public long RejectedLines { get; set; }
        public string Sha256 { get; set; } = "";
    }

    private readonly struct ParseResult
    {
        public ParseResult(string hostname, string compactLabel, string reason)
        {
            Hostname = hostname;
            CompactLabel = compactLabel;
            Reason = reason;
        }

        public string Hostname { get; }
        public string CompactLabel { get; }
        public string Reason { get; }
    }

    private static int Main(string[] args)
    {
        try
        {
            Options options = ParseOptions(args);
            ValidateOptions(options);

            DateTimeOffset startedAt = DateTimeOffset.UtcNow;
            Directory.CreateDirectory(options.OutputDirectory);

            string timestamp = DateTime.Now.ToString("yyyyMMdd-HHmmss");
            string buildDirectory = Path.Combine(
                options.OutputDirectory,
                "compiled-build-" + timestamp
            );
            string chunkDirectory = Path.Combine(buildDirectory, "chunks");

            Directory.CreateDirectory(buildDirectory);
            Directory.CreateDirectory(chunkDirectory);

            string labelsPath = Path.Combine(buildDirectory, "labels.txt");
            string rejectedPath = Path.Combine(buildDirectory, "rejected-lines.tsv");
            string pslOnlyPath = Path.Combine(
                buildDirectory,
                "public-suffix-only.tsv"
            );
            string metadataPath = Path.Combine(buildDirectory, "metadata.json");

            Console.WriteLine("Loading Public Suffix List...");
            PslRules psl = LoadPublicSuffixList(options.PublicSuffixList);
            Console.WriteLine(
                "Loaded {0:N0} Public Suffix List rules.",
                psl.Count
            );

            var chunk = new HashSet<string>(StringComparer.Ordinal);
            var chunkPaths = new List<string>();
            var sourceReports = new List<SourceMetadata>();
            var idn = new IdnMapping();

            long totalLines = 0;
            long totalAccepted = 0;
            long totalRejected = 0;
            long publicSuffixOnly = 0;
            long duplicateLabels = 0;
            int chunkNumber = 0;

            using (StreamWriter rejectedWriter = NewWriter(rejectedPath))
using (StreamWriter pslOnlyWriter = NewWriter(pslOnlyPath))
{
    rejectedWriter.WriteLine("reason\tsource\tline\toriginal");
    pslOnlyWriter.WriteLine("hostname\tpublicSuffix\tsource\tline");

    foreach (string inputPath in options.InputFiles)
            {
                Console.WriteLine("Reading: " + inputPath);

                long sourceLines = 0;
                long sourceAccepted = 0;
            foreach (
                StreamReader reader in readers
            )
            {
                reader.Dispose();
            }
        }

        return (
            uniqueLabels,
            duplicateLabels
        );
    }

    private static StreamWriter NewWriter(
        string path
    )
    {
        return new StreamWriter(
            path,
            false,
            new UTF8Encoding(false),
            1024 * 1024
        );
    }

    private static string EscapeField(
        string value
    )
    {
        return value
            .Replace('\t', ' ')
            .Replace('\r', ' ')
            .Replace('\n', ' ');
    }

    private static string ComputeSha256(
        string path
    )
    {
        using SHA256 sha256 =
            SHA256.Create();

        using FileStream stream =
            new FileStream(
                path,
                FileMode.Open,
                FileAccess.Read,
                FileShare.Read,
                1024 * 1024,
                FileOptions.SequentialScan
            );

        byte[] hash =
            sha256.ComputeHash(stream);

        var builder =
            new StringBuilder(
                hash.Length * 2
            );

        foreach (byte value in hash)
        {
            builder.Append(
                value.ToString(
                    "x2",
                    CultureInfo.InvariantCulture
                )
            );
        }

        return builder.ToString();
    }
}