1use urlencoding::encode as urlencode;
2use urlparse::{Query, Url, parse_qs, urlparse, urlunparse};
3
4fn partial_urlencode(value: &str) -> String {
5    // The urlencode from the urlencoding library goes further than I like,
6    // and also URL encodes ASCII digits. Reverse that stuff.
7    urlencode(value)
8        .replace("%30", "0")
9        .replace("%31", "1")
10        .replace("%32", "2")
11        .replace("%33", "3")
12        .replace("%34", "4")
13        .replace("%35", "5")
14        .replace("%36", "6")
15        .replace("%37", "7")
16        .replace("%38", "8")
17        .replace("%39", "9")
18}
19
20/// Re-encode a query string for Rust
21fn encode_querystring(query: Query) -> Option<String> {
22    let mut query_components: Vec<String> = vec![];
23    for (key, value) in query {
24        for v in value.iter() {
25            query_components.push(format!("{}={}", key, partial_urlencode(v)));
26        }
27    }
28    if query_components.len() > 0 {
29        Some(query_components.join("&"))
30    } else {
31        None
32    }
33}
34
35/// Strip tracking junk and URL suffixes.
36pub fn clean_url(url: &str) -> String {
37    let mut parsed_url = urlparse(url);
38
39    // Always get the desktop version of Twitter URLs
40    if parsed_url.netloc == "mobile.twitter.com" {
41        parsed_url.netloc = String::from("twitter.com");
42    }
43
44    // Always get the non-mobile version of nytimes.com URLs
45    if parsed_url.netloc == "mobile.nytimes.com" {
46        parsed_url.netloc = String::from("nytimes.com");
47    }
48
49    // Remove any tracking junk from Amazon URLs so they're not a
50    // ridiculous length
51    if (parsed_url.netloc == "www.amazon.co.uk") || (parsed_url.netloc == "smile.amazon.co.uk") {
52        parsed_url.query = None;
53        let mut new_path = String::from("");
54        for component in parsed_url.path.split("ref=") {
55            new_path = String::from(component);
56            break;
57        }
58        parsed_url.path = new_path;
59    }
60
61    // Strip tracking junk from Medium, Mashable and Buzzfeed
62    if (parsed_url.netloc == "medium.com")
63        || (parsed_url.netloc == "www.buzzfeed.com")
64        || (parsed_url.netloc == "mashable.com")
65    {
66        parsed_url.fragment = None;
67    }
68
69    // Strip tracking junk from TikTok URLs
70    if parsed_url.netloc == "www.tiktok.com" {
71        parsed_url.query = None;
72    }
73
74    // Remove '#notes' from Tumblr URLs
75    if parsed_url.netloc.ends_with("tumblr.com") {
76        remove_query_params(
77            &mut parsed_url,
78            &["_branch_match_id", "_branch_referrer", "source"],
79        );
80
81        parsed_url.fragment = match parsed_url.fragment {
82            Some(fragment) => {
83                if fragment == "notes" {
84                    None
85                } else {
86                    Some(fragment)
87                }
88            }
89            None => None,
90        };
91    }
92
93    // Remove &feature=youtu.be from YouTube URLs
94    if parsed_url.netloc.ends_with("youtube.com") {
95        remove_query_params(&mut parsed_url, &["app", "feature"]);
96    }
97
98    // Remove any tracking parameters which:
99    //
100    //    * Come from UTM (Urchin Tracking Module)
101    //    * Come from Cloudflare tracking
102    //    * Come from HubSpot
103    //
104    parsed_url.query = match parsed_url.query {
105        Some(qs) => {
106            let mut query = parse_qs(&qs);
107            let ignore_keys: Vec<_> = query
108                .keys()
109                .filter(|key| {
110                    key.starts_with("utm_") || key.starts_with("__cf") || key.starts_with("hsa")
111                })
112                .map(|k| k.clone())
113                .collect();
114            for key in ignore_keys {
115                query.remove(&key);
116            }
117            encode_querystring(query)
118        }
119        None => None,
120    };
121
122    // Tidy up the query and anchor links in modules on docs.python.org
123    if parsed_url.netloc == "docs.python.org" {
124        // If this is a module page, scrap any module- fragment.
125        parsed_url.fragment = match parsed_url.fragment {
126            Some(fragment) => {
127                if fragment.starts_with("module-") {
128                    None
129                } else {
130                    Some(fragment)
131                }
132            }
133            None => None,
134        };
135
136        // Scrap the highlight
137        remove_query_param(&mut parsed_url, "highlight");
138    }
139
140    // If I'm on a GitHub pull request and looking at the files tab,
141    // link to the top of the pull request.
142    if parsed_url.netloc == "github.com" {
143        if parsed_url.path.ends_with("/files") {
144            parsed_url.path = parsed_url.path.replace("/files", "");
145        }
146    }
147
148    // Remove tracking query parameters from telegraph.co.uk URLs
149    // https://github.com/alexwlchan/safari.rs/issues/48
150    if parsed_url.netloc == "www.telegraph.co.uk" {
151        remove_query_param(&mut parsed_url, "WT.mc_id");
152    }
153
154    // Remove tracking parameters from Etsy URLs.
155    if parsed_url.netloc == "www.etsy.com" {
156        remove_query_params(
157            &mut parsed_url,
158            &[
159                "awc",
160                "click_key",
161                "click_sum",
162                "content_source",
163                "crt",
164                "dd",
165                "frs",
166                "logging_key",
167                "ls",
168                "pro",
169                "source",
170                "sts",
171            ],
172        );
173
174        parsed_url.query = match parsed_url.query {
175            Some(qs) => {
176                let mut query = parse_qs(&qs);
177                let utm_keys: Vec<_> = query
178                    .keys()
179                    .filter(|key| {
180                        key.starts_with("ga_")
181                            || key.starts_with("ref")
182                            || key.starts_with("organic_search_click")
183                    })
184                    .map(|k| k.clone())
185                    .collect();
186                for key in utm_keys {
187                    query.remove(&key);
188                }
189                encode_querystring(query)
190            }
191            None => None,
192        };
193    }
194
195    // Un-mobile-ify blogspot links.
196    if parsed_url.netloc.ends_with("blogspot.com") {
197        remove_query_param(&mut parsed_url, "m");
198    }
199
200    // Un-mobile-ify blogspot links.
201    if parsed_url.netloc == "www.redbubble.com" {
202        remove_query_params(&mut parsed_url, &["asc", "ref"]);
203    }
204
205    // Always remove the _ga Google Analytics tracking parameter.
206    remove_query_param(&mut parsed_url, "_ga");
207
208    // Always remove the Google Click ID tracking parameters.
209    remove_query_param(&mut parsed_url, "gclid");
210
211    // Remove tracking parameters from stacks.wellcomecollection.org URLs,
212    // which are from Medium.
213    if parsed_url.netloc == "stacks.wellcomecollection.org" {
214        remove_query_param(&mut parsed_url, "source");
215    }
216
217    // Remove tracking parameters from Wordery URLs.
218    if parsed_url.netloc == "wordery.com" {
219        remove_query_param(&mut parsed_url, "cTrk");
220    }
221
222    // Remove sharing/referral parameter from Twitter URLs.
223    if parsed_url.netloc == "twitter.com" {
224        remove_query_params(&mut parsed_url, &["cxt", "ref_src", "ref_url", "s"]);
225    }
226
227    // Remove the scheme/host from localhost:5757 links, which is the
228    // local instance of my website. I just want the path/query.
229    if parsed_url.netloc == "localhost:5757" {
230        parsed_url.netloc = "".to_string();
231        parsed_url.scheme = "".to_string();
232        return urlunparse(parsed_url).replace("://", "");
233    }
234
235    // Convert links to questions on Stack Overflow to insert my sharing
236    // link for the Announcer badge.
237    // This code is autogenerated by the `se_referral_autogen.py` script.
238    fix_se_referral(&mut parsed_url, "academia.stackexchange.com", "7658");
239    fix_se_referral(&mut parsed_url, "anime.stackexchange.com", "7674");
240    fix_se_referral(&mut parsed_url, "apple.stackexchange.com", "14295");
241    fix_se_referral(&mut parsed_url, "area51.stackexchange.com", "47881");
242    fix_se_referral(&mut parsed_url, "askubuntu.com", "265738");
243    fix_se_referral(&mut parsed_url, "aviation.stackexchange.com", "1372");
244    fix_se_referral(&mut parsed_url, "bicycles.stackexchange.com", "17362");
245    fix_se_referral(&mut parsed_url, "biology.stackexchange.com", "16115");
246    fix_se_referral(&mut parsed_url, "bricks.stackexchange.com", "445");
247    fix_se_referral(&mut parsed_url, "chemistry.stackexchange.com", "5443");
248    fix_se_referral(&mut parsed_url, "chess.stackexchange.com", "2564");
249    fix_se_referral(&mut parsed_url, "christianity.stackexchange.com", "10196");
250    fix_se_referral(&mut parsed_url, "codegolf.stackexchange.com", "13285");
251    fix_se_referral(&mut parsed_url, "codereview.stackexchange.com", "36525");
252    fix_se_referral(&mut parsed_url, "cogsci.stackexchange.com", "7973");
253    fix_se_referral(
254        &mut parsed_url,
255        "communitybuilding.stackexchange.com",
256        "372",
257    );
258    fix_se_referral(&mut parsed_url, "cooking.stackexchange.com", "25134");
259    fix_se_referral(&mut parsed_url, "crypto.stackexchange.com", "1185");
260    fix_se_referral(&mut parsed_url, "diy.stackexchange.com", "25263");
261    fix_se_referral(&mut parsed_url, "dsp.stackexchange.com", "8360");
262    fix_se_referral(&mut parsed_url, "earthscience.stackexchange.com", "352");
263    fix_se_referral(&mut parsed_url, "elementaryos.stackexchange.com", "3586");
264    fix_se_referral(&mut parsed_url, "english.stackexchange.com", "22597");
265    fix_se_referral(&mut parsed_url, "gaming.stackexchange.com", "73524");
266    fix_se_referral(&mut parsed_url, "gardening.stackexchange.com", "813");
267    fix_se_referral(&mut parsed_url, "gis.stackexchange.com", "26054");
268    fix_se_referral(&mut parsed_url, "graphicdesign.stackexchange.com", "19347");
269    fix_se_referral(&mut parsed_url, "law.stackexchange.com", "2488");
270    fix_se_referral(&mut parsed_url, "lifehacks.stackexchange.com", "11245");
271    fix_se_referral(&mut parsed_url, "linguistics.stackexchange.com", "2722");
272    fix_se_referral(&mut parsed_url, "math.stackexchange.com", "24160");
273    fix_se_referral(&mut parsed_url, "matheducators.stackexchange.com", "661");
274    fix_se_referral(&mut parsed_url, "mathematica.stackexchange.com", "5190");
275    fix_se_referral(&mut parsed_url, "mathoverflow.net", "38734");
276    fix_se_referral(&mut parsed_url, "mechanics.stackexchange.com", "14629");
277    fix_se_referral(&mut parsed_url, "meta.stackexchange.com", "226928");
278    fix_se_referral(&mut parsed_url, "money.stackexchange.com", "13518");
279    fix_se_referral(&mut parsed_url, "movies.stackexchange.com", "9285");
280    fix_se_referral(
281        &mut parsed_url,
282        "networkengineering.stackexchange.com",
283        "16668",
284    );
285    fix_se_referral(&mut parsed_url, "opensource.stackexchange.com", "2909");
286    fix_se_referral(&mut parsed_url, "parenting.stackexchange.com", "14304");
287    fix_se_referral(&mut parsed_url, "patents.stackexchange.com", "3882");
288    fix_se_referral(&mut parsed_url, "philosophy.stackexchange.com", "16838");
289    fix_se_referral(&mut parsed_url, "photo.stackexchange.com", "29700");
290    fix_se_referral(&mut parsed_url, "physics.stackexchange.com", "38614");
291    fix_se_referral(&mut parsed_url, "politics.stackexchange.com", "5590");
292    fix_se_referral(&mut parsed_url, "productivity.stackexchange.com", "1990");
293    fix_se_referral(&mut parsed_url, "puzzling.stackexchange.com", "4692");
294    fix_se_referral(&mut parsed_url, "rpg.stackexchange.com", "22823");
295    fix_se_referral(&mut parsed_url, "salesforce.stackexchange.com", "36215");
296    fix_se_referral(&mut parsed_url, "scifi.stackexchange.com", "3567");
297    fix_se_referral(&mut parsed_url, "security.stackexchange.com", "9814");
298    fix_se_referral(&mut parsed_url, "serverfault.com", "206273");
299    fix_se_referral(&mut parsed_url, "skeptics.stackexchange.com", "5416");
300    fix_se_referral(
301        &mut parsed_url,
302        "softwareengineering.stackexchange.com",
303        "94977",
304    );
305    fix_se_referral(&mut parsed_url, "space.stackexchange.com", "1003");
306    fix_se_referral(&mut parsed_url, "sqa.stackexchange.com", "7301");
307    fix_se_referral(&mut parsed_url, "stackapps.com", "21515");
308    fix_se_referral(&mut parsed_url, "stackoverflow.com", "1558022");
309    fix_se_referral(&mut parsed_url, "stats.stackexchange.com", "32450");
310    fix_se_referral(&mut parsed_url, "superuser.com", "243137");
311    fix_se_referral(&mut parsed_url, "tex.stackexchange.com", "9668");
312    fix_se_referral(&mut parsed_url, "travel.stackexchange.com", "12415");
313    fix_se_referral(&mut parsed_url, "unix.stackexchange.com", "43183");
314    fix_se_referral(&mut parsed_url, "ux.stackexchange.com", "9976");
315    fix_se_referral(&mut parsed_url, "webapps.stackexchange.com", "45296");
316    fix_se_referral(&mut parsed_url, "webmasters.stackexchange.com", "35749");
317    fix_se_referral(&mut parsed_url, "workplace.stackexchange.com", "14106");
318    fix_se_referral(&mut parsed_url, "worldbuilding.stackexchange.com", "2575");
319    fix_se_referral(&mut parsed_url, "writers.stackexchange.com", "11018");
320
321    urlunparse(parsed_url)
322}
323
324/// Remove a query parameter from a URL.
325fn remove_query_param(parsed_url: &mut Url, query_param: &str) {
326    remove_query_params(parsed_url, &[query_param]);
327}
328
329/// Remove multiple query parameters from a URL.
330fn remove_query_params(parsed_url: &mut Url, query_params: &[&str]) {
331    parsed_url.query = match parsed_url.query {
332        Some(ref qs) => {
333            let mut query = parse_qs(qs);
334            for param in query_params {
335                query.remove(*param);
336            }
337            encode_querystring(query)
338        }
339        None => None,
340    };
341}
342
343/// Turn a URL into a Stack Exchange referral link.
344///
345/// - `parsed_url` - the `Url` structure returned by urlparse
346/// - `hostname` - hostname of the SE site in question
347/// - `user_id` - your user ID on the SE site
348///
349fn fix_se_referral(parsed_url: &mut Url, hostname: &str, user_id: &str) {
350    if parsed_url.netloc != hostname {
351        return;
352    }
353
354    // A question URL is of the form
355    //
356    //    http://stackoverflow.com/questions/:question_id/:question_title
357    //
358    if !parsed_url.path.starts_with("/questions") {
359        return;
360    }
361
362    // Take a copy of the original path to get around ownership rules
363    let original_path = parsed_url.path.to_owned();
364
365    let new_path = match original_path.split("/").nth(2) {
366        Some(path_component) => {
367            // Check it's a number
368            match path_component.parse::<i32>() {
369                Ok(q_id) => {
370                    // Check if there's an answer fragment
371                    match parsed_url.fragment.to_owned() {
372                        Some(fragment) => match fragment.parse::<i32>() {
373                            Ok(ans_id) => Some(format!("/a/{}/{}", ans_id, user_id)),
374                            Err(_) => None,
375                        },
376                        None => Some(format!("/q/{}/{}", q_id, user_id)),
377                    }
378                }
379                Err(_) => None,
380            }
381        }
382        None => None,
383    };
384
385    // If we got something interesting, update the URL.
386    match new_path {
387        Some(p) => {
388            parsed_url.path = p;
389            parsed_url.fragment = None;
390        }
391        None => (),
392    };
393}
394
395#[cfg(test)]
396mod clean_url_tests {
397    use crate::clean_url::clean_url;
398
399    macro_rules! clean_url_tests {
400        ($($name:ident: $value:expr,)*) => {
401            $(
402                #[test]
403                fn $name() {
404                    let (input, expected) = $value;
405                    assert_eq!(expected, clean_url(input));
406                }
407            )*
408        }
409    }
410
411    clean_url_tests! {
412        etsy_bear: (
413                "https://www.etsy.com/uk/listing/721435625/bear-cross-stitch-pattern-pdf-bear?ls=s&ga_order=most_relevant&ga_search_type=all&ga_view_type=gallery&ga_search_query=bear+cross+stitch+pattern&ref=sr_gallery-1-12&sts=1&dd=1&content_source=00ca7d91-b721-4f6f-8b8e-128c10da15f7%253ALT6e4e75eacc959cc45062f0ebc0d7ede920b198fe&organic_search_click=1&logging_key=00ca7d91-b721-4f6f-8b8e-128c10da15f7%3ALT6e4e75eacc959cc45062f0ebc0d7ede920b198fe",
414                "https://www.etsy.com/uk/listing/721435625/bear-cross-stitch-pattern-pdf-bear"
415        ),
416
417        etsy_readme: (
418            "https://www.etsy.com/uk/listing/689034321/pride-flag-planets-complete-set-cross?ls=s&ga_order=most_relevant&ga_search_type=all&ga_view_type=gallery&ga_search_query=mathysphere+pride+planets&ref=sr_gallery-1-3&dd=1&content_source=750fbda2-0f23-409a-b29d-9714dc4e0ba3%253ALTcadfd6ad1b51d9f2c43bf9fc61aa37ee3ee5cc68&organic_search_click=1&logging_key=750fbda2-0f23-409a-b29d-9714dc4e0ba3%3ALTcadfd6ad1b51d9f2c43bf9fc61aa37ee3ee5cc68",
419            "https://www.etsy.com/uk/listing/689034321/pride-flag-planets-complete-set-cross"
420        ),
421
422        self_localhost_homepage: (
423            "http://localhost:5757/",
424            "/"
425        ),
426
427        self_localhost_page: (
428            "http://localhost:5757/book-reviews/",
429            "/book-reviews/"
430        ),
431
432        twitter_mobile: (
433            "https://mobile.twitter.com/Breaking911/status/822589852596191235",
434            "https://twitter.com/Breaking911/status/822589852596191235"
435        ),
436
437        regular_twitter: (
438            "https://twitter.com/WholesomeMeme/status/846421658835537921",
439            "https://twitter.com/WholesomeMeme/status/846421658835537921"
440        ),
441
442        twitter_with_tracking_param: (
443            "https://twitter.com/b0rk/status/1627763836479479811?cxt=HHwWhoCxubil_ZYtAAAA",
444            "https://twitter.com/b0rk/status/1627763836479479811"
445        ),
446
447        amazon_product: (
448            "https://www.amazon.co.uk/dp/B01DFKBL68/ref=gw_aucc_comb_AB_clean_2?pf_rd_r=N1MF2PWADRHS55427ETG&pf_rd_p=c11a2c11-c670-46ff-87a1-c1eef4fad652",
449            "https://www.amazon.co.uk/dp/B01DFKBL68/"
450        ),
451
452        amazon_smile_product: (
453            "https://smile.amazon.co.uk/dp/B01DFKBL68/ref=gw_aucc_comb_AB_clean_2?pf_rd_r=N1MF2PWADRHS55427ETG&pf_rd_p=c11a2c11-c670-46ff-87a1-c1eef4fad652",
454            "https://smile.amazon.co.uk/dp/B01DFKBL68/"
455        ),
456
457        medium_with_tracking: (
458            "https://medium.com/@anildash/forget-why-its-time-to-get-to-work-c49ac5f0da20#.sjyskxdsz",
459            "https://medium.com/@anildash/forget-why-its-time-to-get-to-work-c49ac5f0da20"
460        ),
461
462        tumblr_with_notes: (
463            "http://azurelunatic.tumblr.com/post/155525051123/things-about-hufflepuffs-539#notes",
464            "http://azurelunatic.tumblr.com/post/155525051123/things-about-hufflepuffs-539"
465        ),
466
467        tumblr_with_tracking: (
468            "https://www.tumblr.com/tightleish/709085976781324289?_branch_referrer=H4sIAAAAAAAAAwXBAQqAIAwAwB8509Vmv6kQpyhFzvD53Ynq03eAQ42OdtbXXHcDzUm0xtwFyAbLa6CNePEOHQeYOGL6Cpbp6Qe%2F0rcpQAAAAA%3D%3D&source=share&_branch_match_id=1137383385327673172",
469            "https://www.tumblr.com/tightleish/709085976781324289"
470        ),
471
472        buzzfeed_with_tracking: (
473            "https://www.buzzfeed.com/katienotopoulos/the-end-of-apple-man#.biqmkzz8kn",
474            "https://www.buzzfeed.com/katienotopoulos/the-end-of-apple-man"
475        ),
476
477        mashable_with_tracking: (
478            "http://mashable.com/2016/03/21/apple-liam-recycling-robot/#b9y4lv3m4qqX",
479            "http://mashable.com/2016/03/21/apple-liam-recycling-robot/"
480        ),
481
482        youtube_plain_video: (
483            "https://www.youtube.com/watch?v=zB4I68XVPzQ",
484            "https://www.youtube.com/watch?v=zB4I68XVPzQ",
485        ),
486
487        youtube_with_timestamp: (
488            "https://www.youtube.com/watch?v=D68cUzqcTrg&feature=youtu.be#t=1m",
489            "https://www.youtube.com/watch?v=D68cUzqcTrg#t=1m",
490        ),
491
492        youtube_with_trailing_feature: (
493            "https://www.youtube.com/watch?v=PbJqNa0_Oz0&feature=youtu.be",
494            "https://www.youtube.com/watch?v=PbJqNa0_Oz0",
495        ),
496
497        youtube_with_leading_feature: (
498            "https://www.youtube.com/watch?feature=youtu.be&v=oPo4n9tBPsk",
499            "https://www.youtube.com/watch?v=oPo4n9tBPsk",
500        ),
501
502        youtube_with_only_feature: (
503            "https://www.youtube.com/watch?feature=youtu.be",
504            "https://www.youtube.com/watch",
505        ),
506
507        youtube_channel: (
508            "https://www.youtube.com/user/TheQIElves",
509            "https://www.youtube.com/user/TheQIElves"
510        ),
511
512        youtube_with_app: (
513            "https://www.youtube.com/watch?app=desktop&v=51HbzsFhh04",
514            "https://www.youtube.com/watch?v=51HbzsFhh04"
515        ),
516
517        single_utm_tracker: (
518            "https://example.com?utm_medium=social",
519            "https://example.com",
520        ),
521
522        multiple_utm_tracker: (
523            "https://example.com?utm_medium=social&utm_source=twitter",
524            "https://example.com",
525        ),
526
527        multiple_utm_tracker_with_others: (
528            "https://example.com?utm_medium=social&foo=bar&utm_source=twitter",
529            "https://example.com?foo=bar",
530        ),
531
532        url_with_spaces: (
533            "https://example.com?foo=bar%20baz",
534            "https://example.com?foo=bar%20baz"
535        ),
536
537        url_with_numerals: (
538            "https://example.com?foo=bar0baz",
539            "https://example.com?foo=bar0baz"
540        ),
541
542        python_docs_bare: (
543            "https://docs.python.org/3.5/library/subprocess.html",
544            "https://docs.python.org/3.5/library/subprocess.html"
545        ),
546
547        python_docs_with_anchor: (
548            "https://docs.python.org/3.5/library/subprocess.html#subprocess.run",
549            "https://docs.python.org/3.5/library/subprocess.html#subprocess.run"
550        ),
551
552        python_docs_with_highlight: (
553            "https://docs.python.org/3.5/library/subprocess.html?highlight=subprocess",
554            "https://docs.python.org/3.5/library/subprocess.html"
555        ),
556
557        python_docs_with_highlight_anchor: (
558            "https://docs.python.org/3.5/library/subprocess.html?highlight=subprocess#subprocess.run",
559            "https://docs.python.org/3.5/library/subprocess.html#subprocess.run"
560        ),
561
562        python_docs_with_highlight_module_anchor: (
563            "https://docs.python.org/3.5/library/subprocess.html?highlight=subprocess#module-subprocess",
564            "https://docs.python.org/3.5/library/subprocess.html"
565        ),
566
567        stack_overflow_non_question: (
568            "http://stackoverflow.com/questions/tagged/html+regex",
569            "http://stackoverflow.com/questions/tagged/html+regex"
570        ),
571
572        stack_overflow_question: (
573            "http://stackoverflow.com/questions/1732348/regex-match-open-tags-except-xhtml-self-contained-tags",
574            "http://stackoverflow.com/q/1732348/1558022"
575        ),
576
577        stack_overflow_answer: (
578            "http://stackoverflow.com/questions/82831/how-do-i-check-whether-a-file-exists-using-python#82852",
579            "http://stackoverflow.com/a/82852/1558022"
580        ),
581
582        sff_se_non_question: (
583            "https://scifi.stackexchange.com/review",
584            "https://scifi.stackexchange.com/review"
585        ),
586
587        sff_se_question: (
588            "https://scifi.stackexchange.com/questions/58980/how-did-lupin-forget-there-was-a-full-moon",
589            "https://scifi.stackexchange.com/q/58980/3567"
590        ),
591
592        sff_se_answer: (
593            "https://scifi.stackexchange.com/questions/39201/which-owls-did-fred-and-george-weasley-achieve/39218#39218",
594            "https://scifi.stackexchange.com/a/39218/3567"
595        ),
596
597        telegraph_bare: (
598            "http://www.telegraph.co.uk/news/2017/06/09/ruth-davidson-planning-scottish-tory-breakaway-challenges-theresa/",
599            "http://www.telegraph.co.uk/news/2017/06/09/ruth-davidson-planning-scottish-tory-breakaway-challenges-theresa/"
600        ),
601
602        strip_telegraph_tracking: (
603            "http://www.telegraph.co.uk/news/2017/06/09/ruth-davidson-planning-scottish-tory-breakaway-challenges-theresa/?WT.mc_id=tmg_share_tw",
604            "http://www.telegraph.co.uk/news/2017/06/09/ruth-davidson-planning-scottish-tory-breakaway-challenges-theresa/"
605        ),
606
607        se_comments: (
608            "https://stackoverflow.com/questions/406230/regular-expression-to-match-a-line-that-doesnt-contain-a-word#comment9209422_406230",
609            "https://stackoverflow.com/questions/406230/regular-expression-to-match-a-line-that-doesnt-contain-a-word#comment9209422_406230"
610        ),
611
612        ga_tracking_parameter: (
613            "https://example.org?_ga=1234",
614            "https://example.org"
615        ),
616
617        ga_tracking_parameter_and_others: (
618            "https://example.org?_ga=1234&foo=bar",
619            "https://example.org?foo=bar"
620        ),
621
622        nytimes_mobile_url: (
623            "https://mobile.nytimes.com/2017/02/24/style/modern-love-when-your-greatest-romance-is-friendship.html",
624            "https://nytimes.com/2017/02/24/style/modern-love-when-your-greatest-romance-is-friendship.html"
625        ),
626
627        nytimes_non_mobile_url: (
628            "https://nytimes.com/2017/02/24/style/modern-love-when-your-greatest-romance-is-friendship.html",
629            "https://nytimes.com/2017/02/24/style/modern-love-when-your-greatest-romance-is-friendship.html"
630        ),
631
632        github_pr: (
633            "https://github.com/wellcometrust/platform/pull/1892",
634            "https://github.com/wellcometrust/platform/pull/1892"
635        ),
636
637        github_pr_with_files: (
638            "https://github.com/wellcometrust/platform/pull/1892/files",
639            "https://github.com/wellcometrust/platform/pull/1892"
640        ),
641
642        etsy_link: (
643            "https://www.etsy.com/uk/listing/473409127/space-sampler-cross-stitch-pattern-pdf?ga_order=most_relevant&ref=sr_gallery-1-3&ga_view_type=gallery&organic_search_click=1&ga_search_type=all&ga_search_query=space%20sampler",
644            "https://www.etsy.com/uk/listing/473409127/space-sampler-cross-stitch-pattern-pdf"
645        ),
646
647        etsy_section_link: (
648            "https://www.etsy.com/uk/shop/Vitraaze?source=aw&section_id=17807678&awc=6091_1577546787_d9f1b918042bbd02fe406d61dae715e7",
649            "https://www.etsy.com/uk/shop/Vitraaze?section_id=17807678"
650        ),
651
652        etsy_pro_link: (
653            "https://www.etsy.com/uk/listing/704478659/gothic-cathedral-window-ornate-cross?pro=1",
654            "https://www.etsy.com/uk/listing/704478659/gothic-cathedral-window-ornate-cross"
655        ),
656
657        etsy_link_with_tracking: (
658            "https://www.etsy.com/uk/listing/731582894/personalised-song-lyric-or-poetry?frs=1",
659            "https://www.etsy.com/uk/listing/731582894/personalised-song-lyric-or-poetry",
660        ),
661
662        etsy_link_with_tmore_tracking: (
663            "https://www.etsy.com/uk/listing/1487193769/progress-goose-pdf-gay-pride-month-cross?click_sum=15f8aef5&click_key=d478a4a346e8318cc55887564ddfef8f5687fb60%3A1487193769&crt=1",
664            "https://www.etsy.com/uk/listing/1487193769/progress-goose-pdf-gay-pride-month-cross",
665        ),
666
667        blogspot: (
668            "https://example.blogspot.com/2020/01/01/my_first_post.html",
669            "https://example.blogspot.com/2020/01/01/my_first_post.html"
670        ),
671
672        blogspot_mobile_link: (
673            "https://example.blogspot.com/2020/01/01/my_first_post.html?m=1",
674            "https://example.blogspot.com/2020/01/01/my_first_post.html"
675        ),
676
677        redbubble_plain: (
678            "https://www.redbubble.com/people/m1sfire/works/42386806-pixel-pride-genderfluid",
679            "https://www.redbubble.com/people/m1sfire/works/42386806-pixel-pride-genderfluid"
680        ),
681
682        redbubble_referrer: (
683            "https://www.redbubble.com/people/m1sfire/works/42386806-pixel-pride-genderfluid?ref=recent-owner",
684            "https://www.redbubble.com/people/m1sfire/works/42386806-pixel-pride-genderfluid"
685        ),
686
687        redbubble_asc: (
688            "https://www.redbubble.com/people/m1sfire/works/42386806-pixel-pride-genderfluid?asc=u",
689            "https://www.redbubble.com/people/m1sfire/works/42386806-pixel-pride-genderfluid"
690        ),
691
692        stacks_medium_referrer: (
693            "https://stacks.wellcomecollection.org/a-sprinkling-of-azure-6cef6e150fb2?source=collection_home---4------0———————————",
694            "https://stacks.wellcomecollection.org/a-sprinkling-of-azure-6cef6e150fb2"
695        ),
696
697        wordery_with_tracking_param: (
698            "https://wordery.com/digging-for-words-angela-burke-kunkel-9781984892638?cTrk=MTgwMTkyNTkxfDVmYjgzYzU5YzQ4Mjc6MToxOjVmYjgzYzE4ZTdhNzQ4LjczMzU3NDQxOjVhMzBiMGYx",
699            "https://wordery.com/digging-for-words-angela-burke-kunkel-9781984892638"
700        ),
701
702        twitter_with_share_link: (
703            "https://twitter.com/ScotRail/status/1355928580895731713?s=20",
704            "https://twitter.com/ScotRail/status/1355928580895731713"
705        ),
706
707        twitter_with_ref: (
708            "https://twitter.com/thingskatedid/status/1386077306381242371?ref_url=https%3A%2F%2Fwww.benweintraub.com%2F&ref_src=twsrc%5Etfw%7Ctwcamp%5Etweetembed%7Ctwterm%5E1386077306381242371%7Ctwgr%5E13dd6945ce97e163ca6ca5d28b54fc562b161b2f%7Ctwcon%5Es1_",
709            "https://twitter.com/thingskatedid/status/1386077306381242371"
710        ),
711
712        twitter_with_cxt: (
713            "https://twitter.com/b0rk/status/1627763836479479811?cxt=HHwWhoCxubil_ZYtAAAA",
714            "https://twitter.com/b0rk/status/1627763836479479811"
715        ),
716
717        url_with_cloudflare_query_param: (
718            "https://example.org/page?__cf_chl_jschl_tk__=1",
719            "https://example.org/page"
720        ),
721
722        url_with_hubspot_query_param: (
723            "https://imgix.com/pricing?hsa_grp=131223639264&hsa_acc=8534109361&gclid=CjwKCAiA-8SdBhBGEiwAWdgtcHVTpIo3PdadyKLfI4Wv3CdUUhcRISS0QNoitcBtZeJxiCJ_ZKowvhoCg68QAvD_BwE&hsa_src=g&hsa_cam=13763501966&hsa_mt=b&hsa_net=adwords&hsa_ad=598338680789&hsa_tgt=kwd-340899810185&hsa_ver=3&hsa_kw=imgix%20pricing",
724            "https://imgix.com/pricing"
725        ),
726
727        url_with_google_click_id_query_param: (
728            "https://imgix.com/pricing?gclid=CjwKCAiA-8SdBhBGEiwAWdgtcHVTpIo3PdadyKLfI4Wv3CdUUhcRISS0QNoitcBtZeJxiCJ_ZKowvhoCg68QAvD_BwE",
729            "https://imgix.com/pricing"
730        ),
731
732        tiktok_with_tracking_junk: (
733            "https://www.tiktok.com/@example/video/1234567890?sec_user_id=ABCDEFGHIJ&u_code=dgfffl6mjl6be3&share_app_id=1233&timestamp=1631110682",
734            "https://www.tiktok.com/@example/video/1234567890"
735        ),
736    }
737}