1const fs = require('fs');
2const https = require('https');
3
4// Write text to a file in the `out` directory.
5//
6// This takes an `options` object with two parameters:
7//
8//    - `filename` -- the name of the file to write
9//    - `contents` -- the text to write to the file
10//
11function writeToFile(options) {
12  let filePath = `out/${options.filename}`;
13
14  fs.mkdir('out', { recursive: true }, (err) => {
15    if (err) {
16      console.error('Error creating `out` directory:', err);
17      process.exit(1);
18    }
19  });
20
21  fs.writeFile(filePath, options.contents, (err) => {
22    if (err) {
23      console.error('Error writing file:', err);
24      process.exit(1);
25    }
26  });
27}
28
29// Format a number of bytes as a human-readable string.
30//
31// Example: naturalsize(1234) ~> "1.21 kB"
32function naturalSize(byteCount) {
33  return `${(byteCount / 1024).toFixed(2)} kB`;
34}
35
36// Left-pad a string with spaces for consistent indentation.
37function leftPad(str, length) {
38  while (str.length < length) {
39    str = ' ' + str;
40  }
41
42  return str;
43}
44
45// Parse command-line arguments.
46//
47// The script takes one or two arguments:
48//
49//  * the URL to fetch (required)
50//  * a label for the downloaded files (optional)
51//
52const args = process.argv.slice(2);
53
54let url = '';
55let label = '';
56
57if (args.length === 0) {
58  console.error("Usage: measure.js URL [LABEL]");
59  process.exit(1);
60} else if (args.length === 1) {
61  url = args[0];
62  label = "export";
63} else if (args.length === 2) {
64  url = args[0];
65  label = args[1];
66} else {
67  console.error("Usage: measure.js URL [LABEL]");
68  process.exit(1);
69}
70
71// Actually fetch the URL, and save the HTML
72//
73// Note: I add a custom User-Agent because CloudFront seems to reject fetches that
74// come from Node's builtin HTTP library.
75const options = {
76  headers: {
77    'User-Agent': 'Mozilla/5.0 (Android 4.4; Mobile; rv:41.0) Gecko/41.0 Firefox/41.0',
78  }
79};
80
81https.get(url, options, (res) => {
82  let html = '';
83
84  res.on('data', (chunk) => {
85    html += chunk;
86  });
87
88  // We've got the whole HTML file.  Parse it, and save the results.
89  res.on('end', () => {
90    let htmlByteCount = Buffer.byteLength(html, 'utf8');
91    console.log(`HTML       = ${leftPad(naturalSize(htmlByteCount), 10)}`);
92
93    let nextData = html
94        .split('<script id="__NEXT_DATA__" type="application/json">')[1]
95        .split("</script>")[0];
96
97    let nextDataByteCount = Buffer.byteLength(nextData, 'utf8');
98    console.log(`NEXT_DATA = ${leftPad(naturalSize(nextDataByteCount), 10)} (${(nextDataByteCount / htmlByteCount * 100).toFixed(1)}%)`);
99
100    console.log();
101
102    writeToFile({ filename: `${label}.html`, contents: html });
103    console.log(`Saved HTML to out/${label}.html`);
104
105    writeToFile({ filename: `${label}.json`, contents: nextData });
106    console.log(`Saved JSON to out/${label}.json`);
107  });
108
109}).on('error', (err) => {
110  console.error('Error fetching the URL: ', err);
111  process.exit(1);
112});