Add cheerio to getRSNFormat and getHiscoresPage

2025-10-15 10:19:04 +00:00 · 2019-06-10 13:48:49 -04:00
parent f776e5e936
commit 2fd1c4f9b1
5 changed files with 213 additions and 63 deletions
--- a/README.md
+++ b/README.md
@@ -1,64 +1,82 @@
 # OSRS .json Hiscores
+
 **The Oldschool Runescape API wrapper that does more!**
+
 ## What it does
+
 The official hiscores API for Oldschool Runescape (OSRS) returns CSV.
 This wrapper converts it to json and provides extra information about the given player. By comparing player info it infers the player's game mode, as well as any previous modes (de-ironed or died as a hardcore ironman).

 An additional function is provided which screen-scrapes the OSRS leaderboards and returns the list of players as json
+
 ## Installation
+
 With npm:
+
 ```
 $ npm install osrs-json-hiscores
 ```
+
 ## How to use
+
 Install the package via npm and then import it into your project:
+
 ```javascript
-import hiscores from 'osrs-json-hiscores'
+import hiscores from 'osrs-json-hiscores';
 ```
+
 Once you import it you can call the functions asynchronously:
+
 ```javascript
-hiscores.getStats('Lynx Titan', 'full')
+hiscores
+  .getStats('Lynx Titan', 'full')
  .then(res => console.log(res))
-  .catch(err => console.error(err))
+  .catch(err => console.error(err));
 ```
+
 `getStats` will return a `full` player object with game mode by default, but it will also accept any of the following game modes:

-Game mode | Param
--- | :-:
-Regular | `main`
-Ironman | `iron`
-Hardcore Ironman | `hc`
-Ultimate Ironman | `ult`
-Deadman Mode | `dmm`
-Seasonal Deadman | `sdmm`
-DMM Tournament | `dmmt`
+| Game mode        | Param  |
+| ---------------- | :----: |
+| Regular          | `main` |
+| Ironman          | `iron` |
+| Hardcore Ironman |  `hc`  |
+| Ultimate Ironman | `ult`  |
+| Deadman Mode     | `dmm`  |
+| Seasonal Deadman | `sdmm` |
+| DMM Tournament   | `dmmt` |

 `getHiscores` requires a game mode and optionally a category and page:
+
 ```javascript
-hiscores.getHiscores('main', 'overall', 1)
+hiscores
+  .getHiscores('main', 'overall', 1)
  .then(res => console.log(res))
-  .catch(err => console.error(err))
+  .catch(err => console.error(err));
 ```
+
 The default values for category and page are `overall` and `1` respectively.
 Categories include all OSRS skills as well as:
+
 ### Clue Scrolls

-Type | Param
--- | :-:
-All | `allclues`
-Easy | `easyclues`
-Medium | `mediumclues`
-Hard | `hardclues`
-Elite | `eliteclues`
-Master | `masterclues`
+| Type     |      Param      |
+| -------- | :-------------: |
+| All      |   `allclues`    |
+| Beginner | `beginnerclues` |
+| Easy     |   `easyclues`   |
+| Medium   |  `mediumclues`  |
+| Hard     |   `hardclues`   |
+| Elite    |  `eliteclues`   |
+| Master   |  `masterclues`  |

 ### Minigames

-Minigame | Param
--- | :-:
-Bounty Hunter (Rogue) | `roguebh`
-Bounty Hunter (Hunter) | `hunterbh`
-Last Man Standing | `lms`
+| Minigame               |   Param    |
+| ---------------------- | :--------: |
+| Bounty Hunter (Rogue)  | `roguebh`  |
+| Bounty Hunter (Hunter) | `hunterbh` |
+| Last Man Standing      |   `lms`    |

 ## What you'll get

@@ -72,7 +90,7 @@ Last Man Standing | `lms`
  deironed: false,
  main: {
    stats: {
-      overall: {rank: 1, level: 2277, xp: 5,000,000,000},
+      overall: {rank: 1, level: 2277, xp: 4600000000},
      attack: {},
      defence: {},
      ...
@@ -91,9 +109,9 @@ Last Man Standing | `lms`

 ```javascript
 [
-  {mode: 'main', category: 'overall', rank: 1, rsn: 'Lynx Titan', level: 2277, xp: 5,000,000,000},
+  {mode: 'main', category: 'overall', rank: 1, rsn: 'Lynx Titan', level: 2277, xp: 4600000000},
  {},
  {},
  ...
 ]
-```
+```
--- a/hiscores.js
+++ b/hiscores.js
@@ -1,4 +1,5 @@
 const axios = require('axios');
+const cheerio = require('cheerio');

 const URLs = {
    main: 'http://services.runescape.com/m=hiscore_oldschool/',
@@ -40,6 +41,18 @@ const URLs = {
    ],
    clues: ['all', 'beginner', 'easy', 'medium', 'hard', 'elite', 'master'],
    bh: ['rouge', 'hunter'],
+    other: [
+      'hunterbh',
+      'roguebh',
+      'lms',
+      'allclues',
+      'beginnerclues',
+      'easyclues',
+      'mediumclues',
+      'hardclues',
+      'eliteclues',
+      'masterclues',
+    ],
  },
  validModes = ['full', 'main', 'iron', 'hc', 'ult', 'dmm', 'sdmm', 'dmmt'];

@@ -105,8 +118,11 @@ async function getPlayerStats(rsn, mode) {
        axios(URLs.ult + URLs.stats + encodeURIComponent(rsn)).catch(
          res => res
        ),
+        getRSNFormat(rsn),
      ]);

+      player.rsn = otherResponses.pop();
+
      for (let res of otherResponses) {
        responses.push(res);
      }
@@ -233,33 +249,33 @@ async function getHiscoresPage(mode, category, page) {
    '&page=' +
    page;

+  const players = [];
  const response = await axios(url);
-  let players = [],
-    element = document.createElement('html');
-  element.innerHTML = await response.text();
-  const playersHTML = element.querySelectorAll('.personal-hiscores__row');
+  const $ = cheerio.load(response.data);
+  const playersHTML = $('.personal-hiscores__row').toArray();

  for (let player of playersHTML) {
-    const attributes = player.querySelectorAll('td');
+    const attributes = player.children.filter(node => node.name === 'td');
+    console.log();
    let playerInfo = {
      mode: mode,
      category: category,
-      rank: attributes[0].innerHTML.slice(1, -1),
-      rsn: attributes[1].childNodes[1].innerHTML.replace(/\uFFFD/g, ' '),
+      rank: attributes[0].children[0].data.slice(1, -1),
+      rsn: attributes[1].children[1].children[0].data.replace(/\uFFFD/g, ' '),
    };

    hiscores.skills.includes(category.toLowerCase())
      ? (playerInfo = Object.assign(
          {
-            level: attributes[2].innerHTML.slice(1, -1),
-            xp: attributes[3].innerHTML.slice(1, -1),
+            level: attributes[2].children[0].data.slice(1, -1),
+            xp: attributes[3].children[0].data.slice(1, -1),
          },
          playerInfo
        ))
-      : (playerInfo.score = attributes[2].innerHTML.slice(1, -1));
+      : (playerInfo.score = attributes[2].children[0].data.slice(1, -1));

    if (mode === 'hc') {
-      playerInfo.dead = attributes[1].childElementCount > 1;
+      playerInfo.dead = attributes[1].children.length > 1;
    }

    players.push(playerInfo);
@@ -280,16 +296,17 @@ async function getHiscoresPage(mode, category, page) {
 * @returns {string}   The player's formatted username.
 */
 async function getRSNFormat(rsn) {
-  const url = URLs.main + URLs.scores + 'table=0&user=' + rsn;
+  const url =
+    URLs.main + URLs.scores + 'table=0&user=' + encodeURIComponent(rsn);

-  const response = await axios(url);
-  let element = document.createElement('html');
-  element.innerHTML = await response.text();
-  const cells = element.querySelectorAll('[style="color:#AA0022;"]');
-
-  if (cells.length >= 2) {
-    return cells[1].innerHTML.replace(/\uFFFD/g, ' ');
-  } else {
+  try {
+    const response = await axios(url);
+    const $ = cheerio.load(response.data);
+    return $('[style="color:#AA0022;"]')[1].children[0].data.replace(
+      /\uFFFD/g,
+      ' '
+    );
+  } catch {
    throw Error('Player not found');
  }
 }
--- a/hiscores.test.js
+++ b/hiscores.test.js
@@ -5,10 +5,11 @@ test('Get Lynx Titans stats', async done => {
    expect(data.main.stats.overall.level).toBe('2277');
    expect(data.main.stats.overall.rank).toBe('1');
    expect(data.main.stats.overall.xp).toBe('4600000000');
+    expect(data.rsn).toBe('Lynx Titan');
    done();
  };

-  hiscores.getStats('Lynx Titan', 'main').then(callback);
+  hiscores.getStats('lYnX tiTaN').then(callback);
 });

 test('Ensure correct lengths', async done => {
--- a/package-lock.json
+++ b/package-lock.json
@@ -1,6 +1,6 @@
 {
  "name": "osrs-json-hiscores",
-  "version": "1.1.0",
+  "version": "1.1.2",
  "lockfileVersion": 1,
  "requires": true,
  "dependencies": {
@@ -449,6 +449,11 @@
        "@types/istanbul-lib-report": "*"
      }
    },
+    "@types/node": {
+      "version": "12.0.7",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-12.0.7.tgz",
+      "integrity": "sha512-1YKeT4JitGgE4SOzyB9eMwO0nGVNkNEsm9qlIt1Lqm/tG2QEiSMTD4kS3aO6L+w5SClLVxALmIBESK6Mk5wX0A=="
+    },
    "@types/stack-utils": {
      "version": "1.0.1",
      "resolved": "https://registry.npmjs.org/@types/stack-utils/-/stack-utils-1.0.1.tgz",
@@ -751,6 +756,11 @@
        "tweetnacl": "^0.14.3"
      }
    },
+    "boolbase": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz",
+      "integrity": "sha1-aN/1++YMUes3cl6p4+0xDcwed24="
+    },
    "brace-expansion": {
      "version": "1.1.11",
      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
@@ -883,6 +893,29 @@
        "supports-color": "^5.3.0"
      }
    },
+    "cheerio": {
+      "version": "1.0.0-rc.3",
+      "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.0.0-rc.3.tgz",
+      "integrity": "sha512-0td5ijfUPuubwLUu0OBoe98gZj8C/AA+RW3v67GPlGOrvxWjZmBXiBCRU+I8VEiNyJzjth40POfHiz2RB3gImA==",
+      "requires": {
+        "css-select": "~1.2.0",
+        "dom-serializer": "~0.1.1",
+        "entities": "~1.1.1",
+        "htmlparser2": "^3.9.1",
+        "lodash": "^4.15.0",
+        "parse5": "^3.0.1"
+      },
+      "dependencies": {
+        "parse5": {
+          "version": "3.0.3",
+          "resolved": "https://registry.npmjs.org/parse5/-/parse5-3.0.3.tgz",
+          "integrity": "sha512-rgO9Zg5LLLkfJF9E6CCmXlSE4UVceloys8JrFqCcHloC3usd/kJCyPDwH2SOlzix2j3xaP9sUX3e8+kvkuleAA==",
+          "requires": {
+            "@types/node": "*"
+          }
+        }
+      }
+    },
    "ci-info": {
      "version": "2.0.0",
      "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-2.0.0.tgz",
@@ -1039,6 +1072,22 @@
        "which": "^1.2.9"
      }
    },
+    "css-select": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/css-select/-/css-select-1.2.0.tgz",
+      "integrity": "sha1-KzoRBTnFNV8c2NMUYj6HCxIeyFg=",
+      "requires": {
+        "boolbase": "~1.0.0",
+        "css-what": "2.1",
+        "domutils": "1.5.1",
+        "nth-check": "~1.0.1"
+      }
+    },
+    "css-what": {
+      "version": "2.1.3",
+      "resolved": "https://registry.npmjs.org/css-what/-/css-what-2.1.3.tgz",
+      "integrity": "sha512-a+EPoD+uZiNfh+5fxw2nO9QwFa6nJe2Or35fGY6Ipw1R3R4AGz1d1TEZrCegvw2YTmZ0jXirGYlzxxpYSHwpEg=="
+    },
    "cssom": {
      "version": "0.3.6",
      "resolved": "https://registry.npmjs.org/cssom/-/cssom-0.3.6.tgz",
@@ -1181,6 +1230,20 @@
      "integrity": "sha512-xLqpez+Zj9GKSnPWS0WZw1igGocZ+uua8+y+5dDNTT934N3QuY1sp2LkHzwiaYQGz60hMq0pjAshdeXm5VUOEw==",
      "dev": true
    },
+    "dom-serializer": {
+      "version": "0.1.1",
+      "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-0.1.1.tgz",
+      "integrity": "sha512-l0IU0pPzLWSHBcieZbpOKgkIn3ts3vAh7ZuFyXNwJxJXk/c4Gwj9xaTJwIDVQCXawWD0qb3IzMGH5rglQaO0XA==",
+      "requires": {
+        "domelementtype": "^1.3.0",
+        "entities": "^1.1.1"
+      }
+    },
+    "domelementtype": {
+      "version": "1.3.1",
+      "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-1.3.1.tgz",
+      "integrity": "sha512-BSKB+TSpMpFI/HOxCNr1O8aMOTZ8hT3pM3GQ0w/mWRmkhEDSFJkkyzz4XQsBV44BChwGkrDfMyjVD0eA2aFV3w=="
+    },
    "domexception": {
      "version": "1.0.1",
      "resolved": "https://registry.npmjs.org/domexception/-/domexception-1.0.1.tgz",
@@ -1190,6 +1253,23 @@
        "webidl-conversions": "^4.0.2"
      }
    },
+    "domhandler": {
+      "version": "2.4.2",
+      "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-2.4.2.tgz",
+      "integrity": "sha512-JiK04h0Ht5u/80fdLMCEmV4zkNh2BcoMFBmZ/91WtYZ8qVXSKjiw7fXMgFPnHcSZgOo3XdinHvmnDUeMf5R4wA==",
+      "requires": {
+        "domelementtype": "1"
+      }
+    },
+    "domutils": {
+      "version": "1.5.1",
+      "resolved": "https://registry.npmjs.org/domutils/-/domutils-1.5.1.tgz",
+      "integrity": "sha1-3NhIiib1Y9YQeeSMn3t+Mjc2gs8=",
+      "requires": {
+        "dom-serializer": "0",
+        "domelementtype": "1"
+      }
+    },
    "ecc-jsbn": {
      "version": "0.1.2",
      "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz",
@@ -1209,6 +1289,11 @@
        "once": "^1.4.0"
      }
    },
+    "entities": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/entities/-/entities-1.1.2.tgz",
+      "integrity": "sha512-f2LZMYl1Fzu7YSBKg+RoROelpOaNrcGmE9AZubeDfrCEia483oW4MI4VyFd5VNHIgQ/7qm1I0wUHK1eJnn2y2w=="
+    },
    "error-ex": {
      "version": "1.3.2",
      "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.2.tgz",
@@ -2267,6 +2352,31 @@
        "whatwg-encoding": "^1.0.1"
      }
    },
+    "htmlparser2": {
+      "version": "3.10.1",
+      "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-3.10.1.tgz",
+      "integrity": "sha512-IgieNijUMbkDovyoKObU1DUhm1iwNYE/fuifEoEHfd1oZKZDaONBSkal7Y01shxsM49R4XaMdGez3WnF9UfiCQ==",
+      "requires": {
+        "domelementtype": "^1.3.1",
+        "domhandler": "^2.3.0",
+        "domutils": "^1.5.1",
+        "entities": "^1.1.1",
+        "inherits": "^2.0.1",
+        "readable-stream": "^3.1.1"
+      },
+      "dependencies": {
+        "readable-stream": {
+          "version": "3.4.0",
+          "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.4.0.tgz",
+          "integrity": "sha512-jItXPLmrSR8jmTRmRWJXCnGJsfy85mB3Wd/uINMXA65yrnFo0cPClFIUWzo2najVNSl+mx7/4W8ttlLWJe99pQ==",
+          "requires": {
+            "inherits": "^2.0.3",
+            "string_decoder": "^1.1.1",
+            "util-deprecate": "^1.0.1"
+          }
+        }
+      }
+    },
    "http-signature": {
      "version": "1.2.0",
      "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz",
@@ -2316,8 +2426,7 @@
    "inherits": {
      "version": "2.0.3",
      "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz",
-      "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=",
-      "dev": true
+      "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4="
    },
    "invariant": {
      "version": "2.2.4",
@@ -3227,8 +3336,7 @@
    "lodash": {
      "version": "4.17.11",
      "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz",
-      "integrity": "sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg==",
-      "dev": true
+      "integrity": "sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg=="
    },
    "lodash.sortby": {
      "version": "4.7.0",
@@ -3515,6 +3623,14 @@
        "path-key": "^2.0.0"
      }
    },
+    "nth-check": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-1.0.2.tgz",
+      "integrity": "sha512-WeBOdju8SnzPN5vTUJYxYUxLeXpCaVP5i5e0LF8fg7WORF2Wd7wFX/pk0tYZk7s8T+J7VLy0Da6J1+wCT0AtHg==",
+      "requires": {
+        "boolbase": "~1.0.0"
+      }
+    },
    "number-is-nan": {
      "version": "1.0.1",
      "resolved": "https://registry.npmjs.org/number-is-nan/-/number-is-nan-1.0.1.tgz",
@@ -4091,8 +4207,7 @@
    "safe-buffer": {
      "version": "5.1.2",
      "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
-      "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==",
-      "dev": true
+      "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g=="
    },
    "safe-regex": {
      "version": "1.1.0",
@@ -4518,7 +4633,6 @@
      "version": "1.1.1",
      "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
      "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
-      "dev": true,
      "requires": {
        "safe-buffer": "~5.1.0"
      }
@@ -4796,8 +4910,7 @@
    "util-deprecate": {
      "version": "1.0.2",
      "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
-      "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=",
-      "dev": true
+      "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8="
    },
    "util.promisify": {
      "version": "1.0.0",
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
 {
  "name": "osrs-json-hiscores",
-  "version": "1.1.0",
+  "version": "1.1.3",
  "description": "The Oldschool Runescape API wrapper that does more!",
  "main": "hiscores.js",
  "scripts": {
@@ -27,7 +27,8 @@
  },
  "homepage": "https://github.com/maxswa/osrs-json-hiscores#readme",
  "dependencies": {
-    "axios": "^0.19.0"
+    "axios": "^0.19.0",
+    "cheerio": "^1.0.0-rc.3"
  },
  "devDependencies": {
    "jest": "^24.8.0"