Fixes #22 adding User-Agent request header to bypass Incapsula protection of OSRS hiscore pages

package.json

@@ -37,7 +37,8 @@
   "homepage": "https://github.com/maxswa/osrs-json-hiscores#readme",
   "dependencies": {
     "axios": "^0.21.1",
-    "jsdom": "^16.3.0"
+    "jsdom": "^16.3.0",
+    "useragent-generator": "^1.1.0"
   },
   "devDependencies": {
     "@types/jest": "^24.0.14",

src/hiscores.ts

@@ -1,4 +1,3 @@
-import axios from 'axios';
 import {
   Player,
   Activity,
@@ -26,6 +25,7 @@ import {
   numberFromElement,
   rsnFromElement,
   getActivityPageURL,
+  httpGet,
   BOSSES,
 } from './utils';
 import { JSDOM } from 'jsdom';
@@ -39,12 +39,12 @@ export async function getStats(rsn: string): Promise<Player> {
     throw Error('RSN must be between 1 and 12 characters');
   }
 
-  const mainRes = await axios(getStatsURL('main', rsn));
+  const mainRes = await httpGet(getStatsURL('main', rsn));
   if (mainRes.status === 200) {
     const otherResponses = await Promise.all([
-      axios(getStatsURL('ironman', rsn)).catch(err => err),
-      axios(getStatsURL('hardcore', rsn)).catch(err => err),
-      axios(getStatsURL('ultimate', rsn)).catch(err => err),
+      httpGet(getStatsURL('ironman', rsn)).catch(err => err),
+      httpGet(getStatsURL('hardcore', rsn)).catch(err => err),
+      httpGet(getStatsURL('ultimate', rsn)).catch(err => err),
       getRSNFormat(rsn).catch(() => undefined),
     ]);
 
@@ -120,7 +120,7 @@ export async function getStatsByGamemode(
   } else if (!GAMEMODES.includes(mode)) {
     throw Error('Invalid game mode');
   }
-  const response = await axios(getStatsURL(mode, rsn));
+  const response = await httpGet(getStatsURL(mode, rsn));
   if (response.status !== 200) {
     throw Error('Player not found');
   }
@@ -143,7 +143,7 @@ export async function getSkillPage(
   }
   const url = getSkillPageURL(mode, skill, page);
 
-  const response = await axios(url);
+  const response = await httpGet(url);
   const dom = new JSDOM(response.data);
   const playersHTML = dom.window.document.querySelectorAll(
     '.personal-hiscores__row'
@@ -183,7 +183,7 @@ export async function getActivityPage(
   }
   const url = getActivityPageURL(mode, activity, page);
 
-  const response = await axios(url);
+  const response = await httpGet(url);
   const dom = new JSDOM(response.data);
   const playersHTML = dom.window.document.querySelectorAll(
     '.personal-hiscores__row'
@@ -218,7 +218,7 @@ export async function getRSNFormat(rsn: string): Promise<string> {
 
   const url = getPlayerTableURL('main', rsn);
   try {
-    const response = await axios(url);
+    const response = await httpGet(url);
     const dom = new JSDOM(response.data);
     const spans = dom.window.document.querySelectorAll(
       'span[style="color:#AA0022;"]'

src/utils/helpers.ts

@@ -1,3 +1,4 @@
+import axios from 'axios';
 import { Gamemode, SkillName, ActivityName } from '../types';
 import {
   GAMEMODE_URL,
@@ -6,6 +7,7 @@ import {
   SKILLS,
   ACTIVITIES,
 } from './constants';
+const ua = require('useragent-generator');
 
 export const getStatsURL = (gamemode: Gamemode, rsn: string) =>
   `${GAMEMODE_URL[gamemode]}${STATS_URL}${encodeURIComponent(rsn)}`;
@@ -45,3 +47,12 @@ export const rsnFromElement = (el: Element | null) => {
   const { innerHTML } = el || {};
   return innerHTML?.replace(/\uFFFD/g, ' ') || '';
 };
+
+export const httpGet = (url: string) => {
+  return axios.get(url, {
+    headers: {
+      // without User-Agent header requests may be rejected by DDoS protection mechanism
+      'User-Agent': ua.firefox(80)
+    }
+  });
+};

yarn.lock

@@ -3528,6 +3528,11 @@ normalize-url@^4.1.0:
   resolved "https://registry.yarnpkg.com/normalize-url/-/normalize-url-4.5.0.tgz#453354087e6ca96957bd8f5baf753f5982142129"
   integrity sha512-2s47yzUxdexf1OhyRi4Em83iQk0aPvwTddtFz4hnSSw9dCEsLEGf6SwIO8ss/19S9iBb5sJaOuTvTGDeZI00BQ==
 
+normalize-version@^1.0.5:
+  version "1.0.5"
+  resolved "https://registry.yarnpkg.com/normalize-version/-/normalize-version-1.0.5.tgz#a6a2b9002dc6fa2e5f15ec2f0b2c0284fb499712"
+  integrity sha1-pqK5AC3G+i5fFewvCywChPtJlxI=
+
 np@^5.0.3:
   version "5.2.1"
   resolved "https://registry.yarnpkg.com/np/-/np-5.2.1.tgz#037bc41a6702fa20ec002fc24f36ebeaa2b318a2"
@@ -4337,6 +4342,13 @@ scoped-regex@^2.0.0:
   resolved "https://registry.yarnpkg.com/scoped-regex/-/scoped-regex-2.1.0.tgz#7b9be845d81fd9d21d1ec97c61a0b7cf86d2015f"
   integrity sha512-g3WxHrqSWCZHGHlSrF51VXFdjImhwvH8ZO/pryFH56Qi0cDsZfylQa/t0jCzVQFNbNvM00HfHjkDPEuarKDSWQ==
 
+semver-closest@^0.1.0:
+  version "0.1.2"
+  resolved "https://registry.yarnpkg.com/semver-closest/-/semver-closest-0.1.2.tgz#ede8d4d5fb04303bb0c334fff69d288cce7fc7db"
+  integrity sha512-Q6qk0bPNlK5zG62mWFC8L0Qc6OJX76XRWxiPgZyrh98IZTL3HPErgUlPfCyrAPsHVpU+YP4lf5Mz+LzpId91Og==
+  dependencies:
+    semver "^5.4.1"
+
 semver-diff@^2.0.0:
   version "2.1.0"
   resolved "https://registry.yarnpkg.com/semver-diff/-/semver-diff-2.1.0.tgz#4bbb8437c8d37e4b0cf1a68fd726ec6d645d6d36"
@@ -5063,6 +5075,15 @@ use@^3.1.0:
   resolved "https://registry.yarnpkg.com/use/-/use-3.1.1.tgz#d50c8cac79a19fbc20f2911f56eb973f4e10070f"
   integrity sha512-cwESVXlO3url9YWlFW/TA9cshCEhtu7IKJ/p5soJ/gGpj7vbvFrAY/eIioQ6Dw23KjZhYgiIo8HOs1nQ2vr/oQ==
 
+useragent-generator@^1.1.0:
+  version "1.1.0"
+  resolved "https://registry.yarnpkg.com/useragent-generator/-/useragent-generator-1.1.0.tgz#acf38287432a17980cf3cd1759ed33709abae762"
+  integrity sha1-rPOCh0MqF5gM880XWe0zcJq652I=
+  dependencies:
+    normalize-version "^1.0.5"
+    semver "^5.4.1"
+    semver-closest "^0.1.0"
+
 util.promisify@^1.0.0:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/util.promisify/-/util.promisify-1.0.1.tgz#6baf7774b80eeb0f7520d8b81d07982a59abbaee"