From cf8a4cc26ebb8d942cbebb5d5fa733286fd20891 Mon Sep 17 00:00:00 2001 From: molo-pl Date: Tue, 12 Jan 2021 15:37:19 +0100 Subject: [PATCH] Fixes #22 adding User-Agent request header to bypass Incapsula protection of OSRS hiscore pages --- package.json | 3 ++- src/hiscores.ts | 18 +++++++++--------- src/utils/helpers.ts | 11 +++++++++++ yarn.lock | 21 +++++++++++++++++++++ 4 files changed, 43 insertions(+), 10 deletions(-) diff --git a/package.json b/package.json index a6f9904..d3cc9c4 100644 --- a/package.json +++ b/package.json @@ -37,7 +37,8 @@ "homepage": "https://github.com/maxswa/osrs-json-hiscores#readme", "dependencies": { "axios": "^0.21.1", - "jsdom": "^16.3.0" + "jsdom": "^16.3.0", + "useragent-generator": "^1.1.0" }, "devDependencies": { "@types/jest": "^24.0.14", diff --git a/src/hiscores.ts b/src/hiscores.ts index 906d7e8..f291e52 100644 --- a/src/hiscores.ts +++ b/src/hiscores.ts @@ -1,4 +1,3 @@ -import axios from 'axios'; import { Player, Activity, @@ -26,6 +25,7 @@ import { numberFromElement, rsnFromElement, getActivityPageURL, + httpGet, BOSSES, } from './utils'; import { JSDOM } from 'jsdom'; @@ -39,12 +39,12 @@ export async function getStats(rsn: string): Promise { throw Error('RSN must be between 1 and 12 characters'); } - const mainRes = await axios(getStatsURL('main', rsn)); + const mainRes = await httpGet(getStatsURL('main', rsn)); if (mainRes.status === 200) { const otherResponses = await Promise.all([ - axios(getStatsURL('ironman', rsn)).catch(err => err), - axios(getStatsURL('hardcore', rsn)).catch(err => err), - axios(getStatsURL('ultimate', rsn)).catch(err => err), + httpGet(getStatsURL('ironman', rsn)).catch(err => err), + httpGet(getStatsURL('hardcore', rsn)).catch(err => err), + httpGet(getStatsURL('ultimate', rsn)).catch(err => err), getRSNFormat(rsn).catch(() => undefined), ]); @@ -120,7 +120,7 @@ export async function getStatsByGamemode( } else if (!GAMEMODES.includes(mode)) { throw Error('Invalid game mode'); } - const response = await axios(getStatsURL(mode, rsn)); + const response = await httpGet(getStatsURL(mode, rsn)); if (response.status !== 200) { throw Error('Player not found'); } @@ -143,7 +143,7 @@ export async function getSkillPage( } const url = getSkillPageURL(mode, skill, page); - const response = await axios(url); + const response = await httpGet(url); const dom = new JSDOM(response.data); const playersHTML = dom.window.document.querySelectorAll( '.personal-hiscores__row' @@ -183,7 +183,7 @@ export async function getActivityPage( } const url = getActivityPageURL(mode, activity, page); - const response = await axios(url); + const response = await httpGet(url); const dom = new JSDOM(response.data); const playersHTML = dom.window.document.querySelectorAll( '.personal-hiscores__row' @@ -218,7 +218,7 @@ export async function getRSNFormat(rsn: string): Promise { const url = getPlayerTableURL('main', rsn); try { - const response = await axios(url); + const response = await httpGet(url); const dom = new JSDOM(response.data); const spans = dom.window.document.querySelectorAll( 'span[style="color:#AA0022;"]' diff --git a/src/utils/helpers.ts b/src/utils/helpers.ts index 38bcd7b..ae605d4 100644 --- a/src/utils/helpers.ts +++ b/src/utils/helpers.ts @@ -1,3 +1,4 @@ +import axios from 'axios'; import { Gamemode, SkillName, ActivityName } from '../types'; import { GAMEMODE_URL, @@ -6,6 +7,7 @@ import { SKILLS, ACTIVITIES, } from './constants'; +const ua = require('useragent-generator'); export const getStatsURL = (gamemode: Gamemode, rsn: string) => `${GAMEMODE_URL[gamemode]}${STATS_URL}${encodeURIComponent(rsn)}`; @@ -45,3 +47,12 @@ export const rsnFromElement = (el: Element | null) => { const { innerHTML } = el || {}; return innerHTML?.replace(/\uFFFD/g, ' ') || ''; }; + +export const httpGet = (url: string) => { + return axios.get(url, { + headers: { + // without User-Agent header requests may be rejected by DDoS protection mechanism + 'User-Agent': ua.firefox(80) + } + }); +}; diff --git a/yarn.lock b/yarn.lock index 15b0c4a..e7c55ef 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3528,6 +3528,11 @@ normalize-url@^4.1.0: resolved "https://registry.yarnpkg.com/normalize-url/-/normalize-url-4.5.0.tgz#453354087e6ca96957bd8f5baf753f5982142129" integrity sha512-2s47yzUxdexf1OhyRi4Em83iQk0aPvwTddtFz4hnSSw9dCEsLEGf6SwIO8ss/19S9iBb5sJaOuTvTGDeZI00BQ== +normalize-version@^1.0.5: + version "1.0.5" + resolved "https://registry.yarnpkg.com/normalize-version/-/normalize-version-1.0.5.tgz#a6a2b9002dc6fa2e5f15ec2f0b2c0284fb499712" + integrity sha1-pqK5AC3G+i5fFewvCywChPtJlxI= + np@^5.0.3: version "5.2.1" resolved "https://registry.yarnpkg.com/np/-/np-5.2.1.tgz#037bc41a6702fa20ec002fc24f36ebeaa2b318a2" @@ -4337,6 +4342,13 @@ scoped-regex@^2.0.0: resolved "https://registry.yarnpkg.com/scoped-regex/-/scoped-regex-2.1.0.tgz#7b9be845d81fd9d21d1ec97c61a0b7cf86d2015f" integrity sha512-g3WxHrqSWCZHGHlSrF51VXFdjImhwvH8ZO/pryFH56Qi0cDsZfylQa/t0jCzVQFNbNvM00HfHjkDPEuarKDSWQ== +semver-closest@^0.1.0: + version "0.1.2" + resolved "https://registry.yarnpkg.com/semver-closest/-/semver-closest-0.1.2.tgz#ede8d4d5fb04303bb0c334fff69d288cce7fc7db" + integrity sha512-Q6qk0bPNlK5zG62mWFC8L0Qc6OJX76XRWxiPgZyrh98IZTL3HPErgUlPfCyrAPsHVpU+YP4lf5Mz+LzpId91Og== + dependencies: + semver "^5.4.1" + semver-diff@^2.0.0: version "2.1.0" resolved "https://registry.yarnpkg.com/semver-diff/-/semver-diff-2.1.0.tgz#4bbb8437c8d37e4b0cf1a68fd726ec6d645d6d36" @@ -5063,6 +5075,15 @@ use@^3.1.0: resolved "https://registry.yarnpkg.com/use/-/use-3.1.1.tgz#d50c8cac79a19fbc20f2911f56eb973f4e10070f" integrity sha512-cwESVXlO3url9YWlFW/TA9cshCEhtu7IKJ/p5soJ/gGpj7vbvFrAY/eIioQ6Dw23KjZhYgiIo8HOs1nQ2vr/oQ== +useragent-generator@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/useragent-generator/-/useragent-generator-1.1.0.tgz#acf38287432a17980cf3cd1759ed33709abae762" + integrity sha1-rPOCh0MqF5gM880XWe0zcJq652I= + dependencies: + normalize-version "^1.0.5" + semver "^5.4.1" + semver-closest "^0.1.0" + util.promisify@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/util.promisify/-/util.promisify-1.0.1.tgz#6baf7774b80eeb0f7520d8b81d07982a59abbaee"