From cf8a4cc26ebb8d942cbebb5d5fa733286fd20891 Mon Sep 17 00:00:00 2001 From: molo-pl Date: Tue, 12 Jan 2021 15:37:19 +0100 Subject: [PATCH 1/2] Fixes #22 adding User-Agent request header to bypass Incapsula protection of OSRS hiscore pages --- package.json | 3 ++- src/hiscores.ts | 18 +++++++++--------- src/utils/helpers.ts | 11 +++++++++++ yarn.lock | 21 +++++++++++++++++++++ 4 files changed, 43 insertions(+), 10 deletions(-) diff --git a/package.json b/package.json index a6f9904..d3cc9c4 100644 --- a/package.json +++ b/package.json @@ -37,7 +37,8 @@ "homepage": "https://github.com/maxswa/osrs-json-hiscores#readme", "dependencies": { "axios": "^0.21.1", - "jsdom": "^16.3.0" + "jsdom": "^16.3.0", + "useragent-generator": "^1.1.0" }, "devDependencies": { "@types/jest": "^24.0.14", diff --git a/src/hiscores.ts b/src/hiscores.ts index 906d7e8..f291e52 100644 --- a/src/hiscores.ts +++ b/src/hiscores.ts @@ -1,4 +1,3 @@ -import axios from 'axios'; import { Player, Activity, @@ -26,6 +25,7 @@ import { numberFromElement, rsnFromElement, getActivityPageURL, + httpGet, BOSSES, } from './utils'; import { JSDOM } from 'jsdom'; @@ -39,12 +39,12 @@ export async function getStats(rsn: string): Promise { throw Error('RSN must be between 1 and 12 characters'); } - const mainRes = await axios(getStatsURL('main', rsn)); + const mainRes = await httpGet(getStatsURL('main', rsn)); if (mainRes.status === 200) { const otherResponses = await Promise.all([ - axios(getStatsURL('ironman', rsn)).catch(err => err), - axios(getStatsURL('hardcore', rsn)).catch(err => err), - axios(getStatsURL('ultimate', rsn)).catch(err => err), + httpGet(getStatsURL('ironman', rsn)).catch(err => err), + httpGet(getStatsURL('hardcore', rsn)).catch(err => err), + httpGet(getStatsURL('ultimate', rsn)).catch(err => err), getRSNFormat(rsn).catch(() => undefined), ]); @@ -120,7 +120,7 @@ export async function getStatsByGamemode( } else if (!GAMEMODES.includes(mode)) { throw Error('Invalid game mode'); } - const response = await axios(getStatsURL(mode, rsn)); + const response = await httpGet(getStatsURL(mode, rsn)); if (response.status !== 200) { throw Error('Player not found'); } @@ -143,7 +143,7 @@ export async function getSkillPage( } const url = getSkillPageURL(mode, skill, page); - const response = await axios(url); + const response = await httpGet(url); const dom = new JSDOM(response.data); const playersHTML = dom.window.document.querySelectorAll( '.personal-hiscores__row' @@ -183,7 +183,7 @@ export async function getActivityPage( } const url = getActivityPageURL(mode, activity, page); - const response = await axios(url); + const response = await httpGet(url); const dom = new JSDOM(response.data); const playersHTML = dom.window.document.querySelectorAll( '.personal-hiscores__row' @@ -218,7 +218,7 @@ export async function getRSNFormat(rsn: string): Promise { const url = getPlayerTableURL('main', rsn); try { - const response = await axios(url); + const response = await httpGet(url); const dom = new JSDOM(response.data); const spans = dom.window.document.querySelectorAll( 'span[style="color:#AA0022;"]' diff --git a/src/utils/helpers.ts b/src/utils/helpers.ts index 38bcd7b..ae605d4 100644 --- a/src/utils/helpers.ts +++ b/src/utils/helpers.ts @@ -1,3 +1,4 @@ +import axios from 'axios'; import { Gamemode, SkillName, ActivityName } from '../types'; import { GAMEMODE_URL, @@ -6,6 +7,7 @@ import { SKILLS, ACTIVITIES, } from './constants'; +const ua = require('useragent-generator'); export const getStatsURL = (gamemode: Gamemode, rsn: string) => `${GAMEMODE_URL[gamemode]}${STATS_URL}${encodeURIComponent(rsn)}`; @@ -45,3 +47,12 @@ export const rsnFromElement = (el: Element | null) => { const { innerHTML } = el || {}; return innerHTML?.replace(/\uFFFD/g, ' ') || ''; }; + +export const httpGet = (url: string) => { + return axios.get(url, { + headers: { + // without User-Agent header requests may be rejected by DDoS protection mechanism + 'User-Agent': ua.firefox(80) + } + }); +}; diff --git a/yarn.lock b/yarn.lock index 15b0c4a..e7c55ef 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3528,6 +3528,11 @@ normalize-url@^4.1.0: resolved "https://registry.yarnpkg.com/normalize-url/-/normalize-url-4.5.0.tgz#453354087e6ca96957bd8f5baf753f5982142129" integrity sha512-2s47yzUxdexf1OhyRi4Em83iQk0aPvwTddtFz4hnSSw9dCEsLEGf6SwIO8ss/19S9iBb5sJaOuTvTGDeZI00BQ== +normalize-version@^1.0.5: + version "1.0.5" + resolved "https://registry.yarnpkg.com/normalize-version/-/normalize-version-1.0.5.tgz#a6a2b9002dc6fa2e5f15ec2f0b2c0284fb499712" + integrity sha1-pqK5AC3G+i5fFewvCywChPtJlxI= + np@^5.0.3: version "5.2.1" resolved "https://registry.yarnpkg.com/np/-/np-5.2.1.tgz#037bc41a6702fa20ec002fc24f36ebeaa2b318a2" @@ -4337,6 +4342,13 @@ scoped-regex@^2.0.0: resolved "https://registry.yarnpkg.com/scoped-regex/-/scoped-regex-2.1.0.tgz#7b9be845d81fd9d21d1ec97c61a0b7cf86d2015f" integrity sha512-g3WxHrqSWCZHGHlSrF51VXFdjImhwvH8ZO/pryFH56Qi0cDsZfylQa/t0jCzVQFNbNvM00HfHjkDPEuarKDSWQ== +semver-closest@^0.1.0: + version "0.1.2" + resolved "https://registry.yarnpkg.com/semver-closest/-/semver-closest-0.1.2.tgz#ede8d4d5fb04303bb0c334fff69d288cce7fc7db" + integrity sha512-Q6qk0bPNlK5zG62mWFC8L0Qc6OJX76XRWxiPgZyrh98IZTL3HPErgUlPfCyrAPsHVpU+YP4lf5Mz+LzpId91Og== + dependencies: + semver "^5.4.1" + semver-diff@^2.0.0: version "2.1.0" resolved "https://registry.yarnpkg.com/semver-diff/-/semver-diff-2.1.0.tgz#4bbb8437c8d37e4b0cf1a68fd726ec6d645d6d36" @@ -5063,6 +5075,15 @@ use@^3.1.0: resolved "https://registry.yarnpkg.com/use/-/use-3.1.1.tgz#d50c8cac79a19fbc20f2911f56eb973f4e10070f" integrity sha512-cwESVXlO3url9YWlFW/TA9cshCEhtu7IKJ/p5soJ/gGpj7vbvFrAY/eIioQ6Dw23KjZhYgiIo8HOs1nQ2vr/oQ== +useragent-generator@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/useragent-generator/-/useragent-generator-1.1.0.tgz#acf38287432a17980cf3cd1759ed33709abae762" + integrity sha1-rPOCh0MqF5gM880XWe0zcJq652I= + dependencies: + normalize-version "^1.0.5" + semver "^5.4.1" + semver-closest "^0.1.0" + util.promisify@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/util.promisify/-/util.promisify-1.0.1.tgz#6baf7774b80eeb0f7520d8b81d07982a59abbaee" From 55abd9f800816185adc8c24466dd270113dae473 Mon Sep 17 00:00:00 2001 From: molo-pl Date: Tue, 12 Jan 2021 23:29:25 +0100 Subject: [PATCH 2/2] Fixes #22 adding User-Agent request header - post review changes --- src/@types/useragent-generator.d.ts | 51 +++++++++++++++++++++++++++++ src/utils/helpers.ts | 2 +- 2 files changed, 52 insertions(+), 1 deletion(-) create mode 100644 src/@types/useragent-generator.d.ts diff --git a/src/@types/useragent-generator.d.ts b/src/@types/useragent-generator.d.ts new file mode 100644 index 0000000..6e1af3a --- /dev/null +++ b/src/@types/useragent-generator.d.ts @@ -0,0 +1,51 @@ +declare module 'useragent-generator' { + /******************** + * Google Chrome * + /*******************/ + export function chrome(opt: number | string | { version: string, os?: string }): string; + export namespace chrome { + function androidPhone(opt: number | string | { version: string, androidVersion?: string, device?: string }): string; + function androidTablet(opt: number | string | { version: string, androidVersion?: string, device?: string }) + : string; + function androidWebview(opt: number | string | { androidVersion: string, chromeVersion?: string, device?: string }) + : string; + function chromecast(opt: number | string | { version: string }): string; + function iOS(opt: number | string | { iOSVersion: string, chromeVersion?: string, device?: string }): string; + } + export function chromium(opt: number | string | { version: string, os?: string }): string; + /*************** + * Firefox * + /*************/ + export function firefox(opt: number | string | { version: string, os?: string }): string; + export namespace firefox { + function androidPhone(opt: number | string | { version: string, androidVersion?: string, device?: string }): string; + function androidTablet(opt: number | string | { version: string, androidVersion?: string, device?: string }) + : string; + function iOS(opt: number | string | { iOSVersion: string, device?: string }): string; + } + /************** + * Safari * + /************/ + export function safari(opt: number | string | { version: string, os?: string }): string; + export namespace safari { + function iOS(opt: number | string | { iOSVersion: string, safariVersion?: string, device?: string }): string; + function iOSWebview(opt: number | string | { iOSVersion: string, safariVersion?: string, device?: string }): string; + } + /*********************** + * Internet Explorer * + /*********************/ + export function ie(opt: number | string | { version: string, os?: string }): string; + export namespace ie { + function windowsPhone(opt: number | string | { version: string, device?: string }): string; + } + /********************** + * Microsoft Edge * + /********************/ + export function edge(opt: number | string | { version: string, chromeVersion?: string, os?: string }): string; + /************************ + * Search Engine Bots * + /**********************/ + export function googleBot(opt?: number | string | { version?: string }): string; + export function bingBot(opt?: number | string | { version?: string }): string; + export function yahooBot(): string; +} diff --git a/src/utils/helpers.ts b/src/utils/helpers.ts index ae605d4..ad28c03 100644 --- a/src/utils/helpers.ts +++ b/src/utils/helpers.ts @@ -1,4 +1,5 @@ import axios from 'axios'; +import * as ua from 'useragent-generator'; import { Gamemode, SkillName, ActivityName } from '../types'; import { GAMEMODE_URL, @@ -7,7 +8,6 @@ import { SKILLS, ACTIVITIES, } from './constants'; -const ua = require('useragent-generator'); export const getStatsURL = (gamemode: Gamemode, rsn: string) => `${GAMEMODE_URL[gamemode]}${STATS_URL}${encodeURIComponent(rsn)}`;