Scraping By (securemsg; dosl7_common; window[“bobcmn”]; window[“blobfp”])

If you are here, you’ve probably been frustrated trying to run your web scraping tool on some site and instead of receiving the proper content, you’ve been given a bunch of obscured javascript courtesy of F5 Networks. There have been several people facing this same problem on Stack Overflow, so I thought I would explain exactly what’s going on here and how I solved it for my application.

When the initial GET request is sent to the server on first contact, if no cookies are sent, the server replies with the obscured javascript page. This javascript executes in a way that is meant to be complicated to the user and difficult to reverse engineer, but its purpose is to set a couple of unique cookies and refresh the page once the cookies are set. Upon a second GET or POST request with the proper cookies, the true content of the page is revealed.

To solve this issue, you can either reverse-engineer the javascript code and set the cookies yourself or you can setup a headless browser to emulate the real screen a user would be viewing through.

I followed Christopher Su’s Instructions to get Selenium and Google Chrome running on my AWS Ubuntu instance and was up and running in no time. You could spin a micro instance up with AWS for virtually nothing.

During setup, I did receive the error when compiling chrome “dpkg: dependency problems prevent configuration of google-chrome-stable” but the installation proceeded flawlessly at with the next command.

<html><head><meta http-equiv="Pragma" content="no-cache"/>
<meta http-equiv="Expires" content="-1"/>
<meta http-equiv="CacheControl" content="no-cache"/>
<meta http-equiv="X-UA-Compatible" content="IE=edge"/>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>
<link rel="shortcut icon" href="data:;base64,iVBORw0KGgo="/>

<script>

(function(){
 var securemsg;
 var dosl7_common;

window["bobcmn"] = "1111101011101020000000220000000520000000020b0a201e200000096300000000300000000300000006/TSPD/300000008TSPD_101300000005https200000000200000000";

window.ugR=!!window.ugR;try{(function(){try{var JJ,lJ,oJ=1,SJ=1,IJ=1,jJ=1,Jl=1,zl=1;for(var Zl=0;Zl<lJ;++Zl)oJ+=2,SJ+=2,IJ+=2,jJ+=2,Jl+=2,zl+=3;JJ=oJ+SJ+IJ+jJ+Jl+zl;window.ZJ===JJ&&(window.ZJ=++JJ)}catch(Il){window.ZJ=JJ}var jl=!0;function JL(J){J&&(jl=!1,document.cookie="brav=ad");return jl}function zL(){}JL(window[zL.name]===zL);JL("function"!==typeof ie9rgb4);JL(/\x3c/.test(function(){return"\x3c"})&!/x3d/.test(function(){return"'x3'+'d';"}));
var jL=window.attachEvent||/mobi/i.test(window["\x6e\x61vi\x67a\x74\x6f\x72"]["\x75\x73e\x72A\x67\x65\x6et"]),lo=+new Date+6E5,Lo,oo,zo=setTimeout,Zo=jL?3E4:6E3;function So(){if(!document.querySelector)return!0;var J=+new Date,O=J>lo;if(O)return JL(!1);O=oo&&Lo+Zo<J;O=JL(O);Lo=J;oo||(oo=!0,zo(function(){oo=!1},1));return O}So();var io=[17795081,27611931586,1558153217];
function Io(J){J="string"===typeof J?J:J.toString(36);var O=window[J];if(!O.toString)return;var s=""+O;window[J]=function(J,s){oo=!1;return O(J,s)};window[J].toString=function(){return s}}for(var jo=0;jo<io.length;++jo)Io(io[jo]);JL(!1!==window.ugR);
(function JO(){if(!So())return;var O=!1;function s(O){for(var Z=0;O--;)Z+=S(document.documentElement,null);return Z}function S(O,Z){var s="vi";Z=Z||new _;return _J(O,function(O){O.setAttribute("data-"+s,Z.Zz());return S(O,Z)},null)}function _(){this.jo=1;this.io=0;this.Sj=this.jo;this.c=null;this.Zz=function(){this.c=this.io+this.Sj;if(!isFinite(this.c))return this.reset(),this.Zz();this.io=this.Sj;this.Sj=this.c;this.c=null;return this.Sj};this.reset=function(){this.jo++;this.io=0;this.Sj=this.jo}}
var I=!1;function Z(O,Z){if(!So())return;var s=document.createElement(O);Z=Z||document.body;Z.appendChild(s);s&&s.style&&(s.style.display="none");So()}function OJ(Z,s){if(!So())return;s=s||Z;var S="|";function _(O){O=O.split(S);var Z=[];for(var s=0;s<O.length;++s){var I="",LJ=O[s].split(",");for(var OJ=0;OJ<LJ.length;++OJ)I+=LJ[OJ][OJ];Z.push(I)}return Z}var OJ=0,_J="datalist,details,embed,figure,hrimg,strong,article,formaddress|audio,blockquote,area,source,input|canvas,form,link,tbase,option,details,article";
_J.split(S);_J=_(_J);_J=new RegExp(_J.join(S),"g");while(_J.exec(Z))_J=new RegExp((""+new Date)[8],"g"),O&&(I=So()),++OJ;return So()?s(OJ&&1):void 0}function _J(O,s,S){if(!So())return;(S=S||I)&&Z("div",O);O=O.children;var _=0;for(var OJ in O){S=O[OJ];try{S instanceof HTMLElement&&(s(S),++_)}catch(_J){}}return So()?_:void 0}OJ(JO,s);So()})();var lO=96;window.Jo={Lo:"08b88de4c7017800793eb49f2553b27bf271379cf4fd0f9c8f1b2a72b740b891cadcffe4557e5800ca9c278000864e45a154bde7be35c3f18fce5440734054f7ca1559c993f413201c3c672014497b3c5baba38fea707379707c7483d5d671a9136c2520b9f7d82aaa7e636b54f030b51498fc7660a6e7a766aa914cb95cb740"};function l(J){return 514>J}
function L(J){var O=arguments.length,s=[];for(var S=1;S<O;++S)s.push(arguments[S]-J);return String.fromCharCode.apply(String,s)}function z(J,O){J+=O;return J.toString(36)}(function oO(O){return O?0:oO(O)*oO(O)})(So());})();}catch(x){document.cookie='brav=oex'+x;}finally{ie9rgb4=void(0);};function ie9rgb4(a,b){return a>>b>>0};

})();

</script>

<script type="text/javascript" src="/TSPD/08e428ff3bab2000194e3c2ae57d3b01a1d989eaaee49d1af8b17d27b43c392a782fc5c62daf4447?type=8"></script>

<script>

(function(){
 var securemsg;
 var dosl7_common;

window["blobfp"] = "11111111102000003e829f42915a30000004a9696b997bf2ee01b00586fceca00001c2080748f36141180ef2668f9f0a8762fd6198ead61300000020http://re.security.f5aas.com/re/";


})();

</script>
<noscript>Please enable JavaScript to view the page content.</noscript>
</head><body>
</body></html>
Obfuscated Javascript Block Page Code
Obfuscated Javascript Block Page Code

9 thoughts on “Scraping By (securemsg; dosl7_common; window[“bobcmn”]; window[“blobfp”])”

  1. 你也可以用nodejs配合另一个组件来解决这个问题,而不用安装Chrome

    [Translated: You can also use nodejs with another component to solve this problem, without installing Chrome ]

    Reply
    • Isn’t possible to bypass this with a headless browser, only emulating a real user interaction. You can do this with webdriver or a “real” browser and a little bit of automation.

      Reply
  2. Seria possível o colega acima dar um exemplo? Realizei os testes e nada deu certo. Infelizmente ainda frustrado.

    Translation: Would it be possible for the above colleague to give an example? I did the tests and nothing worked. Unfortunately still frustrated.

    Reply

Leave a Comment