From c4fa727c547b3bcd210e2b3eed90ec0d644805b2 Mon Sep 17 00:00:00 2001 From: Sean Kessler Date: Mon, 4 Mar 2024 19:28:21 -0500 Subject: [PATCH] Fix Feeds. InsiderTransactions, ETFHoldings, Yahoo Price Feed --- App.config | 2 +- MarketData.v12.suo | Bin 625152 -> 630272 bytes .../DataAccess/InsiderTransactionDA.cs | 54 ++ MarketDataLib/Helper/MarketDataHelper.cs | 647 ++++++++++-------- MarketDataLib/Integration/HttpNetRequest.cs | 208 ++++-- MarketDataLib/Utility/FeedParser.cs | 6 + MarketDataLib/Utility/Sections.cs | 31 + 7 files changed, 596 insertions(+), 352 deletions(-) diff --git a/App.config b/App.config index deee84b..6149667 100644 --- a/App.config +++ b/App.config @@ -16,7 +16,7 @@ - + diff --git a/MarketData.v12.suo b/MarketData.v12.suo index 8ac9d02d1753669cfc38c4dfe984bacc29a314ee..e799a7555cb53e07c5309e8f66dc74b1b05c049e 100644 GIT binary patch delta 10089 zcmd5?3wTY(zTdN+vto-RA|fK$A~aHoy(1!s*gFp;9*t*}R1-o=iHPX&2-)<>*Q1fe z{yUEAs-}x7ZG>iAoY+!QMcvTQRvisheH<#3R?+1CXD9J!dwNcL&Ue1M^8NPw*UYS$ z`M+n@igRTb=bkP_R)=6}x_nHg93%q}^$iUTyjDa6A<_^sLLyj)4@d|ZA>e(#ZL@9> z?{wcEmZ#*LBHmc*WcqwzKKZMDT^K2K!pHO#Ln6p&eU3OxGquJ0HcfTDAA8|gc*C!` zh#yxhZtc1%N_wyqL5Aw7Ql~b!l7>KR>W$cheTQN>Twf^l4H%7g7QEv^BlaPNBc8EPsn#eh16iNNEKsL{ITbF~5pL34&XKzS1lENltL=YpQ$)%Xbk65LHce z2eEt)aR_l3aRl)`;wYl|d0rmJ`wtK`h!cpDhz}7TA#_U%f8E;RrQ+y_uIVU{#Z3Z= zK%T5fzlP1w1`W0{Otf^>f@^@Q&B!ttoE&+vwKd=|^UH9cfQ`XbVu5T7Bi zn<*B{b6ED^G=e{q)KnVSb{XM~;PpQ=m7M0+CG3~m)VHw)e~C#j?bLtNlK-D7ghSlp zW!i}YZ}1D=5Fz+IiWadOZztinuVDEV(fzDbY#$)G2$79=4ZEDdvH;5t*ljY>6A{Z0 zX1rgC<>&hA)?UPR+k*APGtX0hWUMoS2W9=*nQF@A z(XOdNIfjwuB!T&lkjU-f*{5y(S=h1xN9GoD0BK%=e;Nsw`d}?2q>*@G5rL=ENC=q< ztI|k!$rro9!4NUr%wyVQ)Pd)r8)>AQC+a{%s5sCYn;Mqk5L2-ICV@qRNf*)puZ4=^ zb?2YLw}-DjoyxPSC+=BQBq)Lph$}R;K$sCN5#9)%F?buFnwKze6B#RY#XBh3M5arg zI{(+nMcrA8W7pnRw_CoeB^$n`uH)i<${*+M;K|3;aQ(Pgs5o(5=58?m12K&X*v6;l z?ehQP0_gT1zU?Y(4~wfO+buV0hjR&U54(DX==RqKgZ-H74|T7q6sjd9fGr$Co*7S) zbmFkWmSeJ?k9q6P+sXZ&+gn!kR^|tVIVw0RNgMX$9$_^p-%NyH5Nb#gm2pD8&+shD z2j@ zQ@*vZAoyr%3XbcaN7$_sL=GCCjn(};Q<+SCJu!NL$zh5y4M4}}h4-l_nJHuZqN%%xr7=<76r4AB>-TcKJ}f}y194iAwI5;cU*a6ZOxty zpA0BgH;me-FTU~X{58dC&6`1BnT0ALfk!g`xvY5duHllJ5ubLTRzx}rU(*Q);D zoGegq*@Q^lDtPQ33!|XArxeU)&KFh? zSB-GH<)J3-Ip_1iwnGeUd8Da|tGVxrS%<+rPL!CmNVwgmIXFSZLD35wok=%6|4jO? zhbePom}=w$j-}GRtglNr(4yQ+un5djBGk4q5(EJ{exmkc{Zg+{q7oWS$>*`78M$M0~`|yYQ^w>B_SLxd5F* z#lhY@djE1tfXYisH>eAjy;+}Nv5L5M3Hd~VIlF{H*9|Jcja@R; z!Ez@!a#XdjB#T%PWq1h!$pObeVF=_V$X+gx1KG@UaYc)k_xZGLGk5wI9of8+7!$;3 z-8mIKH$$u@Ek41V!SfYftl^JhCy9k+p!7Vs#LRxsMb*pol@P9Xi4B6pD)PK1Df8ow zC-|RY*Q-bi0=}!I)~xk<@ltD74N37N2upJxWd5M|u;;nvBNtq&AihJ|IQSsK@ zRKP^pY^)Ilqe>LYYs7=Z|~n!=M8HF@?CEc_Gk zv_J~L5l1q?XQmVk3-^o3t$rBLQ`+509=_O|7!+E0goiKo*?zGDah(zKNhQoVBQ_8> z%=uI-l=t$S%YJxI{GNEfjPxQzA;Rg!>Q9J*;Hnk#;t?PrQftTv zl}d~-NiZ&m0?)u5_{6`IffLYEGgiz>{*NYLr7;2Pi{^B>D@vXzSXcxyi9%IK65xKa zKgBp`;&647;?=QybB4bELN{q1IJ-#^5D_H>Qto=Z4-AZw;=vvzIgQ(>k{Ib;fzBjQ z=EajlthArFyri9X=`*0A{83KMwH5L5^y$15~M$^hOn@5`3xzaA_)@2N;Hw#cgk+j zn1;>gdf^Tn7!N|E3p0rNn*jvrSsjvZ?!lEmaSBSSgxcL5!kRB7aUw6otL5VzmzplSFB2G zouW)6P&rgu1Em*G^6N)R@od;dDU+~qRgytq#EbG2Hiar{h;dNz0wIMKgY&9jX6w8> zPyUZIpUpomof5RQC(~cxFU_KXoRCJc(i2j8fRUgGP26+leXXzx*suqA0DW)#cwO05WMm(e6}Zc(3P zn_p7Kh>*EUrHmFUH)Qa8h(mw3E!1=fu?BNF;eIHy|t~?Cso#j9{dQ=T(wVM@-4Cne_7W8?E#Ice? zN`?rfesT`WsaA?r$UBS1c;mPl&73ZWL@oQulyDomWFj)e(@ zQUcUh zA~6J2fSPk?R6%a#0?u|xc?qh|DZy-$q>dF~>_sU;pE=N%_4ZQVRZ*fsG}MikDT_O* z93qCF>IJ#as$md47N<=NQ7ctAK28mS%m=6iD)LbkNna}!1kOd0FenNWa39b`t!M|i zw{Xwnoupn>jqh;PmBDf*%S=<(cQu|?sc};gj76&8x~AHheF+ZA1gh2(5VEvxtlKgb znK6H^CbGn0KH-R7GLpe6W|TFm4`kj@0-)|CC4|LpQIDXx)ho!|4i&lcS*gUaxSi@D z5sKed!eHE563fQe)y>4%gL1f)k#u-ez78q#aF19uVX|m|II3w|3iYp6t{{8YpW=GfqR`|WqtOm zxN~yI!@yE3m|=oH^=CN;RhJBbZ=%#KZ>qIi=Q#1OHcvSiemQ1aFxtETK%@ zPry%?{g_{^`VE27|CZCB?mNW^_H*bPx!)-v#^)-!+g~Kb^*1E}M)}H_`ugks;GTmg zpi^I}U5H+NxC>Y{Tz8+VY88gHPvl7W#wzE)gzIQld&<=mV%$&#qY_2BA(Bjl`ps$> zd-kSUK#VG^t5z`h;Q)aw_oRw}Y@MHsYCfmB32gBp!BFo{6^IRzgTUfX$H28}c?kGW zIe=ZepqkO|aws}Py}Cg(zQ)m#o{%Skb*JnDZv=;93(`u zF&9zKaNNypic*}+GFHS$VWu|-B;8PAz`GOXi&5XGTZrLXuW zQo+c;eq}jasInXYrI9q2xq|34l)@k6r`gk)bQwark67j*} z7>sq+>9m>{1M&9UU?GHsdg{z zDH`C~O)4gge9lhZALIex)168zZ5%Bp##}1kiBN9CUz(iFuiuc&s(s%KGOd5FRl_?( z?j<>=WfCMtG7>VT-;J(E+Tdd9@W#Ky)Z#U;hXE-YZAX)^d~wAdCh z%vw0K+KGNx|BMs_r8$_&^GoDRs2z-NQqbZ8*(n$8BA35OodouML!+SfE2_d@-=dws za+wCg@?+H3wGP9S|9tdWV?LE!rPKlIkJ0|%ETvBP^cd~v>8dVmoEz-hs2zqYf}N!3 zi)X~PNJq&`#e?`{Q?y~=^E;^BF?n8AUUtR|4};mrU`AGMZuF#?Ia8)SfxHfI^hi$W zVM~qb9p5V{DmJ!}WCc4yQI~*ntcz zeoX5xe3h+}@My3f!F=`k+w>CxdqTAi;By(J;J=mjCC|gYJ!B>;-ARuNFfve!0!tNU z+;v-#!?C5OTv3_{nx^$TTud!nYB(_(l*FF zgf^aVl$xRLEu^><$x?Xv7>#jNqP$+*Mse$KfKqT@pboMSYDbF}zWfko|AIONa9p7y z;Lp2hG|5>-$Oys6;n7f8n=nf)*G8A;B z%uBO_aRt@#;McSbG#o&+WKEg|i&M!cSnh*S_>B{EbdxC`qn%@aqfOm!O%(0i4!&Q| z?$CW3^=&kEODHbEW8cRP(01I29%wXj^F2mB4ICSV45;piZ_P1tDe%;b+A58^^z7WM zyxHKmO54K&Ewpyfa1ag6XD0^6UoaZz>8A}ApPW2>qdh=F=M2|^3n%b*-<~Vb{S?yt z-9*YAP5rAn`iGkFURdCDHz-BG%Ozr4?#?i)3}d~=Ct=CwG!eS*rCqoO1^>(u0Ufb@ z=evVMPnw;ZH*;DOSN9Yt9-nzc;!0%BQ|KA?-w+?TF1;LgVoJwJeKID_1p8m9yS$w5 zT4wE~b7!|$SXjUK9l;Um<#i{6|O}^c6E47zj#8?}g^D7Y$d|PM% z{6q4cOE!VKgXYGgP)jKF*NUK_22<~$wpuUWO8zy+a|OCfn3fNvTQSTo_16L*<0=i8>Yh2Lz_fZA!R)vA*!#Ch(SAcrgG|Fr zgYnyz=Us*=-Q>EC7QgR%Iue9>YB5j6_q3Bt(@k^mdnwo>H78jPi$A4VP^3%IP}G2) z;k=3VRS>Bq!G&DGn-%>?6A4`Wo_f194ZltEV|8C6Ruc?CQy>fF`?Ih~?|9xTS1`df z2ah3T}i3 z`DwkN|4|wm@Bsfr$uFnXos2ZTfHE~2wuAgMaACzS#0@Aa5d-@Fj!1T|^~1KeXj`wl zgLn+g_kIJr&$P5;?l0U%8-vdWZMA6MV!oa9T#@e2kbdZXOSAE;=H)T6*7GuE&*Hn7 z>9f(l8=qs04WV5f*p#r$Sr;8ysK#*AKby_GuN~Bicv8|umxHC#~f}5M& zht0v_mG(}(v z)ozk|rXQu%;fs$o@)$_Jr3_;wW3&v_$VT(e)F@Pr*MeLHnuYb5rd7#CzGjA`94&wa zOxL~^jpsF?JYQRFc0H%%wgmi6$0G+Xd#*aGJxGSZzCx{Q*P*5?+^yx9B5*I8@7cNK zn4Z9YMR?3)!InIvXPfxf=oVlbr_FFVwbh>S)-;XBOY83$&x?;Y6Oxu{6|R$9>>tvp zcfb1a4_x5P7Q9#0Yj_sF^BwTtLU}$LqHR#1*+OT$YwfWVnSZMHB)KloeW{j*5tJu9 SX~>{qgP~}tmfO6}k^cwJnd~e8 delta 9163 zcmdT}3tUxIx<6|_*9JshA|mM#5y=P-h=z!IKyu8~kV+kM4Dm7Q5FtLUIzEopm}6Z- zww=$+3?15(2e%&F|0p)_Z;5 z_h0Kxh%s*E+X}KyeJv8`y|;Ly?ZK=WBhUMygW zje5xO<#U$VK|Hhfy;$jJ3@eKS($>J|GJ|In-jNS{gs~mG&z7BW@UV@!qcJL z_b6=Ltw~qgxk0%hmGFto)F8E222HqWH-uVX53m<_36Kq5M!L`bQE+>^Bv`X&2QOVw zmYezk7Fq(Ru=w*xb);vo0JOg*JuOrDd>kmdMy^c#0A+QVxLA#Ym&)d&=ZeyB#+S8py+Tp2vShGDewWIqA>hDASPNWBrj>8xXvIDt4 zU_X``qyIOA`|amaJ9asZsviTlqof#VInpsm9{|=sz60s_?H{B@YX078!uEtuK7(g7 z8gAZ}y;x!MH^f)lvQ^&D0u^Vha?-X7o3Y;Jci zZD8xqac3}7wq1%>w#T16#r$g&_jdQ?hT&1;&~`n#+yuM~`MB&GewMmwv8b@?$;0N+qC_^98lN&b)n^&eV*U{J(0#UcDk@!Wc^S;) zd1fKSk=8Op2+~$SC?M+)6I)VwQrm-0jk><@%2P#}B$FNW*7@p9Oo=0x#I5NRQrzi`KQXW{KKH=4CMIqRrexo95{$;(o5IW6m&D@mDHt;SNz6!u^JF zq*E{W=mg*=qTrLwjw%90ii57fw3@=>IlZ+JRl{!YLT1MOl^e;?%C zNPmQMTcCXbiBB7shvWtI>b*p$ zGh6LJwNF~1bg&0YGaCm;a=FOdtM(4t8NcoIt3BSo^Cyem*_hk6^8gC_oR|3LC>&I^ zdPR(PeZ7IsJe1GD+~`Tlxo{b2@NSVq=Be&PrA`Qx0B8@d0EoG#MwI2w%h2_~# zhP2nd5w~wpj;t9+gBZ z8eN9OVrXVZ^&yH&wK&TmrYRoUd|HnP>-B7pbwLk|wfb*}nKgXoM?a_Ry{wIxypgS9 zloqRJiL4r>u8n`5@>#gF*cZ=S&}eFa$S0j+N<{h3v4RZsBzq!Txs^(Os)dR^=aiRk zsm@j3VR0KzhLt|HDQm-9kB%zNdNx>|R9`aPiBT1M+#NA;v0)-J?AuRSxt_l|W$B4nb!3Qi+>IxzCQnv7i3WY!>3JVTB+zYP=@mvy@ZJS*Zs7KRm7Y%kZNQ+@tUcEmA~Q zs2@bO(*j4@7u+-g`H&WVvJL^7dRS*EMK0_Ts870~U-AvI9494^_XN^Zr0F-b$+$t5 z?K1=UeqSl?kGvPi26BJ_nk&bZ3*mNPAn*fV5O4=D7`PJ{0t^G>z;e^&LCz24??yVR zMGv-a#k97RZYp6^Tcm`oDARg&0>R{~sq849;y@=U%b-z$ybKzpNWTmiE2ecXuRs7h zp*Aq*c{M~{A5QW@D!E&arTO*hfY#65wds?N@5bEy?44UrzIO4a_M(eX;_Z6%Cg$uz z^!uiuy+i#Qf_C*0wUJTdM{1aJ664$uzN^)%JP@&6a<+CdqCFB&rU6RIs3xP?qm+79 ziR@X1sujRqzy^MSCXb-n4|x+(slR;IApaEbLzF#D)6Od0+2geCtTMh+9ZJiAWx&h8 zK43o}d%cRZdBvhXlsDDcb&7}Ww?A%LXe>nOs&4k&j|TJ3m`+|uTCP2>V6$`a>1owe zoMTW6C+AIHJ2?X*%h9jd(}EvgQ8vndK)oa`orC?IgXq?%&R{EO=3(_3mN@fGbz#sv zIWT4c`J4Y+y`toXh%1*>L;V-t>qOLB>UqU^N7bZ9ob#%Z{$204-}L?{UqdKtj2e|7 z&){6-q>kVHx@%7e{#V^hZ_wZpw209R=Mz<_ZdS__*8SHjw603aO;+quv|JU%OcCse z`ezm)T_mvpR$d7Bf~fhLTK=_8tb8g0ouQp%)tVNs*a_)zf9KuT{@KMa5yrJfxda$X z3)ZqMIdwjABZ&oRku)Y)tNd~zCqz@^3)(+$&;BQm_XsY0nGV56JU-?Gs1Q zE2(OU5k&6$O-+Pu)SGnLHCNBQ{?^fNtrjPWtMqc_q+!r%-JtwK+Dk2ij)cc8k$MwA`MU2u?L`uGPIFiov z*P|%wKg=YN)u7j@&Lk`~t-U^qs$N8yPcwWf#dp>sXmw|-B4w&}Sf%w#%&B53H~gwo zthw#-Et%x2)Z=K`GBZ|`WEo}5cK>l1H7qrw#M)e=kWt=9eUg|o*jUY|)->YC{enJ= zd`(J-SUUps&T`G;lo|@{y{zr0Q)jg}dUuSrmAs>HH>=pK6~KH3%M*DEj4DP=SG6@_ zO`SGDrNs~GY1F}n%6Si?YQy4@8gpLNEGmpPlST3p;|Qbj8m*TI8>giyV%Js8V50b_ zRvt}hFXLSz&hEy8WDtTJzu_CU@c!!blt}&Vl z9?>$Xc|Yb|ea@JK_V<|?R3eO6@;qb2`|H8j*`F9ov7l)h7=Fb#%baaei5HC=s+y`L zQ9&?9e7M?}r&D93zDaD_Ym5L1f1spL<9MBmUfg_*QArn_)5!#VpcuB#*k)2gXT2YJ z^L4ivTW_onr^Ble%VUXEq(aN)00##&y?fuHkyGugzBd@C|K$& z6Sm*$HiwB}RpvIhf4bbcv&}k_8qczBB6+cib)U&pazxHHX@wkBzWpqf4N`t4Uq#E! zbFC>YSn6FZ2P+)Oqv(R)aEtWy<^e|e*?I((>^4J%d#~BRsOmi}nl6?rQKH*E^B8>L z!yCqx%TO@um|4f@VGHeBd=*=1zKiq51otjv=*2YlG>EqAjG2oSZ`RX9nqwYRoz1YG zFItbb&)39@j2-5F=AR2k^XJXW%y}2rjh(@~eK z>^DYf3zY?^oW{s)8H4FUBab2XpAADy-fynbsdT7WB#O?PD;dQ{^IW_AHMf{>&|G7Z zcaR<-B95CEpnC`1E4&xXTG;6!12#P&ZF+|DcuJ24rAj(r*~N`!IgX0fI{bJN7I8j- z1|^mb^4_7Oh!LN`f#iwTI#9s{BZ@-X@py5M!Z%}KKQWh+r!5wiQ)YUoVTYMakB6!u z6dKH%DDAwNDKbx)`&3d^^K|mI;fZ2&D0i6i4rVBrq)+e*yvRL#)|`i>K5lq~CzjVU zD(#>TlDj8bjBPU4tF-L}W2_k4kFSRo59)>FIgX9L^_W>iB@x)~6)#}#g??%_DOB4R zXJY+%?C*kRvw}H;xKha#T0a;R$orvMCF+LoEaqt3pxs>Tn^FlsD>yCR;VhL4CHeWM*8Xl<&bWK6)wZdStCkB4ddrwx`n6;>&i7Tcq*@E zP99gBQVf=}P@g4c-NTn*0bTWOqT4JEs#iUX7!x{=H!+GR<38aV!4dNIKEtPyuQ$I( z)GXjRU`D16&y10t3FD?go7QrV=vBa9WBwje}@DE{Rsb z-xcpp=CRnvH%WRe=Ac&**0d3yJ|d>U!_g1(b8V?$w2Dol8s0!pzR9~%!BIZaUk3%=KhFD5)YH6E zbtOk&IR7CZK~+b2H~(gyP4QcJTMFBWSmN8veKhVU_mX!D{vEo=yHi-K70z<)JwrW| zS88YpdH6&*1>fH$x>CO9>*de>^7V3B=P$lq`nPd!a9TzWPtWvTx24bDP2oFvcXM3n zPbSXDT|j$UD_&BX_yX387EQGJE5Ds!vxF4#))~FTpSJTqFnaWDXt0F#>|zg!s%m~% zq59*z9aSIXX|(D+C~;~Z??-Q4;H`vrC$DAKO|pg`pf`^5dnsx+|7G7_qk6Y7Hf5*vj_PIR{X>s zmQMw*nQ`L3>Ug-Kj2rJi$`@?#zQVz$Caei&4&C6}50*9_=jp8?hhEop+>CL9R@>QG z{him*viT^_p=Wku>wJ|-il^ZThwn9MhyM(4Oq8!mF3;wjkTEnNoz{dxFJ&t#|*f8fKAYYX5#aXapNIXeG1I3N&73i%XgYn^#cD@k9jy*j`vI?|6*X#i$~%nD^qztEHWKzGB=;8Z z^uQ%PGot0^%76`O{Sb!J1D&nx>PBcr!*}!SziirhU97J3c%|BfF7{?7&F^A$p@F}H zgB!2#1mS}%6s_^0Ge*k-I?H#X>rcb(&JZMzq5$~=lf*B&npFEa&lcG?TMsEzmt@Vi zJ&H+7n|TKFwfr1XJ6sPFxNJ>e{$$HXwJ9)rR4*-x>YI78@Lt7)l+RequSZ$E7!Av` z()@j_+E!mO7PGkMSD9Kns%hr2PL^dj+)`*vsf9aMsMW~)`(U5X5Ap#NTE{~v^eFFs zeY?EzA@|}p7}(YESc?BOoJFmhcn`UQTCuP{5N!q3CK{5J{6rT{Ck1NQR_E(>UR?4Iyb-)n;N=UQS{ky z9CaYawCfyTMdbY*j|=}A0f=4YZQ**_bFbF1#U;Y~G0rf?>`qEs`ISD(f3w_K5U&5* z+f(bN4t0H%4diQvLt5xidkv0iVZsyT)+i7miM4QGqH9dRNYX%Ga)6q~TWRDOjH`Rb zC~Lp!D3)TU-isS-FY6uE(JbQzcF;YuwTSXR8zLshI@?MTp*r6(6k&|AuINsorP=ur z2(<-PgXt7woYK{}zRtHY#gHOvhw7B#x4oPh79ww{b*{C4rL~GmU(piDcg)bJL#7!a zmfBVrvrB&(M+t{jP;!0) continue; - // HtmlNodeCollection dataColumns=rows[index].SelectNodes(".//td"); - // if(dataColumns.Count<2) continue; - // String market=dataColumns[0].InnerText.Trim().ToUpper(); - // market=market.Replace("&","&"); - // String quoteHtml=dataColumns[1].InnerHtml; - // List sections=Sections.GetSections(quoteHtml); - // if(3!=sections.Count)continue; - // String changeValue=sections[0]; - // String changePercent=sections[2]; - // premarketElement.Market=market; - // premarketElement.ChangePercent=FeedParser.ParseValue(changePercent); - // premarketElement.ChangeValue=FeedParser.ParseValue(changeValue); - // premarketElement.Timestamp=timestamp; - // premarketElements.Add(premarketElement); - // } - // if(0==premarketElements.Count) - // { - // MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetPremarketData, Did not find any results for request '{0}'.",strRequest)); - // } - // return premarketElements; - // } - // catch(Exception exception) - // { - // MDTrace.WriteLine(LogLevel.DEBUG,exception); - // return null; - // } - // finally - // { - // if(null!=memoryStream) memoryStream.Close(); - // if(null!=httpNetResponse) httpNetResponse.Dispose(); - // } - //} - //private static bool ParsePremarketElementsBySection(String strResponseString,PremarketElements premarketElements) - //{ - // if(null==strResponseString)return false; - // List sections=Sections.GetSections(strResponseString); - // if(null==sections||0==sections.Count)return false; - // DateTime timestamp=DateTime.Now; - // PremarketElement premarketElement=null; - - // String changePercent=Sections.ScanSectionsFindStartWithReturnIndexAfter(sections,"S&P 500 Futures",8); - // String changeValue=Sections.ScanSectionsFindStartWithReturnIndexAfter(sections,"S&P 500 Futures",4); - // if(!(null==changePercent||null==changeValue)) - // { - // premarketElement=new PremarketElement(); - // premarketElement.Market="S&P"; - // premarketElement.ChangePercent=FeedParser.ParseValue(changePercent); - // premarketElement.ChangeValue=FeedParser.ParseValue(changeValue); - // premarketElement.Timestamp=timestamp; - // premarketElements.Add(premarketElement); - // } - - // changePercent=Sections.ScanSectionsFindStartWithReturnIndexAfter(sections,"DOW Futures",8); - // changeValue=Sections.ScanSectionsFindStartWithReturnIndexAfter(sections,"DOW Futures",4); - // if(!(null==changePercent||null==changeValue)) - // { - // premarketElement=new PremarketElement(); - // premarketElement.Market="DOW"; - // premarketElement.ChangePercent=FeedParser.ParseValue(changePercent); - // premarketElement.ChangeValue=FeedParser.ParseValue(changeValue); - // premarketElement.Timestamp=timestamp; - // premarketElements.Add(premarketElement); - // } - - // changePercent=Sections.ScanSectionsFindStartWithReturnIndexAfter(sections,"Nasdaq Futures",8); - // changeValue=Sections.ScanSectionsFindStartWithReturnIndexAfter(sections,"Nasdaq Futures",4); - // if(!(null==changePercent||null==changeValue)) - // { - // premarketElement=new PremarketElement(); - // premarketElement.Market="NASDAQ"; - // premarketElement.ChangePercent=FeedParser.ParseValue(changePercent); - // premarketElement.ChangeValue=FeedParser.ParseValue(changeValue); - // premarketElement.Timestamp=timestamp; - // premarketElements.Add(premarketElement); - // } - - // return true; - //} // ****************************************************************************************************************************************************************************** // ************************************************************************** Z A C K S E A R N I N G S A N N O U N C E M E N T S ******************************************* // ****************************************************************************************************************************************************************************** @@ -691,11 +573,11 @@ namespace MarketData.Helper if(!companyProfile.IsEquity)sb.Append("https://api.nasdaq.com/api/quote/").Append(symbol).Append("/dividends?assetclass=etf"); else sb.Append("https://api.nasdaq.com/api/quote/").Append(symbol).Append("/dividends?assetclass=stocks"); strRequest = sb.ToString(); - httpNetResponse = HttpNetRequest.GetRequestNoEncodingV4(strRequest, cookieCollection, webProxy); MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Requesting {0}",strRequest)); + httpNetResponse = HttpNetRequest.GetRequestNoEncodingV4(strRequest, cookieCollection, webProxy); // httpNetResponse = HttpNetRequest.GetRequestNoEncodingV5(strRequest, 5000, webProxy); // httpNetResponse = HttpNetRequest.GetRequestNoEncodingV5A(strRequest, 10000, webProxy); - //httpNetResponse = HttpNetRequest.GetRequestNoEncodingV7(strRequest); +// httpNetResponse = HttpNetRequest.GetRequestNoEncodingV3A(strRequest); if(!httpNetResponse.Success) { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode)); @@ -1646,6 +1528,68 @@ namespace MarketData.Helper // *************************************************************************************************************************************************************************************** // ********************************************************************** E T F H O L D I N G S - Y A H O O F I N A N C E ********************************************************* // *************************************************************************************************************************************************************************************** + //public static ETFHoldings GetETFHoldings(String etfSymbol) + //{ + // MemoryStream memoryStream = null; + // ETFHoldings etfHoldings = new ETFHoldings(); + // HttpNetResponse httpNetResponse=null; + // DateTime modified=DateTime.Now; + + // try + // { + // StringBuilder sb = new StringBuilder(); + // String strRequest; + // etfSymbol = etfSymbol.ToUpper(); + // sb.Append("http://finance.yahoo.com/q/hl?s=").Append(etfSymbol).Append("+Holdings"); + // strRequest = sb.ToString(); + + // WebProxy webProxy=HttpNetRequest.GetProxy("GetETFHoldings"); + // MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetETFHoldings:{0}",strRequest)); + // httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5A(strRequest,DEFAULT_TIMEOUT_MS,webProxy); + // if(!httpNetResponse.Success) + // { + // MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode)); + // return null; + // } + // String strResponse=Utility.KeepAfter(httpNetResponse.ResponseString,"/* -- Data -- */"); + // strResponse=Utility.KeepAfterLast(strResponse,"\"holdings\""); + // List> items=LocateJSONItems(strResponse); + // if(null==items||0==items.Count)return null; + // for(int index=0;indexitems.Count)break; + // List symbolPair=items[index]; + // List holdingNamePair=items[index+1]; + // List percentOfAssetsPair=items[index+2]; + // if(!symbolPair[0].Equals("symbol"))continue; + // if(!holdingNamePair[0].Equals("holdingName"))continue; + // if(!percentOfAssetsPair[0].Equals("fmt"))continue; + // if(String.IsNullOrEmpty(symbolPair[1]))continue; + // ETFHolding etfHolding = new ETFHolding(); + // etfHolding.ETFSymbol = etfSymbol; + // etfHolding.HoldingSymbolShareClass=null; + // etfHolding.HoldingCompanyName=holdingNamePair[1]; + // etfHolding.HoldingSymbolShareClass=symbolPair[1]; + // etfHolding.HoldingSymbol=symbolPair[1]; + // etfHolding.PercentOfAssets=FeedParser.ParseValue(percentOfAssetsPair[1]); + // etfHolding.Modified=modified; + // etfHoldings.Add(etfHolding); + // } + // return etfHoldings; + // } + // catch (Exception exception) + // { + // MDTrace.WriteLine(LogLevel.DEBUG,exception); + // return null; + // } + // finally + // { + // if (null != memoryStream) memoryStream.Close(); + // if(null!=httpNetResponse)httpNetResponse.Dispose(); + // } + //} + + public static ETFHoldings GetETFHoldings(String etfSymbol) { MemoryStream memoryStream = null; @@ -1658,40 +1602,39 @@ namespace MarketData.Helper StringBuilder sb = new StringBuilder(); String strRequest; etfSymbol = etfSymbol.ToUpper(); - sb.Append("http://finance.yahoo.com/q/hl?s=").Append(etfSymbol).Append("+Holdings"); + sb.Append("https://finance.yahoo.com/quote/").Append(etfSymbol).Append("/holdings"); strRequest = sb.ToString(); WebProxy webProxy=HttpNetRequest.GetProxy("GetETFHoldings"); MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetETFHoldings:{0}",strRequest)); -// httpNetResponse=HttpNetRequest.GetRequestNoEncoding(strRequest,webProxy); - httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5(strRequest,DEFAULT_TIMEOUT_MS,webProxy); + httpNetResponse=HttpNetRequest.GetRequestNoEncodingV3C(strRequest,webProxy); +// httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5A(strRequest,DEFAULT_TIMEOUT_MS,webProxy); if(!httpNetResponse.Success) { MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode)); return null; } - String strResponse=Utility.KeepAfter(httpNetResponse.ResponseString,"/* -- Data -- */"); - strResponse=Utility.KeepAfterLast(strResponse,"\"holdings\""); - List> items=LocateJSONItems(strResponse); - if(null==items||0==items.Count)return null; - for(int index=0;index sections = Sections.GetAllItemsInSections(httpNetResponse.ResponseString,"table"); + if(null == sections || 0==sections.Count)return null; + + byte[] streamBytes=Encoding.ASCII.GetBytes(sections[0]); + memoryStream=new MemoryStream(streamBytes); + HtmlDocument htmlDocument=new HtmlDocument(); + htmlDocument.Load(memoryStream); + HtmlNodeCollection rows = htmlDocument.DocumentNode.SelectNodes(".//td"); + if(null==rows || 0==rows.Count)return null; + for(int rowIndex=0;rowIndexitems.Count)break; - List symbolPair=items[index]; - List holdingNamePair=items[index+1]; - List percentOfAssetsPair=items[index+2]; - if(!symbolPair[0].Equals("symbol"))continue; - if(!holdingNamePair[0].Equals("holdingName"))continue; - if(!percentOfAssetsPair[0].Equals("fmt"))continue; - if(String.IsNullOrEmpty(symbolPair[1]))continue; + HtmlNode node = rows[rowIndex]; ETFHolding etfHolding = new ETFHolding(); etfHolding.ETFSymbol = etfSymbol; - etfHolding.HoldingSymbolShareClass=null; - etfHolding.HoldingCompanyName=holdingNamePair[1]; - etfHolding.HoldingSymbolShareClass=symbolPair[1]; - etfHolding.HoldingSymbol=symbolPair[1]; - etfHolding.PercentOfAssets=FeedParser.ParseValue(percentOfAssetsPair[1]); - etfHolding.Modified=modified; + etfHolding.HoldingSymbolShareClass = null; + etfHolding.HoldingCompanyName = rows[rowIndex].InnerText; + etfHolding.HoldingSymbolShareClass = rows[rowIndex+1].InnerText; + etfHolding.HoldingSymbol = rows[rowIndex+1].InnerText; + etfHolding.PercentOfAssets = FeedParser.ParseValue(rows[rowIndex+2].InnerText); + etfHolding.Modified = modified; etfHoldings.Add(etfHolding); } return etfHoldings; @@ -2265,50 +2208,6 @@ namespace MarketData.Helper } } - - //private static AnalystPriceTarget GetAnalystPriceTargetYahoo(String symbol) - //{ - // AnalystPriceTarget analystPriceTarget = null; - // HttpNetResponse httpNetResponse=null; - // try - // { - // StringBuilder sb = new StringBuilder(); - // String strRequest; - // symbol = symbol.ToUpper(); - // sb.Append("http://finance.yahoo.com/q/ao?s=").Append(symbol).Append("+Analyst+Opinion"); - // WebProxy webProxy=HttpNetRequest.GetProxy("GetAnalystPriceTargetYahoo"); - // strRequest=sb.ToString(); - // MDTrace.WriteLine(LogLevel.DEBUG, String.Format("GetAnalystPriceTargetYahoo: {0}", strRequest)); - // httpNetResponse=HttpNetRequest.GetRequestNoEncodingV5(strRequest,DEFAULT_TIMEOUT_MS,webProxy); - // if(!httpNetResponse.Success) - // { - // MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode)); - // return null; - // } - // analystPriceTarget=new AnalystPriceTarget(); - // analystPriceTarget.Symbol=symbol; - // analystPriceTarget.Date=DateTime.Now.Date; - // String strResponse=Utility.KeepAfter(httpNetResponse.ResponseString,"/* -- Data -- */"); - // analystPriceTarget.HighTargetPrice=FeedParser.ParseValue(Utility.Find(strResponse,"\"targetHighPrice\":{\"raw\":",',')); - // analystPriceTarget.MeanTargetPrice=FeedParser.ParseValue(Utility.Find(strResponse,"\"targetMeanPrice\":{\"raw\":",',')); - // analystPriceTarget.LowTargetPrice=FeedParser.ParseValue(Utility.Find(strResponse,"\"targetLowPrice\":{\"raw\":",',')); - // analystPriceTarget.MedianTargetPrice=FeedParser.ParseValue(Utility.Find(strResponse,"\"targetMedianPrice\":{\"raw\":",',')); - // if(double.IsNaN(analystPriceTarget.HighTargetPrice)&& - // double.IsNaN(analystPriceTarget.MeanTargetPrice)&& - // double.IsNaN(analystPriceTarget.LowTargetPrice)&& - // double.IsNaN(analystPriceTarget.MedianTargetPrice))return null; - // return analystPriceTarget; - // } - // catch (Exception exception) - // { - // MDTrace.WriteLine(LogLevel.DEBUG,exception); - // return null; - // } - // finally - // { - // if(null!=httpNetResponse)httpNetResponse.Dispose(); - // } - //} // **************************************************************************************************************************************************************************** // ************************************************************ M O R N I N G S T A R H I S T O R I C A L D A T A V 2 B E G I N ******************************************* // **************************************************************************************************************************************************************************** @@ -2804,15 +2703,6 @@ namespace MarketData.Helper return currency; } } - //public static String GetMStarSecurityId(String symbol,String responseString) - //{ - // String securityIdentifier=GetMStarSecurityId_1(responseString); - // if(null!=securityIdentifier)return securityIdentifier; - // securityIdentifier=GetMStarSecurityId_2(symbol,responseString); - // if(null!=securityIdentifier)return securityIdentifier; - // return null; - //} - public static String GetMStarSecurityId(String symbol,String responseString) { @@ -2838,23 +2728,6 @@ namespace MarketData.Helper //byId:{\"0P000003MU\":$} // AAPL:0P000000GY - //private static String GetMStarSecurityId_2(String responseString) - //{ - // try - // { - // string token="byId"; - // int index=responseString.IndexOf(token); - // if(-1==index)return null; - // string securityIdentifier=responseString.Substring(index+token.Length); - // securityIdentifier=Utility.BetweenString(securityIdentifier,"\"","\""); - // return securityIdentifier; - // } - // catch(Exception exception) - // { - // MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetMStarSecurityId encountered an exception:{0}",exception.ToString())); - // return null; - // } - //} // 5","0P000003MU","MIDD",60 private static String GetMStarSecurityId_2(String symbol,String responseString) @@ -4424,7 +4297,7 @@ namespace MarketData.Helper } // ***************************************************************************************************************************************************************************** -// **************************************************************************** B A L A N C E S H E E T - N A S D A Q **************************************************** +// **************************************************************************** C U R R E N C Y C O N V E R S I O N **************************************************** // ***************************************************************************************************************************************************************************** public static CurrencyConversionCollection GetCurrencyConversion(String sourceCurrency,DateTime conversionDate) { @@ -4711,7 +4584,8 @@ namespace MarketData.Helper strRequest = sb.ToString(); MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetFundamental {0}",strRequest)); WebProxy webProxy=HttpNetRequest.GetProxy("GetFundamentalEx"); - httpNetResponse=HttpNetRequest.GetRequestNoEncoding(strRequest,webProxy); +// httpNetResponse=HttpNetRequest.GetRequestNoEncoding(strRequest,webProxy); + httpNetResponse=HttpNetRequest.GetRequestNoEncodingV3C(strRequest,webProxy); // user agents refreshed in this version //MDTrace.WriteLine(LogLevel.DEBUG,String.Format("{0}:{1}",(int)httpNetResponse.StatusCode,httpNetResponse.StatusCode)); if(!httpNetResponse.Success) { @@ -4890,10 +4764,12 @@ namespace MarketData.Helper MDTrace.WriteLine(LogLevel.DEBUG,String.Format("*** ALL SOURCES FAILED TO RETRIEVE PRICE FOR SYMBOL *** {0}",symbol)); return null; } +// *************************************************************************************************************** +// ******************** L A T E S T P R I C E R E T R I E V A L F O R I N T R A - D A Y F E E D ******** +// *************************************************************************************************************** -// This should be proxied through TOR because Yahoo tracks IP address. -// This is used in the intra-day price feed - private static Price GetLatestPriceYahoo(String symbol) + + public static Price GetLatestPriceYahoo(String symbol) { HttpNetResponse httpNetResponse=null; try @@ -4904,15 +4780,16 @@ namespace MarketData.Helper sb.Append("https://finance.yahoo.com/quote/").Append(symbol).Append("/?=").Append(symbol); strRequest=sb.ToString(); WebProxy webProxy=HttpNetRequest.GetProxy("GetLatestPriceYahoo"); - httpNetResponse=HttpNetRequest.GetRequestNoEncoding(strRequest,webProxy); + httpNetResponse=HttpNetRequest.GetRequestNoEncodingV3A(strRequest, webProxy); if(!httpNetResponse.Success) { - MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode)); + MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetLatestPriceYahoo: Request:{0} failed with status {1}",httpNetResponse.Request,httpNetResponse.StatusCode)); return null; } Price price=new Price(); price.Symbol=symbol.ToUpper(); price.Date=DateTime.Now; + price.Close=price.AdjClose=price.Open=price.High=price.Low=double.NaN; price.Source=Price.PriceSource.Yahoo; return BuildPriceFromResponse(httpNetResponse.ResponseString,price); } @@ -4926,8 +4803,250 @@ namespace MarketData.Helper if(null!=httpNetResponse) httpNetResponse.Dispose(); } } + + public static Price BuildPriceFromResponse(String httpNetResponse,Price price) + { + if(httpNetResponse.Contains("HTML_TAG_END")) + { + price = BuildPriceFromSections(httpNetResponse, price); + } + else + { + price = BuildPriceFromTable(httpNetResponse, price); + } + return price; + } + + private static Price BuildPriceFromSections(String httpNetResponse,Price price) + { + try + { + String strResponse=Utility.KeepAfter(httpNetResponse,"HTML_TAG_END"); + List sections = null; + + sections = Sections.GetAllItemsInSections(httpNetResponse, "fin-streamer"); + + String marker ="data-symbol=\""+price.Symbol+"\""; + List dataSections = sections.Where(x => x.Contains(marker)).ToList(); + dataSections = dataSections.Where(x => x.Contains(" data-field=\"regularMarketPrice\"")).ToList(); + if(dataSections.Count>0) + { + dataSections = Sections.GetSections(dataSections[0]); + if(null!=dataSections && dataSections.Count>0) + { + price.Close=price.AdjClose=FeedParser.ParseValue(dataSections[1]); + } + } + + marker = " x.StartsWith(marker)).ToList(); + for(int index=0;index sections = null; + double bid = double.NaN; + double bidContracts = double.NaN; + double ask = double.NaN; + double askContracts = double.NaN; + + String symbol=price.Symbol.ToUpper(); + byte[] streamBytes = Encoding.ASCII.GetBytes(httpNetResponse); + memoryStream = new MemoryStream(streamBytes); + HtmlDocument htmlDocument = new HtmlDocument(); + htmlDocument.Load(memoryStream); + + HtmlNodeCollection table = htmlDocument.DocumentNode.SelectNodes("//*[@class=\"W(100%)\"]"); + if(null == table || 0 == table.Count)return null; + HtmlNodeCollection rows = table[0].SelectNodes(".//tr"); + if (null == rows || 0 == rows.Count) return null; + for(int index=0;index< rows.Count; index++) + { + int itemIndex=0; + String itemValue=null; + String innerHtml = rows[index].InnerHtml; + sections=Sections.GetSections(innerHtml); + + if(Sections.FindInSections(sections, "Previous Close",0, ref itemIndex, true)) + { + itemValue = sections[itemIndex+3]; + } + else if(Sections.FindInSections(sections, "Open",0, ref itemIndex, true)) + { + itemValue = sections[itemIndex+3]; + price.Open=FeedParser.ParseValue(itemValue); + } + else if(Sections.FindInSections(sections, "Bid",0, ref itemIndex, true)) + { + itemValue = sections[itemIndex+3]; + String[] bidContractSplit = itemValue.Split('x'); + bid=FeedParser.ParseValue(bidContractSplit[0]); + bidContracts=FeedParser.ParseValue(bidContractSplit[1]); + } + else if(Sections.FindInSections(sections, "Ask",0, ref itemIndex, true)) + { + itemValue = sections[itemIndex+3]; + String[] askContractSplit = itemValue.Split('x'); + ask = FeedParser.ParseValue(askContractSplit[0]); + askContracts = FeedParser.ParseValue(askContractSplit[1]); + } + else if(Sections.FindInSections(sections, "Day's Range",0, ref itemIndex, true)) + { + itemValue = sections[itemIndex+3]; + String[] subItems=itemValue.Split('-'); + price.Low=FeedParser.ParseValue(subItems[0]); + price.High=FeedParser.ParseValue(subItems[1]); + } + else if(Sections.FindInSections(sections, "52 Week Range",0, ref itemIndex, true)) + { + itemValue = sections[itemIndex+3]; + } + else if(Sections.FindInSections(sections, "Volume",0, ref itemIndex, true)) + { + itemValue = sections[itemIndex+4]; + price.Volume=FeedParser.ParseValueLong(itemValue); + } + else if(Sections.FindInSections(sections, "Avg. Volume",0, ref itemIndex, true)) + { + itemValue = sections[itemIndex+3]; + } + } + + if (double.IsNaN(price.Close)) + { + sections = Sections.GetAllItemsInSections(httpNetResponse, "fin-streamer"); + String marker = "data-symbol=\"" + price.Symbol + "\""; + sections = sections.Where(x => x.Contains(marker)).ToList(); + sections = sections.Where(x => x.Contains(" data-field=\"regularMarketPrice\"")).ToList(); + if (sections.Count > 0) + { + sections = Sections.GetSections(sections[0]); + if (null != sections && sections.Count > 0) + { + price.Close = price.AdjClose = FeedParser.ParseValue(sections[0]); + } + } + } + + if(double.IsNaN(price.Close)) + { + double bidAskPrice=CalculateBidAskPrice(price.Symbol,bid,bidContracts,ask,askContracts); + if(!double.IsNaN(bidAskPrice))price.Close=price.AdjClose=bidAskPrice; + if(double.IsNaN(price.Close) && 0!=price.High && 0!=price.Low)price.Close=price.AdjClose=(price.High+price.Low)/2.00; + } + + CheckPrice(price); + return price; + } + catch(Exception exception) + { + MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Exception: {0}",exception.ToString())); + return null; + } + finally + { + if(null!=memoryStream) + { + memoryStream.Close(); + memoryStream.Dispose(); + memoryStream=null; + } + } + } + + private static double CalculateBidAskPrice(String symbol,double bid,double bidContracts,double ask, double askContracts) + { + if(double.IsNaN(bid) || double.IsNaN(bidContracts) || double.IsNaN(ask) || double.IsNaN(askContracts) || 0==askContracts || 0==bidContracts)return double.NaN; + double totalContracts = bidContracts + askContracts; + double bidWeight = bidContracts/totalContracts; + double askWeight = askContracts/totalContracts; + double weightedBid = bid*bidWeight; + double weightedAsk = ask*askWeight; + MDTrace.WriteLine(LogLevel.DEBUG,String.Format("{0} Bid:{1} x {2} Ask:{3} x {4} Price:{5}",symbol,Utility.FormatCurrency(bid,2),Utility.FormatNumber(bidContracts,0,true),Utility.FormatCurrency(ask,2),Utility.FormatNumber(askContracts,0,true),Utility.FormatCurrency(weightedBid + weightedAsk,2))); + return weightedBid + weightedAsk; + } + + public static void CheckPrice(Price price) + { + if(double.IsNaN(price.Close)&&!double.IsNaN(price.High)&&!double.IsNaN(price.Low)) + { + price.Close=price.AdjClose=(price.High+price.Low)/2.00; + } + if(!double.IsNaN(price.High)&&!double.IsNaN(price.Low)&&0.00!=price.High&&0.00!=price.Low) + { // if the close price is garbage set it to the midpoint of the high and the low + if(price.Closeprice.High||double.IsNaN(price.Close))price.Close=(price.High+price.Low)/2.00; + } + if(double.IsNaN(price.AdjClose))price.AdjClose=price.Close; + if(double.IsNaN(price.Open))price.Open=0.00; + if(double.IsNaN(price.Low))price.Low=0.00; + if(double.IsNaN(price.High))price.High=0.00; + if(0!=price.Close&&0==price.Open&&0==price.High&&0==price.Low) + { + price.Open=price.High=price.Low=price.Close; + } + } + // This is used in the intra-day price feed. - private static Price GetLatestPriceBigCharts(String symbol) + public static Price GetLatestPriceBigCharts(String symbol) { HttpNetResponse httpNetResponse=null; MemoryStream memoryStream=null; @@ -5123,6 +5242,12 @@ namespace MarketData.Helper if(null!=memoryStream){memoryStream.Close();memoryStream.Dispose();} } } +// ******************************************************************************************************************************************************************************* +// ******************************************************************************************************************************************************************************* +// ******************************************************************************************************************************************************************************* + + + // ******************************************************************************************************************************************************************************* // ************************************************************** H I S T O R I C A L P R I C E S - B I G C H A R T S ******************************************************** // ******************************************************************************************************************************************************************************* @@ -5349,88 +5474,6 @@ namespace MarketData.Helper // ****************************************************************************************************************************************************************************************************************** -// This function attempts to locate the quote data within the raw data returned from Yahoo. It should be interpreted as intraday data because -// the market date cannot be located within the raw data response. - public static Price BuildPriceFromResponse(String httpNetResponse,Price price) - { - String strResponse=Utility.KeepAfter(httpNetResponse,"/* -- Data -- */"); - String symbol=price.Symbol.ToUpper(); - bool isMutualFund=false; - String findPattern1="\"quoteType\":\"MUTUALFUND\",\"symbol\":"+"\""+symbol+"\""; - String findPattern2="\"quoteType\":\"EQUITY\",\"symbol\":"+"\""+symbol+"\""; - String findPattern3="\"quoteType\":\"ETF\",\"symbol\":"+"\""+symbol+"\""; - String findPattern4="\"quoteType\":\"INDEX\",\"symbol\":"+"\""+symbol+"\""; - strResponse=Utility.KeepAfter(httpNetResponse,findPattern1); - if(null!=strResponse)isMutualFund=true; - if(null==strResponse)strResponse=Utility.KeepAfter(httpNetResponse,findPattern2); - if(null==strResponse)strResponse=Utility.KeepAfter(httpNetResponse,findPattern3); - if(null==strResponse)strResponse=Utility.KeepAfter(httpNetResponse,findPattern4); - if(null==strResponse)return null; - strResponse=Utility.KeepBefore(strResponse,"MOST_ACTIVE_TITLE"); - price.Open=FeedParser.ParseValue(Utility.FindFirst(strResponse,"\"regularMarketOpen\":{\"raw\":",',')); - price.High=FeedParser.ParseValue(Utility.FindFirst(strResponse,"\"regularMarketDayHigh\":{\"raw\":",',')); - price.Low=FeedParser.ParseValue(Utility.FindFirst(strResponse,"\"regularMarketDayLow\":{\"raw\":",',')); - price.Close=FeedParser.ParseValue(Utility.FindFirst(Utility.KeepAfterLast(strResponse,"\"regularMarketDayLow\":{\"raw\":"),"\"regularMarketPrice\":{\"raw\":",',')); - price.Volume=FeedParser.ParseValueLong(Utility.FindFirst(strResponse,"\"regularMarketVolume\":{\"raw\":",',')); - if(double.NaN.Equals(price.Close))price.Close=FeedParser.ParseValue(Utility.FindFirst(strResponse,"\"regularMarketPrice\":{\"raw\":",',')); - if(double.IsNaN(price.Close)) // try to find raw quote data - { - String strQuote=Utility.KeepAfter(httpNetResponse,"",'<')); - else if(null==strQuote&&httpNetResponse.Contains("data-reactid=\"38\">As of")) - { - strQuote=Utility.KeepAfter(httpNetResponse,"class=\"Trsdu(0.3s) Fw(b) Fz(36px) Mb(-4px) D(ib)\""); - if(null!=strQuote)price.Close=FeedParser.ParseValue(Utility.FindFirst(strQuote,">",'<')); - } - if(!double.IsNaN(price.Close))MDTrace.WriteLine(LogLevel.DEBUG,String.Format("Found quote data for {0}->{1}",symbol,Utility.FormatCurrency(price.Close))); - } - if(double.IsNaN(price.Close)) - { - double bid=FeedParser.ParseValue(Utility.FindFirst(strResponse,"\"bid\":{\"raw\":",',')); - double ask=FeedParser.ParseValue(Utility.FindFirst(strResponse,"\"ask\":{\"raw\":",',')); - if(!double.IsNaN(bid)&&!double.IsNaN(ask))price.Close=(bid+ask)/2.00; - } - if(double.IsNaN(price.Close)&&!double.IsNaN(price.High)&&!double.IsNaN(price.Low)) - { - price.Close=price.AdjClose=(price.High+price.Low)/2.00; - } - if(double.IsNaN(price.Open)&&double.IsNaN(price.High)&&double.IsNaN(price.Low)&&double.IsNaN(price.Close)) - { -// if this is a mutual fund then the regularMarketPreviousClose may actually be the close from two days ago and not last night. Goto BigCharts in this case - if(isMutualFund)price=GetPriceAsOf(symbol,DateTime.Now); - else price.Close=FeedParser.ParseValue(Utility.FindFirst(strResponse,"\"regularMarketPreviousClose\":{\"raw\":",',')); - if(null==price||double.IsNaN(price.Close)) - { - MDTrace.WriteLine(LogLevel.DEBUG,String.Format("*** No closing price for {0}",symbol)); - return null; - } - price.Open=price.High=price.Low=price.AdjClose=price.Close; - return price; - } - if(!double.IsNaN(price.High)&&!double.IsNaN(price.Low)&&0.00!=price.High&&0.00!=price.Low) - { // if the close price is garbage set it to the midpoint of the high and the low - if(price.Closeprice.High||double.IsNaN(price.Close))price.Close=(price.High+price.Low)/2.00; - } - price.AdjClose=price.Close; - if(double.IsNaN(price.Open)&&double.IsNaN(price.High)&&double.IsNaN(price.Low)&&double.IsNaN(price.Close)) - { - MDTrace.WriteLine(LogLevel.DEBUG,String.Format("*** No open,high,low,close price for {0}",price.Symbol)); - return null; - } - if(double.IsNaN(price.Open))price.Open=0.00; - if(double.IsNaN(price.Low))price.Low=0.00; - if(double.IsNaN(price.High))price.High=0.00; - if(0!=price.Close&&0==price.Open&&0==price.High&&0==price.Low) - { - price.Open=price.High=price.Low=price.Close; - } - return price; - } // **************************************************************************************************************************************************************************************** // **************************************************************************** E N D D A T A F E E D S ********************************************************************************* // **************************************************************************************************************************************************************************************** diff --git a/MarketDataLib/Integration/HttpNetRequest.cs b/MarketDataLib/Integration/HttpNetRequest.cs index 3aa35cc..1aac6e5 100644 --- a/MarketDataLib/Integration/HttpNetRequest.cs +++ b/MarketDataLib/Integration/HttpNetRequest.cs @@ -688,6 +688,97 @@ namespace MarketData.Integration MDTrace.WriteLine(LogLevel.VERBOSE,"GetRequestNoEncodingV3B[LEAVE]"); } } + public static HttpNetResponse GetRequestNoEncodingV3C(String strRequest,WebProxy webProxy=null) + { + HttpWebResponse webResponse=null; + WebException lastWebException=null; + ServicePointManager.Expect100Continue=true; + ServicePointManager.SecurityProtocol=SecurityProtocolType.Ssl3|SecurityProtocolType.Tls12|SecurityProtocolType.Tls11|SecurityProtocolType.Tls; + Random random=new Random(); + + try + { + MDTrace.WriteLine(LogLevel.VERBOSE,String.Format("GetRequestNoEncodingV3A[ENTER]{0}",strRequest)); + String[] userAgents= + { + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.3", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.3.1 Safari/605.1.1", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.6 Safari/605.1.1", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.3" + }; + int MAX_RETRIES=5; + int TIMEOUT_BETWEEN_RETRIES=1000; + int charCount=0; + byte[] buffer=new byte[8192]; + StringBuilder sb=new StringBuilder(); + + for(int count=0;count=userAgents.Length) index=userAgents.Length-1; + String userAgent=userAgents[index]; + webRequest.UserAgent=userAgent; + + try + { + webResponse=(HttpWebResponse)webRequest.GetResponse(); + Stream responseStream=webResponse.GetResponseStream(); + if(webResponse.ContentEncoding.ToLower().Contains("gzip")) + { + responseStream=new GZipStream(responseStream,CompressionMode.Decompress); + StreamReader reader=new StreamReader(responseStream,Encoding.Default); + sb.Append(reader.ReadToEnd()); + reader.Close(); + } + else if(webResponse.ContentEncoding.ToLower().Contains("deflate")) + { + responseStream=new DeflateStream(responseStream,CompressionMode.Decompress); + StreamReader reader=new StreamReader(responseStream,Encoding.Default); + sb.Append(reader.ReadToEnd()); + reader.Close(); + } + else + { + while(true) + { + charCount=responseStream.Read(buffer,0,buffer.Length); + if(0==charCount) break; + sb.Append(Encoding.ASCII.GetString(buffer,0,charCount)); + } + } + webResponse.Close(); + return new HttpNetResponse(sb.ToString(),strRequest,webResponse,true); + } + catch(WebException webException) + { + if(webException.Message.Contains("(404) Not Found")) return new HttpNetResponse(webResponse,strRequest,false,null==lastWebException?"":lastWebException.Message); + lastWebException=webException; + Thread.Sleep(TIMEOUT_BETWEEN_RETRIES); + } + } + MDTrace.WriteLine(LogLevel.DEBUG,String.Format("General failure with {0}",lastWebException.Message)); + return new HttpNetResponse(webResponse,strRequest,false,lastWebException.Message); + } + catch(Exception exception) + { + return new HttpNetResponse(webResponse,strRequest,false,exception.Message); + } + finally + { + MDTrace.WriteLine(LogLevel.VERBOSE,"GetRequestNoEncodingV3A[LEAVE]"); + } + } public static HttpNetResponse GetRequestNoEncodingV4(String strRequest,CookieCollection cookieCollection=null,WebProxy webProxy=null) { HttpWebResponse webResponse=null; @@ -796,6 +887,7 @@ namespace MarketData.Integration MDTrace.WriteLine(LogLevel.VERBOSE,"GetRequestNoEncodingV4[LEAVE]"); } } + public static HttpNetResponse GetRequestNoEncodingV5(String strRequest,int webRequestTimeoutMS,WebProxy webProxy=null) { HttpWebResponse webResponse = null; @@ -844,55 +936,73 @@ namespace MarketData.Integration MDTrace.WriteLine(LogLevel.VERBOSE, "GetRequestNoEncodingV5[LEAVE]"); } } -// public static HttpNetResponse GetRequestNoEncodingV5A(String strRequest,int webRequestTimeoutMS,WebProxy webProxy=null) -// { -// HttpWebResponse webResponse = null; -// try -// { -// MDTrace.WriteLine(LogLevel.VERBOSE, String.Format("GetRequestNoEncodingV5[ENTER]{0}", strRequest)); -// int charCount = 0; -// byte[] buffer = new byte[8192]; -// StringBuilder sb = new StringBuilder(); -// bool expect100Condition = ServicePointManager.Expect100Continue; -// SecurityProtocolType securityProtocolType = ServicePointManager.SecurityProtocol; -// ServicePointManager.Expect100Continue = true; -// ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12; -// HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(new Uri(strRequest)); -// if(null!=webProxy)webRequest.Proxy=webProxy; -// webRequest.Timeout = webRequestTimeoutMS; -// webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5"); -// webRequest.Headers.Add("Accept-Encoding: *"); -// webRequest.Accept = "image/gif, image/jpeg, image/pjpeg, image/pjpeg, application/x-shockwave-flash, application/x-ms-application, application/x-ms-xbap, application/vnd.ms-xpsdocument, application/xaml+xml, application/vnd.ms-excel, application/vnd.ms-powerpoint, application/msword, */*"; -//// webRequest.UserAgent = "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR 3.0.04506.648; .NET CLR 3.5.21022; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; MS-RTC LM 8; .NET4.0C; .NET4.0E; InfoPath.3)"; -// webRequest.UserAgent = GetUserAgent(true); -// webRequest.KeepAlive = true; -// webRequest.CookieContainer = new CookieContainer(); -// webResponse = (HttpWebResponse)webRequest.GetResponse(); -// Stream responseStream = webResponse.GetResponseStream(); -// while (true) -// { -// charCount = responseStream.Read(buffer, 0, buffer.Length); -// if (0 == charCount) break; -// sb.Append(Encoding.ASCII.GetString(buffer, 0, charCount)); -// } -// webResponse.Close(); -// ServicePointManager.Expect100Continue = expect100Condition; -// ServicePointManager.SecurityProtocol = securityProtocolType; -// return new HttpNetResponse(sb.ToString(), strRequest, webResponse, webResponse.Cookies, true); -// } -// catch (WebException webException) -// { -// return new HttpNetResponse((HttpWebResponse)webException.Response, strRequest, false, webException.Message); -// } -// catch (Exception exception) -// { -// return new HttpNetResponse(webResponse, strRequest, false, exception.Message); -// } -// finally -// { -// MDTrace.WriteLine(LogLevel.VERBOSE, "GetRequestNoEncodingV5[LEAVE]"); -// } -// } + + public static HttpNetResponse GetRequestNoEncodingV5A(String strRequest,int webRequestTimeoutMS,WebProxy webProxy=null) + { + HttpWebResponse webResponse = null; + try + { + Random random = new Random(); + String[] userAgents= + { + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.0", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:82.0) Gecko/20100101 Firefox/82.0" + }; + + MDTrace.WriteLine(LogLevel.VERBOSE, String.Format("GetRequestNoEncodingV5[ENTER]{0}", strRequest)); + int charCount = 0; + byte[] buffer = new byte[8192]; + StringBuilder sb = new StringBuilder(); + bool expect100Condition = ServicePointManager.Expect100Continue; + SecurityProtocolType securityProtocolType = ServicePointManager.SecurityProtocol; + ServicePointManager.Expect100Continue = true; + ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12; + HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(new Uri(strRequest)); + if(null!=webProxy)webRequest.Proxy=webProxy; + webRequest.Timeout = webRequestTimeoutMS; + webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5"); + webRequest.Headers.Add("Accept-Encoding: None"); + webRequest.Accept = "image/gif, image/jpeg, image/pjpeg, image/pjpeg, application/x-shockwave-flash, application/x-ms-application, application/x-ms-xbap, application/vnd.ms-xpsdocument, application/xaml+xml, application/vnd.ms-excel, application/vnd.ms-powerpoint, application/msword, */*"; +// webRequest.UserAgent = "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR 3.0.04506.648; .NET CLR 3.5.21022; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; MS-RTC LM 8; .NET4.0C; .NET4.0E; InfoPath.3)"; + webRequest.KeepAlive = true; + webRequest.CookieContainer = new CookieContainer(); + + double randomNumber=random.NextDouble(); // number between 0 and 1 + int index=(int)(randomNumber*((double)userAgents.Length)); + if(index>=userAgents.Length) index=userAgents.Length-1; + String userAgent=userAgents[index]; + webRequest.UserAgent=userAgent; + + + webResponse = (HttpWebResponse)webRequest.GetResponse(); + Stream responseStream = webResponse.GetResponseStream(); + while (true) + { + charCount = responseStream.Read(buffer, 0, buffer.Length); + if (0 == charCount) break; + sb.Append(Encoding.ASCII.GetString(buffer, 0, charCount)); + } + webResponse.Close(); + ServicePointManager.Expect100Continue = expect100Condition; + ServicePointManager.SecurityProtocol = securityProtocolType; + return new HttpNetResponse(sb.ToString(), strRequest, webResponse, webResponse.Cookies, true); + } + catch (WebException webException) + { + return new HttpNetResponse((HttpWebResponse)webException.Response, strRequest, false, webException.Message); + } + catch (Exception exception) + { + return new HttpNetResponse(webResponse, strRequest, false, exception.Message); + } + finally + { + MDTrace.WriteLine(LogLevel.VERBOSE, "GetRequestNoEncodingV5[LEAVE]"); + } + } + + + public static HttpNetResponse GetRequestNoEncodingV6(String strRequest, int webRequestTimeoutMS) { HttpWebResponse webResponse = null; diff --git a/MarketDataLib/Utility/FeedParser.cs b/MarketDataLib/Utility/FeedParser.cs index d70b937..c0bee2f 100644 --- a/MarketDataLib/Utility/FeedParser.cs +++ b/MarketDataLib/Utility/FeedParser.cs @@ -231,6 +231,12 @@ namespace MarketData.Utils multiplier = -1.00; } if (strText.Equals("-")) return double.NaN; + if (strText[strText.Length - 1].Equals('T')) + { + strText = strText.Replace("T", ""); + value = double.Parse(strText); + value *= 1000000000000; + } if (strText[strText.Length - 1].Equals('B')) { strText = strText.Replace("B", ""); diff --git a/MarketDataLib/Utility/Sections.cs b/MarketDataLib/Utility/Sections.cs index 2fc1965..3d50c66 100644 --- a/MarketDataLib/Utility/Sections.cs +++ b/MarketDataLib/Utility/Sections.cs @@ -3,6 +3,7 @@ using System.Collections.Generic; using System.Linq; using System.Text; using System.Threading.Tasks; +using MarketData.Utils; namespace MarketDataLib.Utility { @@ -28,6 +29,24 @@ namespace MarketDataLib.Utility if(String.IsNullOrEmpty(strContainingString))return null; return strContainingString; } + + public static List GetAllItemsInSections(String strInput, String sectionName) + { + int searchIndex = 0; + List sectionItems = new List(); + + while (true) + { + String itemsInSection = GetItemsInSection(strInput, sectionName, ref searchIndex); + if(null==itemsInSection)break; + sectionItems.Add(itemsInSection); + searchIndex++; + } + return sectionItems; + } + + + public static List GetSections(String strInput) { try @@ -36,6 +55,7 @@ namespace MarketDataLib.Utility int index=0; String strLiteral=null; + if(null==strInput)return null; while(index", "<"); + return strItem; + } + private static String ScanSection(String strInput, ref int index) { try