using System;
using System.IO;
using System.Net;
using System.Text;
using System.Threading;
using System.IO.Compression;
using System.Configuration;
// Filename: HttpRequest.cs
// Author:Sean Kessler

namespace IPMonitor
{
  public class HttpNetResponse : IDisposable
  {
    public enum TypeResponse {Stream,String}; 
    public String ResponseString{get;set;}
    public MemoryStream ResponseStream{get;set;}
    public CookieCollection CookieCollection{get;set;}

    public HttpNetResponse()
    {
      Success=false;
    }
    public HttpNetResponse(String responseString,String request,HttpWebResponse webResponse,CookieCollection cookieCollection,bool success)
    {
      ResponseString=responseString;
      ResponseType=TypeResponse.String;
      Success=success;
      Request=request;
      CookieCollection=cookieCollection;
      if(null!=webResponse)StatusCode=webResponse.StatusCode;
    }
    public HttpNetResponse(String responseString,String request,HttpWebResponse webResponse,bool success)
    {
      ResponseString=responseString;
      ResponseType=TypeResponse.String;
      Success=success;
      Request=request;
      if(null!=webResponse)StatusCode=webResponse.StatusCode;
    }
    public HttpNetResponse(MemoryStream responseStream,String request,HttpWebResponse webResponse,bool success)
    {
      ResponseStream=responseStream;
      ResponseType=TypeResponse.String;
      Success=success;
      Request=request;
      if(null!=webResponse)StatusCode=webResponse.StatusCode;
    }
    public HttpNetResponse(HttpWebResponse webResponse,String request,bool success,String errorMessage)
    {
      ResponseType=TypeResponse.String;
      Success=success;
      Request=request;
      if(null!=webResponse)StatusCode=webResponse.StatusCode;
      ErrorMessage=errorMessage;
    }
    public Stream ResponseStringToStream()
    {
      MemoryStream stream = new MemoryStream();
      StreamWriter writer = new StreamWriter(stream);
      writer.Write(ResponseString);
      writer.Flush();
      stream.Position = 0;
      return stream;
    }
    public String ResponseStreamToString
    {
      get
      {
        if(!Success)return null;
        return System.Text.Encoding.UTF8.GetString(ResponseStream.ToArray());
      }
    }
    public TypeResponse ResponseType{get;set;}
    public bool Success{get;set;}
    public String ErrorMessage{get;set;}
    public HttpStatusCode StatusCode{get;set;}
    public String Request{get;set;}
    public bool IsNetErrorResponse
    {
      get
      {
        return (int)StatusCode>=400?true:false;
      }
    }
    public void Dispose()
    {
      if(null!=ResponseStream)
      {
        ResponseStream.Close();
        ResponseStream.Dispose();
        ResponseStream=null;
      }
    }
  }
// *********************************************************************************************************************************************************************

  public class HttpNetRequest
  {
    public static int REQUEST_TIMEOUT = 300000;
    private HttpNetRequest()
    {
    }
		public static HttpNetResponse GetRequestStreamZIP(String strRequest)
		{
      HttpWebResponse webResponse=null;
      try
      {
        MDTrace.WriteLine(LogLevel.VERBOSE,String.Format("GetRequestStreamZIP[ENTER]{0}",strRequest));
        int charCount = 0;
        byte[] buffer = new byte[8192];
        StringBuilder sb = new StringBuilder();
        HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(new Uri(strRequest));
        webRequest.Timeout = REQUEST_TIMEOUT;
        webRequest.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
        webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5");
				webRequest.Headers.Add("Accept-Encoding: gzip, deflate");
        webRequest.UserAgent = "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:52.0) Gecko/20100101 Firefox/52.0";
        webRequest.KeepAlive = true;
        webResponse = (HttpWebResponse)webRequest.GetResponse();
        Stream responseStream = webResponse.GetResponseStream();
				MemoryStream memoryStream=new MemoryStream();
        while (true)
        {
          charCount = responseStream.Read(buffer, 0, buffer.Length);
					if(charCount<=0)break;
					memoryStream.Write(buffer,0,charCount);
        }
        webResponse.Close();
				memoryStream.Flush();
        return new HttpNetResponse(memoryStream,strRequest,webResponse,true);
      }
      catch (WebException webException)
      {
          return new HttpNetResponse((HttpWebResponse)webException.Response,strRequest,false,webException.Message);
      }
      catch (Exception exception)
      {
        return new HttpNetResponse(webResponse,strRequest,false,exception.Message);
      }
      finally
      {                                                                     
        MDTrace.WriteLine(LogLevel.VERBOSE,"GetRequestStreamZIP[LEAVE]");
      }
		}
		public static HttpNetResponse GetRequestStreamZIPV2(String strRequest)
		{
      HttpWebResponse webResponse=null;
      try
      {
        MDTrace.WriteLine(LogLevel.VERBOSE,String.Format("GetRequestStreamZIPV2[ENTER]{0}",strRequest));
        int charCount = 0;
        byte[] buffer = new byte[8192];
        StringBuilder sb = new StringBuilder();
        HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(new Uri(strRequest));
        webRequest.Timeout = REQUEST_TIMEOUT;
        webRequest.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
        webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5");
				webRequest.Headers.Add("Accept-Encoding: gzip, deflate");
        webRequest.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:59.0) Gecko/20100101 Firefox/59.0";
        webRequest.KeepAlive = true;
        webRequest.CookieContainer=new CookieContainer();
        webResponse = (HttpWebResponse)webRequest.GetResponse();
        Stream responseStream = webResponse.GetResponseStream();
				MemoryStream memoryStream=new MemoryStream();
        while (true)
        {
          charCount = responseStream.Read(buffer, 0, buffer.Length);
					if(charCount<=0)break;
					memoryStream.Write(buffer,0,charCount);
        }
        webResponse.Close();
				memoryStream.Flush();
        return new HttpNetResponse(memoryStream,strRequest,webResponse,true);
      }
      catch (WebException webException)
      {
          return new HttpNetResponse((HttpWebResponse)webException.Response,strRequest,false,webException.Message);
      }
      catch (Exception exception)
      {
        return new HttpNetResponse(webResponse,strRequest,false,exception.Message);
      }
      finally
      {
        MDTrace.WriteLine(LogLevel.VERBOSE,"GetRequestStreamZIPV2[LEAVE]");
      }
		}
		public static HttpNetResponse GetRequestStreamCSV(String strRequest,CookieCollection cookieCollection=null, WebProxy webProxy=null)
		{
      HttpWebResponse webResponse=null;
      try
      {
        ServicePointManager.Expect100Continue=true;
        ServicePointManager.SecurityProtocol=SecurityProtocolType.Ssl3|SecurityProtocolType.Tls12|SecurityProtocolType.Tls11|SecurityProtocolType.Tls;
        MDTrace.WriteLine(LogLevel.VERBOSE,String.Format("GetRequestStreamCSV[ENTER]{0}",strRequest));
        int charCount = 0;
        byte[] buffer = new byte[8192];
        StringBuilder sb = new StringBuilder();
        HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(new Uri(strRequest));
        if(null!=webProxy)
        {
          webRequest.Proxy=webProxy;
        }
        if(null!=cookieCollection)
        {
          webRequest.CookieContainer=new CookieContainer();
          webRequest.CookieContainer.Add(cookieCollection);
        }
        webRequest.Timeout = REQUEST_TIMEOUT;
        webRequest.Accept = "text/csv,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
        webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5");
				webRequest.Headers.Add("Accept-Encoding: gzip, deflate");    // 
        webRequest.UserAgent = "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:52.0) Gecko/20100101 Firefox/52.0";
        webRequest.KeepAlive = true;
        webResponse = (HttpWebResponse)webRequest.GetResponse();
        Stream responseStream = webResponse.GetResponseStream();

        if(webResponse.ContentEncoding.ToLower().Contains("gzip"))
        {
          responseStream = new GZipStream(responseStream, CompressionMode.Decompress);          
          StreamReader reader = new StreamReader(responseStream, Encoding.Default);
          sb.Append(reader.ReadToEnd());
          reader.Close();
        }
        else if(webResponse.ContentEncoding.ToLower().Contains("deflate"))
        {
          responseStream = new DeflateStream(responseStream, CompressionMode.Decompress);
          StreamReader reader = new StreamReader(responseStream, Encoding.Default);
          sb.Append(reader.ReadToEnd());
          reader.Close();
        }
        else
        {
          while (true)
          {
            charCount = responseStream.Read(buffer, 0, buffer.Length);
            if (0 == charCount) break;
            sb.Append(Encoding.ASCII.GetString(buffer, 0, charCount));
          }
        }
        webResponse.Close();
        return new HttpNetResponse(sb.ToString(),strRequest,webResponse,true);
      }
      catch (WebException webException)
      {
          return new HttpNetResponse((HttpWebResponse)webException.Response,strRequest,false,webException.Message);
      }
      catch (Exception exception)
      {
        return new HttpNetResponse(webResponse,strRequest,false,exception.Message);
      }
      finally
      {
        MDTrace.WriteLine(LogLevel.VERBOSE,"GetRequestStreamCSV[LEAVE]");
      }
		}

    public static HttpNetResponse GetRequestNoEncoding(Uri uriRequest,WebProxy webProxy=null)
    {
      HttpWebResponse webResponse=null;
      try
      {
				ServicePointManager.Expect100Continue = true;
        ServicePointManager.SecurityProtocol=SecurityProtocolType.Ssl3|SecurityProtocolType.Tls12|SecurityProtocolType.Tls11|SecurityProtocolType.Tls;
			  MDTrace.WriteLine(LogLevel.VERBOSE,String.Format("GetRequestNoEncoding[ENTER]{0}",uriRequest.ToString()));
        int charCount = 0;
        byte[] buffer = new byte[8192];
        StringBuilder sb = new StringBuilder();
        HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(uriRequest);
        if(null!=webProxy)webRequest.Proxy=webProxy;
        webRequest.Timeout = REQUEST_TIMEOUT;
        webRequest.Accept = "text/html, text/csv";
        webRequest.ContentType="text/csv; charset=utf-8";
        webRequest.Headers.Add("Accept-Encoding: None");
        webRequest.Headers.Add("Accept-Language: en-US");
        webRequest.UserAgent = "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:52.0) Gecko/20100101 Firefox/52.0";
        webRequest.KeepAlive = true;
        webResponse = (HttpWebResponse)webRequest.GetResponse();
        Stream responseStream = webResponse.GetResponseStream();
        while (true)
        {
          charCount = responseStream.Read(buffer, 0, buffer.Length);
          if (0 >= charCount) break;
          sb.Append(Encoding.ASCII.GetString(buffer, 0, charCount));
        }
        webResponse.Close();
        return new HttpNetResponse(sb.ToString(),uriRequest.ToString(),webResponse,true);
      }
      catch (WebException webException)
      {
          return new HttpNetResponse((HttpWebResponse)webException.Response,uriRequest.ToString(),false,webException.Message);
      }
      catch (Exception exception)
      {
        return new HttpNetResponse(webResponse,uriRequest.ToString(),false,exception.Message);
      }
      finally
      {
        MDTrace.WriteLine(LogLevel.VERBOSE,"GetRequestNoEncoding[LEAVE]");
      }
    }


    public static HttpNetResponse GetRequestNoEncoding(String strRequest,WebProxy webProxy=null)
    {
      HttpWebResponse webResponse=null;
      try
      {
				ServicePointManager.Expect100Continue = true;
        ServicePointManager.SecurityProtocol=SecurityProtocolType.Ssl3|SecurityProtocolType.Tls12|SecurityProtocolType.Tls11|SecurityProtocolType.Tls;
			  MDTrace.WriteLine(LogLevel.VERBOSE,String.Format("GetRequestNoEncoding[ENTER]{0}",strRequest));
        int charCount = 0;
        byte[] buffer = new byte[8192];
        StringBuilder sb = new StringBuilder();
        HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(new Uri(strRequest));
        if(null!=webProxy)webRequest.Proxy=webProxy;
        webRequest.Timeout = REQUEST_TIMEOUT;
        webRequest.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8";
        webRequest.ContentType="text/csv; charset=utf-8";
        webRequest.Headers.Add("Accept-Encoding: gzip, deflate, br");
        webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5");
        webRequest.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:123.0) Gecko/20100101 Firefox/123.0";
        webRequest.KeepAlive = true;
        webResponse = (HttpWebResponse)webRequest.GetResponse();
        Stream responseStream = webResponse.GetResponseStream();
        while (true)
        {
          charCount = responseStream.Read(buffer, 0, buffer.Length);
          if (0 >= charCount) break;
          sb.Append(Encoding.ASCII.GetString(buffer, 0, charCount));
        }
        webResponse.Close();
        return new HttpNetResponse(sb.ToString(),strRequest,webResponse,true);
      }
      catch (WebException webException)
      {
          return new HttpNetResponse((HttpWebResponse)webException.Response,strRequest,false,webException.Message);
      }
      catch (Exception exception)
      {
        return new HttpNetResponse(webResponse,strRequest,false,exception.Message);
      }
      finally
      {
        MDTrace.WriteLine(LogLevel.VERBOSE,"GetRequestNoEncoding[LEAVE]");
      }
    }

    public static HttpNetResponse GetRequestNoEncodingV1(String strRequest,WebProxy webProxy=null)
    {
      ServicePointManager.Expect100Continue=true;
      ServicePointManager.SecurityProtocol=SecurityProtocolType.Ssl3|SecurityProtocolType.Tls12|SecurityProtocolType.Tls11|SecurityProtocolType.Tls;
      HttpWebResponse webResponse=null;
      try
      {
        MDTrace.WriteLine(LogLevel.VERBOSE,String.Format("GetRequestNoEncoding[ENTER]{0}",strRequest));
        int charCount=0;
        byte[] buffer=new byte[8192];
        StringBuilder sb=new StringBuilder();
        HttpWebRequest webRequest=(HttpWebRequest)WebRequest.Create(new Uri(strRequest));
				if(null!=webProxy)webRequest.Proxy=webProxy;
        webRequest.Timeout=REQUEST_TIMEOUT;
        webRequest.Accept="text/html, text/csv";
        webRequest.ContentType="text/csv; charset=utf-8";
        webRequest.Headers.Add("Accept-Encoding: None");
        webRequest.Headers.Add("Accept-Language: en-US");
        webRequest.UserAgent="Mozilla/5.0 (Windows NT 10.0; WOW64; rv:52.0) Gecko/20100101 Firefox/52.0";
        webRequest.KeepAlive=true;
        webResponse=(HttpWebResponse)webRequest.GetResponse();
        Stream responseStream=webResponse.GetResponseStream();
        while(true)
        {
          charCount=responseStream.Read(buffer,0,buffer.Length);
          if(0>=charCount) break;
          sb.Append(Encoding.ASCII.GetString(buffer,0,charCount));
        }
        webResponse.Close();
        return new HttpNetResponse(sb.ToString(),strRequest,webResponse,true);
      }
      catch(WebException webException)
      {
        return new HttpNetResponse((HttpWebResponse)webException.Response,strRequest,false,webException.Message);
      }
      catch(Exception exception)
      {
        return new HttpNetResponse(webResponse,strRequest,false,exception.Message);
      }
      finally
      {
        MDTrace.WriteLine(LogLevel.VERBOSE,"GetRequestNoEncoding[LEAVE]");
      }
    }
    public static HttpNetResponse GetRequestCSV(String strRequest,String referer)
    {
      HttpWebResponse webResponse=null;
      try
      {
        MDTrace.WriteLine(LogLevel.VERBOSE,String.Format("GetRequestCSV[ENTER]{0}",strRequest));
        int charCount = 0;
        byte[] buffer = new byte[8192];
        StringBuilder sb = new StringBuilder();
        HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(new Uri(strRequest));
        webRequest.Timeout = REQUEST_TIMEOUT;
        webRequest.Accept="text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
        webRequest.ContentType="text/csv; charset=utf-8";
        webRequest.Headers.Add("Accept-Encoding: identity");   // "gzip, deflate"
        webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5");   
        webRequest.Headers.Add("Upgrade-Insecure-Requests: 1");
        webRequest.UserAgent="Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:66.0) Gecko/20100101 Firefox/66.0";
        webRequest.Referer=referer;
        webRequest.KeepAlive = true;
        webResponse = (HttpWebResponse)webRequest.GetResponse();
        Stream responseStream = webResponse.GetResponseStream();
        while (true)
        {
          charCount = responseStream.Read(buffer, 0, buffer.Length);
          if (0 == charCount) break;
          sb.Append(Encoding.ASCII.GetString(buffer, 0, charCount));
        }
        webResponse.Close();
        return new HttpNetResponse(sb.ToString(),strRequest,webResponse,true);
      }
      catch (WebException webException)
      {
          return new HttpNetResponse((HttpWebResponse)webException.Response,strRequest,false,webException.Message);
      }
      catch (Exception exception)
      {
        return new HttpNetResponse(webResponse,strRequest,false,exception.Message);
      }
      finally
      {
        MDTrace.WriteLine(LogLevel.VERBOSE,"GetRequestCSV[LEAVE]");
      }
    }
    public static HttpNetResponse GetRequestNoEncodingV2(String strRequest)
    {
      HttpWebResponse webResponse=null;
      try
      {
        MDTrace.WriteLine(LogLevel.VERBOSE,String.Format("GetRequestNoEncodingV2[ENTER]{0}",strRequest));
        int charCount = 0;
        byte[] buffer = new byte[8192];
        StringBuilder sb = new StringBuilder();
        HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(new Uri(strRequest));
        webRequest.Timeout = REQUEST_TIMEOUT;
        webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5");
        webRequest.Headers.Add("Accept-Encoding: None");
        webRequest.UserAgent = "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:52.0) Gecko/20100101 Firefox/52.0";
        webRequest.Accept = "image/gif, image/jpeg, image/pjpeg, image/pjpeg, application/x-shockwave-flash, application/x-ms-application, application/x-ms-xbap, application/vnd.ms-xpsdocument, application/xaml+xml, application/vnd.ms-excel, application/vnd.ms-powerpoint, application/msword, */*";
        webRequest.KeepAlive = true;
        webResponse = (HttpWebResponse)webRequest.GetResponse();
        Stream responseStream = webResponse.GetResponseStream();
        while (true)
        {
          charCount = responseStream.Read(buffer, 0, buffer.Length);
          if (0 == charCount) break;
          sb.Append(Encoding.ASCII.GetString(buffer, 0, charCount));
        }
        webResponse.Close();
        return new HttpNetResponse(sb.ToString(),strRequest,webResponse,true);
      }
      catch (WebException webException)
      {
          return new HttpNetResponse((HttpWebResponse)webException.Response,strRequest,false,webException.Message);
      }
      catch (Exception exception)
      {
        return new HttpNetResponse(webResponse,strRequest,false,exception.Message);
      }
      finally
      {
        MDTrace.WriteLine(LogLevel.VERBOSE,"GetRequestNoEncodingV2[LEAVE]");
      }
    }
// I am using this code specifically on the seeking alpha web site.  it seems as though seeking alpha has implemeted some user agent based bot prevention mechanism on their website
// to prevent scrapers.  What I do here is to choose from a set of commonly used user agents and then randomly choose a user agent to code into the request.  retry logic up to 3 times
// and handling of 404 (not found)
    public static HttpNetResponse GetRequestNoEncodingV3(String strRequest,String referer=null)
    {
      HttpWebResponse webResponse=null;
      WebException lastWebException=null;
      
      try
      {
        MDTrace.WriteLine(LogLevel.VERBOSE,String.Format("GetRequestNoEncodingV3[ENTER]{0}",strRequest));
        Random random=new Random();
        String[] userAgents=
        {
          "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537."
        };
        int MAX_RETRIES=5;
        int TIMEOUT_BETWEEN_RETRIES=1000;
        int charCount = 0;
        byte[] buffer = new byte[8192];
        StringBuilder sb = new StringBuilder();

        for(int count=0;count<MAX_RETRIES;count++)
        {
          CookieContainer cookieContainer=new CookieContainer();
          HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(new Uri(strRequest));
          webRequest.CookieContainer=cookieContainer;
          webRequest.Timeout = REQUEST_TIMEOUT;
          webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5");
          webRequest.Headers.Add("Accept-Encoding: gzip,deflate");
          webRequest.Accept = "text/html";
          String userAgent=userAgents[random.Next(0,userAgents.Length-1)];
          userAgent+=random.Next(1,57).ToString();
          webRequest.UserAgent=userAgent;
          if(null!=referer)webRequest.Referer=referer;
          try
          {
            webResponse = (HttpWebResponse)webRequest.GetResponse();
            Stream responseStream = webResponse.GetResponseStream();
            if(webResponse.ContentEncoding.ToLower().Contains("gzip"))
            {
              responseStream = new GZipStream(responseStream, CompressionMode.Decompress);          
              StreamReader reader = new StreamReader(responseStream, Encoding.Default);
              sb.Append(reader.ReadToEnd());
              reader.Close();
            }
            else if(webResponse.ContentEncoding.ToLower().Contains("deflate"))
            {
              responseStream = new DeflateStream(responseStream, CompressionMode.Decompress);
              StreamReader reader = new StreamReader(responseStream, Encoding.Default);
              sb.Append(reader.ReadToEnd());
              reader.Close();
            }
            else
            {
              while (true)
              {
                charCount = responseStream.Read(buffer, 0, buffer.Length);
                if (0 == charCount) break;
                sb.Append(Encoding.ASCII.GetString(buffer, 0, charCount));
              }
            }
            webResponse.Close();
            return new HttpNetResponse(sb.ToString(),strRequest,webResponse,true);
          }
          catch (WebException webException)
          {
            if(webException.Message.Contains("(404) Not Found"))
            {
              return new HttpNetResponse((HttpWebResponse)webException.Response, strRequest, false, webException.Message);
            }
            lastWebException=webException;
            Thread.Sleep(TIMEOUT_BETWEEN_RETRIES);
          }
        }
        MDTrace.WriteLine(LogLevel.DEBUG,String.Format("General failure with {0}",lastWebException.Message));
        return new HttpNetResponse(webResponse,strRequest,false,lastWebException.Message);
      }
      catch (Exception exception)
      {
        return new HttpNetResponse(webResponse,strRequest,false,exception.Message);
      }
      finally
      {
        MDTrace.WriteLine(LogLevel.VERBOSE,"GetRequestNoEncodingV3[LEAVE]");
      }
    }
// https://query1.finance.yahoo.com/v7/finance/chart/MIDD?period1=1606172400&period2=1606172400&interval=1d&events=history
    public static HttpNetResponse GetRequestNoEncodingV3A(String strRequest,WebProxy webProxy=null)
    {
      HttpWebResponse webResponse=null;
      WebException lastWebException=null;
      ServicePointManager.Expect100Continue=true;
      ServicePointManager.SecurityProtocol=SecurityProtocolType.Ssl3|SecurityProtocolType.Tls12|SecurityProtocolType.Tls11|SecurityProtocolType.Tls;
      Random random=new Random();

      try
      {
        MDTrace.WriteLine(LogLevel.VERBOSE,String.Format("GetRequestNoEncodingV3A[ENTER]{0}",strRequest));
        String[] userAgents=
        {
          "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.0",
          "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:82.0) Gecko/20100101 Firefox/82.0"
        };
        int MAX_RETRIES=5;
        int TIMEOUT_BETWEEN_RETRIES=1000;
        int charCount=0;
        byte[] buffer=new byte[8192];
        StringBuilder sb=new StringBuilder();

        for(int count=0;count<MAX_RETRIES;count++)
        {
          CookieContainer cookieContainer=new CookieContainer();
          HttpWebRequest webRequest=(HttpWebRequest)WebRequest.Create(new Uri(strRequest));
          if(null!=webProxy)webRequest.Proxy=webProxy;
          webRequest.ProtocolVersion=HttpVersion.Version10;
          webRequest.CookieContainer=cookieContainer;
          webRequest.KeepAlive=false;
          webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5");
          webRequest.Headers.Add("Accept-Encoding: 	gzip, deflate, br");
          webRequest.Accept="application/json,text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8";

          double randomNumber=random.NextDouble();    // number between 0 and 1
          int index=(int)(randomNumber*((double)userAgents.Length));
          if(index>=userAgents.Length) index=userAgents.Length-1;
          String userAgent=userAgents[index];
          webRequest.UserAgent=userAgent;

          try
          {
            webResponse=(HttpWebResponse)webRequest.GetResponse();
            Stream responseStream=webResponse.GetResponseStream();
            if(webResponse.ContentEncoding.ToLower().Contains("gzip"))
            {
              responseStream=new GZipStream(responseStream,CompressionMode.Decompress);
              StreamReader reader=new StreamReader(responseStream,Encoding.Default);
              sb.Append(reader.ReadToEnd());
              reader.Close();
            }
            else if(webResponse.ContentEncoding.ToLower().Contains("deflate"))
            {
              responseStream=new DeflateStream(responseStream,CompressionMode.Decompress);
              StreamReader reader=new StreamReader(responseStream,Encoding.Default);
              sb.Append(reader.ReadToEnd());
              reader.Close();
            }
            else
            {
              while(true)
              {
                charCount=responseStream.Read(buffer,0,buffer.Length);
                if(0==charCount) break;
                sb.Append(Encoding.ASCII.GetString(buffer,0,charCount));
              }
            }
            webResponse.Close();
            return new HttpNetResponse(sb.ToString(),strRequest,webResponse,true);
          }
          catch(WebException webException)
          {
            if(webException.Message.Contains("(404) Not Found"))
            {
              return new HttpNetResponse((HttpWebResponse)webException.Response, strRequest, false, webException.Message);
            }
            lastWebException=webException;
            Thread.Sleep(TIMEOUT_BETWEEN_RETRIES);
          }
        }
        MDTrace.WriteLine(LogLevel.DEBUG,String.Format("General failure with {0}",lastWebException.Message));
        return new HttpNetResponse(webResponse,strRequest,false,lastWebException.Message);
      }
      catch(Exception exception)
      {
        return new HttpNetResponse(webResponse,strRequest,false,exception.Message);
      }
      finally
      {
        MDTrace.WriteLine(LogLevel.VERBOSE,"GetRequestNoEncodingV3A[LEAVE]");
      }
    }
//  This one uses the random user agent generator
    public static HttpNetResponse GetRequestNoEncodingV3B(String strRequest,CookieCollection cookieCollection=null,WebProxy webProxy=null)
    {
      HttpWebResponse webResponse=null;
      WebException lastWebException=null;
      ServicePointManager.Expect100Continue=true;
      ServicePointManager.SecurityProtocol=SecurityProtocolType.Ssl3|SecurityProtocolType.Tls12|SecurityProtocolType.Tls11|SecurityProtocolType.Tls;

      try
      {
        MDTrace.WriteLine(LogLevel.VERBOSE,String.Format("GetRequestNoEncodingV3B[ENTER]{0}",strRequest));
        int MAX_RETRIES=5;
        int TIMEOUT_BETWEEN_RETRIES=1000;
        int charCount=0;
        byte[] buffer=new byte[8192];
        StringBuilder sb=new StringBuilder();

        for(int count=0;count<MAX_RETRIES;count++)
        {
          CookieContainer cookieContainer=new CookieContainer();
          HttpWebRequest webRequest=(HttpWebRequest)WebRequest.Create(new Uri(strRequest));
          if(null!=webProxy) webRequest.Proxy=webProxy;
          if(null!=cookieCollection)
          {
            webRequest.CookieContainer=new CookieContainer();
            webRequest.CookieContainer.Add(cookieCollection);
          }
          webRequest.ProtocolVersion=HttpVersion.Version10;
          webRequest.KeepAlive=false;
          webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5");
          webRequest.Headers.Add("Accept-Encoding: 	gzip, deflate, br");
          webRequest.Accept="application/json,text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8";
          String userAgent=UserAgent.GetInstance().GetUserAgent();
          webRequest.UserAgent=userAgent;
          try
          {
            webResponse=(HttpWebResponse)webRequest.GetResponse();
            Stream responseStream=webResponse.GetResponseStream();
            if(webResponse.ContentEncoding.ToLower().Contains("gzip"))
            {
              responseStream=new GZipStream(responseStream,CompressionMode.Decompress);
              StreamReader reader=new StreamReader(responseStream,Encoding.Default);
              sb.Append(reader.ReadToEnd());
              reader.Close();
            }
            else if(webResponse.ContentEncoding.ToLower().Contains("deflate"))
            {
              responseStream=new DeflateStream(responseStream,CompressionMode.Decompress);
              StreamReader reader=new StreamReader(responseStream,Encoding.Default);
              sb.Append(reader.ReadToEnd());
              reader.Close();
            }
            else
            {
              while(true)
              {
                charCount=responseStream.Read(buffer,0,buffer.Length);
                if(0==charCount) break;
                sb.Append(Encoding.ASCII.GetString(buffer,0,charCount));
              }
            }
            webResponse.Close();
            return new HttpNetResponse(sb.ToString(),strRequest,webResponse,true);
          }
          catch(WebException webException)
          {
            if(webException.Message.Contains("(404) Not Found"))
            {
              return new HttpNetResponse((HttpWebResponse)webException.Response, strRequest, false, webException.Message);
            }
            lastWebException=webException;
            Thread.Sleep(TIMEOUT_BETWEEN_RETRIES);
          }
        }
        MDTrace.WriteLine(LogLevel.DEBUG,String.Format("General failure with {0} for request {1}",lastWebException.Message,strRequest));
        return new HttpNetResponse(webResponse,strRequest,false,lastWebException.Message);
      }
      catch(Exception exception)
      {
        return new HttpNetResponse(webResponse,strRequest,false,exception.Message);
      }
      finally
      {
        MDTrace.WriteLine(LogLevel.VERBOSE,"GetRequestNoEncodingV3B[LEAVE]");
      }
    }
    public static HttpNetResponse GetRequestNoEncodingV3C(String strRequest,WebProxy webProxy=null)
    {
      HttpWebResponse webResponse=null;
      WebException lastWebException=null;
      ServicePointManager.Expect100Continue=true;
      ServicePointManager.SecurityProtocol=SecurityProtocolType.Ssl3|SecurityProtocolType.Tls12|SecurityProtocolType.Tls11|SecurityProtocolType.Tls;
      Random random=new Random();

      try
      {
        MDTrace.WriteLine(LogLevel.VERBOSE,String.Format("GetRequestNoEncodingV3A[ENTER]{0}",strRequest));
        String[] userAgents=
        {
          "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.3",
          "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.3.1 Safari/605.1.1",
          "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.6 Safari/605.1.1",
          "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.3"
        };
        int MAX_RETRIES=5;
        int TIMEOUT_BETWEEN_RETRIES=1000;
        int charCount=0;
        byte[] buffer=new byte[8192];
        StringBuilder sb=new StringBuilder();

        for(int count=0;count<MAX_RETRIES;count++)
        {
          CookieContainer cookieContainer=new CookieContainer();
          HttpWebRequest webRequest=(HttpWebRequest)WebRequest.Create(new Uri(strRequest));
          if(null!=webProxy)webRequest.Proxy=webProxy;
          webRequest.ProtocolVersion=HttpVersion.Version10;
          webRequest.CookieContainer=cookieContainer;
          webRequest.KeepAlive=false;
          webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5");
          webRequest.Headers.Add("Accept-Encoding: 	gzip, deflate, br");
          webRequest.Accept="application/json,text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8";

          double randomNumber=random.NextDouble();    // number between 0 and 1
          int index=(int)(randomNumber*((double)userAgents.Length));
          if(index>=userAgents.Length) index=userAgents.Length-1;
          String userAgent=userAgents[index];
          webRequest.UserAgent=userAgent;

          try
          {
            webResponse=(HttpWebResponse)webRequest.GetResponse();
            Stream responseStream=webResponse.GetResponseStream();
            if(webResponse.ContentEncoding.ToLower().Contains("gzip"))
            {
              responseStream=new GZipStream(responseStream,CompressionMode.Decompress);
              StreamReader reader=new StreamReader(responseStream,Encoding.Default);
              sb.Append(reader.ReadToEnd());
              reader.Close();
            }
            else if(webResponse.ContentEncoding.ToLower().Contains("deflate"))
            {
              responseStream=new DeflateStream(responseStream,CompressionMode.Decompress);
              StreamReader reader=new StreamReader(responseStream,Encoding.Default);
              sb.Append(reader.ReadToEnd());
              reader.Close();
            }
            else
            {
              while(true)
              {
                charCount=responseStream.Read(buffer,0,buffer.Length);
                if(0==charCount) break;
                sb.Append(Encoding.ASCII.GetString(buffer,0,charCount));
              }
            }
            webResponse.Close();
            return new HttpNetResponse(sb.ToString(),strRequest,webResponse,true);
          }
          catch(WebException webException)
          {
            if(webException.Message.Contains("(404) Not Found"))
            {
               return new HttpNetResponse((HttpWebResponse)webException.Response, strRequest, false, webException.Message);
            }
            lastWebException=webException;
            Thread.Sleep(TIMEOUT_BETWEEN_RETRIES);
          }
        }
        MDTrace.WriteLine(LogLevel.DEBUG,String.Format("General failure with {0}",lastWebException.Message));
        return new HttpNetResponse(webResponse,strRequest,false,lastWebException.Message);
      }
      catch(Exception exception)
      {
        return new HttpNetResponse(webResponse,strRequest,false,exception.Message);
      }
      finally
      {
        MDTrace.WriteLine(LogLevel.VERBOSE,"GetRequestNoEncodingV3C[LEAVE]");
      }
    }
    public static HttpNetResponse GetRequestNoEncodingV4(String strRequest,CookieCollection cookieCollection=null,WebProxy webProxy=null)
    {
      HttpWebResponse webResponse=null;
      try
      {
        MDTrace.WriteLine(LogLevel.VERBOSE,String.Format("GetRequestNoEncodingV4[ENTER]{0}",strRequest));
        int charCount = 0;
        byte[] buffer = new byte[8192];
        StringBuilder sb = new StringBuilder();
        bool expect100Condition=ServicePointManager.Expect100Continue;
        SecurityProtocolType securityProtocolType= ServicePointManager.SecurityProtocol;
        ServicePointManager.Expect100Continue = true;
        ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12;
        HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(new Uri(strRequest));
        if(null!=webProxy) webRequest.Proxy=webProxy;
        webRequest.Timeout = REQUEST_TIMEOUT;
        webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5");
        webRequest.Headers.Add("Accept-Encoding: None");
        webRequest.Accept = "image/gif, image/jpeg, image/pjpeg, image/pjpeg, application/x-shockwave-flash, application/x-ms-application, application/x-ms-xbap, application/vnd.ms-xpsdocument, application/xaml+xml, application/vnd.ms-excel, application/vnd.ms-powerpoint, application/msword, */*";
        webRequest.UserAgent = "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR 3.0.04506.648; .NET CLR 3.5.21022; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; MS-RTC LM 8; .NET4.0C; .NET4.0E; InfoPath.3)";
        webRequest.KeepAlive = true;
        webRequest.CookieContainer=new CookieContainer();
        if(null!=cookieCollection)webRequest.CookieContainer.Add(cookieCollection);
        webResponse = (HttpWebResponse)webRequest.GetResponse();
        Stream responseStream = webResponse.GetResponseStream();
        while (true)
        {
          charCount = responseStream.Read(buffer, 0, buffer.Length);
          if (0 == charCount) break;
          sb.Append(Encoding.ASCII.GetString(buffer, 0, charCount));
        }
        webResponse.Close();
        ServicePointManager.Expect100Continue = expect100Condition;
        ServicePointManager.SecurityProtocol = securityProtocolType;
        return new HttpNetResponse(sb.ToString(),strRequest,webResponse,webResponse.Cookies,true);
      }
      catch (WebException webException)
      {
          return new HttpNetResponse((HttpWebResponse)webException.Response,strRequest,false,webException.Message);
      }
      catch (Exception exception)
      {
        return new HttpNetResponse(webResponse,strRequest,false,exception.Message);
      }
      finally
      {
        MDTrace.WriteLine(LogLevel.VERBOSE,"GetRequestNoEncodingV4[LEAVE]");
      }
    }
    public static HttpNetResponse GetRequestNoEncodingV4A(String strRequest,CookieCollection cookieCollection=null,WebProxy webProxy=null)
    {
      HttpWebResponse webResponse=null;
      try
      {
        MDTrace.WriteLine(LogLevel.VERBOSE,String.Format("GetRequestNoEncodingV4[ENTER]{0}",strRequest));
        int charCount=0;
        byte[] buffer=new byte[8192];
        StringBuilder sb=new StringBuilder();
        bool expect100Condition=ServicePointManager.Expect100Continue;
        SecurityProtocolType securityProtocolType=ServicePointManager.SecurityProtocol;
        ServicePointManager.Expect100Continue=true;
        ServicePointManager.SecurityProtocol=SecurityProtocolType.Tls12;
        HttpWebRequest webRequest=(HttpWebRequest)WebRequest.Create(new Uri(strRequest));
        if(null!=webProxy)webRequest.Proxy=webProxy;
        webRequest.Timeout=REQUEST_TIMEOUT;
        webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5");
        webRequest.Headers.Add("Accept-Encoding: gzip, deflate, br");
        webRequest.Accept="text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8";
        webRequest.UserAgent="Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:82.0) Gecko/20100101 Firefox/82.0";
        webRequest.KeepAlive=true;
        webRequest.CookieContainer=new CookieContainer();
        if(null!=cookieCollection) webRequest.CookieContainer.Add(cookieCollection);
        webResponse=(HttpWebResponse)webRequest.GetResponse();
        Stream responseStream=webResponse.GetResponseStream();
        if(webResponse.ContentEncoding.ToLower().Contains("gzip"))
        {
          responseStream=new GZipStream(responseStream,CompressionMode.Decompress);
          StreamReader reader=new StreamReader(responseStream,Encoding.Default);
          sb.Append(reader.ReadToEnd());
          reader.Close();
        }
        else
        {
          while(true)
          {
            charCount=responseStream.Read(buffer,0,buffer.Length);
            if(0==charCount) break;
            sb.Append(Encoding.ASCII.GetString(buffer,0,charCount));
          }
        }
        webResponse.Close();
        ServicePointManager.Expect100Continue=expect100Condition;
        ServicePointManager.SecurityProtocol=securityProtocolType;
        return new HttpNetResponse(sb.ToString(),strRequest,webResponse,webResponse.Cookies,true);
      }
      catch(WebException webException)
      {
        return new HttpNetResponse((HttpWebResponse)webException.Response,strRequest,false,webException.Message);
      }
      catch(Exception exception)
      {
        return new HttpNetResponse(webResponse,strRequest,false,exception.Message);
      }
      finally
      {
        MDTrace.WriteLine(LogLevel.VERBOSE,"GetRequestNoEncodingV4[LEAVE]");
      }
    }

    public static HttpNetResponse GetRequestNoEncodingV5(String strRequest,int webRequestTimeoutMS,WebProxy webProxy=null)
    {
      HttpWebResponse webResponse = null;
      try
      {
        MDTrace.WriteLine(LogLevel.VERBOSE, String.Format("GetRequestNoEncodingV5[ENTER]{0}", strRequest));
        int charCount = 0;
        byte[] buffer = new byte[8192];
        StringBuilder sb = new StringBuilder();
        bool expect100Condition = ServicePointManager.Expect100Continue;
        SecurityProtocolType securityProtocolType = ServicePointManager.SecurityProtocol;
        ServicePointManager.Expect100Continue = true;
        ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12;
        HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(new Uri(strRequest));
        if(null!=webProxy)webRequest.Proxy=webProxy;
        webRequest.Timeout = webRequestTimeoutMS;
        webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5");
        webRequest.Headers.Add("Accept-Encoding: None");
        webRequest.Accept = "image/gif, image/jpeg, image/pjpeg, image/pjpeg, application/x-shockwave-flash, application/x-ms-application, application/x-ms-xbap, application/vnd.ms-xpsdocument, application/xaml+xml, application/vnd.ms-excel, application/vnd.ms-powerpoint, application/msword, */*";
        webRequest.UserAgent = "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR 3.0.04506.648; .NET CLR 3.5.21022; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; MS-RTC LM 8; .NET4.0C; .NET4.0E; InfoPath.3)";
        webRequest.KeepAlive = true;
        webRequest.CookieContainer = new CookieContainer();
        webResponse = (HttpWebResponse)webRequest.GetResponse();
        Stream responseStream = webResponse.GetResponseStream();
        while (true)
        {
          charCount = responseStream.Read(buffer, 0, buffer.Length);
          if (0 == charCount) break;
          sb.Append(Encoding.ASCII.GetString(buffer, 0, charCount));
        }
        webResponse.Close();
        ServicePointManager.Expect100Continue = expect100Condition;
        ServicePointManager.SecurityProtocol = securityProtocolType;
        return new HttpNetResponse(sb.ToString(), strRequest, webResponse, webResponse.Cookies, true);
      }
      catch (WebException webException)
      {
        return new HttpNetResponse((HttpWebResponse)webException.Response, strRequest, false, webException.Message);
      }
      catch (Exception exception)
      {
        return new HttpNetResponse(webResponse, strRequest, false, exception.Message);
      }
      finally
      {
        MDTrace.WriteLine(LogLevel.VERBOSE, "GetRequestNoEncodingV5[LEAVE]");
      }
    }

    public static HttpNetResponse GetRequestNoEncodingV5A(String strRequest,int webRequestTimeoutMS,WebProxy webProxy=null)
    {
      HttpWebResponse webResponse = null;
      try
      {
        Random random = new Random();
        String[] userAgents=
        {
          "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.0",
          "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:82.0) Gecko/20100101 Firefox/82.0"
        };
        
        MDTrace.WriteLine(LogLevel.VERBOSE, String.Format("GetRequestNoEncodingV5[ENTER]{0}", strRequest));
        int charCount = 0;
        byte[] buffer = new byte[8192];
        StringBuilder sb = new StringBuilder();
        bool expect100Condition = ServicePointManager.Expect100Continue;
        SecurityProtocolType securityProtocolType = ServicePointManager.SecurityProtocol;
        ServicePointManager.Expect100Continue = true;
        ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12;
        HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(new Uri(strRequest));
        if(null!=webProxy)webRequest.Proxy=webProxy;
        webRequest.Timeout = webRequestTimeoutMS;
        webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5");
        webRequest.Headers.Add("Accept-Encoding: None");
        webRequest.Accept = "image/gif, image/jpeg, image/pjpeg, image/pjpeg, application/x-shockwave-flash, application/x-ms-application, application/x-ms-xbap, application/vnd.ms-xpsdocument, application/xaml+xml, application/vnd.ms-excel, application/vnd.ms-powerpoint, application/msword, */*";
//        webRequest.UserAgent = "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR 3.0.04506.648; .NET CLR 3.5.21022; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; MS-RTC LM 8; .NET4.0C; .NET4.0E; InfoPath.3)";
        webRequest.KeepAlive = true;
        webRequest.CookieContainer = new CookieContainer();

        double randomNumber=random.NextDouble();    // number between 0 and 1
        int index=(int)(randomNumber*((double)userAgents.Length));
        if(index>=userAgents.Length) index=userAgents.Length-1;
        String userAgent=userAgents[index];
        webRequest.UserAgent=userAgent;   


        webResponse = (HttpWebResponse)webRequest.GetResponse();
        Stream responseStream = webResponse.GetResponseStream();
        while (true)
        {
          charCount = responseStream.Read(buffer, 0, buffer.Length);
          if (0 == charCount) break;
          sb.Append(Encoding.ASCII.GetString(buffer, 0, charCount));
        }
        webResponse.Close();
        ServicePointManager.Expect100Continue = expect100Condition;
        ServicePointManager.SecurityProtocol = securityProtocolType;
        return new HttpNetResponse(sb.ToString(), strRequest, webResponse, webResponse.Cookies, true);
      }
      catch (WebException webException)
      {
        return new HttpNetResponse((HttpWebResponse)webException.Response, strRequest, false, webException.Message);
      }
      catch (Exception exception)
      {
        return new HttpNetResponse(webResponse, strRequest, false, exception.Message);
      }
      finally
      {
        MDTrace.WriteLine(LogLevel.VERBOSE, "GetRequestNoEncodingV5[LEAVE]");
      }
    }

    public static HttpNetResponse GetRequestNoEncodingV5B(String strRequest,int webRequestTimeoutMS,WebProxy webProxy=null,bool useRandomUserAgent=false,CookieCollection cookieCollection=null)
    {
      HttpWebResponse webResponse = null;
      try
      {
        MDTrace.WriteLine(LogLevel.VERBOSE, String.Format("GetRequestNoEncodingV5[ENTER]{0}", strRequest));
        int charCount = 0;
        byte[] buffer = new byte[8192];
        StringBuilder sb = new StringBuilder();
        bool expect100Condition = ServicePointManager.Expect100Continue;
        SecurityProtocolType securityProtocolType = ServicePointManager.SecurityProtocol;
        ServicePointManager.Expect100Continue = true;
        ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12;
        HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(new Uri(strRequest));
        if(null!=webProxy)webRequest.Proxy=webProxy;
        webRequest.Timeout = webRequestTimeoutMS;
        webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5");
        webRequest.Headers.Add("Accept-Encoding: None");
        webRequest.Accept = "image/gif, image/jpeg, image/pjpeg, image/pjpeg, application/x-shockwave-flash, application/x-ms-application, application/x-ms-xbap, application/vnd.ms-xpsdocument, application/xaml+xml, application/vnd.ms-excel, application/vnd.ms-powerpoint, application/msword, */*";
//        if(useRandomUserAgent)webRequest.UserAgent=GetUserAgent();                            
        if (useRandomUserAgent) webRequest.UserAgent = UserAgent.GetInstance().GetUserAgent();
        else webRequest.UserAgent = "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR 3.0.04506.648; .NET CLR 3.5.21022; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; MS-RTC LM 8; .NET4.0C; .NET4.0E; InfoPath.3)";
        webRequest.KeepAlive = true;
        webRequest.CookieContainer = new CookieContainer();
        if(null!=cookieCollection)webRequest.CookieContainer.Add(cookieCollection);
        webResponse = (HttpWebResponse)webRequest.GetResponse();
        Stream responseStream = webResponse.GetResponseStream();
        while (true)
        {
          charCount = responseStream.Read(buffer, 0, buffer.Length);
          if (0 == charCount) break;
          sb.Append(Encoding.ASCII.GetString(buffer, 0, charCount));
        }
        webResponse.Close();
        ServicePointManager.Expect100Continue = expect100Condition;
        ServicePointManager.SecurityProtocol = securityProtocolType;
        return new HttpNetResponse(sb.ToString(), strRequest, webResponse, webResponse.Cookies, true);
      }
      catch (WebException webException)
      {
        return new HttpNetResponse((HttpWebResponse)webException.Response, strRequest, false, webException.Message);
      }
      catch (Exception exception)
      {
        return new HttpNetResponse(webResponse, strRequest, false, exception.Message);
      }
      finally
      {
        MDTrace.WriteLine(LogLevel.VERBOSE, "GetRequestNoEncodingV5[LEAVE]");
      }
    }

    public static HttpNetResponse GetRequestNoEncodingV5C(String strRequest,String userName, String password)
    {
      HttpWebResponse webResponse = null;
      try
      {
        MDTrace.WriteLine(LogLevel.VERBOSE, String.Format("GetRequestNoEncodingV5C[ENTER]{0}", strRequest));
        int charCount = 0;
        byte[] buffer = new byte[8192];
        StringBuilder sb = new StringBuilder();
        bool expect100Condition = ServicePointManager.Expect100Continue;
        SecurityProtocolType securityProtocolType = ServicePointManager.SecurityProtocol;
        ServicePointManager.Expect100Continue = true;
        ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12;
        HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(new Uri(strRequest));
        webRequest.Credentials=new NetworkCredential(userName,password);
        webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5");
        webRequest.Headers.Add("Accept-Encoding: None");
        webRequest.Accept = "image/gif, image/jpeg, image/pjpeg, image/pjpeg, application/x-shockwave-flash, application/x-ms-application, application/x-ms-xbap, application/vnd.ms-xpsdocument, application/xaml+xml, application/vnd.ms-excel, application/vnd.ms-powerpoint, application/msword, */*";
        webRequest.KeepAlive = true;
        webResponse = (HttpWebResponse)webRequest.GetResponse();
        Stream responseStream = webResponse.GetResponseStream();
        while (true)
        {
          charCount = responseStream.Read(buffer, 0, buffer.Length);
          if (0 == charCount) break;
          sb.Append(Encoding.ASCII.GetString(buffer, 0, charCount));
        }
        webResponse.Close();
        ServicePointManager.Expect100Continue = expect100Condition;
        ServicePointManager.SecurityProtocol = securityProtocolType;
        return new HttpNetResponse(sb.ToString(), strRequest, webResponse, webResponse.Cookies, true);
      }
      catch (WebException webException)
      {
        return new HttpNetResponse((HttpWebResponse)webException.Response, strRequest, false, webException.Message);
      }
      catch (Exception exception)
      {
        return new HttpNetResponse(webResponse, strRequest, false, exception.Message);
      }
      finally
      {
        MDTrace.WriteLine(LogLevel.VERBOSE, "GetRequestNoEncodingV5C[LEAVE]");
      }
    }


    public static HttpNetResponse GetRequestNoEncodingV6(String strRequest, int webRequestTimeoutMS)
    {
      HttpWebResponse webResponse = null;
      try
      {
        MDTrace.WriteLine(LogLevel.VERBOSE, String.Format("GetRequestNoEncodingV6[ENTER]{0}", strRequest));
        int charCount = 0;
        byte[] buffer = new byte[8192];
        StringBuilder sb = new StringBuilder();
        bool expect100Condition = ServicePointManager.Expect100Continue;
        SecurityProtocolType securityProtocolType = ServicePointManager.SecurityProtocol;
        ServicePointManager.Expect100Continue = true;
        ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12;
        HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(new Uri(strRequest));
        webRequest.Timeout = webRequestTimeoutMS;
        webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5");
        webRequest.Headers.Add("Accept-Encoding: gzip, deflate, br");
        webRequest.Accept = "image/gif, image/jpeg, image/pjpeg, image/pjpeg, application/x-shockwave-flash, application/x-ms-application, application/x-ms-xbap, application/vnd.ms-xpsdocument, application/xaml+xml, application/vnd.ms-excel, application/vnd.ms-powerpoint, application/msword, */*";
        webRequest.UserAgent = "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR 3.0.04506.648; .NET CLR 3.5.21022; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; MS-RTC LM 8; .NET4.0C; .NET4.0E; InfoPath.3)";
        webRequest.KeepAlive = true;
        webRequest.CookieContainer = new CookieContainer();
        webResponse = (HttpWebResponse)webRequest.GetResponse();
        Stream responseStream = webResponse.GetResponseStream();
        while (true)
        {
          charCount = responseStream.Read(buffer, 0, buffer.Length);
          if (0 == charCount) break;
          sb.Append(Encoding.ASCII.GetString(buffer, 0, charCount));
        }
        webResponse.Close();
        ServicePointManager.Expect100Continue = expect100Condition;
        ServicePointManager.SecurityProtocol = securityProtocolType;
        return new HttpNetResponse(sb.ToString(), strRequest, webResponse, webResponse.Cookies, true);
      }
      catch (WebException webException)
      {
        return new HttpNetResponse((HttpWebResponse)webException.Response, strRequest, false, webException.Message);
      }
      catch (Exception exception)
      {
        return new HttpNetResponse(webResponse, strRequest, false, exception.Message);
      }
      finally
      {
        MDTrace.WriteLine(LogLevel.VERBOSE, "GetRequestNoEncodingV6[LEAVE]");
      }
    }

    public static HttpNetResponse GetRequestNoEncodingV7(String strRequest,WebProxy webProxy=null,CookieCollection cookieCollection=null)
    {
      HttpWebResponse webResponse = null;
      try
      {
        MDTrace.WriteLine(LogLevel.VERBOSE, String.Format("GetRequestNoEncodingV7[ENTER]{0}", strRequest));
        int charCount = 0;
        byte[] buffer = new byte[8192];
        StringBuilder sb = new StringBuilder();
        bool expect100Condition = ServicePointManager.Expect100Continue;
        SecurityProtocolType securityProtocolType = ServicePointManager.SecurityProtocol;
        ServicePointManager.Expect100Continue = true;
        ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12;
        HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(new Uri(strRequest));
        if(null!=webProxy)webRequest.Proxy=webProxy;
        webRequest.KeepAlive=true;
        webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5");
        webRequest.Headers.Add("Accept-Encoding: gzip, deflate, br");
        webRequest.Headers.Add("Upgrade-Insecure-Requests: 1");
        webRequest.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8";
        webRequest.UserAgent = " Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:76.0) Gecko/20100101 Firefox/76.0";
        webRequest.KeepAlive = true;
        webRequest.CookieContainer = new CookieContainer();
        if(null!=cookieCollection)webRequest.CookieContainer.Add(cookieCollection);
        webResponse = (HttpWebResponse)webRequest.GetResponse();
        Stream responseStream = webResponse.GetResponseStream();
        if (webResponse.ContentEncoding.ToLower().Contains("gzip"))
        {
          responseStream = new GZipStream(responseStream, CompressionMode.Decompress);
          StreamReader reader = new StreamReader(responseStream, Encoding.Default);
          sb.Append(reader.ReadToEnd());
          reader.Close();
        }
        else if (webResponse.ContentEncoding.ToLower().Contains("deflate"))
        {
          responseStream = new DeflateStream(responseStream, CompressionMode.Decompress);
          StreamReader reader = new StreamReader(responseStream, Encoding.Default);
          sb.Append(reader.ReadToEnd());
          reader.Close();
        }
        else
        {
          while (true)
          {
            charCount = responseStream.Read(buffer, 0, buffer.Length);
            if (0 == charCount) break;
            sb.Append(Encoding.ASCII.GetString(buffer, 0, charCount));
          }
        }

        webResponse.Close();
        ServicePointManager.Expect100Continue = expect100Condition;
        ServicePointManager.SecurityProtocol = securityProtocolType;
        return new HttpNetResponse(sb.ToString(), strRequest, webResponse, webResponse.Cookies, true);
      }
      catch (WebException webException)
      {
        return new HttpNetResponse((HttpWebResponse)webException.Response, strRequest, false, webException.Message);
      }
      catch (Exception exception)
      {
        return new HttpNetResponse(webResponse, strRequest, false, exception.Message);
      }
      finally
      {
        MDTrace.WriteLine(LogLevel.VERBOSE, "GetRequestNoEncodingV7[LEAVE]");
      }
    }
// This is specifically taylored to Morningstar site for retrieval of historical data
    public static HttpNetResponse GetRequestNoEncodingMStar(String strRequest,WebProxy webProxy=null)
    {
      HttpWebResponse webResponse = null;
      try
      {
        MDTrace.WriteLine(LogLevel.VERBOSE, String.Format("GetRequestNoEncodingMStar[ENTER]{0}", strRequest));
        int charCount = 0;
        byte[] buffer = new byte[8192];
        StringBuilder sb = new StringBuilder();
        bool expect100Condition = ServicePointManager.Expect100Continue;
        SecurityProtocolType securityProtocolType = ServicePointManager.SecurityProtocol;
        ServicePointManager.Expect100Continue = true;
        ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12;
        HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(new Uri(strRequest));
        webRequest.KeepAlive=true;
        webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5");
        webRequest.Headers.Add("Accept-Encoding: gzip, deflate, br");
        webRequest.Headers.Add("DNT: 1");
        webRequest.Headers.Add("Sec-Fetch-Dest:empty");
        webRequest.Headers.Add("Sec-Fetch-Mode:cors");
        webRequest.Headers.Add("Sec-Fetch-Site:same-site");
        webRequest.Headers.Add("X-SAL-ContentType:e7FDDltrTy+tA2HnLovvGL0LFMwT+KkEptGju5wXVTU=");
        webRequest.Headers.Add("X-API-RequestId: d2ffea42-4a07-4fa7-fa0d-b307facd81fb");
        webRequest.Headers.Add("ApiKey:lstzFDEOhfFNMLikKa0am9mgEKLBl49T");
        webRequest.Accept = "*/*";
        webRequest.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36";
        webRequest.Host="api-global.morningstar.com";
        webRequest.Referer="https://www.morningstar.com/stocks/xnas/aapl/performance";

        if(null!=webProxy)webRequest.Proxy=webProxy;

        webRequest.CookieContainer = new CookieContainer();
        webResponse = (HttpWebResponse)webRequest.GetResponse();
        Stream responseStream = webResponse.GetResponseStream();
        if (webResponse.ContentEncoding.ToLower().Contains("gzip"))
        {
          responseStream = new GZipStream(responseStream, CompressionMode.Decompress);
          StreamReader reader = new StreamReader(responseStream, Encoding.Default);
          sb.Append(reader.ReadToEnd());
          reader.Close();
        }
        else if (webResponse.ContentEncoding.ToLower().Contains("deflate"))
        {
          responseStream = new DeflateStream(responseStream, CompressionMode.Decompress);
          StreamReader reader = new StreamReader(responseStream, Encoding.Default);
          sb.Append(reader.ReadToEnd());
          reader.Close();
        }
        else
        {
          while (true)
          {
            charCount = responseStream.Read(buffer, 0, buffer.Length);
            if (0 == charCount) break;
            sb.Append(Encoding.ASCII.GetString(buffer, 0, charCount));
          }
        }

        webResponse.Close();
        ServicePointManager.Expect100Continue = expect100Condition;
        ServicePointManager.SecurityProtocol = securityProtocolType;
        return new HttpNetResponse(sb.ToString(), strRequest, webResponse, webResponse.Cookies, true);
      }
      catch (WebException webException)
      {
        return new HttpNetResponse((HttpWebResponse)webException.Response, strRequest, false, webException.Message);
      }
      catch (Exception exception)
      {
        return new HttpNetResponse(webResponse, strRequest, false, exception.Message);
      }
      finally
      {
        MDTrace.WriteLine(LogLevel.VERBOSE, "GetRequestNoEncodingMStar[LEAVE]");
      }
    }

    /// <summary>
    /// Get the proxy for the specified service or null if it is not congfigured to use the proxy
    /// </summary>
    ///<param name="serviceName">The service name</param>
    public static WebProxy GetProxy(String serviceName)
    {
      try
      {
        String proxyAddress=ConfigurationManager.AppSettings["proxy_address"];
        if(null==proxyAddress) return null;
        String useServiceProxy=ConfigurationManager.AppSettings["proxy_"+serviceName];
        if(null==useServiceProxy||false==bool.Parse(useServiceProxy)) return null;
        String[] addressParts=proxyAddress.Split(':');
        SocketControl socketControl=null;
        try
        {
          socketControl=new SocketControl(addressParts[1].Replace("//",null),int.Parse(addressParts[2]));
          if(!socketControl.IsConnected())throw new Exception("Unreachable destination "+proxyAddress);
        }
        catch(Exception exception)
        {
          MDTrace.WriteLine(LogLevel.DEBUG,String.Format("****** ERROR: CANNOT REACH PROXY {0} FOR SERVICE {1}...{2}",proxyAddress,serviceName,exception.ToString()));
          return null;
        }
        socketControl.Close();
        MDTrace.WriteLine(LogLevel.DEBUG,String.Format("USING PROXY:{0} FOR {1}",proxyAddress,serviceName));
        Uri newUri=new Uri(proxyAddress);
        WebProxy proxy=new WebProxy();
        proxy.Address=newUri;
        return proxy;
      }
      catch(Exception exception)
      {
        MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetProxy: Exception:{0}",exception.ToString()));
        return null;
      }
    }

		/// <summary>
		/// Get the proxy.  This is only to be used as part of unit tests to test out the TOR Proxy.  User code should use the GetProxy(serviceName) variation
		/// </summary>
    public static WebProxy GetProxy()
    {
        String proxyAddress=ConfigurationManager.AppSettings["proxy_address"];
        String[] addressParts=proxyAddress.Split(':');
        SocketControl socketControl=null;
        try
        {
          socketControl=new SocketControl(addressParts[1].Replace("//",null),int.Parse(addressParts[2]));
          if(!socketControl.IsConnected())throw new Exception("Unreachable destination "+proxyAddress);
        }
        catch(Exception exception)
        {
          MDTrace.WriteLine(LogLevel.DEBUG,String.Format("****** ERROR: CANNOT REACH PROXY. {0} {1} ",proxyAddress,exception.ToString()));
          return null;
        }
        socketControl.Close();
        MDTrace.WriteLine(LogLevel.DEBUG,String.Format("USING PROXY:{0} ",proxyAddress));
        Uri newUri=new Uri(proxyAddress);
        WebProxy proxy=new WebProxy();
        proxy.Address=newUri;
        return proxy;
    }
  }  
}