Home>

There is a link code for a specific site. Through it, I receive data that I manage to change or modify in the process, and then I send these changed data back to the site. Everything would be fine, but the process of work itself, it is desirable to speed up and optimize it. And to do this, I need to somehow implement gzip compression for the POST method and gzip decompression for the GET method:

namespace SoftWARE
{
    using System;
    using System.Collections.Generic;
    using System.IO;
    using System.Net;
    using System.Runtime.InteropServices;
    using System.Configuration;
    using System.Text;
    public class Connecter
    {
        public CookieContainer cookies= new CookieContainer ();
        public string OriginUrl= System.Configuration.ConfigurationManager.AppSettings ["OriginUrl"];
        public HttpWebRequest GetNewRequest (string targetUrl, CookieContainer SessionCookieContainer)
        {
            HttpWebRequest request= (HttpWebRequest) WebRequest.Create (targetUrl);
            request.CookieContainer= SessionCookieContainer;
            request.AllowAutoRedirect= false;
            return request;
        }
        public HttpWebResponse MakeRequest (HttpWebRequest request, CookieContainer SessionCookieContainer, Dictionary <
string, string >
 parameters= null, int TypeRequest= 0)
        {
            request.UserAgent= "Mozilla /5.0 (Windows NT 10.0; WOW64) AppleWebKit /537.36 (KHTML, like Gecko) Chrome /45.0.2454.85 Safari /537.36 OPR /32.0.1948.25Accept: * /*";
            request.Accept= "text /html, application /xhtml + xml, application /xml; q= 0.9, * /*; q= 0.8";
            request.Proxy= new WebProxy ();
            System.Net.ServicePointManager.Expect100Continue= false;
            System.Net.ServicePointManager.UseNagleAlgorithm= false;
            request.ServicePoint.ConnectionLimit= 100;
            ServicePointManager.FindServicePoint (new Uri (this.OriginUrl));
            request.KeepAlive= true;
            request.AutomaticDecompression= DecompressionMethods.GZip | DecompressionMethods.Deflate;
            request.Headers ["Origin"]= this.OriginUrl;
            if (TypeRequest== 1)
            {
                request.Headers ["X-Requested-With"]= "XMLHttpRequest";
            }
            if (TypeRequest== 2)
            {
                request.Headers ["Upgrade-Insecure-Requests"]= "1";
            }
            request.CookieContainer= SessionCookieContainer;
            request.AllowAutoRedirect= false;
            if (parameters!= null)
            {
                request.Method= "POST";
                request.ContentType= "application /x-www-form-urlencoded";
                string s= "";
                foreach (KeyValuePair <
string, string >
 pair in parameters)
                {
                    if (s.Length== 0)
                    {
                        s += $ "{pair.Key}= {pair.Value}";
                    }
                    else
                    {
                        s += $ "&
{pair.Key}= {pair.Value} ";
                    }
                }
                byte [] bytes= Encoding.UTF8.GetBytes (s);
                using (Stream stream= request.GetRequestStream ())
                {
                    stream.Write (bytes, 0, bytes.Length);
                }
            }
            else
            {
                request.Method= "GET";
            }
            HttpWebResponse response= request.GetResponse () as HttpWebResponse;
            SessionCookieContainer.Add (response.Cookies);
            while (response.StatusCode== HttpStatusCode.Found)
            {
                response.Close ();
                request= this.GetNewRequest (response.Headers ["Location"],
 SessionCookieContainer);
                response= (HttpWebResponse) request.GetResponse ();SessionCookieContainer.Add (response.Cookies); }
            return response;
        }
    }
}

HttpWebRequest is deprecated, use HttpClient. Important! We do not recommend using HttpWebRequest for new development. Use the System.Net.Http.HttpClient class instead.

aepot2021-09-27 12:47:22

It is interesting! And if it's not difficult, of course, is it possible to somehow describe in detail how exactly these changes can be made to this code?

Виктор Химан2021-09-27 12:59:06

In .NET 6 coming November, WebRequest and WebClient are already declared Obsolete. So yes, use HttpClient.

Alexander Petrov2021-09-27 13:03:40
  • Answer # 1

    HttpWebRequestdeprecated, useHttpClient...

    :

    Important! We do not recommend usingHttpWebRequestfor a new development. Use insteadSystem.Net.Http.HttpClientClass.

    Here, I rewrote your code underHttpClient, tried to preserve your logic as much as possible.

    public class Connecter
    {
        private readonly HttpClient _client;
        public string OriginUrl {get; }
        public CookieContainer Cookies {get; }
        public Connecter (CookieContainer cookies= null)
        {
            OriginUrl= ConfigurationManager.AppSettings ["OriginUrl"];
            HttpClientHandler handler= new HttpClientHandler ()
            {
                AutomaticDecompression= DecompressionMethods.All //enable support for Content-Encoding compression: gzip, defalte, br -no need to add the header manually
                //here you can set a bunch of different connection parameters
            };
            if (cookies!= null)
                handler.CookieContainer= cookies;
            Cookies= handler.CookieContainer;
            _client= new HttpClient (handler)
            {
                BaseAddress= new Uri (OriginUrl),
                //DefaultRequestVersion= HttpVersion.Version20 //HTTP /2 support is available in .NET Core 3.1, .NET 5 and newer
            };
            _client.DefaultRequestHeaders.UserAgent.ParseAdd ("Mozilla /5.0 (Windows NT 10.0; WOW64) AppleWebKit /537.36 (KHTML, like Gecko) Chrome /45.0.2454.85 Safari /537.36 OPR /32.0.1948.25");
            _client.DefaultRequestHeaders.Accept.ParseAdd ("text /html, application /xhtml + xml, application /xml; q= 0.9, * /*; q= 0.8");
        }
        public Task <
    string >
     GetAsync (string url)
        {
            return _client.GetStringAsync (url);
        }
        public async Task <
    string >
     PostFormAsync (string url, Dictionary <
    string, string >
     parameters, int TypeRequest= 0)
        {
            using HttpRequestMessage request= new HttpRequestMessage (HttpMethod.Post, url)
            {
                Content= new FormUrlEncodedContent (parameters)
            };
            request.Headers.Add ("Origin", OriginUrl);
            if (TypeRequest== 1)
            {
                request.Headers.Add ("X-Requested-With", "XMLHttpRequest");
            }
            if (TypeRequest== 2)
            {
                request.Headers.Add ("Upgrade-Insecure-Requests", "1");
            }
            using HttpResponseMessage response= await _client.SendAsync (request);
            return await response.Content.ReadAsStringAsync ();
        }
    }
    

    The point of enabling compression is that you tell the server that you support decompression. The server will compress its response if it wishes.HttpClientwill unpack the answer automatically.

    With cookiesHttpClientworks like a browser, that is, each subsequent request takes into account the cookies added by the server in the previous request. That is, outside of this class, you can save and load cookies, for example, when starting and ending an application.

    I threw out unnecessary protocol settings, since they either do not affect anything, or coincide with the default settings.

    Use it like this.

    For example, there is a linkhttps: //my.api/post_methodandhttps: //my.api/get_method... In this caseOriginUrlit should behttps: //my.api/ , and requests should be sent like this.

    string getResponseText= await connecter.GetAsync ("get_method");
    string postResponseText= await connecter.PostFormAsync ("post_method", data, 1);
    

    You also had a syntax error inUserAgent, I have corrected. Well, of course, this design will work faster, and in .NET Core /.NET 5 -generally fast, because it uses a new HTTP engineSocketsHttpHandler, .NET Framework 4.x uses an old engine, so there may not be any noticeable speed gain.

  • Answer # 2

    HttpWebRequestdeprecated, useHttpClient...

    :

    Important! We do not recommend usingHttpWebRequestfor a new development. Use insteadSystem.Net.Http.HttpClientClass.

    Here, I rewrote your code underHttpClient, tried to preserve your logic as much as possible.

    public class Connecter
    {
        private readonly HttpClient _client;
        public string OriginUrl {get; }
        public CookieContainer Cookies {get; }
        public Connecter (CookieContainer cookies= null)
        {
            OriginUrl= ConfigurationManager.AppSettings ["OriginUrl"];
            HttpClientHandler handler= new HttpClientHandler ()
            {
                AutomaticDecompression= DecompressionMethods.All //enable support for Content-Encoding compression: gzip, defalte, br -no need to add the header manually
                //here you can set a bunch of different connection parameters
            };
            if (cookies!= null)
                handler.CookieContainer= cookies;
            Cookies= handler.CookieContainer;
            _client= new HttpClient (handler)
            {
                BaseAddress= new Uri (OriginUrl),
                //DefaultRequestVersion= HttpVersion.Version20 //HTTP /2 support is available in .NET Core 3.1, .NET 5 and newer
            };
            _client.DefaultRequestHeaders.UserAgent.ParseAdd ("Mozilla /5.0 (Windows NT 10.0; WOW64) AppleWebKit /537.36 (KHTML, like Gecko) Chrome /45.0.2454.85 Safari /537.36 OPR /32.0.1948.25");
            _client.DefaultRequestHeaders.Accept.ParseAdd ("text /html, application /xhtml + xml, application /xml; q= 0.9, * /*; q= 0.8");
        }
        public Task <
    string >
     GetAsync (string url)
        {
            return _client.GetStringAsync (url);
        }
        public async Task <
    string >
     PostFormAsync (string url, Dictionary <
    string, string >
     parameters, int TypeRequest= 0)
        {
            using HttpRequestMessage request= new HttpRequestMessage (HttpMethod.Post, url)
            {
                Content= new FormUrlEncodedContent (parameters)
            };
            request.Headers.Add ("Origin", OriginUrl);
            if (TypeRequest== 1)
            {
                request.Headers.Add ("X-Requested-With", "XMLHttpRequest");
            }
            if (TypeRequest== 2)
            {
                request.Headers.Add ("Upgrade-Insecure-Requests", "1");
            }
            using HttpResponseMessage response= await _client.SendAsync (request);
            return await response.Content.ReadAsStringAsync ();
        }
    }
    

    The point of enabling compression is that you tell the server that you support decompression. The server will compress its response if it wishes.HttpClientwill unpack the answer automatically.

    With cookiesHttpClientworks like a browser, that is, each subsequent request takes into account the cookies added by the server in the previous request. That is, outside of this class, you can save and load cookies, for example, when starting and ending an application.

    I threw out unnecessary protocol settings, since they either do not affect anything, or coincide with the default settings.

    Use it like this.

    For example, there is a linkhttps: //my.api/post_methodandhttps: //my.api/get_method... In this caseOriginUrlit should behttps: //my.api/ , and requests should be sent like this.

    string getResponseText= await connecter.GetAsync ("get_method");
    string postResponseText= await connecter.PostFormAsync ("post_method", data, 1);
    

    You also had a syntax error inUserAgent, I have corrected. Well, of course, this design will work faster, and in .NET Core /.NET 5 -generally fast, because it uses a new HTTP engineSocketsHttpHandler, .NET Framework 4.x uses an old engine, so there may not be any noticeable speed gain.