digitalmars.D.learn - std.net.curl : Performance
- Vino (47/47) Nov 09 2020 Hi All,
- Daniel Kozak (3/50) Nov 09 2020 Just delete it
- rinfz (14/15) Nov 09 2020 The only curl option you need to set within the loop is the
- rinfz (11/26) Nov 09 2020 In fact, you don't need url in there either since it's not
- Daniel Kozak (3/32) Nov 09 2020 In fact he does not need foreach. Because he use it on empty result
- Vino (68/105) Nov 09 2020 Hi All,
Hi All,
Request your help to on how to improve the performance of the
below code.
import std.conv: to;
import std.net.curl : get, HTTP, CurlOption;
import std.parallelism: parallel;
import std.range: chain, only;
import std.typecons: Tuple, tuple;
void main ()
{
Array!(Tuple!(int,string)) apidata;
Row[] result;
string apihost = "abc.com"; int apiport = 1830; string apiuri =
/getdata;
string apiuser = "user"; string apipass = "pass";
foreach(i, k; parallel(result,1))
{
string url = chain(apihost, only(':'), to!string(apiport),
apiuri).to!string;
string usrpass = chain(apiuser, only(':'),
apipass).to!string;
auto https = HTTP();
https.handle.set(CurlOption.buffersize, 512000);
https.handle.set(CurlOption.userpwd, usrpass);
https.handle.set(CurlOption.connecttimeout, 600);
https.handle.set(CurlOption.tcp_nodelay, 1);
https.handle.set(CurlOption.http_version, 2);
https.handle.set(CurlOption.sslversion, 1;
https.handle.set(CurlOption.use_ssl, 3);
https.handle.set(CurlOption.ssl_verifypeer, 0);
https.handle.set(CurlOption.url, url);
https.method(HTTP.Method.get);
https.StatusLine st;
https.onReceiveStatusLine = (https.StatusLine st) { if
(st.code != 200) { throw new Exception(st.reason); } };
ubyte[] content;
https.onReceive = (ubyte[] data) { content ~= data; return
data.length; };
https.perform();
scope(failure) { https.shutdown; exit(-4); } scope(exit)
https.shutdown;
apidata.insert(tuple(seq, cast(string) content));
}
return apidata[].sort;
}
From,
Vino.B
Nov 09 2020
Daniel Kozak <kozzi11 gmail.com>
Just delete it
On Mon, Nov 9, 2020 at 9:00 PM Vino via Digitalmars-d-learn <
digitalmars-d-learn puremagic.com> wrote:
Hi All,
Request your help to on how to improve the performance of the
below code.
import std.conv: to;
import std.net.curl : get, HTTP, CurlOption;
import std.parallelism: parallel;
import std.range: chain, only;
import std.typecons: Tuple, tuple;
void main ()
{
Array!(Tuple!(int,string)) apidata;
Row[] result;
string apihost = "abc.com"; int apiport = 1830; string apiuri =
/getdata;
string apiuser = "user"; string apipass = "pass";
foreach(i, k; parallel(result,1))
{
string url = chain(apihost, only(':'), to!string(apiport),
apiuri).to!string;
string usrpass = chain(apiuser, only(':'),
apipass).to!string;
auto https = HTTP();
https.handle.set(CurlOption.buffersize, 512000);
https.handle.set(CurlOption.userpwd, usrpass);
https.handle.set(CurlOption.connecttimeout, 600);
https.handle.set(CurlOption.tcp_nodelay, 1);
https.handle.set(CurlOption.http_version, 2);
https.handle.set(CurlOption.sslversion, 1;
https.handle.set(CurlOption.use_ssl, 3);
https.handle.set(CurlOption.ssl_verifypeer, 0);
https.handle.set(CurlOption.url, url);
https.method(HTTP.Method.get);
https.StatusLine st;
https.onReceiveStatusLine = (https.StatusLine st) { if
(st.code != 200) { throw new Exception(st.reason); } };
ubyte[] content;
https.onReceive = (ubyte[] data) { content ~= data; return
data.length; };
https.perform();
scope(failure) { https.shutdown; exit(-4); } scope(exit)
https.shutdown;
apidata.insert(tuple(seq, cast(string) content));
}
return apidata[].sort;
}
From,
Vino.B
Nov 09 2020
On Monday, 9 November 2020 at 19:55:07 UTC, Vino wrote:...The only curl option you need to set within the loop is the CurlOption.url. So your foreach block should look more like: foreach (...) { string url = chain(apihost, only(':'), to!string(apiport), apiuri).to!string; https.handle.set(CurlOption.url, url); https.perform(); scope(failure) exit(-4); scope(exit) https.shutdown; apidata.insert(tuple(seq, cast(string) content)); content = []; } Every other line can be placed before the foreach.
Nov 09 2020
On Monday, 9 November 2020 at 20:40:59 UTC, rinfz wrote:On Monday, 9 November 2020 at 19:55:07 UTC, Vino wrote:In fact, you don't need url in there either since it's not dependent on loop variables, and you don't need the scope guards if this is running in main (you can move them out of the loop too). foreach (...) { https.handle.set(CurlOption.url, url); https.perform(); apidata.insert(tuple(seq, cast(string) content)); content = []; }...The only curl option you need to set within the loop is the CurlOption.url. So your foreach block should look more like: foreach (...) { string url = chain(apihost, only(':'), to!string(apiport), apiuri).to!string; https.handle.set(CurlOption.url, url); https.perform(); scope(failure) exit(-4); scope(exit) https.shutdown; apidata.insert(tuple(seq, cast(string) content)); content = []; } Every other line can be placed before the foreach.
Nov 09 2020
On Mon, Nov 9, 2020 at 9:50 PM rinfz via Digitalmars-d-learn < digitalmars-d-learn puremagic.com> wrote:On Monday, 9 November 2020 at 20:40:59 UTC, rinfz wrote:In fact he does not need foreach. Because he use it on empty resultOn Monday, 9 November 2020 at 19:55:07 UTC, Vino wrote:In fact, you don't need url in there either since it's not dependent on loop variables, and you don't need the scope guards if this is running in main (you can move them out of the loop too). foreach (...) { https.handle.set(CurlOption.url, url); https.perform(); apidata.insert(tuple(seq, cast(string) content)); content = []; }...The only curl option you need to set within the loop is the CurlOption.url. So your foreach block should look more like: foreach (...) { string url = chain(apihost, only(':'), to!string(apiport), apiuri).to!string; https.handle.set(CurlOption.url, url); https.perform(); scope(failure) exit(-4); scope(exit) https.shutdown; apidata.insert(tuple(seq, cast(string) content)); content = []; } Every other line can be placed before the foreach.
Nov 09 2020
On Monday, 9 November 2020 at 20:57:33 UTC, Daniel Kozak wrote:On Mon, Nov 9, 2020 at 9:50 PM rinfz via Digitalmars-d-learn < digitalmars-d-learn puremagic.com> wrote:Hi All, The reason that the above code is within foreach loop is because there are several api's and each of the api's have separate configuration. These configuration are maintained in a table in MySQL database, the result contains the configuration details for each of these api's, below is the complete code. Sequence The code collects the data from several api's whose output is json. The output of the api's are parsed, filtered, validated and stored in a array using another function.(using asdf json parser) The data stores in array is then inserted into the table in MySQL database.(using hunt-database). Code: auto getData () { Array!(Tuple!(int,string)) apidata; Row[] result; result = getApiconf(); \\ fetch's the api configuration from the database table. foreach(i, k; parallel(result,1)) { string apihost = result[i][0].get!(string); int apiport = result[i][1].get!(int); string apiuser = result[i][2].get!(string); string apipass = result[i][3].get!(string); int seq = result[i][4].get!int; string apiuri = result[i][5].get!(string); int connecttimeout = result[i][6].get!(int); int tcp_nodelay = result[i][7].get!(int); int http_version = result[i][8].get!(int); int sslversion = result[i][9].get!(int); int use_ssl = result[i][10].get!(int); int ssl_verifypeer = result[i][11].get!(int); string url = chain(apihost, only(':'), to!string(apiport), apiuri).to!string; string usrpass = chain(apiuser, only(':'), apipass).to!string; auto https = HTTP(); https.handle.set(CurlOption.buffersize, 512000); https.handle.set(CurlOption.userpwd, usrpass); https.handle.set(CurlOption.connecttimeout, connecttimeout); https.handle.set(CurlOption.tcp_nodelay, tcp_nodelay); https.handle.set(CurlOption.http_version, http_version); https.handle.set(CurlOption.sslversion, sslversion); https.handle.set(CurlOption.use_ssl, use_ssl); https.handle.set(CurlOption.ssl_verifypeer, ssl_verifypeer); https.handle.set(CurlOption.url, url); https.method(HTTP.Method.get); https.StatusLine st; https.onReceiveStatusLine = (https.StatusLine st) { if (st.code != 200) { throw new Exception(st.reason); } }; ubyte[] content; https.onReceive = (ubyte[] data) { content ~= data; return data.length; }; https.perform(); scope(failure) { https.shutdown; exit(-4); } scope(exit) https.shutdown; apidata.insert(tuple(seq, cast(string) content)); } return apidata[].sort; } From, Vino.BOn Monday, 9 November 2020 at 20:40:59 UTC, rinfz wrote:In fact he does not need foreach. Because he use it on empty resultOn Monday, 9 November 2020 at 19:55:07 UTC, Vino wrote:In fact, you don't need url in there either since it's not dependent on loop variables, and you don't need the scope guards if this is running in main (you can move them out of the loop too). foreach (...) { https.handle.set(CurlOption.url, url); https.perform(); apidata.insert(tuple(seq, cast(string) content)); content = []; }...The only curl option you need to set within the loop is the CurlOption.url. So your foreach block should look more like: foreach (...) { string url = chain(apihost, only(':'), to!string(apiport), apiuri).to!string; https.handle.set(CurlOption.url, url); https.perform(); scope(failure) exit(-4); scope(exit) https.shutdown; apidata.insert(tuple(seq, cast(string) content)); content = []; } Every other line can be placed before the foreach.
Nov 09 2020