digitalmars.D.learn - libcurl acting differently to curl.exe
I'm trying to use libcurl in D to download a page that requires logging in first. At the moment though, I can't even get the logging in working. I tried with curl.exe, got it working, and used the --libcurl command to export C code that I then turned into (I think) equivalent D code. The problem is that when I POST the login form, curl.exe gets a HTTP 200 to the correct page, whereas libcurl gets a HTTP 302 back to the login page, which is the same behaviour I noticed when cookies weren't being saved/reused. I've tried with the HTTP struct functions, and also setting curl options explicitly like the C code does. Is there something I've missed? import std.file; import std.net.curl; import std.stdio; void main() { getLogon(); postLogon(); getIndex(); } void getLogon() { auto http = HTTP(); http.onReceive = (ubyte[] data) { /+ drop +/ std.file.write("logon1.html", data); /+writeln(cast(char[])(data)); stdout.flush;+/ return data.length; }; http.handle.set(CurlOption.tcp_nodelay, 1); http.handle.set(CurlOption.buffersize, 102400); http.handle.set(CurlOption.noprogress, 1); //http.handle.set(CurlOption.useragent, "curl/7.57.0"); http.handle.set(CurlOption.maxredirs, 50); //http.handle.set(CurlOption.cainfo, "C:\\Users\\Josh\\Downloads\\curl-7.57.0-win64-mingw\\bin\\curl-ca-bundle.crt"); http.handle.set(CurlOption.cookiejar, "cookie.dat"); http.handle.set(CurlOption.cookiefile, "cookie.dat"); http.handle.set(CurlOption.verbose, 1); http.handle.set(CurlOption.url, "https://foo.com/logon.php"); http.method(HTTP.Method.get); http.perform(); } void doLogon() { auto http = HTTP(); http.onReceive = (ubyte[] data) { /+ drop +/ std.file.write("logon2.html", data); /+writeln(cast(char[])(data)); stdout.flush;+/ return data.length; }; http.handle.set(CurlOption.tcp_nodelay, 1); http.handle.set(CurlOption.buffersize, 102400); http.handle.set(CurlOption.noprogress, 1); //http.handle.set(CurlOption.useragent, "curl/7.57.0"); http.handle.set(CurlOption.maxredirs, 50); //http.handle.set(CurlOption.cainfo, "C:\\Users\\Josh\\Downloads\\curl-7.57.0-win64-mingw\\bin\\curl-ca-bundle.crt"); http.handle.set(CurlOption.cookiejar, "cookie.dat"); http.handle.set(CurlOption.cookiefile, "cookie.dat"); http.handle.set(CurlOption.verbose, 1); http.handle.set(CurlOption.url, "https://foo.com/logon.php"); http.handle.set(CurlOption.postfields, "username=user&password=pass&Logon=submit"); http.handle.set(CurlOption.postfieldsize_large, 52); http.method(HTTP.Method.post); http.perform(); } void getIndex() { auto http = HTTP(); http.onReceive = (ubyte[] data) { /+ drop +/ std.file.write("index.html", data); /+writeln(cast(char[])(data)); stdout.flush;+/ return data.length; }; http.handle.set(CurlOption.tcp_nodelay, 1); http.handle.set(CurlOption.buffersize, 102400); http.handle.set(CurlOption.noprogress, 1); //http.handle.set(CurlOption.useragent, "curl/7.57.0"); http.handle.set(CurlOption.maxredirs, 50); //http.handle.set(CurlOption.cainfo, "C:\\Users\\Josh\\Downloads\\curl-7.57.0-win64-mingw\\bin\\curl-ca-bundle.crt"); http.handle.set(CurlOption.cookiejar, "cookie.dat"); http.handle.set(CurlOption.cookiefile, "cookie.dat"); http.handle.set(CurlOption.verbose, 1); http.handle.set(CurlOption.url, "https://foo.com/index.php"); http.method(HTTP.Method.get); http.perform(); } libcurl verbose dump: https://pastebin.com/Sq60CLHV curl.exe verbose dump: https://pastebin.com/KBDDNq9k Thanks, Josh
Dec 10 2017
The POST C code was: /********* Sample code generated by the curl command line tool ********** * All curl_easy_setopt() options are documented at: * https://curl.haxx.se/libcurl/c/curl_easy_setopt.html ************************************************************************/ #include <curl/curl.h> int main(int argc, char *argv[]) { CURLcode ret; CURL *hnd; hnd = curl_easy_init(); curl_easy_setopt(hnd, CURLOPT_BUFFERSIZE, 102400L); curl_easy_setopt(hnd, CURLOPT_URL, "https://foo.com/logon.php"); curl_easy_setopt(hnd, CURLOPT_POSTFIELDS, "username=user&password=pass&Logon=submit"); curl_easy_setopt(hnd, CURLOPT_POSTFIELDSIZE_LARGE, (curl_off_t)52); curl_easy_setopt(hnd, CURLOPT_USERAGENT, "curl/7.57.0"); curl_easy_setopt(hnd, CURLOPT_MAXREDIRS, 50L); curl_easy_setopt(hnd, CURLOPT_HTTP_VERSION, (long)CURL_HTTP_VERSION_2TLS); curl_easy_setopt(hnd, CURLOPT_CAINFO, "C:\\Users\\Josh\\Downloads\\curl-7.57.0-win64-mingw\\bin\\curl-ca-bundle.crt"); curl_easy_setopt(hnd, CURLOPT_SSH_KNOWNHOSTS, "C:\\Users\\Josh\\AppData\\Roaming/_ssh/known_hosts"); curl_easy_setopt(hnd, CURLOPT_COOKIEFILE, "cookie.dat"); curl_easy_setopt(hnd, CURLOPT_COOKIEJAR, "cookie.dat"); curl_easy_setopt(hnd, CURLOPT_VERBOSE, 1L); curl_easy_setopt(hnd, CURLOPT_TCP_KEEPALIVE, 1L); /* Here is a list of options the curl code used that cannot get generated as source easily. You may select to either not use them or implement them yourself. CURLOPT_WRITEDATA set to a objectpointer CURLOPT_INTERLEAVEDATA set to a objectpointer CURLOPT_WRITEFUNCTION set to a functionpointer CURLOPT_READDATA set to a objectpointer CURLOPT_READFUNCTION set to a functionpointer CURLOPT_SEEKDATA set to a objectpointer CURLOPT_SEEKFUNCTION set to a functionpointer CURLOPT_ERRORBUFFER set to a objectpointer CURLOPT_STDERR set to a objectpointer CURLOPT_DEBUGFUNCTION set to a functionpointer CURLOPT_DEBUGDATA set to a objectpointer CURLOPT_HEADERFUNCTION set to a functionpointer CURLOPT_HEADERDATA set to a objectpointer */ ret = curl_easy_perform(hnd); curl_easy_cleanup(hnd); hnd = NULL; return (int)ret; } /**** End of sample code ****/
Dec 10 2017
On Monday, 11 December 2017 at 03:53:25 UTC, Josh wrote:The POST C code was: /********* Sample code generated by the curl command line tool ********** * All curl_easy_setopt() options are documented at: * https://curl.haxx.se/libcurl/c/curl_easy_setopt.htmlMaybe off-topic but you can try http://code.dlang.org/packages/requests.
Dec 11 2017