digitalmars.D.learn - request assistance resolving a std.net.curl segmentation fault
- anonymouse (72/72) May 19 2023 What am I doing wrong here?
- kdevel (6/9) May 19 2023 According to [1] this line must read
- anonymouse (2/11) May 19 2023 Thank you so much.
- anonymouse (2/11) May 19 2023 Thank you so much.
- Danny Arends (39/111) May 19 2023 You're running the whole thing in a while(TRUE) loop, recreating
- anonymouse (13/24) May 19 2023 The reason I used a while loop was to detect loss of internet
- kdevel (23/28) May 20 2023 What if the internet connection is not re-established within an
- anonymouse (5/21) May 24 2023 I am the interactive user but I'm usually not on site to monitor
What am I doing wrong here?
```D
import std.net.curl: Curl, CurlOption, CurlException;
import std.file: exists;
import std.stdio: File, writefln;
import core.thread: Thread;
void downloadFile(string url, string filename)
{
while (true) {
try {
File fp;
if (filename.exists())
fp.open(filename, "a");
else
fp.open(filename, "w");
Curl curl;
curl.initialize();
curl.onProgress = delegate int(size_t dltotal, size_t
dlnow, size_t ultotal, size_t ulnow)
{
writefln("Progress: %s of %s", dlnow, dltotal);
return 0;
};
curl.set(CurlOption.url, url~filename);
curl.set(CurlOption.resume_from_large, fp.size());
// Start the download
curl.set(CurlOption.writedata, &fp);
curl.perform();
// Close the file
fp.close();
writefln("Download as %s complete.", filename);
break;
} catch (CurlException e) {
writefln("Error while downloading: %s", e.msg);
// Wait for a bit before retrying
Thread.sleep(imported!"core.time".seconds(10));
}
}
}
void main()
{
string url =
"https://downloads.dlang.org/releases/2.x/2.103.1/";
string filename = "dmd.2.103.1.dmg";
downloadFile(url, filename);
}
```
Output:
```
./download_file
Progress: 0 of 0
Progress: 0 of 0
Progress: 0 of 0
Progress: 0 of 0
Progress: 0 of 0
Progress: 0 of 0
Progress: 0 of 0
Progress: 0 of 0
Progress: 0 of 0
Progress: 0 of 0
Progress: 0 of 0
Progress: 0 of 0
Progress: 0 of 0
Progress: 0 of 0
Progress: 0 of 0
Progress: 0 of 0
Progress: 0 of 0
Progress: 0 of 0
zsh: segmentation fault ./download_file
```
Thanks.
--anonymouse
May 19 2023
On Friday, 19 May 2023 at 11:07:01 UTC, anonymouse wrote:
What am I doing wrong here?
[...]
curl.set(CurlOption.writedata, &fp);
According to [1] this line must read
```
curl.set(CurlOption.writedata, cast (void *) fp.getFP());
```
[1] https://curl.se/libcurl/c/CURLOPT_WRITEDATA.html
May 19 2023
anonymouse <anony mouse.com> On Friday, 19 May 2023 at 12:28:20 UTC, kdevel wrote:On Friday, 19 May 2023 at 11:07:01 UTC, anonymouse wrote:Thank you so much.What am I doing wrong here? [...] curl.set(CurlOption.writedata, &fp);According to [1] this line must read ``` curl.set(CurlOption.writedata, cast (void *) fp.getFP()); ``` [1] https://curl.se/libcurl/c/CURLOPT_WRITEDATA.html
May 19 2023
On Friday, 19 May 2023 at 12:28:20 UTC, kdevel wrote:On Friday, 19 May 2023 at 11:07:01 UTC, anonymouse wrote:Thank you so much.What am I doing wrong here? [...] curl.set(CurlOption.writedata, &fp);According to [1] this line must read ``` curl.set(CurlOption.writedata, cast (void *) fp.getFP()); ``` [1] https://curl.se/libcurl/c/CURLOPT_WRITEDATA.html
May 19 2023
On Friday, 19 May 2023 at 11:07:01 UTC, anonymouse wrote:
What am I doing wrong here?
```D
import std.net.curl: Curl, CurlOption, CurlException;
import std.file: exists;
import std.stdio: File, writefln;
import core.thread: Thread;
void downloadFile(string url, string filename)
{
while (true) {
try {
File fp;
if (filename.exists())
fp.open(filename, "a");
else
fp.open(filename, "w");
Curl curl;
curl.initialize();
curl.onProgress = delegate int(size_t dltotal,
size_t dlnow, size_t ultotal, size_t ulnow)
{
writefln("Progress: %s of %s", dlnow, dltotal);
return 0;
};
curl.set(CurlOption.url, url~filename);
curl.set(CurlOption.resume_from_large, fp.size());
// Start the download
curl.set(CurlOption.writedata, &fp);
curl.perform();
// Close the file
fp.close();
writefln("Download as %s complete.", filename);
break;
} catch (CurlException e) {
writefln("Error while downloading: %s", e.msg);
// Wait for a bit before retrying
Thread.sleep(imported!"core.time".seconds(10));
}
}
}
void main()
{
string url =
"https://downloads.dlang.org/releases/2.x/2.103.1/";
string filename = "dmd.2.103.1.dmg";
downloadFile(url, filename);
}
```
Output:
```
./download_file
Progress: 0 of 0
Progress: 0 of 0
Progress: 0 of 0
Progress: 0 of 0
Progress: 0 of 0
Progress: 0 of 0
Progress: 0 of 0
Progress: 0 of 0
Progress: 0 of 0
Progress: 0 of 0
Progress: 0 of 0
Progress: 0 of 0
Progress: 0 of 0
Progress: 0 of 0
Progress: 0 of 0
Progress: 0 of 0
Progress: 0 of 0
Progress: 0 of 0
zsh: segmentation fault ./download_file
```
Thanks.
--anonymouse
You're running the whole thing in a while(TRUE) loop, recreating
the curl object re-initiating the transfer and file pointer, etc.
furthermore, the curl.set(CurlOption.writedata,
&fp); doesn't work as you expect..
After fiddling a bit, this works:
```D
import std.net.curl: Curl, CurlOption, CurlException;
import std.file: exists;
import std.stdio: File, writefln;
import core.thread: Thread;
void downloadFile(string url, string filename){
try {
File fp;
fp.open(filename, "w");
Curl curl;
curl.initialize();
curl.onProgress = delegate int(size_t dltotal, size_t dlnow,
size_t ultotal, size_t ulnow){
writefln("Progress: %s of %s", dlnow, dltotal);
return 0;
};
curl.onReceive = (ubyte[] data) { fp.rawWrite(data); return
data.length;};
curl.set(CurlOption.url, url~filename);
// Start the download
curl.perform();
writefln("Download as %s complete.", filename);
} catch (CurlException e) {
writefln("Error while downloading: %s", e.msg);
}
}
void main(){
string url =
"https://downloads.dlang.org/releases/2.x/2.103.1/";
string filename = "dmd.2.103.1.dmg";
downloadFile(url, filename);
}
```
May 19 2023
On Friday, 19 May 2023 at 12:40:29 UTC, Danny Arends wrote:On Friday, 19 May 2023 at 11:07:01 UTC, anonymouse wrote:[SNIP]What am I doing wrong here?You're running the whole thing in a while(TRUE) loop, recreating the curl object re-initiating the transfer and file pointer, etc.The reason I used a while loop was to detect loss of internet connection and resume the process once the connection is re-established. What would have been a better approach?furthermore, the curl.set(CurlOption.writedata, &fp); doesn't work as you expect..The idea was to detect an incomplete download and continue from where it left off. I'm sometimes downloading files 15Gb or greater. Reaching 80% and having to restart the process is a nogo. As I understand it, `CurlOption.writedata` allows me to achieve that goal. Is there a better option to accomplish the same?After fiddling a bit, this works: curl.onReceive = (ubyte[] data) { fp.rawWrite(data); return data.length;};Thank you for your assistance thus far. --anonymouse
May 19 2023
On Friday, 19 May 2023 at 23:36:28 UTC, anonymouse wrote:[...] The reason I used a while loop was to detect loss of internet connection and resume the process once the connection is re-established.What if the internet connection is not re-established within an reasonable amount of time? What if the resource is no longer available on the server (HTTP eror 404 [1])? If there is an interactive user: Wouldn't it be better have the user restart the download at his discretion?What would have been a better approach?That depends on where you want to use that download function. If it is intended to download a full software update of a modern e-vehicle I would suggest not to use such an endless loop. I would limit the retries to a low single-digit number greater than one and of log the event. [1] The 404 or other errors are not detected by default. ``` curl.set(CurlOption.failonerror, 1); ``` must be set. In case of error an exception is thrown. This unfortunately does not contain the required error information. It seems that one must supply an error buffer ``` ubyte [<?>] buf; curl.set (CurlOption.errorbuffer, buf.ptr); ``` to store that result.
May 20 2023
On Saturday, 20 May 2023 at 09:20:54 UTC, kdevel wrote:What if the internet connection is not re-established within an reasonable amount of time? What if the resource is no longer available on the server (HTTP eror 404 [1])? If there is an interactive user: Wouldn't it be better have the user restart the download at his discretion?I am the interactive user but I'm usually not on site to monitor it while this is happening.Noted.What would have been a better approach?That depends on where you want to use that download function. If it is intended to download a full software update of a modern e-vehicle I would suggest not to use such an endless loop. I would limit the retries to a low single-digit number greater than one and of log the event.``` ubyte [<?>] buf; curl.set (CurlOption.errorbuffer, buf.ptr); ``` to store that result.Okay. Got it. Thank you.
May 24 2023