digitalmars.D.learn - segfault on gc.impl.conservative.gc.Gcx.smallAlloc
- Mengu (182/182) Sep 25 2017 hi all
- Vladimir Panteleev (8/9) Sep 25 2017 This looks suspicious - it is a slice of the memory-mapped file,
- Mengu (5/14) Sep 26 2017 thanks vladimir, i'll look into that.
- Mengu (3/8) Sep 26 2017 hi all
- drug (3/102) Sep 26 2017 not big deal probably, but isn't ~32GB enormous value here? I would
- Mengu (8/12) Sep 26 2017 i could not find out why.
hi all
this following code block [0] is exiting with "terminated by
signal SIGBUS (Misaligned address error)" error. it processes
like 200K rows and then fails. any ideas?
void getHotels() {
import std.parallelism : taskPool;
import std.functional : partial;
auto sunHotels = new SunHotels();
auto destinations = sunHotels.parseDestinations();
auto conn = client.lockConnection();
auto destinationResult = conn.execStatement("SELECT
provider_ref, id FROM hotels_destination", ValueFormat.BINARY);
int[int] destinationIds;
foreach (row; rangify(destinationResult)) {
destinationIds[row["provider_ref"].as!PGinteger] =
row["id"].as!PGinteger;
}
foreach (destination; parallel(destinations)) {
const string destId = to!string(destination.destinationId);
const auto destinationFilePath = getcwd() ~
"/ext/data/hotels/" ~ destId ~ ".xml";
auto xmlFile = new MmFile(destinationFilePath);
auto fileContents = cast(string)xmlFile[0..xmlFile.length];
auto hotels = sunHotels.parseHotelsResult(fileContents);
const destIdInDb = destinationIds[destination.destinationId];
auto sqls = appender!string;
writeln("parsing destination: ", destination.destinationName);
foreach (hotel; parallel(hotels)) {
sqls.put(hotel.generateSql(destIdInDb).data.join);
}
sqls.data.writeln;
delete fileContents;
delete xmlFile;
}
}
this is the full trace i got [1]:
gc.impl.conservative.gc.Gcx.smallAlloc(ubyte, ref ulong, uint) ()
[Current thread is 1 (LWP 100171)]
(gdb) bt full
gc.impl.conservative.gc.Gcx.smallAlloc(ubyte, ref ulong, uint) ()
No symbol table info available.
gc.impl.conservative.gc.ConservativeGC.runLocked!(gc.impl.conservative.gc.ConservativeGC
mallocNoSync(ulong, uint, ref ulong, const(TypeInfo)),
gc.impl.conservative.gc.mallocTime, gc.impl.conservative.gc.numMallocs, ulong,
uint, ulong, const(TypeInfo)).runLocked(ref ulong, ref uint, ref ulong, ref
const(TypeInfo)) ()
No symbol table info available.
gc.impl.conservative.gc.ConservativeGC.qalloc(ulong, uint,
const(TypeInfo)) ()
No symbol table info available.
No symbol table info available.
const(TypeInfo)) ()
No symbol table info available.
std.array.Appender!(immutable(char)[]).Appender.ensureAddable(ulong) (this=...,
nelems=761)
at
/home/search-master/dmd2/freebsd/bin64/../../src/phobos/std/array.d:2929
len = 394
reqlen = 1155
newlen = 1155
u = 0
overflow = false
nbytes = 1155
bi = {base = 0x2b1, size = 140737488337376, attr =
4294949280}
std.array.Appender!(immutable(char)[]).Appender.put!(immutable(char)[]).put(immutable(char)[
).bigDataFun(ulong) (
this=0x7fffffffb9f0, extra=761)
at
/home/search-master/dmd2/freebsd/bin64/../../src/phobos/std/array.d:3023
No locals.
std.array.Appender!(immutable(char)[]).Appender.put!(immutable(char)[]).pu
(immutable(char)[]) (this=...,
---Type <return> to continue, or q <return> to quit---
at
/home/search-master/dmd2/freebsd/bin64/../../src/phobos/std/array.d:3026
bigData = "'"
len = 760
newlen = 34799909888
destinationId=5743) at source/hotel.d:216
sqls = {_data = 0x81d5ca6c0}
sql = {_data = 0x81d5ca6e0}
childSqls = {_data = 0x81d5ca700}
destination.Destination).__foreachbody2(ref hotel.Hotel)
(this=0x7fffffffd2a0,
__applyArg0=...) at source/app.d:211
hotel = {provider_ref = 121475, destinationId = 7931,
resortId = 11313, transfer = 0, roomTypes = {{rooms = {{roomType
= 0x81a351188,
hotelId = 121475, roomId = 5369802, beds =
2, extrabeds = 0, meals = 0x0}}, roomType = "Twin/Double room",
roomTypeId = 21}, {rooms = {{roomType =
0x81a35c108, hotelId = 121475, roomId = 5761375, beds = 2,
extrabeds = 0,
meals = 0x0}}, roomType = "Double room
Queen bed", roomTypeId = 2651}}, reviews = 0x0, distance =
nan(0xc000000000000),
type = "Hotel", name = "Best Western Carriage House Inn
and Suites", addr_1 = "1936 Highway 45 Bypass", addr_2 = 0x0,
zip_code = "38305", city = "Jackson", state = "TN",
country = "United States", country_code = "US",
address = "1936 Highway 45 Bypass 38305 Jackson TN
United States",
mapurl =
"http://www.sunhotels.net/GoogleMaps/showGoogleMap.asp?hotelId=121475&cc=en",
headline = "With a stay at Best Western Carriage House
Inn & Suites in Jackson, you'll be minutes from Casey Jones
Village and close to Old Hickory Mall",
description = "With a stay at Best Western Carriage
House Inn & Suites in Jackson, you'll be minutes from Casey Jones
Village and close to Old Hickory Mall. This hotel is within the
vicinity of International Rock-A"..., resort = "Jackson",
destination = "Jackson (TN)",
images = {{hotelId = 121475, id = 2378342}, {hotelId =
121475, id = 2378343}, {hotelId = 121475, id = 2378344}, {hotelId
= 121475,
id = 2378345}, {hotelId = 121475, id = 2378347},
{hotelId = 121475, id = 2378348}, {hotelId = 121475, id =
2378349}, {
hotelId = 121475, id = 2378350}, {hotelId =
121475, id = 2378351}, {hotelId = 121475, id = 2378352}, {hotelId
= 121475,
id = 2378353}, {hotelId = 121475, id = 2378354},
{hotelId = 121475, id = 2378355}, {hotelId = 121475, id =
2378356}, {
hotelId = 121475, id = 2378357}, {hotelId =
121475, id = 2378359}, {hotelId = 121475, id = 2378360}, {hotelId
= 121475,
id = 2378362}, {hotelId = 121475, id = 2378363},
{hotelId = 121475, id = 2378364}, {hotelId = 121475, id =
2378365}, {
hotelId = 121475, id = 2378366}, {hotelId =
121475, id = 2378368}, {hotelId = 121475, id = 2378369}, {hotelId
= 121475,
id = 2378371}, {hotelId = 121475, id = 2378379},
{hotelId = 121475, id = 2378386}, {hotelId = 121475, id =
2378389}, {
hotelId = 121475, id = 2378395}}, classification
= "2", features = 0x0, timezone = "-06:00", latitude =
"35.660888671875",
longitude = "-88.853157043457031"}
std.parallelism.ParallelForeach!(hotel.Hotel[]).ParallelForeach.opApply(scope
int(ref hotel.Hotel) delegate).doIt() (
this=0x7fffffffd0d0) at
/home/search-master/dmd2/freebsd/bin64/../../src/phobos/std/parallelism.d-mixin-3824:3870
myUnitIndex = 3
start = 3
end = 4
---Type <return> to continue, or q <return> to quit---
__key6543 = 3
__limit6544 = 4
i = 3
__EAX = 0x81d5d4800
__EDX = 1730176101
__handler = 7
__exception_object = 0x6
delegate).run(void() delegate) ()
No symbol table info available.
std.parallelism.Task!(std.parallelism.run, void()
delegate).Task.impl(void*) ()
No symbol table info available.
No symbol table info available.
std.parallelism.submitAndExecute(std.parallelism.TaskPool, scope
void() delegate) ()
No symbol table info available.
std.parallelism.ParallelForeach!(hotel.Hotel[]).ParallelForeach.opApply(scope
int(ref hotel.Hotel) delegate) (
this=..., dg=...) at
/home/search-master/dmd2/freebsd/bin64/../../src/phobos/std/parallelism.d-mixin-3824:3876
workUnitIndex = 5
len = 26
shouldContinue = true
[0] https://pastebin.com/JmsTKU95
[1] https://pastebin.com/v1Yufz7g
Sep 25 2017
On Monday, 25 September 2017 at 21:34:40 UTC, Mengu wrote:
delete fileContents;
This looks suspicious - it is a slice of the memory-mapped file,
not memory on the GC-managed heap, so "delete" is inapplicable to
it. The GC ought to throw an exception when attempting to delete
things not on the GC heap though.
I think the stack trace itself looks like something that should
only happen when the GC's internal data structures are corrupted,
so you may want to investigate in that direction.
Sep 25 2017
Mengu <mengukagan gmail.com> On Tuesday, 26 September 2017 at 00:36:36 UTC, Vladimir Panteleev wrote:On Monday, 25 September 2017 at 21:34:40 UTC, Mengu wrote:thanks vladimir, i'll look into that. i also should mention that everything works as expected on mac os x.delete fileContents;This looks suspicious - it is a slice of the memory-mapped file, not memory on the GC-managed heap, so "delete" is inapplicable to it. The GC ought to throw an exception when attempting to delete things not on the GC heap though. I think the stack trace itself looks like something that should only happen when the GC's internal data structures are corrupted, so you may want to investigate in that direction.
Sep 26 2017
On Monday, 25 September 2017 at 21:34:40 UTC, Mengu wrote:hi all this following code block [0] is exiting with "terminated by signal SIGBUS (Misaligned address error)" error. it processes like 200K rows and then fails. any ideas? [...]hi all does anyone else have any ideas?
Sep 26 2017
26.09.2017 00:34, Mengu пишет:
hi all
this following code block [0] is exiting with "terminated by signal
SIGBUS (Misaligned address error)" error. it processes like 200K rows
and then fails. any ideas?
void getHotels() {
import std.parallelism : taskPool;
import std.functional : partial;
auto sunHotels = new SunHotels();
auto destinations = sunHotels.parseDestinations();
auto conn = client.lockConnection();
auto destinationResult = conn.execStatement("SELECT provider_ref, id
FROM hotels_destination", ValueFormat.BINARY);
int[int] destinationIds;
foreach (row; rangify(destinationResult)) {
destinationIds[row["provider_ref"].as!PGinteger] =
row["id"].as!PGinteger;
}
foreach (destination; parallel(destinations)) {
const string destId = to!string(destination.destinationId);
const auto destinationFilePath = getcwd() ~ "/ext/data/hotels/" ~
destId ~ ".xml";
auto xmlFile = new MmFile(destinationFilePath);
auto fileContents = cast(string)xmlFile[0..xmlFile.length];
auto hotels = sunHotels.parseHotelsResult(fileContents);
const destIdInDb = destinationIds[destination.destinationId];
auto sqls = appender!string;
writeln("parsing destination: ", destination.destinationName);
foreach (hotel; parallel(hotels)) {
sqls.put(hotel.generateSql(destIdInDb).data.join);
}
sqls.data.writeln;
delete fileContents;
delete xmlFile;
}
}
this is the full trace i got [1]:
ref ulong, uint) ()
[Current thread is 1 (LWP 100171)]
(gdb) bt full
ref ulong, uint) ()
No symbol table info available.
gc.impl.conservative.gc.ConservativeGC.runLocked!(gc.impl.conservative.gc.ConservativeGC
mallocNoSync(ulong,
uint, ref ulong, const(TypeInfo)), gc.impl.conservative.gc.mallocTime,
gc.impl.conservative.gc.numMallocs, ulong, uint, ulong,
const(TypeInfo)).runLocked(ref ulong, ref uint, ref ulong, ref
const(TypeInfo)) ()
No symbol table info available.
gc.impl.conservative.gc.ConservativeGC.qalloc(ulong, uint,
const(TypeInfo)) ()
No symbol table info available.
No symbol table info available.
const(TypeInfo)) ()
No symbol table info available.
std.array.Appender!(immutable(char)[]).Appender.ensureAddable(ulong)
(this=..., nelems=761)
at
/home/search-master/dmd2/freebsd/bin64/../../src/phobos/std/array.d:2929
len = 394
reqlen = 1155
newlen = 1155
u = 0
overflow = false
nbytes = 1155
bi = {base = 0x2b1, size = 140737488337376, attr = 4294949280}
std.array.Appender!(immutable(char)[]).Appender.put!(immutable(char)[]).put(immutable(char)[
).bigDataFun(ulong)
(
this=0x7fffffffb9f0, extra=761)
at
/home/search-master/dmd2/freebsd/bin64/../../src/phobos/std/array.d:3023
No locals.
std.array.Appender!(immutable(char)[]).Appender.put!(immutable(char)[]).pu
(immutable(char)[])
(this=...,
---Type <return> to continue, or q <return> to quit---
at
/home/search-master/dmd2/freebsd/bin64/../../src/phobos/std/array.d:3026
bigData = "'"
len = 760
newlen = 34799909888
not big deal probably, but isn't ~32GB enormous value here? I would
check why bigDataFun return this.
Sep 26 2017
On Tuesday, 26 September 2017 at 17:06:28 UTC, drug wrote:26.09.2017 00:34, Mengu пишет:i could not find out why. d certainly needs to improve on freebsd. i don't think devs would take d seriously if it won't work on a major platform. for now, i switched to debian rather than rewriting the app. i had some invalid memory operation errors but i managed to get rid of them. i'll certainly port the app to another language and switch back to freebsd.[...]not big deal probably, but isn't ~32GB enormous value here? I would check why bigDataFun return this.
Sep 26 2017