Frequent internal test runner failure: EndOfStream on powerpc64le-linux #31562

Since the merge of #31403, we've seen this failure frequently on PowerPC. I've only seen it when the test runner is built in release modes. The build mode of the compiler does not seem to matter.

test
mq test-modules
 mq test-std
 mq run test std-native-pwr10-ReleaseSafe-libc failure
thread 947389 panic: internal test runner failure: EndOfStream
/home/ci/.cache/act/4a7b3683960aac35/hostexecutor/lib/compiler/test_runner.zig:68:60: 0x11e18af in main (test)
 return mainServer(init) catch |err| std.debug.panic("internal test runner failure: {t}", .{err});
 ^
???:?:?: 0x3fff94796f0b in ??? (/lib/powerpc64le-linux-gnu/libc.so.6)
???:?:?: 0x3fff9479714b in ??? (/lib/powerpc64le-linux-gnu/libc.so.6)
error: test process unexpectedly terminated with signal ABRT
failed command: /home/ci/.cache/act/4a7b3683960aac35/hostexecutor/zig-local-cache/o/a5a8897b77e0551d7332cca83deffa41/test --cache-dir=/home/ci/.cache/act/4a7b3683960aac35/hostexecutor/zig-local-cache --seed=0x9b11b31b --listen=-

Some more findings:

• It only happens with --listen=-.
• It only happens with -lc.

ReleaseSafe error trace with a slightly modified test runner:

test-std
└─ run test std-native-pwr10-ReleaseSafe-libc failure
[default] (err): EndOfStream
/home/ci/zig-repro/lib/compiler/test_runner.zig:68:9: 0x1202e4f in mainServer (test)
 return mainServer(init);
 ^
/home/ci/zig-repro/lib/compiler/test_runner.zig:68:9: 0x1202e57 in mainServer (test)
 return mainServer(init);
 ^
/home/ci/zig-repro/lib/compiler/test_runner.zig:102:60: 0x1202e1f in mainServer (test)
 defer if (testing.allocator_instance.deinit() == .leak) {
 ^
/home/ci/zig-repro/lib/compiler/test_runner.zig:68:9: 0x1202e27 in main (test)
 return mainServer(init);
 ^
error: test process unexpectedly exited with code 1
failed command: ./.zig-cache/o/164cda2816ad0972e13e50279f45dea7/test --cache-dir=./.zig-cache --seed=0x7dcb0c2f --listen=-

🤨

ReleaseFast error trace, which surprisingly seems more relevant than the ReleaseSafe one:

test-std
└─ run test std-native-pwr10-ReleaseFast-libc failure
[default] (err): EndOfStream
/home/ci/zig-repro/lib/std/Io/Reader.zig:690:33: 0x120905b in readSliceAll (test)
 error.ReadFailed => return error.ReadFailed,
 ^
/home/ci/zig-repro/lib/std/zig/Server.zig:152:5: 0x1209063 in receiveMessage (test)
 return s.in.takeStruct(InMessage.Header, .little);
 ^
/home/ci/zig-repro/lib/std/hash_map.zig:1517:17: 0x1208867 in mainServer (test)
 self.available = 0;
 ^
/home/ci/zig-repro/lib/compiler/test_runner.zig:68:9: 0x120886f in main (test)
 return mainServer(init);
 ^
error: test process unexpectedly exited with code 1
failed command: ./.zig-cache/o/0fda276b7b6b411dcb2dd54333f68294/test --cache-dir=./.zig-cache --seed=0xe6f4bcce --listen=-

Reproduced on Android aarch64, Termux, kernel 4.19.157-perf+, Zig 0.16.0:

echo '{}' | zig test lib/compiler/test_runner.zig -lc --listen=-
error: EndOfStream

Without -lc it works. Same behavior as powerpc64le, but on an unsupported kernel (<5.10). Suggests the issue is not arch-specific, but std.Io.Reader interacting badly with glibc when reading from pipes.

Update: actually fails both with and without -lc on 4.19. So it's not glibc-specific, it's kernel.

@Liveiciee wrote in #31562 (comment):

Reproduced on Android aarch64, Termux, kernel 4.19.157-perf+, Zig 0.16.0:

I'm not sure what that command is supposed to demonstrate; you're deliberately violating the test runner protocol, so it'll fail on any system.

@jacobly's repro from Zulip:

@@ -73,6 +73,7 @@ pub fn main(init: std.process.Init.Minimal) void {

 fn mainServer(init: std.process.Init.Minimal) !void {
 @disableInstrumentation();
+ _ = init;
 var stdin_reader = Io.File.stdin().readerStreaming(runner_threaded_io, &stdin_buffer);
 var stdout_writer = Io.File.stdout().writerStreaming(runner_threaded_io, &stdout_buffer);
 var server = try std.zig.Server.init(.{
@@ -129,12 +130,12 @@ fn mainServer(init: std.process.Init.Minimal) !void {
 },
 .run_test => {
- testing.environ = init.environ;
- testing.allocator_instance = .{};
- testing.io_instance = .init(testing.allocator, .{
- .argv0 = .init(init.args),
- .environ = init.environ,
- });
 log_err_count = 0;
 const index = try server.receiveBody_u32();
 const test_fn = builtin.test_functions[index];
@@ -156,9 +157,10 @@ fn mainServer(init: std.process.Init.Minimal) !void {
 break :s .fail;
 },
 };
- testing.io_instance.deinit();
- const leak_count = testing.allocator_instance.detectLeaks();
- testing.allocator_instance.deinitWithoutLeakChecks();
+ const leak_count = 0;
 try server.serveTestResults(.{
 .index = index,
 .flags = .{
diff --git a/lib/std/Io/Threaded.zig b/lib/std/Io/Threaded.zig
index 14ad2f459f..5cf3c1d695 100644
--- a/lib/std/Io/Threaded.zig
+++ b/lib/std/Io/Threaded.zig
@@ -9641,6 +9641,7 @@ fn fileReadStreamingPosix(file: File, data: []const []u8) File.ReadStreamingErro
 }
 }
+ if (!builtin.is_test and std.os.linux.nanosleep(&.{ .sec = 0, .nsec = std.time.ns_per_ms }, null) != 0) @trap();
 const syscall: Syscall = try .start();
 while (true) {
 const rc = posix.system.readv(file.handle, dest.ptr, @intCast(dest.len));

conststd=@import("std");comptime{for(0..2)|i|_=struct{test{_=i;varparams=std.mem.zeroInit(std.os.linux.io_uring_params,.{});if(std.os.linux.close(@intCast(std.os.linux.io_uring_setup(1,&params)))!=0)@trap();if(std.os.linux.nanosleep(&.{.sec=0,.nsec=25*std.time.ns_per_ms},null)!=0)@trap();}};for(0..2)|i|_=struct{test{_=i;}};}

conststd=@import("std");pubfnbuild(b:*std.Build)void{construn=b.addRunArtifact(b.addTest(.{.root_module=b.createModule(.{.root_source_file=b.path("repro.zig"),.target=b.graph.host,})}));run.has_side_effects=true;b.getInstallStep().dependOn(&run.step);}

This needs to be reduced to a minimal repro, and then (I suspect) reported as a Linux kernel bug.