Skip to content

Commit 3d306d5

Browse files
BCSharpBCSharp
authored
Redesign PythonFileManager (#1729)
* Redesign PythonFileManager * Tweak ref counting API of PythonFileManager * Add some dup tests
1 parent 562cecd commit 3d306d5

8 files changed

Lines changed: 312 additions & 330 deletions

File tree

‎Src/IronPython.Modules/mmap.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -147,8 +147,8 @@ public MmapDefault(CodeContext/*!*/ context, int fileno, long length, string tag
147147
_offset = offset;
148148

149149
PythonContext pContext = context.LanguageContext;
150-
if (pContext.FileManager.TryGetFileFromId(fileno, out PythonIOModule.FileIO fileio)) {
151-
if ((_sourceStream = fileio._readStream as FileStream) == null) {
150+
if (pContext.FileManager.TryGetStreams(fileno, out StreamBox streams)) {
151+
if ((_sourceStream = streams.ReadStream as FileStream) == null) {
152152
throw WindowsError(PythonExceptions._OSError.ERROR_INVALID_HANDLE);
153153
}
154154
} else {

‎Src/IronPython.Modules/msvcrt.cs‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,7 @@ to os.fdopen() to create a file object.
6565
public static int open_osfhandle(CodeContext context, BigInteger os_handle, int flags) {
6666
if ((flags & O_TEXT) != 0) throw new NotImplementedException();
6767
FileStream stream = new FileStream(new SafeFileHandle(new IntPtr((long)os_handle), true), FileAccess.ReadWrite);
68-
return context.LanguageContext.FileManager.AddStream(stream);
68+
return context.LanguageContext.FileManager.Add(new(stream));
6969
}
7070

7171
private static bool TryGetFileHandle(Stream stream, out object handle) {
@@ -96,11 +96,10 @@ private static bool TryGetFileHandle(Stream stream, out object handle) {
9696
Return the file handle for the file descriptor fd. Raises IOError
9797
if fd is not recognized.")]
9898
public static object get_osfhandle(CodeContext context, int fd) {
99-
// TODO: handle other FileManager types?
100-
var pfile = context.LanguageContext.FileManager.GetFileFromId(fd);
99+
var sbox = context.LanguageContext.FileManager.GetStreams(fd);
101100

102101
object handle;
103-
if (TryGetFileHandle(pfile._readStream, out handle)) return handle;
102+
if (TryGetFileHandle(sbox.ReadStream, out handle)) return handle;
104103

105104
return -1;
106105
}

‎Src/IronPython.Modules/nt.cs‎

Lines changed: 39 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -333,14 +333,8 @@ public static void chmod(CodeContext context, object? path, int mode, [ParamDict
333333

334334
public static void close(CodeContext/*!*/ context, int fd) {
335335
PythonFileManager fileManager = context.LanguageContext.FileManager;
336-
if (fileManager.TryGetFileFromId(fd, out PythonIOModule.FileIO? file)) {
337-
file.CloseStreams(fileManager);
338-
fileManager.RemoveObjectOnId(fd);
339-
} else {
340-
Stream stream = fileManager.GetStreamFromId(fd);
341-
fileManager.RemoveObjectOnId(fd);
342-
fileManager.DerefAndCloseIfLast(stream);
343-
}
336+
StreamBox streams = fileManager.GetStreams(fd);
337+
streams.CloseStreams(fileManager);
344338
}
345339

346340
public static void closerange(CodeContext/*!*/ context, int fd_low, int fd_high) {
@@ -356,26 +350,17 @@ public static void closerange(CodeContext/*!*/ context, int fd_low, int fd_high)
356350
public static int dup(CodeContext/*!*/ context, int fd) {
357351
PythonFileManager fileManager = context.LanguageContext.FileManager;
358352

359-
object obj = fileManager.GetObjectFromId(fd); // OSError if fd not valid
360-
if (obj is PythonIOModule.FileIO file) {
361-
var file2 = new PythonIOModule.FileIO(context, file.fileno(context)) { closefd = false };
362-
int fd2 = fileManager.AddFile(file2);
363-
fileManager.EnsureRef(file._readStream);
364-
fileManager.AddRef(file2._readStream);
365-
return fd2;
366-
} else {
367-
var stream = (Stream)obj;
368-
fileManager.EnsureRef(stream);
369-
fileManager.AddRef(stream);
370-
return fileManager.AddStream(stream);
371-
}
353+
StreamBox streams = fileManager.GetStreams(fd); // OSError if fd not valid
354+
fileManager.EnsureRefStreams(streams);
355+
fileManager.AddRefStreams(streams);
356+
return fileManager.Add(new(streams));
372357
}
373358

374359

375360
public static int dup2(CodeContext/*!*/ context, int fd, int fd2) {
376361
PythonFileManager fileManager = context.LanguageContext.FileManager;
377362

378-
object obj = fileManager.GetObjectFromId(fd); // OSError if fd not valid
363+
StreamBox streams = fileManager.GetStreams(fd); // OSError if fd not valid
379364
if (fd == fd2) {
380365
return fd2;
381366
}
@@ -384,24 +369,15 @@ public static int dup2(CodeContext/*!*/ context, int fd, int fd2) {
384369
throw PythonOps.OSError(9, "Bad file descriptor");
385370
}
386371

387-
if (fileManager.TryGetObjectFromId(fd2, out _)) {
372+
if (fileManager.TryGetStreams(fd2, out _)) {
388373
close(context, fd2);
389374
}
390375

391376
// TODO: race condition: `open` or `dup` on another thread may occupy fd2
392377

393-
if (obj is PythonIOModule.FileIO file) {
394-
var file2 = new PythonIOModule.FileIO(context, file.fileno(context)) { closefd = false };
395-
fileManager.AddFile(fd2, file2);
396-
fileManager.EnsureRef(file._readStream);
397-
fileManager.AddRef(file2._readStream);
398-
return fd2;
399-
} else {
400-
var stream = (Stream)obj;
401-
fileManager.EnsureRef(stream);
402-
fileManager.AddRef(stream);
403-
return fileManager.AddStream(fd2, stream);
404-
}
378+
fileManager.EnsureRefStreams(streams);
379+
fileManager.AddRefStreams(streams);
380+
return fileManager.Add(fd2, new(streams));
405381
}
406382

407383
#if FEATURE_PROCESS
@@ -426,18 +402,18 @@ public static object fspath(CodeContext context, [AllowNull] object path)
426402
public static object fstat(CodeContext/*!*/ context, int fd) {
427403
PythonFileManager fileManager = context.LanguageContext.FileManager;
428404

429-
fileManager.TryGetObjectFromId(fd, out object? obj);
430-
if (obj is PythonIOModule.FileIO file) {
431-
if (file.IsConsole) return new stat_result(0x2000);
432-
if (StatStream(file._readStream) is not null and var res) return res;
433-
} else if (obj is Stream stream && StatStream(stream) is not null and var res) {
434-
return res;
405+
if (fileManager.TryGetStreams(fd, out StreamBox? streams)) {
406+
if (streams.IsConsoleStream()) return new stat_result(0x2000);
407+
if (streams.IsStandardIOStream()) return new stat_result(0x1000);
408+
if (StatStream(streams.ReadStream) is not null and var res) return res;
435409
}
436410
return LightExceptions.Throw(PythonOps.OSError(9, "Bad file descriptor"));
437411

438412
static object? StatStream(Stream stream) {
439413
if (stream is FileStream fs) return lstat(fs.Name, new Dictionary<string, object>(1));
414+
#if FEATURE_PIPES
440415
if (stream is PipeStream) return new stat_result(0x1000);
416+
#endif
441417
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) {
442418
if (ReferenceEquals(stream, Stream.Null)) return new stat_result(0x2000);
443419
} else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux) || RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) {
@@ -453,10 +429,10 @@ static bool IsUnixStream(Stream stream) {
453429

454430
public static void fsync(CodeContext context, int fd) {
455431
PythonFileManager fileManager = context.LanguageContext.FileManager;
456-
var pf = fileManager.GetFileFromId(fd);
432+
StreamBox streams = fileManager.GetStreams(fd);
457433
try {
458-
pf.flush(context);
459-
} catch (Exception ex) when (ex is ValueErrorException || ex is IOException) {
434+
streams.Flush();
435+
} catch (IOException) {
460436
throw PythonOps.OSError(9, "Bad file descriptor");
461437
}
462438
}
@@ -512,8 +488,8 @@ static void linkUnix(string src, string dst) {
512488
}
513489

514490
public static bool isatty(CodeContext context, int fd) {
515-
if (context.LanguageContext.FileManager.TryGetFileFromId(fd, out var file))
516-
return file.isatty(context);
491+
if (context.LanguageContext.FileManager.TryGetStreams(fd, out var streams))
492+
return streams.IsConsoleStream();
517493
return false;
518494
}
519495

@@ -568,9 +544,9 @@ public static PythonList listdir(CodeContext context, object? path)
568544
=> listdir(context, ConvertToFsString(context, path, nameof(path)));
569545

570546
public static BigInteger lseek(CodeContext context, int fd, long offset, int whence) {
571-
var file = context.LanguageContext.FileManager.GetFileFromId(fd);
547+
var streams = context.LanguageContext.FileManager.GetStreams(fd);
572548

573-
return file.seek(context, offset, whence);
549+
return streams.ReadStream.Seek(offset, (SeekOrigin)whence);
574550
}
575551

576552
[Documentation("lstat(path, *, dir_fd=None) -> stat_result\n\n" +
@@ -866,7 +842,7 @@ public static object open(CodeContext/*!*/ context, [NotNone] string path, int f
866842
fs = new FileStream(path, fileMode, access, FileShare.ReadWrite, DefaultBufferSize, options);
867843
}
868844

869-
return context.LanguageContext.FileManager.AddFile(new PythonIOModule.FileIO(context, fs) { name = path, closefd = false });
845+
return context.LanguageContext.FileManager.Add(new(fs));
870846
} catch (Exception e) {
871847
throw ToPythonException(e, path);
872848
}
@@ -918,13 +894,11 @@ static Tuple<Stream, Stream> CreatePipeStreamsUnix() {
918894

919895
public static PythonTuple pipe(CodeContext context) {
920896
var pipeStreams = CreatePipeStreams();
921-
922-
var inFile = new PythonIOModule.FileIO(context, pipeStreams.Item1) { closefd = false };
923-
var outFile = new PythonIOModule.FileIO(context, pipeStreams.Item2) { closefd = false };
897+
var manager = context.LanguageContext.FileManager;
924898

925899
return PythonTuple.MakeTuple(
926-
context.LanguageContext.FileManager.AddFile(inFile),
927-
context.LanguageContext.FileManager.AddFile(outFile)
900+
manager.Add(new(pipeStreams.Item1)),
901+
manager.Add(new(pipeStreams.Item2))
928902
);
929903
}
930904
#endif
@@ -946,8 +920,10 @@ public static Bytes read(CodeContext/*!*/ context, int fd, int buffersize) {
946920

947921
try {
948922
PythonContext pythonContext = context.LanguageContext;
949-
var pf = pythonContext.FileManager.GetFileFromId(fd);
950-
return (Bytes)pf.read(context, buffersize);
923+
var streams = pythonContext.FileManager.GetStreams(fd);
924+
if (!streams.ReadStream.CanRead) throw PythonOps.OSError(9, "Bad file descriptor");
925+
926+
return Bytes.Make(streams.Read(buffersize));
951927
} catch (Exception e) {
952928
throw ToPythonException(e);
953929
}
@@ -1600,7 +1576,7 @@ public static void truncate(CodeContext context, int fd, BigInteger length)
16001576
=> ftruncate(context, fd, length);
16011577

16021578
public static void ftruncate(CodeContext context, int fd, BigInteger length)
1603-
=> context.LanguageContext.FileManager.GetFileFromId(fd).truncate(context, length);
1579+
=> context.LanguageContext.FileManager.GetStreams(fd).Truncate((long)length);
16041580

16051581
#if FEATURE_FILESYSTEM
16061582
public static object times() {
@@ -1827,8 +1803,12 @@ public static PythonTuple waitpid(int pid, int options) {
18271803
public static int write(CodeContext/*!*/ context, int fd, [NotNone] IBufferProtocol data) {
18281804
try {
18291805
PythonContext pythonContext = context.LanguageContext;
1830-
var pf = pythonContext.FileManager.GetFileFromId(fd);
1831-
return (int)pf.write(context, data);
1806+
var streams = pythonContext.FileManager.GetStreams(fd);
1807+
using var buffer = data.GetBuffer();
1808+
var bytes = buffer.AsReadOnlySpan();
1809+
if (!streams.WriteStream.CanWrite) throw PythonOps.OSError(9, "Bad file descriptor");
1810+
1811+
return streams.Write(bytes);
18321812
} catch (Exception e) {
18331813
throw ToPythonException(e);
18341814
}

0 commit comments

Comments
 (0)