Newer
Older
ddhx / src / os / mmfile.d
/// A re-imagination and updated version of std.mmfile that features
/// some minor enhancements, such as APIs similar to File.
/// Copyright: dd86k <dd@dax.moe>
/// License: MIT
/// Authors: $(LINK2 github.com/dd86k, dd86k)
module os.mmfile;

import std.stdio : File;
import std.mmfile : MmFile;

// temp
public class OSMmFile : MmFile {
	private void *address;
	this(string path, bool readOnly) {
		super(path,
			readOnly ?
				MmFile.Mode.read :
				MmFile.Mode.readWriteNew,
			0,
			address);
	}
	bool eof, err;
	private long position;
	long seek(long pos) { // only do seek_set for now
		return position = pos;
	}
	long tell() {
		return position;
	}
	ubyte[] read(size_t size) {
		long sz = cast(long)this.length;
		long p2 = position+size;
		eof = p2 > sz;
		if (eof) p2 = sz;
		return cast(ubyte[])this[position..p2];
	}
	/*void seek(long pos) {
		final switch (origin) with (Seek) {
		case start:
			position = pos;
			return 0;
		case current:
			position += pos;
			return 0;
		case end:
			position = size - pos;
			return 0;
		}
	}*/
}

/// The mode the memory mapped file is opened with.
/+enum MmFileMode {
	read,            /// Read existing file
	readWriteNew,    /// Delete existing file, write new file (overwrite)
	readWrite,       /// Read/Write existing file, create if not existing
	readCopyOnWrite, /// Read/Write existing file, copy on write
}

/// Memory-mapped file.
struct OSMmFile2 {
	/// Open a memory-mapped file.
	/// Params:
	///   filename = File path.
	///   mode = mmfile operating mode.
	///   size = 
	this(string filename, MmFileMode mode = Mode.read, ulong size = 0,
		void* address = null, size_t window = 0) {
		
		version (Windows) {
		} else version (linux) {
			int oflag;
			int fmode;

			final switch (mode) with (MmFileMode) {
			case read:
				flags = MAP_SHARED;
				prot = PROT_READ;
				oflag = O_RDONLY;
				fmode = 0;
				break;
			case Mode.readWriteNew:
				assert(size != 0);
				flags = MAP_SHARED;
				prot = PROT_READ | PROT_WRITE;
				oflag = O_CREAT | O_RDWR | O_TRUNC;
				fmode = octal!660;
				break;
			case Mode.readWrite:
				flags = MAP_SHARED;
				prot = PROT_READ | PROT_WRITE;
				oflag = O_CREAT | O_RDWR;
				fmode = octal!660;
				break;
			case Mode.readCopyOnWrite:
				flags = MAP_PRIVATE;
				prot = PROT_READ | PROT_WRITE;
				oflag = O_RDWR;
				fmode = 0;
				break;
			}

			fd = fildes;

			// Adjust size
			stat_t statbuf = void;
			errnoEnforce(fstat(fd, &statbuf) == 0);
			if (prot & PROT_WRITE && size > statbuf.st_size)
			{
			// Need to make the file size bytes big
			lseek(fd, cast(off_t)(size - 1), SEEK_SET);
			char c = 0;
			core.sys.posix.unistd.write(fd, &c, 1);
			}
			else if (prot & PROT_READ && size == 0)
			size = statbuf.st_size;
			this.size = size;

			// Map the file into memory!
			size_t initial_map = (window && 2*window<size)
			? 2*window : cast(size_t) size;
			auto p = mmap(address, initial_map, prot, flags, fd, 0);
			if (p == MAP_FAILED)
			{
			errnoEnforce(false, "Could not map file into memory");
			}
			data = p[0 .. initial_map];
		}
	}

    version (linux) this(File file, Mode mode = Mode.read, ulong size = 0,
	    void* address = null, size_t window = 0)
    {
	// Save a copy of the File to make sure the fd stays open.
	this.file = file;
	this(file.fileno, mode, size, address, window);
    }

    version (linux) private this(int fildes, Mode mode, ulong size,
	    void* address, size_t window)
    {
	int oflag;
	int fmode;

	final switch (mode)
	{
	case Mode.read:
	    flags = MAP_SHARED;
	    prot = PROT_READ;
	    oflag = O_RDONLY;
	    fmode = 0;
	    break;

	case Mode.readWriteNew:
	    assert(size != 0);
	    flags = MAP_SHARED;
	    prot = PROT_READ | PROT_WRITE;
	    oflag = O_CREAT | O_RDWR | O_TRUNC;
	    fmode = octal!660;
	    break;

	case Mode.readWrite:
	    flags = MAP_SHARED;
	    prot = PROT_READ | PROT_WRITE;
	    oflag = O_CREAT | O_RDWR;
	    fmode = octal!660;
	    break;

	case Mode.readCopyOnWrite:
	    flags = MAP_PRIVATE;
	    prot = PROT_READ | PROT_WRITE;
	    oflag = O_RDWR;
	    fmode = 0;
	    break;
	}

	fd = fildes;

	// Adjust size
	stat_t statbuf = void;
	errnoEnforce(fstat(fd, &statbuf) == 0);
	if (prot & PROT_WRITE && size > statbuf.st_size)
	{
	    // Need to make the file size bytes big
	    lseek(fd, cast(off_t)(size - 1), SEEK_SET);
	    char c = 0;
	    core.sys.posix.unistd.write(fd, &c, 1);
	}
	else if (prot & PROT_READ && size == 0)
	    size = statbuf.st_size;
	this.size = size;

	// Map the file into memory!
	size_t initial_map = (window && 2*window<size)
	    ? 2*window : cast(size_t) size;
	auto p = mmap(address, initial_map, prot, flags, fd, 0);
	if (p == MAP_FAILED)
	{
	    errnoEnforce(false, "Could not map file into memory");
	}
	data = p[0 .. initial_map];
    }

    /**
     * Open memory mapped file filename in mode.
     * File is closed when the object instance is deleted.
     * Params:
     *  filename = name of the file.
     *      If null, an anonymous file mapping is created.
     *  mode = access mode defined above.
     *  size =  the size of the file. If 0, it is taken to be the
     *      size of the existing file.
     *  address = the preferred address to map the file to,
     *      although the system is not required to honor it.
     *      If null, the system selects the most convenient address.
     *  window = preferred block size of the amount of data to map at one time
     *      with 0 meaning map the entire file. The window size must be a
     *      multiple of the memory allocation page size.
     * Throws:
     *  - On POSIX, $(REF ErrnoException, std, exception).
     *  - On Windows, $(REF WindowsException, std, windows, syserror).
     */
    this(string filename, Mode mode, ulong size, void* address,
	    size_t window = 0)
    {
	this.filename = filename;
	this.mMode = mode;
	this.window = window;
	this.address = address;

	version (Windows)
	{
	    void* p;
	    uint dwDesiredAccess2;
	    uint dwShareMode;
	    uint dwCreationDisposition;
	    uint flProtect;

	    final switch (mode)
	    {
	    case Mode.read:
		dwDesiredAccess2 = GENERIC_READ;
		dwShareMode = FILE_SHARE_READ;
		dwCreationDisposition = OPEN_EXISTING;
		flProtect = PAGE_READONLY;
		dwDesiredAccess = FILE_MAP_READ;
		break;

	    case Mode.readWriteNew:
		assert(size != 0);
		dwDesiredAccess2 = GENERIC_READ | GENERIC_WRITE;
		dwShareMode = FILE_SHARE_READ | FILE_SHARE_WRITE;
		dwCreationDisposition = CREATE_ALWAYS;
		flProtect = PAGE_READWRITE;
		dwDesiredAccess = FILE_MAP_WRITE;
		break;

	    case Mode.readWrite:
		dwDesiredAccess2 = GENERIC_READ | GENERIC_WRITE;
		dwShareMode = FILE_SHARE_READ | FILE_SHARE_WRITE;
		dwCreationDisposition = OPEN_ALWAYS;
		flProtect = PAGE_READWRITE;
		dwDesiredAccess = FILE_MAP_WRITE;
		break;

	    case Mode.readCopyOnWrite:
		dwDesiredAccess2 = GENERIC_READ | GENERIC_WRITE;
		dwShareMode = FILE_SHARE_READ | FILE_SHARE_WRITE;
		dwCreationDisposition = OPEN_EXISTING;
		flProtect = PAGE_WRITECOPY;
		dwDesiredAccess = FILE_MAP_COPY;
		break;
	    }

	    if (filename != null)
	    {
		hFile = CreateFileW(filename.tempCStringW(),
			dwDesiredAccess2,
			dwShareMode,
			null,
			dwCreationDisposition,
			FILE_ATTRIBUTE_NORMAL,
			cast(HANDLE) null);
		wenforce(hFile != INVALID_HANDLE_VALUE, "CreateFileW");
	    }
	    else
		hFile = INVALID_HANDLE_VALUE;

	    scope(failure)
	    {
		if (hFile != INVALID_HANDLE_VALUE)
		{
		    CloseHandle(hFile);
		    hFile = INVALID_HANDLE_VALUE;
		}
	    }

	    int hi = cast(int)(size >> 32);
	    hFileMap = CreateFileMappingW(hFile, null, flProtect,
		    hi, cast(uint) size, null);
	    wenforce(hFileMap, "CreateFileMapping");
	    scope(failure)
	    {
		CloseHandle(hFileMap);
		hFileMap = null;
	    }

	    if (size == 0 && filename != null)
	    {
		uint sizehi;
		uint sizelow = GetFileSize(hFile, &sizehi);
		wenforce(sizelow != INVALID_FILE_SIZE || GetLastError() != ERROR_SUCCESS,
		    "GetFileSize");
		size = (cast(ulong) sizehi << 32) + sizelow;
	    }
	    this.size = size;

	    size_t initial_map = (window && 2*window<size)
		? 2*window : cast(size_t) size;
	    p = MapViewOfFileEx(hFileMap, dwDesiredAccess, 0, 0,
		    initial_map, address);
	    wenforce(p, "MapViewOfFileEx");
	    data = p[0 .. initial_map];

	    debug (MMFILE) printf("MmFile.this(): p = %p, size = %d\n", p, size);
	}
	else version (Posix)
	{
	    void* p;
	    int oflag;
	    int fmode;

	    final switch (mode)
	    {
	    case Mode.read:
		flags = MAP_SHARED;
		prot = PROT_READ;
		oflag = O_RDONLY;
		fmode = 0;
		break;

	    case Mode.readWriteNew:
		assert(size != 0);
		flags = MAP_SHARED;
		prot = PROT_READ | PROT_WRITE;
		oflag = O_CREAT | O_RDWR | O_TRUNC;
		fmode = octal!660;
		break;

	    case Mode.readWrite:
		flags = MAP_SHARED;
		prot = PROT_READ | PROT_WRITE;
		oflag = O_CREAT | O_RDWR;
		fmode = octal!660;
		break;

	    case Mode.readCopyOnWrite:
		flags = MAP_PRIVATE;
		prot = PROT_READ | PROT_WRITE;
		oflag = O_RDWR;
		fmode = 0;
		break;
	    }

	    if (filename.length)
	    {
		fd = .open(filename.tempCString(), oflag, fmode);
		errnoEnforce(fd != -1, "Could not open file "~filename);

		stat_t statbuf;
		if (fstat(fd, &statbuf))
		{
		    //printf("\tfstat error, errno = %d\n", errno);
		    .close(fd);
		    fd = -1;
		    errnoEnforce(false, "Could not stat file "~filename);
		}

		if (prot & PROT_WRITE && size > statbuf.st_size)
		{
		    // Need to make the file size bytes big
		    .lseek(fd, cast(off_t)(size - 1), SEEK_SET);
		    char c = 0;
		    core.sys.posix.unistd.write(fd, &c, 1);
		}
		else if (prot & PROT_READ && size == 0)
		    size = statbuf.st_size;
	    }
	    else
	    {
		fd = -1;
		flags |= MAP_ANON;
	    }
	    this.size = size;
	    size_t initial_map = (window && 2*window<size)
		? 2*window : cast(size_t) size;
	    p = mmap(address, initial_map, prot, flags, fd, 0);
	    if (p == MAP_FAILED)
	    {
		if (fd != -1)
		{
		    .close(fd);
		    fd = -1;
		}
		errnoEnforce(false, "Could not map file "~filename);
	    }

	    data = p[0 .. initial_map];
	}
	else
	{
	    static assert(0);
	}
    }

    /**
     * Flushes pending output and closes the memory mapped file.
     */
    ~this()
    {
	debug (MMFILE) printf("MmFile.~this()\n");
	unmap();
	data = null;
	version (Windows)
	{
	    wenforce(hFileMap == null || CloseHandle(hFileMap) == TRUE,
		    "Could not close file handle");
	    hFileMap = null;

	    wenforce(!hFile || hFile == INVALID_HANDLE_VALUE
		    || CloseHandle(hFile) == TRUE,
		    "Could not close handle");
	    hFile = INVALID_HANDLE_VALUE;
	}
	else version (Posix)
	{
	    version (linux)
	    {
		if (file !is File.init)
		{
		    // The File destructor will close the file,
		    // if it is the only remaining reference.
		    return;
		}
	    }
	    errnoEnforce(fd == -1 || fd <= 2
		    || .close(fd) != -1,
		    "Could not close handle");
	    fd = -1;
	}
	else
	{
	    static assert(0);
	}
    }

    /* Flush any pending output.
     */
    void flush()
    {
	debug (MMFILE) printf("MmFile.flush()\n");
	version (Windows)
	{
	    FlushViewOfFile(data.ptr, data.length);
	}
	else version (Posix)
	{
	    int i;
	    i = msync(cast(void*) data, data.length, MS_SYNC);   // sys/mman.h
	    errnoEnforce(i == 0, "msync failed");
	}
	else
	{
	    static assert(0);
	}
    }

    /**
     * Gives size in bytes of the memory mapped file.
     */
    @property ulong length() const
    {
	debug (MMFILE) printf("MmFile.length()\n");
	return size;
    }

    /**
     * Forwards `length`.
     */
    alias opDollar = length;

    /**
     * Read-only property returning the file mode.
     */
    Mode mode()
    {
	debug (MMFILE) printf("MmFile.mode()\n");
	return mMode;
    }

    /**
     * Returns entire file contents as an array.
     */
    void[] opSlice()
    {
	debug (MMFILE) printf("MmFile.opSlice()\n");
	return opSlice(0,size);
    }

    /**
     * Returns slice of file contents as an array.
     */
    void[] opSlice(ulong i1, ulong i2)
    {
	debug (MMFILE) printf("MmFile.opSlice(%lld, %lld)\n", i1, i2);
	ensureMapped(i1,i2);
	size_t off1 = cast(size_t)(i1-start);
	size_t off2 = cast(size_t)(i2-start);
	return data[off1 .. off2];
    }

    /**
     * Returns byte at index i in file.
     */
    ubyte opIndex(ulong i)
    {
	debug (MMFILE) printf("MmFile.opIndex(%lld)\n", i);
	ensureMapped(i);
	size_t off = cast(size_t)(i-start);
	return (cast(ubyte[]) data)[off];
    }

    /**
     * Sets and returns byte at index i in file to value.
     */
    ubyte opIndexAssign(ubyte value, ulong i)
    {
	debug (MMFILE) printf("MmFile.opIndex(%lld, %d)\n", i, value);
	ensureMapped(i);
	size_t off = cast(size_t)(i-start);
	return (cast(ubyte[]) data)[off] = value;
    }


    // return true if the given position is currently mapped
    private int mapped(ulong i)
    {
	debug (MMFILE) printf("MmFile.mapped(%lld, %lld, %d)\n", i,start,
		data.length);
	return i >= start && i < start+data.length;
    }

    // unmap the current range
    private void unmap()
    {
	debug (MMFILE) printf("MmFile.unmap()\n");
	version (Windows)
	{
	    wenforce(!data.ptr || UnmapViewOfFile(data.ptr) != FALSE, "UnmapViewOfFile");
	}
	else
	{
	    errnoEnforce(!data.ptr || munmap(cast(void*) data, data.length) == 0,
		    "munmap failed");
	}
	data = null;
    }

    // map range
    private void map(ulong start, size_t len)
    {
	debug (MMFILE) printf("MmFile.map(%lld, %d)\n", start, len);
	void* p;
	if (start+len > size)
	    len = cast(size_t)(size-start);
	version (Windows)
	{
	    uint hi = cast(uint)(start >> 32);
	    p = MapViewOfFileEx(hFileMap, dwDesiredAccess, hi, cast(uint) start, len, address);
	    wenforce(p, "MapViewOfFileEx");
	}
	else
	{
	    p = mmap(address, len, prot, flags, fd, cast(off_t) start);
	    errnoEnforce(p != MAP_FAILED);
	}
	data = p[0 .. len];
	this.start = start;
    }

    // ensure a given position is mapped
    private void ensureMapped(ulong i)
    {
	debug (MMFILE) printf("MmFile.ensureMapped(%lld)\n", i);
	if (!mapped(i))
	{
	    unmap();
	    if (window == 0)
	    {
		map(0,cast(size_t) size);
	    }
	    else
	    {
		ulong block = i/window;
		if (block == 0)
		    map(0,2*window);
		else
		    map(window*(block-1),3*window);
	    }
	}
    }

    // ensure a given range is mapped
    private void ensureMapped(ulong i, ulong j)
    {
	debug (MMFILE) printf("MmFile.ensureMapped(%lld, %lld)\n", i, j);
	if (!mapped(i) || !mapped(j-1))
	{
	    unmap();
	    if (window == 0)
	    {
		map(0,cast(size_t) size);
	    }
	    else
	    {
		ulong iblock = i/window;
		ulong jblock = (j-1)/window;
		if (iblock == 0)
		{
		    map(0,cast(size_t)(window*(jblock+2)));
		}
		else
		{
		    map(window*(iblock-1),cast(size_t)(window*(jblock-iblock+3)));
		}
	    }
	}
    }

private:
	string filename;
	void[] data;
	ulong  start;
	size_t window;
	ulong  size;
	Mode   mMode;
	void*  address;
	version (linux) File file;

	version (Windows) {
		HANDLE hFile = INVALID_HANDLE_VALUE;
		HANDLE hFileMap = null;
		uint dwDesiredAccess;
	} else version (Posix) {
		int fd;
		int prot;
		int flags;
		int fmode;
	} else {
		static assert(0);
	}
}+/