Newer
Older
ddhx / src / os / mmfile.d
/// A re-imagination and updated version of std.mmfile that features
/// some minor enhancements, such as APIs similar to File.
/// Copyright: dd86k <dd@dax.moe>
/// License: MIT
/// Authors: $(LINK2 https://github.com/dd86k, dd86k)
module os.mmfile;

import std.stdio : File;
import std.mmfile : MmFile;

enum MMFlags
{
    exists  = 1,        /// File must exist.
    read    = 1 << 1,   /// Read access.
    write   = 1 << 2,   /// Write access.
}

// temp
public class OSMmFile : MmFile
{
    private void *address;
    
    this(string path, int flags)
    {
        super(path,
            flags & MMFlags.read ? MmFile.Mode.read :
                flags & MMFlags.exists ?
                    MmFile.Mode.readWrite : MmFile.Mode.readWriteNew,
            0,
            address);
    }
    
    bool eof, err;
    private long position;
    long seek(long pos) // only do seek_set for now
    {
        return position = pos;
    }
    long tell()
    {
        return position;
    }
    ubyte[] read(size_t size)
    {
        long sz = cast(long)this.length;
        long p2 = position+size;
        eof = p2 > sz;
        if (eof) p2 = sz;
        return cast(ubyte[])this[position..p2];
    }
    /*void seek(long pos)
    {
        final switch (origin) with (Seek)
        {
        case start:
            position = pos;
            return 0;
        case current:
            position += pos;
            return 0;
        case end:
            position = size - pos;
            return 0;
        }
    }*/
}

/// The mode the memory mapped file is opened with.
/+enum MmFileMode {
    read,            /// Read existing file
    readWriteNew,    /// Delete existing file, write new file (overwrite)
    readWrite,       /// Read/Write existing file, create if not existing
    readCopyOnWrite, /// Read/Write existing file, copy on write
}

/// Memory-mapped file.
struct OSMmFile2 {
    /// Open a memory-mapped file.
    /// Params:
    ///   filename = File path.
    ///   mode = mmfile operating mode.
    ///   size = 
    this(string filename, MmFileMode mode = Mode.read, ulong size = 0,
        void* address = null, size_t window = 0)
{
        
        version (Windows)
{
        } else version (linux)
{
            int oflag;
            int fmode;

            final switch (mode) with (MmFileMode)
{
            case read:
                flags = MAP_SHARED;
                prot = PROT_READ;
                oflag = O_RDONLY;
                fmode = 0;
                break;
            case Mode.readWriteNew:
                assert(size != 0);
                flags = MAP_SHARED;
                prot = PROT_READ | PROT_WRITE;
                oflag = O_CREAT | O_RDWR | O_TRUNC;
                fmode = octal!660;
                break;
            case Mode.readWrite:
                flags = MAP_SHARED;
                prot = PROT_READ | PROT_WRITE;
                oflag = O_CREAT | O_RDWR;
                fmode = octal!660;
                break;
            case Mode.readCopyOnWrite:
                flags = MAP_PRIVATE;
                prot = PROT_READ | PROT_WRITE;
                oflag = O_RDWR;
                fmode = 0;
                break;
            }

            fd = fildes;

            // Adjust size
            stat_t statbuf = void;
            errnoEnforce(fstat(fd, &statbuf) == 0);
            if (prot & PROT_WRITE && size > statbuf.st_size)
            {
            // Need to make the file size bytes big
            lseek(fd, cast(off_t)(size - 1), SEEK_SET);
            char c = 0;
            core.sys.posix.unistd.write(fd, &c, 1);
            }
            else if (prot & PROT_READ && size == 0)
            size = statbuf.st_size;
            this.size = size;

            // Map the file into memory!
            size_t initial_map = (window && 2*window<size)
            ? 2*window : cast(size_t) size;
            auto p = mmap(address, initial_map, prot, flags, fd, 0);
            if (p == MAP_FAILED)
            {
            errnoEnforce(false, "Could not map file into memory");
            }
            data = p[0 .. initial_map];
        }
    }

    version (linux) this(File file, Mode mode = Mode.read, ulong size = 0,
        void* address = null, size_t window = 0)
    {
    // Save a copy of the File to make sure the fd stays open.
    this.file = file;
    this(file.fileno, mode, size, address, window);
    }

    version (linux) private this(int fildes, Mode mode, ulong size,
        void* address, size_t window)
    {
    int oflag;
    int fmode;

    final switch (mode)
    {
    case Mode.read:
        flags = MAP_SHARED;
        prot = PROT_READ;
        oflag = O_RDONLY;
        fmode = 0;
        break;

    case Mode.readWriteNew:
        assert(size != 0);
        flags = MAP_SHARED;
        prot = PROT_READ | PROT_WRITE;
        oflag = O_CREAT | O_RDWR | O_TRUNC;
        fmode = octal!660;
        break;

    case Mode.readWrite:
        flags = MAP_SHARED;
        prot = PROT_READ | PROT_WRITE;
        oflag = O_CREAT | O_RDWR;
        fmode = octal!660;
        break;

    case Mode.readCopyOnWrite:
        flags = MAP_PRIVATE;
        prot = PROT_READ | PROT_WRITE;
        oflag = O_RDWR;
        fmode = 0;
        break;
    }

    fd = fildes;

    // Adjust size
    stat_t statbuf = void;
    errnoEnforce(fstat(fd, &statbuf) == 0);
    if (prot & PROT_WRITE && size > statbuf.st_size)
    {
        // Need to make the file size bytes big
        lseek(fd, cast(off_t)(size - 1), SEEK_SET);
        char c = 0;
        core.sys.posix.unistd.write(fd, &c, 1);
    }
    else if (prot & PROT_READ && size == 0)
        size = statbuf.st_size;
    this.size = size;

    // Map the file into memory!
    size_t initial_map = (window && 2*window<size)
        ? 2*window : cast(size_t) size;
    auto p = mmap(address, initial_map, prot, flags, fd, 0);
    if (p == MAP_FAILED)
    {
        errnoEnforce(false, "Could not map file into memory");
    }
    data = p[0 .. initial_map];
    }

    /**
     * Open memory mapped file filename in mode.
     * File is closed when the object instance is deleted.
     * Params:
     *  filename = name of the file.
     *      If null, an anonymous file mapping is created.
     *  mode = access mode defined above.
     *  size =  the size of the file. If 0, it is taken to be the
     *      size of the existing file.
     *  address = the preferred address to map the file to,
     *      although the system is not required to honor it.
     *      If null, the system selects the most convenient address.
     *  window = preferred block size of the amount of data to map at one time
     *      with 0 meaning map the entire file. The window size must be a
     *      multiple of the memory allocation page size.
     * Throws:
     *  - On POSIX, $(REF ErrnoException, std, exception).
     *  - On Windows, $(REF WindowsException, std, windows, syserror).
     */
    this(string filename, Mode mode, ulong size, void* address,
        size_t window = 0)
    {
    this.filename = filename;
    this.mMode = mode;
    this.window = window;
    this.address = address;

    version (Windows)
    {
        void* p;
        uint dwDesiredAccess2;
        uint dwShareMode;
        uint dwCreationDisposition;
        uint flProtect;

        final switch (mode)
        {
        case Mode.read:
        dwDesiredAccess2 = GENERIC_READ;
        dwShareMode = FILE_SHARE_READ;
        dwCreationDisposition = OPEN_EXISTING;
        flProtect = PAGE_READONLY;
        dwDesiredAccess = FILE_MAP_READ;
        break;

        case Mode.readWriteNew:
        assert(size != 0);
        dwDesiredAccess2 = GENERIC_READ | GENERIC_WRITE;
        dwShareMode = FILE_SHARE_READ | FILE_SHARE_WRITE;
        dwCreationDisposition = CREATE_ALWAYS;
        flProtect = PAGE_READWRITE;
        dwDesiredAccess = FILE_MAP_WRITE;
        break;

        case Mode.readWrite:
        dwDesiredAccess2 = GENERIC_READ | GENERIC_WRITE;
        dwShareMode = FILE_SHARE_READ | FILE_SHARE_WRITE;
        dwCreationDisposition = OPEN_ALWAYS;
        flProtect = PAGE_READWRITE;
        dwDesiredAccess = FILE_MAP_WRITE;
        break;

        case Mode.readCopyOnWrite:
        dwDesiredAccess2 = GENERIC_READ | GENERIC_WRITE;
        dwShareMode = FILE_SHARE_READ | FILE_SHARE_WRITE;
        dwCreationDisposition = OPEN_EXISTING;
        flProtect = PAGE_WRITECOPY;
        dwDesiredAccess = FILE_MAP_COPY;
        break;
        }

        if (filename != null)
        {
        hFile = CreateFileW(filename.tempCStringW(),
            dwDesiredAccess2,
            dwShareMode,
            null,
            dwCreationDisposition,
            FILE_ATTRIBUTE_NORMAL,
            cast(HANDLE) null);
        wenforce(hFile != INVALID_HANDLE_VALUE, "CreateFileW");
        }
        else
        hFile = INVALID_HANDLE_VALUE;

        scope(failure)
        {
        if (hFile != INVALID_HANDLE_VALUE)
        {
            CloseHandle(hFile);
            hFile = INVALID_HANDLE_VALUE;
        }
        }

        int hi = cast(int)(size >> 32);
        hFileMap = CreateFileMappingW(hFile, null, flProtect,
            hi, cast(uint) size, null);
        wenforce(hFileMap, "CreateFileMapping");
        scope(failure)
        {
        CloseHandle(hFileMap);
        hFileMap = null;
        }

        if (size == 0 && filename != null)
        {
        uint sizehi;
        uint sizelow = GetFileSize(hFile, &sizehi);
        wenforce(sizelow != INVALID_FILE_SIZE || GetLastError() != ERROR_SUCCESS,
            "GetFileSize");
        size = (cast(ulong) sizehi << 32) + sizelow;
        }
        this.size = size;

        size_t initial_map = (window && 2*window<size)
        ? 2*window : cast(size_t) size;
        p = MapViewOfFileEx(hFileMap, dwDesiredAccess, 0, 0,
            initial_map, address);
        wenforce(p, "MapViewOfFileEx");
        data = p[0 .. initial_map];

        debug (MMFILE) printf("MmFile.this(): p = %p, size = %d\n", p, size);
    }
    else version (Posix)
    {
        void* p;
        int oflag;
        int fmode;

        final switch (mode)
        {
        case Mode.read:
        flags = MAP_SHARED;
        prot = PROT_READ;
        oflag = O_RDONLY;
        fmode = 0;
        break;

        case Mode.readWriteNew:
        assert(size != 0);
        flags = MAP_SHARED;
        prot = PROT_READ | PROT_WRITE;
        oflag = O_CREAT | O_RDWR | O_TRUNC;
        fmode = octal!660;
        break;

        case Mode.readWrite:
        flags = MAP_SHARED;
        prot = PROT_READ | PROT_WRITE;
        oflag = O_CREAT | O_RDWR;
        fmode = octal!660;
        break;

        case Mode.readCopyOnWrite:
        flags = MAP_PRIVATE;
        prot = PROT_READ | PROT_WRITE;
        oflag = O_RDWR;
        fmode = 0;
        break;
        }

        if (filename.length)
        {
        fd = .open(filename.tempCString(), oflag, fmode);
        errnoEnforce(fd != -1, "Could not open file "~filename);

        stat_t statbuf;
        if (fstat(fd, &statbuf))
        {
            //printf("\tfstat error, errno = %d\n", errno);
            .close(fd);
            fd = -1;
            errnoEnforce(false, "Could not stat file "~filename);
        }

        if (prot & PROT_WRITE && size > statbuf.st_size)
        {
            // Need to make the file size bytes big
            .lseek(fd, cast(off_t)(size - 1), SEEK_SET);
            char c = 0;
            core.sys.posix.unistd.write(fd, &c, 1);
        }
        else if (prot & PROT_READ && size == 0)
            size = statbuf.st_size;
        }
        else
        {
        fd = -1;
        flags |= MAP_ANON;
        }
        this.size = size;
        size_t initial_map = (window && 2*window<size)
        ? 2*window : cast(size_t) size;
        p = mmap(address, initial_map, prot, flags, fd, 0);
        if (p == MAP_FAILED)
        {
        if (fd != -1)
        {
            .close(fd);
            fd = -1;
        }
        errnoEnforce(false, "Could not map file "~filename);
        }

        data = p[0 .. initial_map];
    }
    else
    {
        static assert(0);
    }
    }

    /**
     * Flushes pending output and closes the memory mapped file.
     */
    ~this()
    {
    debug (MMFILE) printf("MmFile.~this()\n");
    unmap();
    data = null;
    version (Windows)
    {
        wenforce(hFileMap == null || CloseHandle(hFileMap) == TRUE,
            "Could not close file handle");
        hFileMap = null;

        wenforce(!hFile || hFile == INVALID_HANDLE_VALUE
            || CloseHandle(hFile) == TRUE,
            "Could not close handle");
        hFile = INVALID_HANDLE_VALUE;
    }
    else version (Posix)
    {
        version (linux)
        {
        if (file !is File.init)
        {
            // The File destructor will close the file,
            // if it is the only remaining reference.
            return;
        }
        }
        errnoEnforce(fd == -1 || fd <= 2
            || .close(fd) != -1,
            "Could not close handle");
        fd = -1;
    }
    else
    {
        static assert(0);
    }
    }

    /* Flush any pending output.
     */
    void flush()
    {
    debug (MMFILE) printf("MmFile.flush()\n");
    version (Windows)
    {
        FlushViewOfFile(data.ptr, data.length);
    }
    else version (Posix)
    {
        int i;
        i = msync(cast(void*) data, data.length, MS_SYNC);   // sys/mman.h
        errnoEnforce(i == 0, "msync failed");
    }
    else
    {
        static assert(0);
    }
    }

    /**
     * Gives size in bytes of the memory mapped file.
     */
    @property ulong length() const
    {
    debug (MMFILE) printf("MmFile.length()\n");
    return size;
    }

    /**
     * Forwards `length`.
     */
    alias opDollar = length;

    /**
     * Read-only property returning the file mode.
     */
    Mode mode()
    {
    debug (MMFILE) printf("MmFile.mode()\n");
    return mMode;
    }

    /**
     * Returns entire file contents as an array.
     */
    void[] opSlice()
    {
    debug (MMFILE) printf("MmFile.opSlice()\n");
    return opSlice(0,size);
    }

    /**
     * Returns slice of file contents as an array.
     */
    void[] opSlice(ulong i1, ulong i2)
    {
    debug (MMFILE) printf("MmFile.opSlice(%lld, %lld)\n", i1, i2);
    ensureMapped(i1,i2);
    size_t off1 = cast(size_t)(i1-start);
    size_t off2 = cast(size_t)(i2-start);
    return data[off1 .. off2];
    }

    /**
     * Returns byte at index i in file.
     */
    ubyte opIndex(ulong i)
    {
    debug (MMFILE) printf("MmFile.opIndex(%lld)\n", i);
    ensureMapped(i);
    size_t off = cast(size_t)(i-start);
    return (cast(ubyte[]) data)[off];
    }

    /**
     * Sets and returns byte at index i in file to value.
     */
    ubyte opIndexAssign(ubyte value, ulong i)
    {
    debug (MMFILE) printf("MmFile.opIndex(%lld, %d)\n", i, value);
    ensureMapped(i);
    size_t off = cast(size_t)(i-start);
    return (cast(ubyte[]) data)[off] = value;
    }


    // return true if the given position is currently mapped
    private int mapped(ulong i)
    {
    debug (MMFILE) printf("MmFile.mapped(%lld, %lld, %d)\n", i,start,
        data.length);
    return i >= start && i < start+data.length;
    }

    // unmap the current range
    private void unmap()
    {
    debug (MMFILE) printf("MmFile.unmap()\n");
    version (Windows)
    {
        wenforce(!data.ptr || UnmapViewOfFile(data.ptr) != FALSE, "UnmapViewOfFile");
    }
    else
    {
        errnoEnforce(!data.ptr || munmap(cast(void*) data, data.length) == 0,
            "munmap failed");
    }
    data = null;
    }

    // map range
    private void map(ulong start, size_t len)
    {
    debug (MMFILE) printf("MmFile.map(%lld, %d)\n", start, len);
    void* p;
    if (start+len > size)
        len = cast(size_t)(size-start);
    version (Windows)
    {
        uint hi = cast(uint)(start >> 32);
        p = MapViewOfFileEx(hFileMap, dwDesiredAccess, hi, cast(uint) start, len, address);
        wenforce(p, "MapViewOfFileEx");
    }
    else
    {
        p = mmap(address, len, prot, flags, fd, cast(off_t) start);
        errnoEnforce(p != MAP_FAILED);
    }
    data = p[0 .. len];
    this.start = start;
    }

    // ensure a given position is mapped
    private void ensureMapped(ulong i)
    {
    debug (MMFILE) printf("MmFile.ensureMapped(%lld)\n", i);
    if (!mapped(i))
    {
        unmap();
        if (window == 0)
        {
        map(0,cast(size_t) size);
        }
        else
        {
        ulong block = i/window;
        if (block == 0)
            map(0,2*window);
        else
            map(window*(block-1),3*window);
        }
    }
    }

    // ensure a given range is mapped
    private void ensureMapped(ulong i, ulong j)
    {
    debug (MMFILE) printf("MmFile.ensureMapped(%lld, %lld)\n", i, j);
    if (!mapped(i) || !mapped(j-1))
    {
        unmap();
        if (window == 0)
        {
        map(0,cast(size_t) size);
        }
        else
        {
        ulong iblock = i/window;
        ulong jblock = (j-1)/window;
        if (iblock == 0)
        {
            map(0,cast(size_t)(window*(jblock+2)));
        }
        else
        {
            map(window*(iblock-1),cast(size_t)(window*(jblock-iblock+3)));
        }
        }
    }
    }

private:
    string filename;
    void[] data;
    ulong  start;
    size_t window;
    ulong  size;
    Mode   mMode;
    void*  address;
    version (linux) File file;

    version (Windows)
    {
        HANDLE hFile = INVALID_HANDLE_VALUE;
        HANDLE hFileMap = null;
        uint dwDesiredAccess;
    }
    else version (Posix)
    {
        int fd;
        int prot;
        int flags;
        int fmode;
    } else {
        static assert(0);
    }
}+/