https://git.reactos.org/?p=reactos.git;a=commitdiff;h=2ba1926037223899fa2052d9a72663f095d37f54
commit 2ba1926037223899fa2052d9a72663f095d37f54 Author: Jérôme Gardou <[email protected]> AuthorDate: Fri Jan 29 18:48:32 2021 +0100 Commit: Jérôme Gardou <[email protected]> CommitDate: Wed Feb 3 09:41:23 2021 +0100 [NTOS:MM][NTOS:CC] Performance improvement again Read files by 64kb chunks instead of page-sized chunks. --- ntoskrnl/cc/copy.c | 48 +--- ntoskrnl/cc/pin.c | 5 - ntoskrnl/cc/view.c | 40 +++- ntoskrnl/include/internal/cc.h | 2 +- ntoskrnl/include/internal/mm.h | 15 +- ntoskrnl/mm/section.c | 499 +++++++++++++++++++++-------------------- 6 files changed, 298 insertions(+), 311 deletions(-) diff --git a/ntoskrnl/cc/copy.c b/ntoskrnl/cc/copy.c index 8ed24f2bd92..cdab94c8313 100644 --- a/ntoskrnl/cc/copy.c +++ b/ntoskrnl/cc/copy.c @@ -504,24 +504,6 @@ CcCopyRead ( CurrentOffset = FileOffset->QuadPart; while(CurrentOffset < ReadEnd) { - if (CurrentOffset >= SharedCacheMap->ValidDataLength.QuadPart) - { - DPRINT1("Zeroing buffer because we are beyond the VDL.\n"); - /* We are beyond what is valid. Just zero this out */ - _SEH2_TRY - { - RtlZeroMemory(Buffer, Length); - } - _SEH2_EXCEPT(CcpCheckInvalidUserBuffer(_SEH2_GetExceptionInformation(), Buffer, Length)) - { - ExRaiseStatus(STATUS_INVALID_USER_BUFFER); - } - _SEH2_END; - - ReadLength += Length; - break; - } - Status = CcRosGetVacb(SharedCacheMap, CurrentOffset, &Vacb); if (!NT_SUCCESS(Status)) { @@ -538,25 +520,15 @@ CcCopyRead ( if (!CcRosEnsureVacbResident(Vacb, Wait, FALSE, VacbOffset, VacbLength)) return FALSE; - /* Do not copy past the section */ - if (CurrentOffset + VacbLength > SharedCacheMap->SectionSize.QuadPart) - CopyLength = SharedCacheMap->SectionSize.QuadPart - CurrentOffset; - if (CopyLength != 0) + _SEH2_TRY { - _SEH2_TRY - { - RtlCopyMemory(Buffer, (PUCHAR)Vacb->BaseAddress + VacbOffset, CopyLength); - } - _SEH2_EXCEPT(CcpCheckInvalidUserBuffer(_SEH2_GetExceptionInformation(), Buffer, VacbLength)) - { - ExRaiseStatus(STATUS_INVALID_USER_BUFFER); - } - _SEH2_END; + RtlCopyMemory(Buffer, (PUCHAR)Vacb->BaseAddress + VacbOffset, CopyLength); } - - /* Zero-out the buffer tail if needed */ - if (CopyLength < VacbLength) - RtlZeroMemory((PUCHAR)Buffer + CopyLength, VacbLength - CopyLength); + _SEH2_EXCEPT(CcpCheckInvalidUserBuffer(_SEH2_GetExceptionInformation(), Buffer, VacbLength)) + { + ExRaiseStatus(STATUS_INVALID_USER_BUFFER); + } + _SEH2_END; ReadLength += VacbLength; @@ -684,10 +656,6 @@ CcCopyWrite ( if (FileObject->Flags & FO_WRITE_THROUGH) CcFlushCache(FileObject->SectionObjectPointer, FileOffset, Length, NULL); - /* Update VDL */ - if (WriteEnd > SharedCacheMap->ValidDataLength.QuadPart) - SharedCacheMap->ValidDataLength.QuadPart = WriteEnd; - return TRUE; } @@ -898,7 +866,7 @@ CcZeroData ( } /* See if we should simply truncate the valid data length */ - if ((StartOffset->QuadPart < SharedCacheMap->ValidDataLength.QuadPart) && (EndOffset->QuadPart > SharedCacheMap->ValidDataLength.QuadPart)) + if ((StartOffset->QuadPart < SharedCacheMap->ValidDataLength.QuadPart) && (EndOffset->QuadPart >= SharedCacheMap->ValidDataLength.QuadPart)) { DPRINT1("Truncating VDL.\n"); SharedCacheMap->ValidDataLength = *StartOffset; diff --git a/ntoskrnl/cc/pin.c b/ntoskrnl/cc/pin.c index af7b3aa71a8..c48f1068042 100644 --- a/ntoskrnl/cc/pin.c +++ b/ntoskrnl/cc/pin.c @@ -543,7 +543,6 @@ CcSetDirtyPinnedData ( IN PLARGE_INTEGER Lsn) { PINTERNAL_BCB iBcb = CONTAINING_RECORD(Bcb, INTERNAL_BCB, PFCB); - PROS_SHARED_CACHE_MAP SharedCacheMap = iBcb->Vacb->SharedCacheMap; CCTRACE(CC_API_DEBUG, "Bcb=%p Lsn=%p\n", Bcb, Lsn); @@ -556,10 +555,6 @@ CcSetDirtyPinnedData ( { CcRosMarkDirtyVacb(iBcb->Vacb); } - - /* Update VDL */ - if (SharedCacheMap->ValidDataLength.QuadPart < (iBcb->PFCB.MappedFileOffset.QuadPart + iBcb->PFCB.MappedLength)) - SharedCacheMap->ValidDataLength.QuadPart = iBcb->PFCB.MappedFileOffset.QuadPart + iBcb->PFCB.MappedLength; } diff --git a/ntoskrnl/cc/view.c b/ntoskrnl/cc/view.c index 591168eeba0..c0f1458287d 100644 --- a/ntoskrnl/cc/view.c +++ b/ntoskrnl/cc/view.c @@ -166,12 +166,12 @@ MmFlushVirtualMemory(IN PEPROCESS Process, NTSTATUS NTAPI CcRosFlushVacb ( - PROS_VACB Vacb) + _In_ PROS_VACB Vacb, + _In_ PIO_STATUS_BLOCK Iosb) { - IO_STATUS_BLOCK Iosb; - SIZE_T FlushSize = VACB_MAPPING_GRANULARITY; NTSTATUS Status; BOOLEAN HaveLock = FALSE; + PROS_SHARED_CACHE_MAP SharedCacheMap = Vacb->SharedCacheMap; CcRosUnmarkDirtyVacb(Vacb, TRUE); @@ -184,7 +184,10 @@ CcRosFlushVacb ( HaveLock = TRUE; } - Status = MmFlushVirtualMemory(NULL, &Vacb->BaseAddress, &FlushSize, &Iosb); + Status = MmFlushSegment(SharedCacheMap->FileObject->SectionObjectPointer, + &Vacb->FileOffset, + VACB_MAPPING_GRANULARITY, + Iosb); if (HaveLock) { @@ -194,6 +197,14 @@ CcRosFlushVacb ( quit: if (!NT_SUCCESS(Status)) CcRosMarkDirtyVacb(Vacb); + else + { + /* Update VDL */ + if (SharedCacheMap->ValidDataLength.QuadPart < (Vacb->FileOffset.QuadPart + VACB_MAPPING_GRANULARITY)) + { + SharedCacheMap->ValidDataLength.QuadPart = Vacb->FileOffset.QuadPart + VACB_MAPPING_GRANULARITY; + } + } return Status; } @@ -285,7 +296,8 @@ CcRosFlushDirtyPages ( continue; } - Status = CcRosFlushVacb(current); + IO_STATUS_BLOCK Iosb; + Status = CcRosFlushVacb(current, &Iosb); SharedCacheMap->Callbacks->ReleaseFromLazyWrite(SharedCacheMap->LazyWriteContext); @@ -308,7 +320,7 @@ CcRosFlushDirtyPages ( ULONG PagesFreed; /* How many pages did we free? */ - PagesFreed = VACB_MAPPING_GRANULARITY / PAGE_SIZE; + PagesFreed = Iosb.Information / PAGE_SIZE; (*Count) += PagesFreed; if (!Wait) @@ -756,7 +768,11 @@ CcRosEnsureVacbResident( if (!NoRead) { - NTSTATUS Status = MmMakePagesResident(NULL, BaseAddress, Length); + PROS_SHARED_CACHE_MAP SharedCacheMap = Vacb->SharedCacheMap; + NTSTATUS Status = MmMakeDataSectionResident(SharedCacheMap->FileObject->SectionObjectPointer, + Vacb->FileOffset.QuadPart + Offset, + Length, + &SharedCacheMap->ValidDataLength); if (!NT_SUCCESS(Status)) ExRaiseStatus(Status); } @@ -935,7 +951,6 @@ CcFlushCache ( } Status = STATUS_SUCCESS; - if (IoStatus) { IoStatus->Information = 0; @@ -953,9 +968,10 @@ CcFlushCache ( if (vacb != NULL) { + IO_STATUS_BLOCK VacbIosb; if (vacb->Dirty) { - Status = CcRosFlushVacb(vacb); + Status = CcRosFlushVacb(vacb, &VacbIosb); if (!NT_SUCCESS(Status)) { goto quit; @@ -966,7 +982,7 @@ CcFlushCache ( CcRosReleaseVacb(SharedCacheMap, vacb, FALSE, FALSE); if (IoStatus) - IoStatus->Information += VACB_MAPPING_GRANULARITY; + IoStatus->Information += VacbIosb.Information; } if (!DirtyVacb) @@ -992,6 +1008,10 @@ CcFlushCache ( if (IoStatus) IoStatus->Information += MmIosb.Information; + + /* Update VDL */ + if (SharedCacheMap->ValidDataLength.QuadPart < FlushEnd) + SharedCacheMap->ValidDataLength.QuadPart = FlushEnd; } if (!NT_SUCCESS(RtlLongLongAdd(FlushStart, VACB_MAPPING_GRANULARITY, &FlushStart))) diff --git a/ntoskrnl/include/internal/cc.h b/ntoskrnl/include/internal/cc.h index 7384d9be0d3..cf57319449b 100644 --- a/ntoskrnl/include/internal/cc.h +++ b/ntoskrnl/include/internal/cc.h @@ -310,7 +310,7 @@ CcMdlWriteComplete2( NTSTATUS NTAPI -CcRosFlushVacb(PROS_VACB Vacb); +CcRosFlushVacb(PROS_VACB Vacb, PIO_STATUS_BLOCK Iosb); NTSTATUS NTAPI diff --git a/ntoskrnl/include/internal/mm.h b/ntoskrnl/include/internal/mm.h index 0243ac55177..cec0d6e13d1 100644 --- a/ntoskrnl/include/internal/mm.h +++ b/ntoskrnl/include/internal/mm.h @@ -1369,13 +1369,6 @@ MmArePagesResident( _In_ PVOID BaseAddress, _In_ ULONG Length); -NTSTATUS -NTAPI -MmMakePagesResident( - _In_ PEPROCESS Process, - _In_ PVOID Address, - _In_ ULONG Length); - NTSTATUS NTAPI MmMakePagesDirty( @@ -1399,6 +1392,14 @@ MmFlushSegment( _In_ ULONG Length, _In_opt_ PIO_STATUS_BLOCK Iosb); +NTSTATUS +NTAPI +MmMakeDataSectionResident( + _In_ PSECTION_OBJECT_POINTERS SectionObjectPointer, + _In_ LONGLONG Offset, + _In_ ULONG Length, + _In_ PLARGE_INTEGER ValidDataLength); + BOOLEAN NTAPI MmPurgeSegment( diff --git a/ntoskrnl/mm/section.c b/ntoskrnl/mm/section.c index 94ea1844d2d..d848e337038 100644 --- a/ntoskrnl/mm/section.c +++ b/ntoskrnl/mm/section.c @@ -1181,126 +1181,234 @@ MiCopyFromUserPage(PFN_NUMBER DestPage, const VOID *SrcAddress) return(STATUS_SUCCESS); } -#ifndef NEWCC static NTSTATUS NTAPI -MiReadPage(PMEMORY_AREA MemoryArea, - LONGLONG SegOffset, - PPFN_NUMBER Page, - BOOLEAN IgnoreSize) -/* - * FUNCTION: Read a page for a section backed memory area. - * PARAMETERS: - * MemoryArea - Memory area to read the page for. - * Offset - Offset of the page to read. - * Page - Variable that receives a page contains the read data. - */ +MmMakeSegmentResident( + _In_ PMM_SECTION_SEGMENT Segment, + _In_ LONGLONG Offset, + _In_ ULONG Length, + _In_opt_ PLARGE_INTEGER ValidDataLength) { + /* Let's use a 64K granularity. */ + LONGLONG RangeStart, RangeEnd; NTSTATUS Status; - IO_STATUS_BLOCK IoStatus; - KEVENT Event; - UCHAR MdlBase[sizeof(MDL) + sizeof(PFN_NUMBER)]; - PMDL Mdl = (PMDL)MdlBase; - PFILE_OBJECT FileObject = MemoryArea->SectionData.Segment->FileObject; - LARGE_INTEGER FileOffset; - KIRQL OldIrql; - PFSRTL_ADVANCED_FCB_HEADER FcbHeader = FileObject->FsContext; - - FileOffset.QuadPart = MemoryArea->SectionData.Segment->Image.FileOffset + SegOffset; - - DPRINT("Reading file at offset %08x:%08x\n", FileOffset.HighPart, FileOffset.LowPart); + PFILE_OBJECT FileObject = Segment->FileObject; - Status = MmRequestPageMemoryConsumer(MC_USER, FALSE, Page); + /* Calculate our range, aligned on 64K if possible. */ + Status = RtlLongLongAdd(Offset, Length, &RangeEnd); + ASSERT(NT_SUCCESS(Status)); if (!NT_SUCCESS(Status)) return Status; - if ((FileOffset.QuadPart > FcbHeader->ValidDataLength.QuadPart) && !IgnoreSize) + RangeStart = Offset - (Offset % _64K); + if (RangeEnd % _64K) + RangeEnd += _64K - (RangeEnd % _64K); + + /* Clamp if needed */ + if (!FlagOn(*Segment->Flags, MM_DATAFILE_SEGMENT)) { - /* Quick path : data is not valid; return a zero-page */ - return STATUS_SUCCESS; + if (RangeEnd > Segment->RawLength.QuadPart) + RangeEnd = Segment->RawLength.QuadPart; } - RtlZeroMemory(MdlBase, sizeof(MdlBase)); - MmInitializeMdl(Mdl, NULL, PAGE_SIZE); - MmBuildMdlFromPages(Mdl, Page); - Mdl->MdlFlags |= MDL_PAGES_LOCKED | MDL_IO_PAGE_READ; + /* Let's gooooooooo */ + for ( ; RangeStart < RangeEnd; RangeStart += _64K) + { + /* First take a look at where we miss pages */ + ULONG ToReadPageBits = 0; + LONGLONG ChunkEnd = RangeStart + _64K; - KeInitializeEvent(&Event, NotificationEvent, FALSE); + if (ChunkEnd > RangeEnd) + ChunkEnd = RangeEnd; - /* Disable APCs */ - KeRaiseIrql(APC_LEVEL, &OldIrql); + MmLockSectionSegment(Segment); + for (LONGLONG ChunkOffset = RangeStart; ChunkOffset < ChunkEnd; ChunkOffset += PAGE_SIZE) + { + LARGE_INTEGER CurrentOffset; - Status = IoPageRead(FileObject, Mdl, &FileOffset, &Event, &IoStatus); - if (Status == STATUS_PENDING) - { - KeWaitForSingleObject(&Event, WrPageIn, KernelMode, FALSE, NULL); - Status = IoStatus.Status; - } + CurrentOffset.QuadPart = ChunkOffset; + ULONG_PTR Entry = MmGetPageEntrySectionSegment(Segment, &CurrentOffset); - if (Mdl->MdlFlags & MDL_MAPPED_TO_SYSTEM_VA) - { - MmUnmapLockedPages (Mdl->MappedSystemVa, Mdl); - } + /* Let any pending read proceed */ + while (MM_IS_WAIT_PTE(Entry)) + { + MmUnlockSectionSegment(Segment); - KeLowerIrql(OldIrql); + KeDelayExecutionThread(KernelMode, FALSE, &TinyTime); - if (Status == STATUS_END_OF_FILE) - { - DPRINT1("Got STATUS_END_OF_FILE at offset %I64d for file %wZ.\n", SegOffset, &FileObject->FileName); - Status = STATUS_SUCCESS; - } + MmLockSectionSegment(Segment); + Entry = MmGetPageEntrySectionSegment(Segment, &CurrentOffset); + } - if ((MemoryArea->VadNode.u.VadFlags.VadType == VadImageMap) - && ((SegOffset + PAGE_SIZE) > MemoryArea->SectionData.Segment->RawLength.QuadPart)) - { - KIRQL OldIrql; - PUCHAR PageMap; + if (Entry != 0) + { + /* There is a page here. Or a swap entry. Or whatever... */ + continue; + } - DPRINT("Zeroing at offset %I64d for file %wZ.\n", SegOffset, &FileObject->FileName); + ToReadPageBits |= 1UL << ((ChunkOffset - RangeStart) >> PAGE_SHIFT); - /* Zero out the end of it */ - PageMap = MiMapPageInHyperSpace(PsGetCurrentProcess(), *Page, &OldIrql); - RtlZeroMemory(PageMap + MemoryArea->SectionData.Segment->RawLength.QuadPart - SegOffset, - PAGE_SIZE - (MemoryArea->SectionData.Segment->RawLength.QuadPart - SegOffset)); - MiUnmapPageInHyperSpace(PsGetCurrentProcess(), PageMap, OldIrql); - } + /* Put a wait entry here */ + MmSetPageEntrySectionSegment(Segment, &CurrentOffset, MAKE_SWAP_SSE(MM_WAIT_ENTRY)); + } + MmUnlockSectionSegment(Segment); - return Status; -} + if (ToReadPageBits == 0) + { + /* Nothing to do for this chunk */ + continue; + } -#else -NTSTATUS -NTAPI -MiReadPage(PMEMORY_AREA MemoryArea, - LONGLONG SegOffset, - PPFN_NUMBER Page) -/* - * FUNCTION: Read a page for a section backed memory area. - * PARAMETERS: - * MemoryArea - Memory area to read the page for. - * Offset - Offset of the page to read. - * Page - Variable that receives a page contains the read data. - */ -{ - MM_REQUIRED_RESOURCES Resources; - NTSTATUS Status; + /* Now perform the actual read */ + LONGLONG ChunkOffset = RangeStart; + while (ChunkOffset < ChunkEnd) + { + /* Move forward if there is a hole */ + ULONG BitSet; + if (!_BitScanForward(&BitSet, ToReadPageBits)) + { + /* Nothing more to read */ + break; + } + ToReadPageBits >>= BitSet; + ChunkOffset += BitSet * PAGE_SIZE; + ASSERT(ChunkOffset < ChunkEnd); - RtlZeroMemory(&Resources, sizeof(MM_REQUIRED_RESOURCES)); + /* Get the range we have to read */ + _BitScanForward(&BitSet, ~ToReadPageBits); + ULONG ReadLength = BitSet * PAGE_SIZE; - Resources.Context = MemoryArea->SectionData.Section->FileObject; - Resources.FileOffset.QuadPart = SegOffset + - MemoryArea->SectionData.Segment->Image.FileOffset; - Resources.Consumer = MC_USER; - Resources.Amount = PAGE_SIZE; + ASSERT(ReadLength <= _64K); - DPRINT("%S, offset 0x%x, len 0x%x, page 0x%x\n", ((PFILE_OBJECT)Resources.Context)->FileName.Buffer, Resources.FileOffset.LowPart, Resources.Amount, Resources.Page[0]); + /* Clamp (This is for image mappings */ + if ((ChunkOffset + ReadLength) > ChunkEnd) + ReadLength = ChunkEnd - ChunkOffset; - Status = MiReadFilePage(MmGetKernelAddressSpace(), MemoryArea, &Resources); - *Page = Resources.Page[0]; - return Status; + ASSERT(ReadLength != 0); + + /* Allocate a MDL */ + PMDL Mdl = IoAllocateMdl(NULL, ReadLength, FALSE, FALSE, NULL); + if (!Mdl) + { + /* Damn. Roll-back. */ + MmLockSectionSegment(Segment); + while (ChunkOffset < ChunkEnd) + { + if (ToReadPageBits & 1) + { + LARGE_INTEGER CurrentOffset; + CurrentOffset.QuadPart = ChunkOffset; + ASSERT(MM_IS_WAIT_PTE(MmGetPageEntrySectionSegment(Segment, &CurrentOffset))); + MmSetPageEntrySectionSegment(Segment, &CurrentOffset, 0); + } + ToReadPageBits >>= 1; + ChunkOffset += PAGE_SIZE; + } + MmUnlockSectionSegment(Segment); + return STATUS_INSUFFICIENT_RESOURCES; + } + + /* Get our pages */ + PPFN_NUMBER Pages = MmGetMdlPfnArray(Mdl); + RtlZeroMemory(Pages, BYTES_TO_PAGES(ReadLength) * sizeof(PFN_NUMBER)); + for (UINT i = 0; i < BYTES_TO_PAGES(ReadLength); i++) + { + /* MmRequestPageMemoryConsumer succeeds or bugchecks */ + (void)MmRequestPageMemoryConsumer(MC_USER, FALSE, &Pages[i]); + } + Mdl->MdlFlags |= MDL_PAGES_LOCKED | MDL_IO_PAGE_READ; + + LARGE_INTEGER FileOffset; + FileOffset.QuadPart = Segment->Image.FileOffset + ChunkOffset; + + /* Clamp to VDL */ + if (ValidDataLength && ((FileOffset.QuadPart + ReadLength) > ValidDataLength->QuadPart)) + { + if (FileOffset.QuadPart > ValidDataLength->QuadPart) + { + /* Great, nothing to read. */ + goto AssignPagesToSegment; + } + + Mdl->Size = (FileOffset.QuadPart + ReadLength) - ValidDataLength->QuadPart; + } + + KEVENT Event; + KeInitializeEvent(&Event, NotificationEvent, FALSE); + + /* Disable APCs */ + KIRQL OldIrql; + KeRaiseIrql(APC_LEVEL, &OldIrql); + + IO_STATUS_BLOCK Iosb; + Status = IoPageRead(FileObject, Mdl, &FileOffset, &Event, &Iosb); + if (Status == STATUS_PENDING) + { + KeWaitForSingleObject(&Event, WrPageIn, KernelMode, FALSE, NULL); + Status = Iosb.Status; + } + + if (Mdl->MdlFlags & MDL_MAPPED_TO_SYSTEM_VA) + { + MmUnmapLockedPages(Mdl->MappedSystemVa, Mdl); + } + + KeLowerIrql(OldIrql); + + if (Status == STATUS_END_OF_FILE) + { + DPRINT1("Got STATUS_END_OF_FILE at offset %I64d for file %wZ.\n", FileOffset.QuadPart, &FileObject->FileName); + Status = STATUS_SUCCESS; + } + + if (!NT_SUCCESS(Status)) + { + /* Damn. Roll back. */ + for (UINT i = 0; i < BYTES_TO_PAGES(ReadLength); i++) + MmReleasePageMemoryConsumer(MC_USER, Pages[i]); + + MmLockSectionSegment(Segment); + while (ChunkOffset < ChunkEnd) + { + if (ToReadPageBits & 1) + { + LARGE_INTEGER CurrentOffset; + CurrentOffset.QuadPart = ChunkOffset; + ASSERT(MM_IS_WAIT_PTE(MmGetPageEntrySectionSegment(Segment, &CurrentOffset))); + MmSetPageEntrySectionSegment(Segment, &CurrentOffset, 0); + } + ToReadPageBits >>= 1; + ChunkOffset += PAGE_SIZE; + } + MmUnlockSectionSegment(Segment); + IoFreeMdl(Mdl);; + return Status; + } + +AssignPagesToSegment: + MmLockSectionSegment(Segment); + + for (UINT i = 0; i < BYTES_TO_PAGES(ReadLength); i++) + { + LARGE_INTEGER CurrentOffset; + CurrentOffset.QuadPart = ChunkOffset + (i * PAGE_SIZE); + + ASSERT(MM_IS_WAIT_PTE(MmGetPageEntrySectionSegment(Segment, &CurrentOffset))); + + MmSetPageEntrySectionSegment(Segment, &CurrentOffset, MAKE_SSE(Pages[i] << PAGE_SHIFT, 0)); + } + + MmUnlockSectionSegment(Segment); + + IoFreeMdl(Mdl); + ToReadPageBits >>= BitSet; + ChunkOffset += BitSet * PAGE_SIZE; + } + } + + return STATUS_SUCCESS; } -#endif static VOID MmAlterViewAttributes(PMMSUPPORT AddressSpace, @@ -1610,85 +1718,57 @@ MmNotPresentFaultSectionView(PMMSUPPORT AddressSpace, if (Entry == 0) { - SWAPENTRY FakeSwapEntry; - - /* - * If the entry is zero (and it can't change because we have - * locked the segment) then we need to load the page. - */ - /* - * Release all our locks and read in the page from disk + * If the entry is zero, then we need to load the page. */ - MmSetPageEntrySectionSegment(Segment, &Offset, MAKE_SWAP_SSE(MM_WAIT_ENTRY)); - MmUnlockSectionSegment(Segment); - MmCreatePageFileMapping(Process, PAddress, MM_WAIT_ENTRY); - MmUnlockAddressSpace(AddressSpace); - if ((Offset.QuadPart >= (LONGLONG)PAGE_ROUND_UP(Segment->RawLength.QuadPart)) && (MemoryArea->VadNode.u.VadFlags.VadType == VadImageMap)) { + /* We are beyond the data which is on file. Just get a new page. */ MI_SET_USAGE(MI_USAGE_SECTION); if (Process) MI_SET_PROCESS2(Process->ImageFileName); if (!Process) MI_SET_PROCESS2("Kernel Section"); - Status = MmRequestPageMemoryConsumer(MC_USER, TRUE, &Page); - if (!NT_SUCCESS(Status)) - { - DPRINT1("MmRequestPageMemoryConsumer failed (Status %x)\n", Status); - } + MmRequestPageMemoryConsumer(MC_USER, FALSE, &Page); + MmSetPageEntrySectionSegment(Segment, &Offset, MAKE_SSE(Page << PAGE_SHIFT, 1)); + MmUnlockSectionSegment(Segment); - } - else - { - DPRINT("Getting fresh page for file %wZ at offset %I64d.\n", &Segment->FileObject->FileName, Offset.QuadPart); - Status = MiReadPage(MemoryArea, Offset.QuadPart, &Page, FALSE); + Status = MmCreateVirtualMapping(Process, PAddress, Attributes, &Page, 1); if (!NT_SUCCESS(Status)) { - DPRINT1("MiReadPage failed (Status %x)\n", Status); + DPRINT1("Unable to create virtual mapping\n"); + KeBugCheck(MEMORY_MANAGEMENT); } - } - if (!NT_SUCCESS(Status)) - { - /* - * FIXME: What do we know in this case? - */ - /* - * Cleanup and release locks - */ - MmLockAddressSpace(AddressSpace); + ASSERT(MmIsPagePresent(Process, PAddress)); + if (Process) + MmInsertRmap(Page, Process, Address); + MiSetPageEvent(Process, Address); DPRINT("Address 0x%p\n", Address); - return(Status); + return(STATUS_SUCCESS); } - /* Lock both segment and process address space while we proceed. */ - MmLockAddressSpace(AddressSpace); - MmLockSectionSegment(Segment); + MmUnlockSectionSegment(Segment); + MmUnlockAddressSpace(AddressSpace); - MmDeletePageFileMapping(Process, PAddress, &FakeSwapEntry); - DPRINT("CreateVirtualMapping Page %x Process %p PAddress %p Attributes %x\n", - Page, Process, PAddress, Attributes); - Status = MmCreateVirtualMapping(Process, - PAddress, - Attributes, - &Page, - 1); + /* The data must be paged in. Lock the file, so that the VDL doesn't get updated behind us. */ + FsRtlAcquireFileExclusive(Segment->FileObject); + + PFSRTL_COMMON_FCB_HEADER FcbHeader = Segment->FileObject->FsContext; + + Status = MmMakeSegmentResident(Segment, Offset.QuadPart, PAGE_SIZE, &FcbHeader->ValidDataLength); + + FsRtlReleaseFile(Segment->FileObject); + + /* Lock address space again */ + MmLockAddressSpace(AddressSpace); if (!NT_SUCCESS(Status)) { - DPRINT1("Unable to create virtual mapping\n"); - KeBugCheck(MEMORY_MANAGEMENT); + /* Damn */ + DPRINT1("Failed to page data in!\n"); + return STATUS_IN_PAGE_ERROR; } - ASSERT(MmIsPagePresent(Process, PAddress)); - if (Process) - MmInsertRmap(Page, Process, Address); - - /* Set this section offset has being backed by our new page. */ - Entry = MAKE_SSE(Page << PAGE_SHIFT, 1); - MmSetPageEntrySectionSegment(Segment, &Offset, Entry); - MmUnlockSectionSegment(Segment); - MiSetPageEvent(Process, Address); - DPRINT("Address 0x%p\n", Address); - return(STATUS_SUCCESS); + /* Everything went fine. Restart the operation */ + return STATUS_MM_RESTART_OPERATION; } else if (IS_SWAP_FROM_SSE(Entry)) { @@ -4458,103 +4538,6 @@ MmArePagesResident( return Ret; } -NTSTATUS -NTAPI -MmMakePagesResident( - _In_ PEPROCESS Process, - _In_ PVOID Address, - _In_ ULONG Length) -{ - PMEMORY_AREA MemoryArea; - PMM_SECTION_SEGMENT Segment; - LARGE_INTEGER SegmentOffset, RangeEnd; - PMMSUPPORT AddressSpace = Process ? &Process->Vm : MmGetKernelAddressSpace(); - - MmLockAddressSpace(AddressSpace); - - MemoryArea = MmLocateMemoryAreaByAddress(AddressSpace, Address); - if (MemoryArea == NULL) - { - DPRINT1("Unable to find memory area at address %p.\n", Address); - MmUnlockAddressSpace(AddressSpace); - return STATUS_NOT_MAPPED_VIEW; - } - - /* Only supported in old Mm for now */ - ASSERT(MemoryArea->Type == MEMORY_AREA_SECTION_VIEW); - /* For file mappings */ - ASSERT(MemoryArea->VadNode.u.VadFlags.VadType != VadImageMap); - - Segment = MemoryArea->SectionData.Segment; - MmLockSectionSegment(Segment); - - SegmentOffset.QuadPart = PAGE_ROUND_DOWN(Address) - MA_GetStartingAddress(MemoryArea) - + MemoryArea->SectionData.ViewOffset.QuadPart; - RangeEnd.QuadPart = PAGE_ROUND_UP((ULONG_PTR)Address + Length) - MA_GetStartingAddress(MemoryArea) - + MemoryArea->SectionData.ViewOffset.QuadPart; - - DPRINT("MmMakePagesResident: Segment %p, 0x%I64x -> 0x%I64x\n", Segment, SegmentOffset.QuadPart, RangeEnd.QuadPart); - - while (SegmentOffset.QuadPart < RangeEnd.QuadPart) - { - ULONG_PTR Entry = MmGetPageEntrySectionSegment(Segment, &SegmentOffset); - - /* Let any pending read proceed */ - while (MM_IS_WAIT_PTE(Entry)) - { - MmUnlockSectionSegment(Segment); - MmUnlockAddressSpace(AddressSpace); - MiWaitForPageEvent(NULL, NULL); - MmLockAddressSpace(AddressSpace); - MmLockSectionSegment(Segment); - Entry = MmGetPageEntrySectionSegment(Segment, &SegmentOffset); - } - - /* We are called from Cc, this can't be backed by the page files */ - ASSERT(!IS_SWAP_FROM_SSE(Entry)); - - /* At this point, there may be a valid page there */ - if (Entry == 0) - { - PFN_NUMBER Page; - NTSTATUS Status; - - /* - * Release all our locks and read in the page from disk - */ - MmSetPageEntrySectionSegment(Segment, &SegmentOffset, MAKE_SWAP_SSE(MM_WAIT_ENTRY)); - MmUnlockSectionSegment(Segment); - MmUnlockAddressSpace(AddressSpace); - - /* FIXME: Read the whole range at once instead of one page at a time */ - /* Ignore file size, as Cc already checked on its side. */ - Status = MiReadPage(MemoryArea, SegmentOffset.QuadPart, &Page, TRUE); - if (!NT_SUCCESS(Status)) - { - /* Reset the Segment entry and fail */ - MmLockSectionSegment(Segment); - MmSetPageEntrySectionSegment(Segment, &SegmentOffset, 0); - MmUnlockSectionSegment(Segment); - MiSetPageEvent(Process, Address); - return Status; - } - - MmLockAddressSpace(AddressSpace); - MmLockSectionSegment(Segment); - - /* We set it with 0 ref count, nobody maps this page yet. */ - MmSetPageEntrySectionSegment(Segment, &SegmentOffset, MAKE_SSE(Page << PAGE_SHIFT, 0)); - MiSetPageEvent(Process, Address); - } - SegmentOffset.QuadPart += PAGE_SIZE; - } - - MmUnlockSectionSegment(Segment); - - MmUnlockAddressSpace(AddressSpace); - return STATUS_SUCCESS; -} - NTSTATUS NTAPI MmRosFlushVirtualMemory( @@ -4723,6 +4706,26 @@ MmPurgeSegment( return TRUE; } +NTSTATUS +NTAPI +MmMakeDataSectionResident( + _In_ PSECTION_OBJECT_POINTERS SectionObjectPointer, + _In_ LONGLONG Offset, + _In_ ULONG Length, + _In_ PLARGE_INTEGER ValidDataLength) +{ + PMM_SECTION_SEGMENT Segment = MiGrabDataSection(SectionObjectPointer); + + /* There must be a segment for this call */ + ASSERT(Segment); + + NTSTATUS Status = MmMakeSegmentResident(Segment, Offset, Length, ValidDataLength); + + MmDereferenceSegment(Segment); + + return Status; +} + NTSTATUS NTAPI MmFlushSegment(
