Hi,
Attached is an update to the patches, after having checked
all my tests run successfully.
Jean-Pierre
Jean-Pierre André wrote:
Hi,
Your tests, leading to the file system getting filled up,
produced situations not seen before.
Could you please check the attached patches.
They care for two different issues :
- replaying some redo actions which were not foreseen,
- locating log blocks more recent than temporary ones.
So far, I have only checked one of your proposed cases
(metadata7_5.gz), so there still might be issues with
the other ones.
Jean-Pierre
Maayan Hanin wrote:
Hello,
On some occasions, I’ve got an NTFS journal containing ops which
ntfsrecover ignores since the Redo/Undo pair doesn’t match what
ntfsrecover expects in the distribute_redosfunction.
Example #1:
Redo op: WriteEndofFileRecordSegment
Actual undo op: Noop
Expected undo op: WriteEndofFileRecordSegment
Example #2:
Redo op: InitializeFileRecordSegment
Actual undo op: DeallocateFileRecordSegment
Expected undo op: Noop
Example #3:
Getting CompensationlogRecord as the undo op, for many different redoops
(DeleteAttribute, CreateAttribute, DeleteIndexEntryAllocation,
WriteEndOfIndexBuffer etc.).
This seems to happen when the partition is full or nearly full.
What is the reason ntfsrecover ignores such cases, instead of just
playing the operations present in the journal?
Worth noting that all examples occurred on a regular Windows 7 machine,
performing trivial operations against the file system.
I have raw partition files available for inspection.
Thanks,
Maayan
--- ntfsprogs/ntfsrecover.c.ref 2017-09-30 10:39:48.603684400 +0200
+++ ntfsprogs/ntfsrecover.c 2017-10-14 08:38:49.781763100 +0200
@@ -3810,6 +3810,60 @@
}
/*
+ * Find the latest log block
+ *
+ * Usually, the latest block is either block 2 or 3 which act as
+ * temporary block before being copied to target location.
+ * However under some unknown condition the block are written
+ * immediately to target location, and we have to scan for the
+ * latest one.
+ * Currently this is not checked for logfile version 2.x which
+ * use a different layout of temporary blocks.
+ */
+
+const struct BUFFER *find_latest_block(CONTEXT *ctx, u32 baseblk,
+ const struct BUFFER *basebuf)
+{
+ le64 offset;
+ leLSN prevlsn;
+ leLSN curlsn;
+ u32 curblk;
+ u32 prevblk;
+ const struct BUFFER *prevbuf;
+ const struct BUFFER *curbuf;
+
+ offset = basebuf->block.record.copy.file_offset;
+ curblk = baseblk;
+ do {
+ if (curblk < BASEBLKS) {
+ prevbuf = basebuf;
+ prevlsn = basebuf->block.record.last_end_lsn;
+ prevblk = baseblk;
+ curblk = le64_to_cpu(offset) >> blockbits;
+ } else {
+ if (optv)
+ printf("block %d is more recent than block
%d\n",
+ (int)curblk, (int)prevblk);
+ prevbuf = curbuf;
+ prevlsn = curlsn;
+ prevblk = curblk;
+ curblk++;
+ if (curblk >= (logfilesz >> blockbits))
+ curblk = (log_major < 2 ? BASEBLKS : BASEBLKS2);
+ }
+ curbuf = read_buffer(ctx, curblk);
+ if (curbuf && (curbuf->block.record.magic == magic_RCRD)) {
+ curlsn = curbuf->block.record.copy.last_lsn;
+ }
+ } while (curbuf
+ && (curbuf->block.record.magic == magic_RCRD)
+ && (le64_to_cpu(curlsn) > le64_to_cpu(prevlsn)));
+ if (optv)
+ printf("Block %d is the latest one\n",(int)prevblk);
+ return (prevbuf);
+}
+
+/*
* Determine the sequencing of blocks (when version >= 2.0)
*
* Blocks 2..17 and 18..33 are temporary blocks being filled until
@@ -4131,6 +4185,12 @@
&nextbuf->block.record);
}
}
+ if (startbuf && opts) {
+ buf = startbuf = find_latest_block(ctx,
+ blk, startbuf);
+ latest_lsn = le64_to_cpu(
+ buf->block.record.last_end_lsn);
+ }
} else {
buf = startbuf = read_buffer(ctx, blk);
nextbuf = (const struct BUFFER*)NULL;
--- ntfsprogs/playlog.c.ref 2017-09-30 10:39:48.672072800 +0200
+++ ntfsprogs/playlog.c 2017-10-14 08:41:01.635953700 +0200
@@ -2165,7 +2165,7 @@
record = (MFT_RECORD*)buffer;
if ((target + length) <= mftrecsz) {
/* write a void mft entry (needed ?) */
- changed = memcmp(buffer + target, data, length)
+ changed = (length && memcmp(buffer + target, data, length))
|| (record->flags & MFT_RECORD_IN_USE);
err = 0;
if (changed) {
@@ -2205,7 +2205,6 @@
data = ((const char*)&action->record)
+ get_undo_offset(&action->record);
length = le16_to_cpu(action->record.undo_length);
-// TODO merge with undo_add_index ?
target = le16_to_cpu(action->record.record_offset)
+ le16_to_cpu(action->record.attribute_offset);
if (optv > 1) {
@@ -2224,7 +2223,9 @@
&& !(length & 7)
&& ((target + length) <= xsize)) {
/* This has to be an idempotent action */
- found = !memcmp(buffer + target, data, length);
+ found = (action->record.undo_operation
+ == const_cpu_to_le16(CompensationlogRecord))
+ || !memcmp(buffer + target, data, length);
err = 0;
if (found) {
/* Remove the entry */
@@ -2239,7 +2240,7 @@
}
if (optv > 1) {
printf("-> INDX record %s\n",
- (found ? "unchanged" : "removed"));
+ (found ? "removed" : "unchanged"));
}
}
return (err);
@@ -2286,7 +2287,9 @@
&& !(length & 7)
&& ((target + length) <= mftrecsz)) {
/* This has to be an idempotent action */
- found = !memcmp(buffer + target, data, length);
+ found = (action->record.undo_operation
+ == const_cpu_to_le16(CompensationlogRecord))
+ || !memcmp(buffer + target, data, length);
err = 0;
/* Only delete if present */
if (found) {
@@ -2626,7 +2629,9 @@
dump(&buffer[target], length);
}
if ((target + length) <= mftrecsz) {
- changed = memcmp(buffer + target, data, length);
+ changed = (action->record.undo_operation
+ == const_cpu_to_le16(CompensationlogRecord))
+ || memcmp(buffer + target, data, length);
err = 0;
if (changed) {
memcpy(buffer + target, data, length);
@@ -4139,8 +4144,10 @@
err = redo_add_root_index(vol, action, buffer);
break;
case ClearBitsInNonResidentBitMap :
- if (action->record.undo_operation
- == const_cpu_to_le16(SetBitsInNonResidentBitMap))
+ if ((action->record.undo_operation
+ == const_cpu_to_le16(SetBitsInNonResidentBitMap))
+ || (action->record.undo_operation
+ == const_cpu_to_le16(CompensationlogRecord)))
err = redo_force_bits(vol, action, buffer);
break;
case CompensationlogRecord :
@@ -4149,28 +4156,38 @@
err = redo_compensate(vol, action, buffer);
break;
case CreateAttribute :
- if (action->record.undo_operation
- == const_cpu_to_le16(DeleteAttribute))
+ if ((action->record.undo_operation
+ == const_cpu_to_le16(DeleteAttribute))
+ || (action->record.undo_operation
+ == const_cpu_to_le16(CompensationlogRecord)))
err = redo_create_attribute(vol, action, buffer);
break;
case DeallocateFileRecordSegment :
- if (action->record.undo_operation
+ if ((action->record.undo_operation
== const_cpu_to_le16(InitializeFileRecordSegment))
+ || (action->record.undo_operation
+ == const_cpu_to_le16(CompensationlogRecord)))
err = redo_delete_file(vol, action, buffer);
break;
case DeleteAttribute :
- if (action->record.undo_operation
- == const_cpu_to_le16(CreateAttribute))
+ if ((action->record.undo_operation
+ == const_cpu_to_le16(CreateAttribute))
+ || (action->record.undo_operation
+ == const_cpu_to_le16(CompensationlogRecord)))
err = redo_delete_attribute(vol, action, buffer);
break;
case DeleteIndexEntryAllocation :
- if (action->record.undo_operation
- == const_cpu_to_le16(AddIndexEntryAllocation))
+ if ((action->record.undo_operation
+ == const_cpu_to_le16(AddIndexEntryAllocation))
+ || (action->record.undo_operation
+ == const_cpu_to_le16(CompensationlogRecord)))
err = redo_delete_index(vol, action, buffer);
break;
case DeleteIndexEntryRoot :
- if (action->record.undo_operation
+ if ((action->record.undo_operation
== const_cpu_to_le16(AddIndexEntryRoot))
+ || (action->record.undo_operation
+ == const_cpu_to_le16(CompensationlogRecord)))
err = redo_delete_root_index(vol, action, buffer);
break;
case InitializeFileRecordSegment :
@@ -4232,8 +4249,10 @@
}
break;
case UpdateResidentValue :
- if (action->record.undo_operation
- == const_cpu_to_le16(UpdateResidentValue))
+ if ((action->record.undo_operation
+ == const_cpu_to_le16(UpdateResidentValue))
+ || (action->record.undo_operation
+ == const_cpu_to_le16(CompensationlogRecord)))
err = redo_update_resident(vol, action, buffer);
break;
case Win10Action37 :
------------------------------------------------------------------------------
Check out the vibrant tech community on one of the world's most
engaging tech sites, Slashdot.org! http://sdm.link/slashdot
_______________________________________________
ntfs-3g-devel mailing list
ntfs-3g-devel@lists.sourceforge.net
https://lists.sourceforge.net/lists/listinfo/ntfs-3g-devel
