why everything so quick?

[mecctor]

As I know the Everything uses the USN log technology, so it can quickly build an index, that is, it can do a fast search. But how can Everything monitor file changes so quickly?

[void]

Everything reads the NTFS Master File Table (MFT) to quickly index your files.
The USN Journal is not used when indexing your files.

Everything uses the USN Journal to keep your indexes up to date.
The system will write file system changes to your USN Journal.
Everything will see these changes immediately and update its database appropriately.

Hopefully you only have a few hundred file system changes a minute.
Everything is very efficient with updating its database and should consume minimal resources.

[mecctor]

thanks a lot. However, I created or deleted some files, then checked the USN_RECORD and found that record->Reason always returned 0. ALSO I DELETE A some FILE just like "ABC.EX", AND I detect USN_RECORD record. The file name it returns is very strange $1a2bd.txt something like this.

[void]

Please make sure your requested version with READ_USN_JOURNAL_DATA is matching your the USN_RECORD structure.

Everything uses version 3.3

[mecctor]

I wrote a piece of code under windows11, VS2019, Platform Toolset= (v142). But I found that record->Reason (USN_RECORD) always returns 0; also, if I delete a file, I monitor that it is a garbled file name with $. For example, if I delete the file "ABC.TXT", I monitor and detect that a file name like "$1a2f5.txt" is returned. After I googled, I found that it is because USN_RECORD has multiple versions, V2, V3, V4, but I don't know how to detect the version number. Here is the code:

Code: Select all

#ifndef VC_EXTRALEAN
#define VC_EXTRALEAN        // 从 Windows 头中排除极少使用的资料
#endif


#include <Windows.h>

#include <stdio.h>
#include <string>
#include <iostream>
#include "usn_test.h"

//#define BUFFER_SIZE (1024 * 1024)
constexpr auto BUFFER_SIZE = sizeof(USN) + 0x100000;
HANDLE drive;
USN maxusn;
DWORDLONG gUsnJournalID;
void delete_usn()
{
    DELETE_USN_JOURNAL_DATA dujd;
    dujd.UsnJournalID = gUsnJournalID;
    dujd.DeleteFlags = USN_DELETE_FLAG_DELETE;
    DWORD br;

    if (DeviceIoControl(drive,
        FSCTL_DELETE_USN_JOURNAL,
        &dujd,
        sizeof(dujd),
        nullptr,
        0,
        &br,
        nullptr)
        )
    {
        CloseHandle(drive);
        return;
    }
    CloseHandle(drive);
    return;
}


template<typename _T_USN_RECORD>
void check_record(_T_USN_RECORD* record)
{

    WCHAR szName[MAX_PATH];

    CopyMemory(szName,
        ((PBYTE)record) + record->FileNameOffset,
        record->FileNameLength);

    szName[record->FileNameLength / sizeof(WCHAR)] = 0;
    std::wcout << szName << L",Reason:" << record->Reason << std::endl;

}

template<>
void check_record(USN_RECORD_V4* record)
{
    WCHAR szName[MAX_PATH];

    CopyMemory(szName,
        ((PBYTE)record) + record->Extents->Offset,
        record->Extents->Length);
    // Let's zero-terminate it
    szName[record->Extents->Length / sizeof(WCHAR)] = 0;
    std::wcout << szName << L",Reason:" << record->Reason << std::endl;
}
bool create_usn()
{
    CREATE_USN_JOURNAL_DATA cujd{};
    cujd.MaximumSize = 0; // 0表示使用默认值  
    cujd.AllocationDelta = 0; // 0表示使用默认值

    DWORD br;
    if (
        DeviceIoControl(drive, // handle to volume
            FSCTL_CREATE_USN_JOURNAL, // dwIoControlCode
            &cujd, // input buffer
            sizeof(cujd), // size of input buffer
            nullptr, // lpOutBuffer
            0, // nOutBufferSize
            &br, // number of bytes returned
            nullptr) // OVERLAPPED structure    
        )
    {
        return true;
    }
    auto&& info = "create usn error. Error code: " + std::to_string(GetLastError());
    fprintf(stderr, "fileSearcherUSN: %s\n", info.c_str());
    return false;
}


template<typename _T_USN_RECORD>
void read_record(DWORDLONG& nextid, void* buffer, _T_USN_RECORD*& record, _T_USN_RECORD*& recordend, const DWORD& bytecount)
{
    nextid = *((DWORDLONG*)buffer);
    //      printf("Next ID: %lu\n", nextid);

    record = (_T_USN_RECORD*)((USN*)buffer + 1);
    recordend = (_T_USN_RECORD*)(((BYTE*)buffer) + bytecount);

    while (record < recordend)
    {
        check_record(record);
        record = (_T_USN_RECORD*)(((BYTE*)record) + record->RecordLength);
    }
}
template<>
void read_record(DWORDLONG& nextid, void* buffer, USN_RECORD_V4*& record, USN_RECORD_V4*& recordend, const DWORD& bytecount)
{
    nextid = *((DWORDLONG*)buffer);
    //      printf("Next ID: %lu\n", nextid);

    record = (USN_RECORD_V4*)((USN*)buffer + 1);
    recordend = (USN_RECORD_V4*)(((BYTE*)buffer) + bytecount);

    while (record < recordend)
    {
        //check_record(record);
        record = (USN_RECORD_V4*)(((BYTE*)record) + record->Header.RecordLength);
    }
}
int main(int argc, char** argv)
{
    //typedef USN_JOURNAL_DATA_V2 USN_JOURNAL_DATA;
#define FILE_END 38
    //typedef MFT_ENUM_DATA MFT_ENUM_DATA_V1;   
    MFT_ENUM_DATA mft_enum_data;
    DWORD bytecount = 1;
    void* buffer;
    USN_RECORD_V3* recordV3;
    USN_RECORD_V3* recordendV3;
    USN_RECORD_V4* recordV4;
    USN_RECORD_V4* recordendV4;
    USN_JOURNAL_DATA* journal;
    DWORDLONG nextid{};
        
    buffer = new BYTE[BUFFER_SIZE];
    if (buffer == NULL)
    {
        printf("VirtualAlloc: %u\n", GetLastError());
        return 0;
    }

    printf("Opening volume.\n");

    drive = CreateFile(L"\\\\?\\h:", GENERIC_READ, FILE_SHARE_DELETE | FILE_SHARE_READ | FILE_SHARE_WRITE, NULL, OPEN_ALWAYS, FILE_FLAG_NO_BUFFERING, NULL);

    if (drive == INVALID_HANDLE_VALUE)
    {
        printf("CreateFile: %u\n", GetLastError());
        return 0;
    }
    if (!create_usn())
    {
        return -1;
    }
    printf("Calling FSCTL_QUERY_USN_JOURNAL\n");

    if (!DeviceIoControl(drive, FSCTL_QUERY_USN_JOURNAL, NULL, 0, buffer, BUFFER_SIZE, &bytecount, NULL))
    {
        printf("FSCTL_QUERY_USN_JOURNAL: %u\n", GetLastError());
        return 0;
    }
    // delete_usn();

    journal = (USN_JOURNAL_DATA*)buffer;
    gUsnJournalID = journal->UsnJournalID;
    printf("UsnJournalID: %lu\n", journal->UsnJournalID);
    printf("FirstUsn: %lu\n", journal->FirstUsn);
    printf("NextUsn: %lu\n", journal->NextUsn);


    maxusn = journal->MaxUsn;

    mft_enum_data.StartFileReferenceNumber = 0;
    mft_enum_data.LowUsn = 0;
    mft_enum_data.HighUsn = maxusn;
    mft_enum_data.MaxMajorVersion = journal->MaxSupportedMajorVersion;
    mft_enum_data.MinMajorVersion = journal->MinSupportedMajorVersion;

    for (;;)
    {
        //      printf("=================================================================\n");
        //      printf("Calling FSCTL_ENUM_USN_DATA\n");

        if (!DeviceIoControl(drive, FSCTL_ENUM_USN_DATA, &mft_enum_data, sizeof(mft_enum_data), buffer, BUFFER_SIZE, &bytecount, NULL))
        {
            auto bRet = GetLastError();
            if (bRet != 38)
            {
                delete_usn();
                break;
            }
            if (bRet == 38)//file to end
            {
                //   break;
                Sleep(1000);//wait for new file change
                auto oldNextUsn = journal->NextUsn;
                if (!DeviceIoControl(drive, FSCTL_QUERY_USN_JOURNAL, NULL, 0, buffer, BUFFER_SIZE, &bytecount, NULL))
                {
                    printf("FSCTL_QUERY_USN_JOURNAL: %u\n", GetLastError());
                    return 0;
                }
                journal = (USN_JOURNAL_DATA*)buffer;
                if (journal->NextUsn > oldNextUsn)//new file have changed
                {
                    mft_enum_data.StartFileReferenceNumber = 0;
                    mft_enum_data.LowUsn = oldNextUsn;
                    mft_enum_data.HighUsn = journal->MaxUsn;
                    mft_enum_data.MaxMajorVersion = journal->MaxSupportedMajorVersion;
                    mft_enum_data.MinMajorVersion = journal->MinSupportedMajorVersion;
                }
                continue;
            }

        }
        //      printf("Bytes returned: %u\n", bytecount);
        if(mft_enum_data.MinMajorVersion == 4)//it's wrong if i determine version by mft_enum_data.MaxMajorVersion
            read_record(nextid, buffer, recordV4, recordendV4, bytecount);
        else
            read_record(nextid, buffer, recordV3, recordendV3, bytecount);

        mft_enum_data.StartFileReferenceNumber = nextid;
    }
    delete[] buffer;
    delete_usn();
}