Home

Resume

Blog

Teikitu


/* =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-==-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
/*  »Project«   Teikitu Gaming System (TgS) (∂)
    »File«      TgS Common - Mem MGR.c
    »Author«    Andrew Aye (EMail: mailto:andrew.aye@gmail.com, Web: http://www.andrewaye.com)
    »Version«   4.51 / »GUID« A9981407-3EC9-42AF-8B6F-8BE6DD919615                                                                                                        */
/*   -------------------------------------------------------------------------------------------------------------------------------------------------------------------- */
/*  Copyright: © 2002-2017, Andrew Aye.  All Rights Reserved.
    This software is free for non-commercial use.  Redistribution and use in source and binary forms, with or without modification, are permitted provided that the
      following conditions are met:
        Redistribution of source code must retain this copyright notice, this list of conditions and the following disclaimers.
        Redistribution in binary form must reproduce this copyright notice, this list of conditions and the following disclaimers in the documentation and other materials
          provided with the distribution.
    The name of the author may not be used to endorse or promote products derived from this software without specific prior written permission.
    The intellectual property rights of the algorithms used reside with Andrew Aye.
    You may not use this software, in whole or in part, in support of any commercial product without the express written consent of the author.
    There is no warranty or other guarantee of fitness of this software for any purpose. It is provided solely "as is".                                                   */
/* =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-==-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */

/* ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- */
/*  Memory Pools: A pool is created for each power of 2 greater than 8, and for the 3 most significant binary values after them (i.e. 10000, 10100, 11000, 11100) - up to */
/* MAXPOOL count. If the OS Page_Size is less than the normal MaxAlloc size number of pools and MaxAlloc are correspondingly modified so that MaxAlloc matches Page_Size. */
/* The reason for this is to reduce internal fragmentation and because no time savings would be incurred. */
/* ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- */


/* == Common ============================================================================================================================================================ */

/* -.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-. */
/*  Internal Platform Functions                                                                                                                                           */
/* -.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-. */

TgEXTN P_TgUINT08                           tgMM_PM_Init_Scratch( C_TgSIZE );
TgEXTN TgVOID                               tgMM_PM_Free_Scratch( PC_TgUINT08 );
TgEXTN TgUINT32                             tgMM_PM_Page_Size( TgVOID );
TgEXTN P_TgVOID                             tgMM_PM_Virtual_Reserve( C_TgSIZE, C_TgBOOL );
TgEXTN P_TgVOID                             tgMM_PM_Virtual_Commit( PC_TgVOID, C_TgSIZE );
TgEXTN TgVOID                               tgMM_PM_Virtual_Free( PC_TgVOID );




/* -.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-. */
/*  File Local Types                                                                                                                                                      */
/* -.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-. */

#define SIZE_OF_MEMPAGE (sizeof(TgMEMPAGE) - 1)
TgTYPE_DECLARE( struct TgMEMPAGE, TgMEMPAGE );

#pragma pack(push,1)
#if TgCOMPILE_MEM_TRACK
struct TgMEMPAGE /* 64 Byte Header Block */
{
    P_TgMEMPAGE                                 m_psBlock_Next, m_psPool_Next;              /* 16 Bytes */
    TgUINT32                                    m_uiBlockSize;                              /*  4 Bytes */
    TgSINT32                                    m_iEndBlock, m_iLastBlock, m_iFreeBlock;    /* 12 Bytes */
    union
    {
        struct
        {
            const TgCHAR **                             apszFile;
            P_TgUINT32                                  auiLine;
        }                                           Pool;
        struct
        {
            CP_TgCHAR                                   pszFile;
            TgUINT32                                    uiLine;
        }                                           Page;
    }                                           m_sTrack;                                  /* 16 Bytes */
    TgUINT32                                    m_uiPad[4];                                /* 16 Bytes */
    TgUINT08                                    m_pData[1];
};
TgCOMPILER_ASSERT( SIZE_OF_MEMPAGE == 64, 0 );
/*# TgCOMPILE_MEM_TRACK */
#else
struct TgMEMPAGE /* 32 Byte Header Block */
{
    P_TgMEMPAGE                                 m_psBlock_Next, m_psPool_Next;
    TgUINT32                                    m_uiBlockSize;
    TgSINT32                                    m_iEndBlock, m_iLastBlock, m_iFreeBlock;
    TgUINT08                                    m_pData[1];
};
TgCOMPILER_ASSERT( SIZE_OF_MEMPAGE == 32, 0 );
/*# TgCOMPILE_MEM_TRACK */
#endif
#pragma pack(pop)




/* -.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-. */
/*  File Local Macros                                                                                                                                                     */
/* -.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-. */

#if TgS_STAT_COMMON
    #define STAT(...)                       __VA_ARGS__
#else
    #define STAT(...)
#endif

#if TgCOMPILE_MEM_TRACK
#define MEM_TRACK_PARAMETERS            , CPC_TgCHAR pszF, C_TgUINT32 uiL
#else
#define MEM_TRACK_PARAMETERS
#endif




/* -.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-. */
/*  File Local Functions and Data                                                                                                                                         */
/* -.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-. */

                                            /* Memory Temporary */
static TgSIZE                               tgMM_Size_Temp( CPC_TgVOID );
static P_TgVOID                             tgMM_Malloc_Temp( C_TgSIZE MEM_TRACK_PARAMETERS );
static TgVOID                               tgMM_Free_Temp( PC_TgVOID MEM_TRACK_PARAMETERS );

                                            /* Virtual Memory */
static TgSIZE                               tgMM_Size_Virtual( CPC_TgVOID );
static P_TgVOID                             tgMM_Malloc_Virtual( C_TgSIZE MEM_TRACK_PARAMETERS );

                                            /* Base OS Allocator */
static P_TgVOID                             tgMM_Malloc_OS( C_TgSIZE uiSize, C_TgBOOL bCommit, C_TgBOOL bTemp MEM_TRACK_PARAMETERS );

#if TgCOMPILE_MEM_FREE_CLEAR
static TgVOID                               tgMM_Free_OS( PC_TgVOID pMem, C_TgBOOL bClearMem );
#else
static TgVOID                               tgMM_Free_OS( PC_TgVOID pMem );
#endif

/*  To make life simple - the allocation granularity and the page granularity are assumed to be the same.  Thus, on the X360 all allocations are made using the large */
/* memory page size option (64K pages). */

static ETgMODULE_STATE                      s_enMem_MGR_State = ETgMODULE_STATE__FREED;

static STg1_MP_CS                           s_csSystemLock; /*« Need to be able to lock during Stats output. */
static STg2_MM_MGR                          s_asAllocator[ETgMM_ALLOCATOR_COUNT];
static TgUINT32                             s_uiPageSize;
static TgSINT32                             s_niMaxPage, s_iMaxBlock;

static P_TgMEMPAGE                          s_aPageOpen[KTgMAX_POOLS], s_aPageClosed[KTgMAX_POOLS];
static STg1_MP_CS                           s_csPageLock[KTgMAX_POOLS];

static P_TgMEMPAGE                          s_pOSPool_Head = 0;

static TgUINT32                             s_uiScratch_Top = 0;

#if TgCOMPILE_THREAD_LOCAL
static TgTLS P_TgUINT08                     tls_pScratchMemory;
static TgTLS TgSIZE                         tls_uiScratchSize;
#endif

#if TgS_STAT_COMMON
static TgUINT32                             s_uiStat_Size_Pool, s_uiStat_Size_OS, s_uiStat_Size_Scratch;
static TgUINT32                             s_nuiStat_Pool, s_nuiStat_OS, s_nuiStat_Scratch;
static TgUINT32                             s_nuiStat_Free_Pool, s_nuiStat_Free_OS, s_nuiStat_Free_Scratch;
static TgUINT32                             s_nuiStat_Max_Pool, s_nuiStat_Max_OS, s_nuiStat_Max_Scratch;
/*# TgS_STAT_COMMON */
#endif




/* -.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-. */
/*  Public Functions                                                                                                                                                      */
/* -.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-. */

/* ---- tgMM_Init_MGR --------------------------------------------------------------------------------------------------------------------------------------------------- */
/* ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- */
TgRESULT tgMM_Init_MGR( TgVOID )
{
    STg2_MM_MGR                         sMGR;
    TgSINT32                            iIdx;

    TgERROR(ETgMODULE_STATE__FREED == s_enMem_MGR_State);
    s_enMem_MGR_State = ETgMODULE_STATE__INITIALIZING;

    memset( s_asAllocator, 0, sizeof( s_asAllocator ) );
    memset( &sMGR, 0, sizeof( STg2_MM_MGR ) );

    sMGR.m_pfnSize = tgMM_Size_Pool;
    sMGR.m_pfnMalloc = tgMM_Malloc_Pool;
    sMGR.m_pfnFree = tgMM_Free_Pool;
    sMGR.m_pfnRealloc = tgMM_Realloc_Pool;
    tgMM_Register_Allocator( ETgMM_ALLOCATOR_POOL, &sMGR );
    sMGR.m_pfnRealloc = nullptr;

    sMGR.m_pfnSize = tgMM_Size_Temp;
    sMGR.m_pfnMalloc = tgMM_Malloc_Temp;
    sMGR.m_pfnFree = tgMM_Free_Temp;
    tgMM_Register_Allocator( ETgMM_ALLOCATOR_TEMP, &sMGR );

    sMGR.m_pfnSize = tgMM_Size_Virtual;
    sMGR.m_pfnMalloc = tgMM_Malloc_Virtual;
    sMGR.m_pfnReserve = tgMM_Reserve_Virtual;
    sMGR.m_pfnCommit = tgMM_Commit_Virtual;
    sMGR.m_pfnFree = tgMM_Free_Virtual;
    tgMM_Register_Allocator( ETgMM_ALLOCATOR_VIRTUAL, &sMGR );

    s_uiPageSize = 0;
    s_niMaxPage = 0;
    s_iMaxBlock = 0;

    memset( s_aPageOpen, 0x00, sizeof( s_aPageOpen ) );
    memset( s_aPageClosed, 0x00, sizeof( s_aPageClosed ) );

#if TgCOMPILE_THREAD_LOCAL
    tls_pScratchMemory = 0;
#endif

    s_pOSPool_Head = 0;

    s_uiScratch_Top = 0;

#if TgS_STAT_COMMON
    s_uiStat_Size_Pool = 0;
    s_uiStat_Size_OS = 0;
    s_uiStat_Size_Scratch = 0;
    s_nuiStat_Pool = 0;
    s_nuiStat_OS = 0;
    s_nuiStat_Scratch = 0;
    s_nuiStat_Free_Pool = 0;
    s_nuiStat_Free_OS = 0;
    s_nuiStat_Free_Scratch = 0;
    s_nuiStat_Max_Pool = 0;
    s_nuiStat_Max_OS = 0;
    s_nuiStat_Max_Scratch = 0;
/*# TgS_STAT_COMMON */
#endif

    /* Set the memory page size and reset the allocation sizes and pool counts if the page size is smaller than expected */

    s_uiPageSize = tgMM_Page_Size();
    s_iMaxBlock = tgCM_MIN_S32( (TgSINT32)1024, s_uiPageSize >> 2 );
    s_niMaxPage = 1 + 4 * (tgPM_BSR_U32( (TgUINT32)s_iMaxBlock ) - KTgMIN_ALLOC_BIT);

    /* Create the Critical Sections (nullptr Function if MP Safe not define-flagged ) */

    TgVERIFY(KTgS_OK == tgCM_MP_CS_Init( &s_csSystemLock ));

    for (iIdx = 0; iIdx < KTgMAX_POOLS; ++iIdx)
    {
        TgVERIFY(KTgS_OK == tgCM_MP_CS_Init( &s_csPageLock[iIdx] ));
    };

    /* Initialize the Scratch Memory space */

#if TgCOMPILE_THREAD_LOCAL
    TgVERIFY(KTgS_OK == tgMM_TL_Init_Scratch( KTgDEFAULT_SCRATCH_SIZE ));
    TgCRITICAL(0u != s_uiPageSize && nullptr != tls_pScratchMemory);
#endif

    /* Sanity Checks */

    TgCRITICAL(tgCM_IS_PW2_U32( s_uiPageSize ) && tgMM_Page_Size() == s_uiPageSize);
    TgCRITICAL(s_iMaxBlock >= KTgMIN_ALLOC_BIT && s_iMaxBlock <= 1024);
    TgCRITICAL(s_niMaxPage >= 0 && s_niMaxPage < KTgMAX_POOLS);

    s_enMem_MGR_State = ETgMODULE_STATE__BOOTED;

#if TgCOMPILE_CONSOLE
    if (tgGB_CMD_Query_Argument_Index( TgT("-//stat/boot/memory") ) >= 0)
    {
        tgMM_Stats( &g_sOutCon );
    };
/*# TgCOMPILE_CONSOLE */
#endif

    return (KTgS_OK);
}


/* ---- tgMM_Boot_MGR --------------------------------------------------------------------------------------------------------------------------------------------------- */
/* ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- */
TgRESULT tgMM_Boot_MGR( TgVOID )
{
    TgERROR_MSG( TgFALSE, TgT("This function should never be executed.") );
    return (KTgS_OK);
}


/* ---- tgMM_Stop_MGR --------------------------------------------------------------------------------------------------------------------------------------------------- */
/* ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- */
TgRESULT tgMM_Stop_MGR( TgVOID )
{
    TgERROR_MSG( TgFALSE, TgT("This function should never be executed.") );
    return (KTgS_OK);
}


/* ---- tgMM_Free_MGR --------------------------------------------------------------------------------------------------------------------------------------------------- */
/* ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- */
TgRESULT tgMM_Free_MGR( TgVOID )
{
    TgSINT32                            iIdx;

    if (ETgMODULE_STATE__FREED == s_enMem_MGR_State)
    {
        return (KTgS_OK);
    };

#if TgCOMPILE_CONSOLE
    if (tgGB_CMD_Query_Argument_Index( TgT("-//stat/free/memory") ) >= 0)
    {
        tgMM_Stats( &g_sOutCon );
    };
/*# TgCOMPILE_CONSOLE */
#endif

    TgERROR(ETgMODULE_STATE__BOOTED == s_enMem_MGR_State);
    s_enMem_MGR_State = ETgMODULE_STATE__FREEING;

    /* Look for unfreed allocation to report */
    for (iIdx = 0; iIdx < KTgMAX_POOLS; ++iIdx)
    {
        P_TgMEMPAGE                         psMemPage;

        for (psMemPage = s_aPageOpen[iIdx]; nullptr != psMemPage; )
        {
            P_TgMEMPAGE                         psNextPage = psMemPage->m_psBlock_Next;

        #if TgCOMPILE_MEM_TRACK
            TgSINT32                            iCheck, iFree;
            TgCHAR                              szBuffer[1024];

            for (iCheck = 0; iCheck < psMemPage->m_iLastBlock; ++iCheck)
            {
                for (iFree = psMemPage->m_iFreeBlock; 0 <= iFree && iFree != iCheck;)
                {
                    TgSCALAR_P_ALIAS_UNION              sUnion;
                    
                    sUnion.m_pui08 = psMemPage->m_pData + (TgUINT32)iFree * psMemPage->m_uiBlockSize;
                    iFree = *(sUnion.m_pi32);
                };

                if (iFree == iCheck)
                {
                    continue;
                };

                tgSZ_PrintF( szBuffer, 1024, TgT("%-16.16s(%-32.32s): Unfreed Memory Allocation from (%s) at (%d).\n"), TgT("Common"),
                    TgT("MM_Free_MGR"), psMemPage->m_sTrack.Pool.apszFile[iCheck], psMemPage->m_sTrack.Pool.auiLine[iCheck] );

                tgPM_DBG_OUT_Write( nullptr, KTgMAX_SIZE, (P_TgUINT08)szBuffer, tgSZ_Length( szBuffer ) );
            };

            if (nullptr != psMemPage->m_sTrack.Pool.apszFile)
            {
                tgMM_PM_Virtual_Free( (P_TgVOID)psMemPage->m_sTrack.Pool.apszFile );
                psMemPage->m_sTrack.Pool.apszFile = nullptr;
            };
            if (nullptr != psMemPage->m_sTrack.Pool.auiLine)
            {
                tgMM_PM_Virtual_Free( psMemPage->m_sTrack.Pool.auiLine );
                psMemPage->m_sTrack.Pool.auiLine = nullptr;
            };
        /*# TgCOMPILE_MEM_TRACK */
        #endif

            tgMM_PM_Virtual_Free( psMemPage );
            psMemPage = psNextPage;
        };

        s_aPageOpen[iIdx] = nullptr;

        TgCRITICAL( nullptr == s_aPageClosed[iIdx] );

        for (psMemPage = s_aPageClosed[iIdx]; nullptr != psMemPage; )
        {
            P_TgMEMPAGE                         psNextPage = psMemPage->m_psBlock_Next;

            tgMM_PM_Virtual_Free( psMemPage );
            psMemPage = psNextPage;
        };

        s_aPageClosed[iIdx] = nullptr;

        /* Free the Critical Sections (nullptr Function if MP Safe not define-flagged ) */
        tgCM_MP_CS_Free( s_csPageLock + iIdx );
    };

#if TgS_STAT_COMMON
    TgCRITICAL( 0u == (TgUINT32)(s_nuiStat_Pool - s_nuiStat_Free_Pool) );
    TgCRITICAL( 0u == (TgUINT32)(s_nuiStat_OS - s_nuiStat_Free_OS) );
    TgCRITICAL( 0u == (TgUINT32)(s_nuiStat_Scratch - s_nuiStat_Free_Scratch) );
/*# TgS_STAT_COMMON */
#endif
    tgCM_MP_CS_Free( &s_csSystemLock );

#if TgS_STAT_COMMON
    TgCRITICAL( 0 == s_uiStat_Size_Scratch );
#endif
#if TgCOMPILE_THREAD_LOCAL
    tgMM_TL_Free_Scratch();
#endif

    s_enMem_MGR_State = ETgMODULE_STATE__FREED;

    return (KTgS_OK);
}


/* ---- tgMM_Update_MGR ------------------------------------------------------------------------------------------------------------------------------------------------- */
/* ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- */
TgRESULT tgMM_Update_MGR( C_TgFLOAT32 UNUSED_PARAM fDT )
{
    return (KTgS_OK);
}


/* ---- tgMM_Query_Init ------------------------------------------------------------------------------------------------------------------------------------------------- */
/* ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- */
TgBOOL tgMM_Query_Init( TgVOID )
{
    return (ETgMODULE_STATE__INITIALIZED <= s_enMem_MGR_State && s_enMem_MGR_State <= ETgMODULE_STATE__STOPPED ? TgTRUE : TgFALSE);
}


/* ---- tgMM_Query_Boot ------------------------------------------------------------------------------------------------------------------------------------------------- */
/* ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- */
TgBOOL tgMM_Query_Boot( TgVOID )
{
    return (ETgMODULE_STATE__BOOTED == s_enMem_MGR_State ? TgTRUE : TgFALSE);
}


/* ---- tgMM_Query_Fixed_Memory ----------------------------------------------------------------------------------------------------------------------------------------- */
/* ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- */
TgSIZE tgMM_Query_Fixed_Memory( TgVOID )
{
    return (0
             + sizeof( s_enMem_MGR_State )
             + sizeof( s_csSystemLock )
             + sizeof( s_asAllocator )
             + sizeof( s_uiPageSize )
             + sizeof( s_niMaxPage )
             + sizeof( s_iMaxBlock )
             + sizeof( s_aPageOpen )
             + sizeof( s_aPageClosed )
             + sizeof( s_csPageLock )
#if TgCOMPILE_THREAD_LOCAL
             + sizeof( tls_pScratchMemory )
#endif
             + sizeof( s_pOSPool_Head )
             + sizeof( s_uiScratch_Top )
#if TgS_STAT_COMMON
             + sizeof( s_uiStat_Size_Pool )
             + sizeof( s_uiStat_Size_OS )
             + sizeof( s_uiStat_Size_Scratch )
             + sizeof( s_nuiStat_Pool )
             + sizeof( s_nuiStat_OS )
             + sizeof( s_nuiStat_Scratch )
             + sizeof( s_nuiStat_Free_Pool )
             + sizeof( s_nuiStat_Free_OS )
             + sizeof( s_nuiStat_Free_Scratch )
             + sizeof( s_nuiStat_Max_Pool )
             + sizeof( s_nuiStat_Max_OS )
             + sizeof( s_nuiStat_Max_Scratch )
/*# TgS_STAT_COMMON */
#endif
    );
}


/* ---- tgMM_Page_Size -------------------------------------------------------------------------------------------------------------------------------------------------- */
/* ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- */
TgUINT32 tgMM_Page_Size( TgVOID )
{
    return (tgMM_PM_Page_Size());
}


/* ---- tgMM_Register_Allocator ------------------------------------------------------------------------------------------------------------------------------------------------- */
/* ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- */
TgVOID tgMM_Register_Allocator( C_ETgMM_ALLOCATOR enAllocator, P_STg2_MM_MGR psMem_MGR )
{
    TgPARAM_CHECK_INDEX( enAllocator, s_asAllocator );

    TgCRITICAL( psMem_MGR && psMem_MGR->m_pfnFree && psMem_MGR->m_pfnMalloc && psMem_MGR->m_pfnSize );

    memcpy( s_asAllocator + enAllocator, psMem_MGR, sizeof( STg2_MM_MGR ) );
}


/* ---- tgMM_Stats ------------------------------------------------------------------------------------------------------------------------------------------------------ */
/* ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- */
#if TgS_STAT_COMMON
TgVOID tgMM_Stats( PC_STg2_Output psOUT )
{
    TgUINT32                            nuiCurrentPool, nuiCurrentOS, nuiCurrentScratch, uiIndex;
    P_TgMEMPAGE                         psMemPage;

    TgERROR(nullptr != psOUT);
    tgCM_MP_CS_Enter_Block( &s_csSystemLock );

    nuiCurrentPool = (TgUINT32)(s_nuiStat_Pool - s_nuiStat_Free_Pool);
    nuiCurrentOS = (TgUINT32)(s_nuiStat_OS - s_nuiStat_Free_OS);
    nuiCurrentScratch = (TgUINT32)(s_nuiStat_Scratch - s_nuiStat_Free_Scratch);

    tgIO_PrintF( psOUT, TgT("              Max        Current     Alloc  \n") );
    tgIO_PrintF( psOUT, TgT("--------    --------    --------    --------\n") );
    tgIO_PrintF( psOUT, TgT("Pool        % 8d    % 8d    % 8d\n"), s_nuiStat_Max_Pool, nuiCurrentPool, s_uiStat_Size_Pool );
    tgIO_PrintF( psOUT, TgT("OS          % 8d    % 8d    % 8d\n"), s_nuiStat_Max_OS, nuiCurrentOS, s_uiStat_Size_OS );
    tgIO_PrintF( psOUT, TgT("Scratch     % 8d    % 8d    % 8d\n"), s_nuiStat_Max_Scratch, nuiCurrentScratch, s_uiStat_Size_Scratch );
    tgIO_PrintF( psOUT, TgT("\n") );

    for (uiIndex = 0; uiIndex < KTgMAX_POOLS; ++uiIndex)
    {
        TgUINT32                            nuiMemPool = 0;
        TgUINT32                            uiBlockSize = 0;

        tgCM_MP_CS_Enter_Block( s_csPageLock + uiIndex );

        for (psMemPage = s_aPageOpen[uiIndex]; nullptr != psMemPage; psMemPage = psMemPage->m_psBlock_Next)
        {
            TgSINT32                            iFree = psMemPage->m_iFreeBlock;

            nuiMemPool += (TgUINT32)psMemPage->m_iLastBlock;
            uiBlockSize = psMemPage->m_uiBlockSize;

            while (0 <= iFree)
            {
                TgSCALAR_P_ALIAS_UNION              sUnion;

                sUnion.m_pui08 = psMemPage->m_pData + (TgUINT32)iFree * psMemPage->m_uiBlockSize;
                iFree = *(sUnion.m_pi32);
                --nuiMemPool;
            };
        };

        for (psMemPage = s_aPageClosed[uiIndex]; nullptr != psMemPage; psMemPage = psMemPage->m_psBlock_Next)
        {
            uiBlockSize = psMemPage->m_uiBlockSize;
            nuiMemPool += (TgUINT32)psMemPage->m_iLastBlock;
        };

        tgIO_PrintF( psOUT, TgT("Pool(% 8d)               % 8d    % 8d\n"), uiBlockSize, nuiMemPool * uiBlockSize, nuiMemPool );

        tgCM_MP_CS_Exit( s_csPageLock + uiIndex );
    };

#if TgCOMPILE_MEM_TRACK
    for (uiIndex = 0; uiIndex < KTgMAX_POOLS; ++uiIndex)
    {
        TgSINT32                            iBlock;

        tgCM_MP_CS_Enter_Block( s_csPageLock + uiIndex );

        for (psMemPage = s_aPageOpen[uiIndex]; nullptr != psMemPage; psMemPage = psMemPage->m_psBlock_Next)
        {
            TgSINT32                            iFree;

            if (!psMemPage->m_sTrack.Pool.apszFile || !psMemPage->m_sTrack.Pool.auiLine)
                continue;

            for (iBlock = 0; iBlock < psMemPage->m_iLastBlock; ++iBlock)
            {
                for (iFree = psMemPage->m_iFreeBlock; 0 <= iFree && iFree != iBlock; )
                {
                    TgSCALAR_P_ALIAS_UNION              sUnion;

                    sUnion.m_pui08 = psMemPage->m_pData + (TgUINT32)iFree * psMemPage->m_uiBlockSize;
                    iFree = *(sUnion.m_pi32);
                };

                if (iFree == iBlock)
                {
                    continue;
                };

                tgIO_PrintF( psOUT, TgT("Block Allocation         % 8d               % 6d %s\n"),
                    psMemPage->m_uiBlockSize, psMemPage->m_sTrack.Pool.auiLine[iBlock], psMemPage->m_sTrack.Pool.apszFile[iBlock] );
            };
        };

        for (psMemPage = s_aPageClosed[uiIndex]; nullptr != psMemPage; psMemPage = psMemPage->m_psBlock_Next)
        {
            for (iBlock = 0; iBlock < psMemPage->m_iLastBlock; ++iBlock)
            {
                if (!psMemPage->m_sTrack.Pool.apszFile || !psMemPage->m_sTrack.Pool.auiLine)
                    continue;

                tgIO_PrintF( psOUT, TgT("Block Allocation         % 8d               % 6d %s\n"),
                    psMemPage->m_uiBlockSize, psMemPage->m_sTrack.Pool.auiLine[iBlock], psMemPage->m_sTrack.Pool.apszFile[iBlock] );
            };
        };

        tgCM_MP_CS_Exit( s_csPageLock + uiIndex );
    };
/*# TgCOMPILE_MEM_TRACK */
#endif

    for (psMemPage = s_pOSPool_Head; nullptr != psMemPage; psMemPage = psMemPage->m_psPool_Next)
    {
    #if TgCOMPILE_MEM_TRACK
        tgIO_PrintF( psOUT, TgT("OS Allocation            % 8d               % 6d %s\n"), psMemPage->m_uiBlockSize,
            psMemPage->m_sTrack.Page.uiLine, psMemPage->m_sTrack.Page.pszFile );
    /*# TgCOMPILE_MEM_TRACK */
    #else
        tgIO_PrintF( psOUT, TgT("OS Allocation            % 8d\n"), psMemPage->m_uiBlockSize );
    /*# TgCOMPILE_MEM_TRACK */
    #endif
    };

    tgCM_MP_CS_Exit( &s_csSystemLock );
}
/*# TgS_STAT_COMMON */
#endif


/* ---- tgMM_Pool_Stats ------------------------------------------------------------------------------------------------------------------------------------------------- */
/* ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- */
#if TgS_STAT_COMMON
TgVOID tgMM_Pool_Stats( C_TgSINT32 iPoolIndex, P_TgSINT32 piSize, P_TgSINT32 pniAlloc )
{
    P_TgMEMPAGE                         psPage;
    TgUINT32                            nuiMemPool = 0;
    TgUINT32                            uiBlockSize = 0;

    tgCM_MP_CS_Enter_Block( &s_csSystemLock );
    tgCM_MP_CS_Enter_Block( s_csPageLock + iPoolIndex );

    for (psPage = s_aPageOpen[iPoolIndex]; nullptr != psPage; psPage = psPage->m_psBlock_Next)
    {
        TgSINT32                            iFree = psPage->m_iFreeBlock;

        nuiMemPool += (TgUINT32)psPage->m_iLastBlock;
        uiBlockSize = psPage->m_uiBlockSize;

        while (0 <= iFree)
        {
            TgSCALAR_P_ALIAS_UNION              sUnion;

            sUnion.m_pui08 = psPage->m_pData + (TgUINT32)iFree * psPage->m_uiBlockSize;
            iFree = *(sUnion.m_pi32);
            --nuiMemPool;
        };
    };

    for (psPage = s_aPageClosed[iPoolIndex]; nullptr != psPage; psPage = psPage->m_psBlock_Next)
    {
        uiBlockSize = psPage->m_uiBlockSize;
        nuiMemPool += (TgUINT32)psPage->m_iLastBlock;
    };

    *piSize = (TgSINT32)(nuiMemPool * uiBlockSize);
    *pniAlloc = (TgSINT32)nuiMemPool;

    tgCM_MP_CS_Exit( s_csPageLock + iPoolIndex );
    tgCM_MP_CS_Exit( &s_csSystemLock );
}
/*# TgS_STAT_COMMON */
#endif


/* ---- tgMM_Size ------------------------------------------------------------------------------------------------------------------------------------------------------- */
/* ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- */
TgSIZE tgMM_Size( C_ETgMM_ALLOCATOR enAllocator, CPC_TgVOID pMem )
{
    TgPARAM_CHECK_INDEX( enAllocator, s_asAllocator );
    return (s_asAllocator[enAllocator].m_pfnSize( pMem ));
}


/* ---- tgMM_Malloc ----------------------------------------------------------------------------------------------------------------------------------------------------- */
/* ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- */
P_TgVOID tgMM_Malloc( C_ETgMM_ALLOCATOR enAllocator, C_TgSIZE uiSize MEM_TRACK_PARAMETERS )
{
    TgPARAM_CHECK_INDEX( enAllocator, s_asAllocator );
    return (s_asAllocator[enAllocator].m_pfnMalloc( uiSize MEM_TRACK_VARS ));
}


/* ---- tgMM_Reserve ---------------------------------------------------------------------------------------------------------------------------------------------------- */
/* ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- */
P_TgVOID tgMM_Reserve( C_ETgMM_ALLOCATOR enAllocator, C_TgSIZE uiSize MEM_TRACK_PARAMETERS )
{
    TgPARAM_CHECK_INDEX( enAllocator, s_asAllocator );
    return (nullptr != s_asAllocator[enAllocator].m_pfnReserve ? s_asAllocator[enAllocator].m_pfnReserve( uiSize MEM_TRACK_VARS ) : nullptr);
}


/* ---- tgMM_Commit ----------------------------------------------------------------------------------------------------------------------------------------------------- */
/* ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- */
P_TgVOID tgMM_Commit( C_ETgMM_ALLOCATOR enAllocator, PC_TgVOID pMem, C_TgSIZE uiSize MEM_TRACK_PARAMETERS )
{
    TgPARAM_CHECK_INDEX( enAllocator, s_asAllocator );
    return (nullptr != s_asAllocator[enAllocator].m_pfnCommit ? s_asAllocator[enAllocator].m_pfnCommit( pMem, uiSize MEM_TRACK_VARS ) : nullptr);
}


/* ---- tgMM_Free ------------------------------------------------------------------------------------------------------------------------------------------------------- */
/* ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- */
TgVOID tgMM_Free( C_ETgMM_ALLOCATOR enAllocator, PC_TgVOID pMem MEM_TRACK_PARAMETERS )
{
    TgPARAM_CHECK_INDEX( enAllocator, s_asAllocator );
    s_asAllocator[enAllocator].m_pfnFree( pMem MEM_TRACK_VARS );
}


/* ---- tgMM_Realloc ---------------------------------------------------------------------------------------------------------------------------------------------------- */
/* ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- */
P_TgVOID tgMM_Realloc( C_ETgMM_ALLOCATOR enAllocator, PC_TgVOID pMem, C_TgSIZE uiSize MEM_TRACK_PARAMETERS )
{
    TgPARAM_CHECK_INDEX( enAllocator, s_asAllocator );
    return (nullptr != s_asAllocator[enAllocator].m_pfnRealloc ? s_asAllocator[enAllocator].m_pfnRealloc( pMem, uiSize MEM_TRACK_VARS ) : nullptr);
}


/* ---- tgMM_Size_Pool -------------------------------------------------------------------------------------------------------------------------------------------------- */
/* ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- */
TgSIZE tgMM_Size_Pool( CPC_TgVOID pMem )
{
    CP_TgMEMPAGE                        psMemPage;

    psMemPage = (CP_TgMEMPAGE)(tgCM_CEL_ALGN_PW2_UPTR( (TgUINTPTR)pMem, (TgUINTPTR)s_uiPageSize ) - s_uiPageSize);

    return (psMemPage->m_uiBlockSize);
}


/* ---- tgMM_Malloc_Pool ------------------------------------------------------------------------------------------------------------------------------------------------ */
/* ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- */
P_TgVOID tgMM_Malloc_Pool( C_TgSIZE uiSize MEM_TRACK_PARAMETERS )
{
    TgSIZE                              uiSizeAlign;
    TgSINT32                            iBitPool, niPool, iPool, idxPage;
    P_TgMEMPAGE                         psMemPage;
    P_TgVOID                            pMemVoid;

    TgERROR(ETgMODULE_STATE__BOOTED == s_enMem_MGR_State);
    TgCRITICAL( s_iMaxBlock >= 0 );

    if (uiSize > (TgUINT32)s_iMaxBlock) /* Use the OS allocator (can always replace later) */
    {
    #if TgCOMPILE_MEM_TRACK
        pMemVoid = tgMM_Malloc_OS( uiSize, TgTRUE, TgFALSE, pszF, uiL );
        TgLOG_MEM( TgT("Mem_MGR"), TgT("Address: % 16p Size: % 16X File: % 128.128s Line % 5d\n"), pMemVoid, uiSize, pszF, uiL );
    #else
        pMemVoid = tgMM_Malloc_OS( uiSize, TgTRUE, TgFALSE );
    #endif

        return (pMemVoid);
    };

    /* Retrieve a Block of Memory from the corresponding pool */

    iBitPool = tgPM_BSR_U32( (TgUINT32)uiSize | KTgMIN_ALLOC );
    uiSizeAlign = tgCM_CEL_ALGN_PW2_UXX( uiSize, ((TgUINT64)1 << (iBitPool - 2)) );
    iBitPool = tgPM_BSR_U32( (TgUINT32)uiSizeAlign | KTgMIN_ALLOC );
    uiSizeAlign = tgCM_CEL_ALGN_PW2_UXX( uiSize, ((TgUINT64)1 << (iBitPool - 2)) );

    TgCRITICAL( uiSizeAlign <= 1024 );
    TgCRITICAL( iBitPool <= 10 );

    niPool = iBitPool - KTgMIN_ALLOC_BIT;
    iPool = tgCM_MAX_S32( 0, (TgSINT32)(uiSizeAlign >> (iBitPool - 2)) - 4 );
    idxPage = niPool * 4 + iPool;

    TgCRITICAL( idxPage < s_niMaxPage );

    tgCM_MP_CS_Enter_Block( &s_csPageLock[idxPage] );

    if (nullptr == s_aPageOpen[idxPage])
    {
        pMemVoid = tgMM_PM_Virtual_Commit( nullptr, s_uiPageSize );

        if (nullptr == pMemVoid)
        {
            TgCRITICAL( TgFALSE );
            return (0);
        };

    #if TgCOMPILE_MEM_MALLOC_CLEAR
        memset( pMemVoid, (int)TgCOMPILE_MEM_MALLOC_PATTERN, s_uiPageSize );
    #endif

        psMemPage = (P_TgMEMPAGE)pMemVoid;

        s_aPageOpen[idxPage] = psMemPage;
        pMemVoid = nullptr;

        psMemPage->m_psBlock_Next = nullptr;
        psMemPage->m_psPool_Next = nullptr;
        psMemPage->m_uiBlockSize = (1 << (iBitPool)) + (TgUINT32)iPool*(1 << (iBitPool - 2));
        psMemPage->m_iEndBlock = (TgSINT32)((s_uiPageSize - SIZE_OF_MEMPAGE) / psMemPage->m_uiBlockSize);
        psMemPage->m_iLastBlock = 0;
        psMemPage->m_iFreeBlock = -1;

    #if TgCOMPILE_MEM_TRACK
        psMemPage->m_sTrack.Pool.apszFile = (const TgCHAR **)tgMM_PM_Virtual_Commit( nullptr, (TgSIZE)psMemPage->m_iEndBlock * sizeof( P_TgCHAR ) );
        if (nullptr != psMemPage->m_sTrack.Pool.apszFile)
            memset( (P_TgVOID)psMemPage->m_sTrack.Pool.apszFile, 0, (TgSIZE)psMemPage->m_iEndBlock * sizeof( P_TgCHAR ) );
        psMemPage->m_sTrack.Pool.auiLine = (P_TgUINT32)tgMM_PM_Virtual_Commit( nullptr, (TgSIZE)psMemPage->m_iEndBlock * sizeof( TgUINT32 ) );
        if (nullptr != psMemPage->m_sTrack.Pool.auiLine)
            memset( psMemPage->m_sTrack.Pool.auiLine, 0, (TgSIZE)psMemPage->m_iEndBlock * sizeof( TgUINT32 ) );
    /*# TgCOMPILE_MEM_TRACK */
    #endif
    };

    psMemPage = s_aPageOpen[idxPage];

    /* Check to see if there are any free blocks in the free list, if not take the first available block */

    if (0 > psMemPage->m_iFreeBlock)
    {
        pMemVoid = (P_TgVOID)(psMemPage->m_pData + psMemPage->m_iLastBlock * (TgSINT32)psMemPage->m_uiBlockSize);
        ++(psMemPage->m_iLastBlock);
    }
    else
    {
        pMemVoid = (P_TgVOID)(psMemPage->m_pData + psMemPage->m_iFreeBlock * (TgSINT32)psMemPage->m_uiBlockSize);
        psMemPage->m_iFreeBlock = *(P_TgSINT32)pMemVoid;
    };

    /* Close the Page if no free memory available in it after this last allocation */

    TgCRITICAL( s_aPageOpen[idxPage]->m_uiBlockSize >= (TgUINT32)uiSize );
    TgCRITICAL( nullptr == s_aPageOpen[idxPage + 1] || s_aPageOpen[idxPage + 1]->m_uiBlockSize > (TgUINT32)uiSize );

    if (psMemPage->m_iLastBlock >= psMemPage->m_iEndBlock && 0 > psMemPage->m_iFreeBlock)
    {
        s_aPageOpen[idxPage] = psMemPage->m_psBlock_Next;
        psMemPage->m_psBlock_Next = s_aPageClosed[idxPage];
        s_aPageClosed[idxPage] = psMemPage;
    };

    STAT( ++s_nuiStat_Pool );
    STAT( s_nuiStat_Max_Pool = tgCM_MAX_U32( s_nuiStat_Max_Pool, (s_nuiStat_Pool - s_nuiStat_Free_Pool) ) );
    STAT( s_uiStat_Size_Pool += psMemPage->m_uiBlockSize );

        tgCM_MP_CS_Exit( &s_csPageLock[idxPage] );

#if TgCOMPILE_MEM_MALLOC_CLEAR
    memset( pMemVoid, (int)TgCOMPILE_MEM_MALLOC_PATTERN, psMemPage->m_uiBlockSize );
#endif

#if TgCOMPILE_MEM_TRACK
    {
        TgSINT32                            iBlock;

        iBlock = (TgSINT32)(((TgUINTPTR)pMemVoid - (TgUINTPTR)psMemPage->m_pData) / psMemPage->m_uiBlockSize);
        TgERROR(iBlock >= 0 && iBlock < psMemPage->m_iEndBlock);
        if (nullptr != psMemPage->m_sTrack.Pool.apszFile) psMemPage->m_sTrack.Pool.apszFile[iBlock] = pszF;
        if (nullptr != psMemPage->m_sTrack.Pool.auiLine) psMemPage->m_sTrack.Pool.auiLine[iBlock] = uiL;
    }
    TgLOG_MEM( TgT("Mem_MGR"), TgT("Address: % 16p Size: % 16X File: % 128.128s Line % 5d\n"), pMemVoid, uiSize, pszF, uiL );
/*# TgCOMPILE_MEM_TRACK */
#endif

    return (pMemVoid);
}


/* ---- tgMM_Free_Pool -------------------------------------------------------------------------------------------------------------------------------------------------- */
/* ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- */
TgVOID tgMM_Free_Pool( PC_TgVOID pMem MEM_TRACK_PARAMETERS )
{
    P_TgMEMPAGE                         psMemPage;

    psMemPage = (P_TgMEMPAGE)(tgCM_CEL_ALGN_PW2_UPTR( (TgUINTPTR)pMem, (TgUINTPTR)s_uiPageSize ) - s_uiPageSize);

    TgERROR(ETgMODULE_STATE__BOOTED == s_enMem_MGR_State);

#if TgCOMPILE_MEM_TRACK
    TgLOG_MEM( TgT("Mem_MGR"), TgT("Address: % 16p Size: % 16X File: % 128.128s Line % 5d\n"), pMem, tgMM_Size_Pool( pMem ), pszF, uiL );
#endif

    if (psMemPage->m_uiBlockSize > (TgUINT32)s_iMaxBlock)
    {
    #if TgCOMPILE_MEM_FREE_CLEAR
        tgMM_Free_OS( pMem, TgTRUE );
    #else
        tgMM_Free_OS( pMem );
    #endif
    }
    else
    {
        TgSINT32                            idxBlock = (TgSINT32)(((P_TgUINT08)pMem - psMemPage->m_pData) / psMemPage->m_uiBlockSize);
        C_TgUINT32                          uiBitPool = (TgUINT32)tgPM_BSR_U32( (TgUINT32)psMemPage->m_uiBlockSize );
        C_TgUINT32                          niPool = uiBitPool - KTgMIN_ALLOC_BIT;
        C_TgUINT32                          uiPool = (psMemPage->m_uiBlockSize >> (uiBitPool - 2)) - 4;
        C_TgUINT32                          idxPage = niPool * 4 + uiPool;

        TgCRITICAL( (P_TgVOID)(psMemPage->m_pData + idxBlock*(TgSINT32)psMemPage->m_uiBlockSize) == pMem );

        tgCM_MP_CS_Enter_Block( &s_csPageLock[idxPage] );

        STAT( s_uiStat_Size_Pool -= psMemPage->m_uiBlockSize );
        STAT( ++s_nuiStat_Free_Pool );

        /* Check to see if this has re-opened a formerly closed page. */

        if (psMemPage->m_iEndBlock == psMemPage->m_iLastBlock && 0 > psMemPage->m_iFreeBlock)
        {
            /* Find this page in the list of closed pages and remove it */
            if (s_aPageClosed[idxPage] != psMemPage)
            {
                P_TgMEMPAGE                        psPage;

                for (psPage = s_aPageClosed[idxPage]; psPage->m_psBlock_Next != psMemPage && 0 != psPage; psPage = psPage->m_psBlock_Next);
                TgCRITICAL( 0 != psPage );

                psPage->m_psBlock_Next = psMemPage->m_psBlock_Next;
            }
            else
            {
                s_aPageClosed[idxPage] = psMemPage->m_psBlock_Next;
            };

            psMemPage->m_psBlock_Next = s_aPageOpen[idxPage];
            s_aPageOpen[idxPage] = psMemPage;
        };

        /* Have to free the tagged block and perform the required merge to end operation */
        TgERROR(idxBlock >= 0 && idxBlock < psMemPage->m_iLastBlock);

    #if TgCOMPILE_MEM_FREE_CLEAR
        memset( pMem, (int)TgCOMPILE_MEM_FREE_PATTERN, psMemPage->m_uiBlockSize );
    #endif

        if (idxBlock + 1 != psMemPage->m_iLastBlock) /*« Block is not the last allocated block - add to free list (no free block merge). */
        {
            ((P_TgSINT32)pMem)[0] = psMemPage->m_iFreeBlock;
            psMemPage->m_iFreeBlock = idxBlock;
        }
        else
        {
            --psMemPage->m_iLastBlock;
        };

        /* Check to see if this is an unneeded open page */

        if ((0 == psMemPage->m_iLastBlock) && (0 != s_aPageOpen[idxPage]->m_psBlock_Next))
        {
            /* Find this page in the list of open pages and remove it */
            if (s_aPageOpen[idxPage] != psMemPage)
            {
                P_TgMEMPAGE                        psPage;

                for (psPage = s_aPageOpen[idxPage]; psPage->m_psBlock_Next != psMemPage && 0 != psPage; psPage = psPage->m_psBlock_Next);
                TgCRITICAL( 0 != psPage );

                psPage->m_psBlock_Next = psMemPage->m_psBlock_Next;
            }
            else
            {
                s_aPageOpen[idxPage] = psMemPage->m_psBlock_Next;
            };

        #if TgCOMPILE_MEM_TRACK
            if (nullptr != psMemPage->m_sTrack.Pool.apszFile)
            {
                tgMM_PM_Virtual_Free( (P_TgVOID)psMemPage->m_sTrack.Pool.apszFile );
                psMemPage->m_sTrack.Pool.apszFile = nullptr;
            };
            if (nullptr != psMemPage->m_sTrack.Pool.auiLine)
            {
                tgMM_PM_Virtual_Free( psMemPage->m_sTrack.Pool.auiLine );
                psMemPage->m_sTrack.Pool.auiLine = nullptr;
            };
        /*# TgCOMPILE_MEM_TRACK */
        #endif

            tgMM_PM_Virtual_Free( psMemPage );
        };

        tgCM_MP_CS_Exit( &s_csPageLock[idxPage] );
    };
}


/* ---- tgMM_Realloc_Pool ----------------------------------------------------------------------------------------------------------------------------------------------- */
/* ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- */
P_TgVOID tgMM_Realloc_Pool( PC_TgVOID pMem, C_TgSIZE uiNewSize MEM_TRACK_PARAMETERS )
{
    P_TgMEMPAGE                         psMemPage;
    P_TgVOID                            pReturn;

    if (nullptr == pMem)
    {
        return (0);
    };

    psMemPage = (PC_TgMEMPAGE)(tgCM_CEL_ALGN_PW2_UPTR( (TgUINTPTR)pMem, (TgUINTPTR)s_uiPageSize ) - s_uiPageSize);

    if (psMemPage->m_uiBlockSize >= uiNewSize)
    {
        return (pMem);
    };

    /* Allocate the New Size, and copy the data from the old Allocation to the new Allocation */

#if TgCOMPILE_MEM_TRACK
    pReturn = tgMM_Malloc_Pool( uiNewSize, pszF, uiL );
#else
    pReturn = tgMM_Malloc_Pool( uiNewSize );
#endif

    if (nullptr == pReturn)
    {
        return (0);
    };

    memcpy( pReturn, pMem, tgCM_MIN_UXX( uiNewSize, psMemPage->m_uiBlockSize ) );

#if TgCOMPILE_MEM_TRACK
    tgMM_Free_Pool( pMem, pszF, uiL );
#else
    tgMM_Free_Pool( pMem );
#endif

    return (pReturn);
}


/* ---- tgMM_Reserve_Virtual -------------------------------------------------------------------------------------------------------------------------------------------- */
/* ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- */
P_TgVOID tgMM_Reserve_Virtual( C_TgSIZE uiSize MEM_TRACK_PARAMETERS )
{
    P_TgUINT32                          pMem;

    TgERROR(ETgMODULE_STATE__BOOTED == s_enMem_MGR_State);

#if TgCOMPILE_MEM_TRACK
    pMem = tgMM_Malloc_OS( uiSize, TgFALSE, TgFALSE, pszF, uiL );
    TgLOG_MEM( TgT("Mem_MGR"), TgT("Address: % 16p Size: % 16X File: % 128.128s Line % 5d\n"), pMem, uiSize, pszF, uiL );
#else
    pMem = tgMM_Malloc_OS( uiSize, TgFALSE, TgFALSE );
#endif

    return (pMem);
}


/* ---- tgMM_Commit_Virtual --------------------------------------------------------------------------------------------------------------------------------------------- */
/* ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- */
P_TgVOID tgMM_Commit_Virtual( PC_TgVOID pMem, C_TgSIZE uiSize MEM_TRACK_PARAMETERS )
{
    P_TgUINT32                          pCheck;

    TgERROR(ETgMODULE_STATE__BOOTED == s_enMem_MGR_State);
    TgERROR(pMem);

    pCheck = tgMM_PM_Virtual_Commit( pMem, uiSize );

#if TgCOMPILE_MEM_TRACK
    TgLOG_MEM( TgT("Mem_MGR"), TgT("Address: % 16p Size: % 16X File: % 128.128s Line % 5d\n"), pMem, uiSize, pszF, uiL );
#endif

    return (pCheck ? pMem : nullptr);
}


/* ---- tgMM_Free_Virtual ----------------------------------------------------------------------------------------------------------------------------------------------- */
/* ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- */
TgVOID tgMM_Free_Virtual( PC_TgVOID pMem MEM_TRACK_PARAMETERS )
{
    TgERROR(ETgMODULE_STATE__BOOTED == s_enMem_MGR_State);

#if TgCOMPILE_MEM_TRACK
    TgLOG_MEM( TgT("Mem_MGR"), TgT("Address: % 16p                        File: % 128.128s Line % 5d\n"), pMem, pszF, uiL );
#endif

#if TgCOMPILE_MEM_FREE_CLEAR
    tgMM_Free_OS( pMem, TgFALSE );
#else
    tgMM_Free_OS( pMem );
#endif
}


/* ---- tgMM_TL_Init_Scratch -------------------------------------------------------------------------------------------------------------------------------------------- */
/* ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- */
#if TgCOMPILE_THREAD_LOCAL
TgRESULT tgMM_TL_Init_Scratch( C_TgSIZE uiSize )
{
    TgCRITICAL( nullptr == tls_pScratchMemory );
    tls_pScratchMemory = tgMM_PM_Init_Scratch( uiSize );
    if (nullptr == tls_pScratchMemory)
    {
        return (KTgE_FAIL);
    }
    tls_uiScratchSize = uiSize;
    return (KTgS_OK);
}
/*# TgCOMPILE_THREAD_LOCAL */
#endif


/* ---- tgMM_TL_Free_Scratch -------------------------------------------------------------------------------------------------------------------------------------------- */
/* ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- */
#if TgCOMPILE_THREAD_LOCAL
TgVOID tgMM_TL_Free_Scratch( TgVOID )
{
    TgCRITICAL( nullptr != tls_pScratchMemory );
    tgMM_PM_Free_Scratch( tls_pScratchMemory );
    tls_uiScratchSize = 0ULL;
}
/*# TgCOMPILE_THREAD_LOCAL */
#endif


/* ---- tgMM_TL_Push_Scratch -------------------------------------------------------------------------------------------------------------------------------------------- */
/* ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- */
#if TgCOMPILE_THREAD_LOCAL
P_TgVOID tgMM_TL_Push_Scratch( TgUINT32 uiSize MEM_TRACK_PARAMETERS )
{
    C_TgUINT32                          uiStack = s_uiScratch_Top;
    TgUINT32                            uiNewStack;
    union
    {
        P_TgUINT08                          p08;
        P_TgUINT32                          p32;
    }                                   sMem;

    uiSize = tgCM_CEL_ALGN_PW2_U32( uiSize, 16u ) + 16u;

    if (s_uiScratch_Top + uiSize > tls_uiScratchSize)
    {
        return (nullptr);
    };

    uiNewStack = s_uiScratch_Top + uiSize;
    s_uiScratch_Top += uiSize;
    sMem.p08 = tls_pScratchMemory + uiNewStack - 16;

    sMem.p32[0] = 0xbaddecaf; /*« Yes, Decaf is bad!! */
    sMem.p32[1] = 0xbaddecaf;
    sMem.p32[2] = 0xbaddecaf;
    sMem.p32[3] = 0xbaddecaf;

    STAT( ++s_nuiStat_Scratch );
    STAT( s_uiStat_Size_Scratch += uiSize );
    STAT( s_nuiStat_Max_Scratch = tgCM_MAX_U32( s_nuiStat_Max_Scratch, (s_nuiStat_Scratch - s_nuiStat_Free_Scratch) ) );

#if TgCOMPILE_MEM_TRACK
    TgLOG_MEM( TgT("Mem_MGR"), TgT("Address: % 16p Size: % 16X File: % 128.128s Line % 5d\n"), sMem.p32, uiSize, pszF, uiL );
/*# TgCOMPILE_MEM_TRACK */
#endif

    return (tls_pScratchMemory + uiStack);
}
/*# TgCOMPILE_THREAD_LOCAL */
#endif


/* ---- tgMM_TL_Pop_Scratch --------------------------------------------------------------------------------------------------------------------------------------------- */
/* ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- */
#if TgCOMPILE_THREAD_LOCAL
TgVOID tgMM_TL_Pop_Scratch( TgUINT32 uiSize MEM_TRACK_PARAMETERS )
{
    union
    {
        P_TgUINT08                          p08;
        P_TgUINT32                          p32;
    }                                   sMem;

    TgCRITICAL( s_uiScratch_Top > uiSize );

#if TgCOMPILE_MEM_TRACK
    TgLOG_MEM( TgT("Mem_MGR"), TgT("Size: % 16X File: % 128.128s Line % 5d\n"), uiSize, pszF, uiL );
#endif

    sMem.p08 = tls_pScratchMemory + s_uiScratch_Top - 16;
    TgCRITICAL( 0xbaddecaf == sMem.p32[0] && 0xbaddecaf == sMem.p32[1] && 0xbaddecaf == sMem.p32[2] && 0xbaddecaf == sMem.p32[3] );

    uiSize = tgCM_CEL_ALGN_PW2_U32( uiSize, 16u ) + 16u;

    s_uiScratch_Top -= uiSize;

    STAT( s_uiStat_Size_Scratch -= uiSize );
    STAT( --s_nuiStat_Scratch );
}
/*# TgCOMPILE_THREAD_LOCAL */
#endif




/* -.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-. */
/*  File Local Functions                                                                                                                                                  */
/* -.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-. */

/* ---- tgMM_Size_Temp -------------------------------------------------------------------------------------------------------------------------------------------------- */
/* ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- */
TgSIZE tgMM_Size_Temp( CPC_TgVOID pMem )
{
    return (tgMM_Size_Pool( pMem ));
}


/* ---- tgMM_Malloc_Temp ------------------------------------------------------------------------------------------------------------------------------------------------ */
/* ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- */
P_TgVOID tgMM_Malloc_Temp( C_TgSIZE uiSize MEM_TRACK_PARAMETERS )
{
    PC_TgVOID pMem = TgMALLOC_POOL( uiSize );

#if TgCOMPILE_MEM_TRACK
    TgLOG_MEM( TgT("Mem_MGR"), TgT("Address: % 16p Size: % 16X File: % 128.128s Line % 5d\n"), pMem, uiSize, pszF, uiL );
#endif

    return (pMem);
}


/* ---- tgMM_Free_Temp -------------------------------------------------------------------------------------------------------------------------------------------------- */
/* ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- */
TgVOID tgMM_Free_Temp( PC_TgVOID pMem MEM_TRACK_PARAMETERS )
{
#if TgCOMPILE_MEM_TRACK
    TgLOG_MEM( TgT("Mem_MGR"), TgT("Address: % 16p Size: % 16X File: % 128.128s Line % 5d\n"), pMem, tgMM_Size_Temp( pMem ), pszF, uiL );
#endif

    TgFREE_POOL( pMem );
}


/* ---- tgMM_Size_Virtual ----------------------------------------------------------------------------------------------------------------------------------------------- */
/* ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- */
TgSIZE tgMM_Size_Virtual( CPC_TgVOID UNUSED_PARAM pMem )
{
    return (0ULL);
}


/* ---- tgMM_Malloc_Virtual --------------------------------------------------------------------------------------------------------------------------------------------- */
/* ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- */
P_TgVOID tgMM_Malloc_Virtual( C_TgSIZE uiSize MEM_TRACK_PARAMETERS )
{
    P_TgVOID pMem = tgMM_Reserve_Virtual( uiSize MEM_TRACK_VARS );
    if (nullptr == pMem)
        return (pMem);
    pMem = tgMM_Commit_Virtual( pMem, uiSize MEM_TRACK_VARS );
    if (nullptr == pMem)
        return (pMem);

#if TgCOMPILE_MEM_TRACK
    TgLOG_MEM( TgT("Mem_MGR"), TgT("Address: % 16p Size: % 16X File: % 128.128s Line % 5d\n"), pMem, uiSize, pszF, uiL );
#endif

    return (pMem);
}


/* ---- tgMM_Malloc_OS -------------------------------------------------------------------------------------------------------------------------------------------------- */
/* ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- */
static P_TgVOID tgMM_Malloc_OS( C_TgSIZE uiSize, C_TgBOOL bCommit, C_TgBOOL bTemp MEM_TRACK_PARAMETERS )
{
    P_TgMEMPAGE                         psMemPage;

    TgERROR(ETgMODULE_STATE__BOOTED == s_enMem_MGR_State);
    TgCRITICAL( uiSize <= KTgMAX_U32 );

    /* Create an OS Allocation for this request */
    psMemPage = (P_TgMEMPAGE)tgMM_PM_Virtual_Reserve( uiSize + SIZE_OF_MEMPAGE, bTemp );

    if (nullptr == psMemPage)
    {
        TgCRITICAL_MSGF( 0, TgT("%-16.16s(%-32.32s): Unable to allocate (%d).\n"), TgT("Mem_MGR"), TgT("tgMM_Malloc_OS"), uiSize );
        return (0);
    };

    tgMM_PM_Virtual_Commit( psMemPage, SIZE_OF_MEMPAGE + (bCommit ? uiSize : 0) );

#if TgCOMPILE_MEM_MALLOC_CLEAR
    memset( psMemPage, (int)TgCOMPILE_MEM_MALLOC_PATTERN, SIZE_OF_MEMPAGE + (bCommit ? uiSize : 0) );
#endif

    psMemPage->m_psBlock_Next = nullptr;
    psMemPage->m_psPool_Next = s_pOSPool_Head;
    psMemPage->m_uiBlockSize = (TgUINT32)uiSize;
    psMemPage->m_iEndBlock = 1;
    psMemPage->m_iLastBlock = 1;
    psMemPage->m_iFreeBlock = -1;

#if TgCOMPILE_MEM_TRACK
    psMemPage->m_sTrack.Page.pszFile = pszF;
    psMemPage->m_sTrack.Page.uiLine = uiL;
/*# TgCOMPILE_MEM_TRACK */
#endif

    s_pOSPool_Head = psMemPage;

    STAT( ++s_nuiStat_OS );
    STAT( s_nuiStat_Max_OS = tgCM_MAX_U32( s_nuiStat_Max_OS, (s_nuiStat_OS - s_nuiStat_Free_OS) ) );
    STAT( s_uiStat_Size_OS += psMemPage->m_uiBlockSize );

    return (psMemPage->m_pData);
}


/* ---- tgMM_Free_OS ---------------------------------------------------------------------------------------------------------------------------------------------------- */
/* ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- */
#if TgCOMPILE_MEM_FREE_CLEAR
static TgVOID tgMM_Free_OS( PC_TgVOID pMem, C_TgBOOL bClearMem )
#else
static TgVOID tgMM_Free_OS( PC_TgVOID pMem )
#endif
{
    P_TgMEMPAGE                         psMemPage;
    TgUINTPTR                           uiMemPageOS;

    TgERROR(ETgMODULE_STATE__BOOTED == s_enMem_MGR_State);

    uiMemPageOS = (TgUINTPTR)pMem - SIZE_OF_MEMPAGE;
    TgCRITICAL( 0 == (uiMemPageOS % s_uiPageSize) );

    psMemPage = (P_TgMEMPAGE)uiMemPageOS;
    if (psMemPage == s_pOSPool_Head)
    {
        s_pOSPool_Head = psMemPage->m_psPool_Next;
    }
    else
    {
        P_TgMEMPAGE                         psSearch;

        for (psSearch = s_pOSPool_Head; nullptr != psSearch->m_psPool_Next; psSearch = psSearch->m_psPool_Next)
        {
            if (psMemPage == psSearch->m_psPool_Next)
            {
                psSearch->m_psPool_Next = psMemPage->m_psPool_Next;
                break;
            };
        };
    };

    STAT( ++s_nuiStat_Free_OS );
    STAT( s_uiStat_Size_OS -= psMemPage->m_uiBlockSize );

#if TgCOMPILE_MEM_FREE_CLEAR
    memset( psMemPage, (int)TgCOMPILE_MEM_FREE_PATTERN, SIZE_OF_MEMPAGE + (bClearMem ? psMemPage->m_uiBlockSize : 0) );
#endif

    tgMM_PM_Virtual_Free( psMemPage );
}