ML_ITK/13/include_cpp/base_utils/PackedMemoryBuilder.hxx

// Copyright 2020 Siemens Digital Industries Software
// ==================================================
// Copyright 2016.
// Siemens Product Lifecycle Management Software Inc.
// All Rights Reserved.
// ==================================================
// Copyright 2020 Siemens Digital Industries Software

/**
    @file

    Provides a template class to help generating packed memory for ITK function and class methods returning OF data.
    It can easily get a little tricky when you return a data structure from an ITK that is more 
    then a fixed array of element like an array of tag_t.

    @note All dynamic memory returned from the ITK functions must be freeable with a single call to MEM_free.
*/

#ifndef PACKED_MEMORY_BUILDER_HXX
#define PACKED_MEMORY_BUILDER_HXX

#include <map>
#include <set>
#include <string.h>
#include <vector>
#include <base_utils/base_utils_errors.h>
#include <base_utils/IFail.hxx>
#include <base_utils/Mem.h>
#include <base_utils/SharedPtr.hxx>
        
namespace Teamcenter
{
    namespace PackedMemory
    {

//! Calculates the alignment size to be used when allocating or creating a pointer reference.
//! In order to not create a page fault exception, a pointer needs to be alligned according to 
//! the current processor.
//! \return The size of data that should be use to make the specified data size fall onto a page boundry.
//! 
inline size_t getAlignedSize(
    size_t datasize     //!< The size of the buffer seeking alignment for.
    )
{
    // This should give us the machine alignment on a pointer.
    const struct { char c; char *ptr; } *pAlignment= 0;
    const size_t alignment= size_t(&pAlignment->ptr);

    return (datasize + alignment - 1) / alignment * alignment;
}

//! This is the default memory allocator used by the Builder.
struct DefaultAllocator
{
    //! Allocates the specified memory suitable for POD objects.
    //!
    //! \return The specified size of Memory allocated using MEM_alloc.
    void * alloc(
        size_t size //!< Size of the memory block requested.
        ) const
    {
        // Forward to MEM_alloc to do the allocation.
        return MEM_alloc((int)size);
    }

    //! Allocates the specified memory suitable for POD objects using Packed Marking.
    //!
    //! \return The specified size of Memory allocated using MEM_alloc_packed.
    void *alloc_packed(
        size_t dataToPackCount,
        MEM_data_to_pack_p_t pDataToPack,
        logical allowOverride
        ) const
    {
        // Forward to MEM_alloc_packed to do the allocation.
        return MEM_alloc_packed(dataToPackCount, pDataToPack, allowOverride);
    }
    
    //! Frees the specified memory address previously allocated by alloc.
    //!
    //! \param[in] pAddress The memory address previously allocated by alloc to be freed.
    void free(
        void *pAddress //!< The address previously allocated by alloc.
        ) const
    {
        // Forward to MEM_alloc to do the allocation.
        MEM_free(pAddress);
    }
};

//! \brief Allows controlling the packing mode when creating the buffer. 
//! 
//! The CreateMode allows different types of packed buffers to be creates. The AllowOverride is 
//! used to not do the marked packing if the SM options has SM_option_override_packing enabled. 
//! If this is set and AllowOverride is specified, no packing will be done and the memory will 
//! be allocated in a unpacked layout (potential memory leaks). 
//! If the NoOverride is specified, the buffer will be packed and special marking will be done
//! on all embedded allocations. the NoMarking option will pack the buffer and no marking is 
//! done on the embedded allocation. This can be used when there is no external exposure to 
//! the packed buffer and there is knowlege that the caller is freeing the buffer correctly.
enum CreateMode
{
    AllowOverride,      //!< The AllowOverride can be use to allow unpacked data to be generated.
    NoOverride,         //!< This generates packed data with special marking.
    NoMarking           //!< Creates a packed with no marking (smallest overhead).
};


//! \brief Provides a template class to help generating packed memory for ITK function and class methods returning OF type data.
//!
//! This class should be able to handle all types of data for generating packed memory for OF parameters. It can easily get a little 
//! tricky when the output data is other then a fixed array of element like an array of tag_t. When other data is referenced from
//! the OF parameter data like and array of strings, the complexity of building this buffer becomes non trivial. There are cases when
//! the returned OF data is a linked list of structures each pointing to the next and terminated with a NULL. This becomes very complex
//! to build and this is where the PackedMemoryBuilder can assist making this quite trivial.
//!
//! \note Contract of the OF parameter: All dynamic memory returned from the ITK functions must be freeable with a single call to MEM_free.
//!
//! Here is an example how memory should <b>NOT</b> be allocated for a OF parameter type:
//!
//! \code
//! typedef struct TestData
//! {
//!     char *StringData1;
//!     char *StringData2;
//!     int   intValue;
//!     tag_t tagValue;
//! } test_data_t, *test_data_p_t; 
//! 
//! |--- Start block ---
//! | 0x23450000 StringData1 -> 0x34560000, StringData2 -> 0x45670000,
//! | 0x23450010 intValue = 12345, tagValue = 23456
//! |---  End block  ---
//!
//! |--- Start block ---
//! | 0x34560000 Some string!
//! |---  End block  ---
//!
//! |--- Start block ---
//! | 0x45670000 Another string!
//! |---  End block  ---
//! \endcode
//!
//! Notice how the string data points to allocations that are made outside of the base memory block 0x23450000. The data referenced
//! with pointers 0x34560000 and 0x45670000 will be leaked if the caller simply calls MEM_free on the structure and not its element. 
//! Note: The caller should be able to make one call to MEM_free on the OF parameter to free the memory.  Otherwise the caller has 
//! to know how the data is constructed and specifically free the elements in the returned data. This doesn't work when auto-generated 
//! code is used like Web Binding and no specialized code is used to free the data. 
//! <p>
//! The appropriate way of returning data is using one memory allocation. All the data referenced is pointing withing this 
//! allocated block. One call to free the memory can be made without leaking referenced data. Here is an example of how the retrieved 
//! data (OF parameter type) would look like in memory:
//!
//! \code
//! typedef struct TestData
//! {
//!     char *StringData1;
//!     char *StringData2;
//!     int   intValue;
//!     tag_t tagValue;
//! } test_data_t, *test_data_p_t; 
//! 
//! |--- Start block ---
//! | 0x12340000 StringData1 -> 0x12340020, StringData2 -> 0x12340030,
//! | 0x23450010 intValue = 12345, tagValue = 23456
//! | 0x12340020 Some string!
//! | 0x12340030 Another string!
//! |---  End block  ---
//! \endcode
//!
//! Notice how the string data points to data inside of the same memory block. The data referenced with pointers 0x12340020 and 
//! 0x12340030 will not be leaked since it is inside of the same allocated block of memory and one call to free can be used.
//! <p>
//! Lets use a more practical example to decribe the issue with a non trivial data type as a OF type parameter:
/**
    @code

    typedef struct KeyValue
    {
        char *key;
        char *value;
    } key_value_t, *key_value_p_t;

    typedef struct SampleData
    {
        int num_key_values;
        key_value_p_t key_values;
    } sample_data_t, *sample_data_p_t;

    int ITK_NotDoingTheRightThing(
        int num_values,                     // <I>  Number of key values to return.
        sample_data_p_t *ppOutputData       // <OF> The allocated and initialized data (must be freed my MEM_free).
        )
    {
        // Allocate the output data. (this ok for a <OF> parameter if no other data is referenced, like and array of tag_t).
        *ppOutputData= (sample_data_p_t)MEM_alloc(sizeof(sample_data_t));

        // Initialize the data to the caller.
        (*ppOutputData)->num_key_values= num_values;
        // Here is the first problem, we need to allocate the array of key value pairs. The OF contract doesn't 
        // allow nedted allocation and this memory will leak if not handled by the caller.
        (*ppOutputData)->key_values= (key_value_p_t)MEM_alloc(sizeof(key_value_t) * num_values);

        for (int i= 0; i < num_values; ++i)
        {
            // The problem gets compounded with extra allocations for each string.
            (*ppOutputData)->key_values[i].key= MEM_string_copy("key");
            (*ppOutputData)->key_values[i].value= MEM_string_copy("value");
        }

        // We are returning a buffer that has (1 + num_values * 2) extra allocation which normally would leak.
        return ITK_ok;
    }

    int ITK_PackedOutput(
        int num_values,                     // <I>  Number of key values to return.
        sample_data_p_t *ppOutputData       // <OF> The allocated and initialized data (must be freed my MEM_free).
        )
    {
        try
        {
            // Create the packed memory builder. Default constructor will create one object of the template type.
            Teamcenter::PackedMemory::Builder<sample_data_t> packedBuilder;

            // The builder now has one element of sample_data_t which is zero initialized. The data 
            // now have to be initialized like normal.
            packedBuilder.get()->num_key_values= num_values;

            // To add extra data, use the addData method and it will allocate the number of element specified.
            key_value_p_t keyValueData= packedBuilder.addData(packedBuilder.get()->key_values, num_values);

            for (int i= 0; i < num_values; ++i)
            {
                // Here we are using the pointer returned from addData to reference the data.
                packedBuilder.addString(keyValueData[i].key, "key");
                // No requirements to use the pointer returned from addData.
                packedBuilder.addString(packedBuilder.get()->key_values[i].value, "value");
            }

            // When you are done initializing the PackedMemoryBuilder, create the buffer for the caller (the <OF> data).
            packedBuilder.createBuffer(ppOutputData);
        }
        catch (const IFail &ifail)
        {
            // The builder can throw an IFail exception on invalid inputs.
            return ifail.ifail();
        }

        return ITK_ok;
    }

    void testPackedData()
    {
        // This is how data should normally be freed using a scope_smptr. If you look at 
        // the implementaion above, there are 5 allocation that is leaked since they are
        // done with independent calls to MEM_alloc.
        scoped_smptr<sample_data_t> pNonPackedDataFromOFParameter;
        ITK_NotDoingTheRightThing(2, &pNonPackedDataFromOFParameter);

        // Same test again using a scoped_smptr to free the memory. In this implementation 
        // above, the PackedMemoryBuilder is used and correctly packs the data into one
        // allocation so that the MEM_free on the pointer will deallocate all the memory.
        scoped_smptr<sample_data_t> pPackedDataFromOFParameter;
        ITK_PackedOutput(2, &pPackedDataFromOFParameter);
    }

    @endcode
*/
template <class T, typename Alloc = DefaultAllocator> 
class Builder
{
public:
    //! \brief Constructs a Builder of the template type having the specified number of elements.
    //!
    //! The Builder will allocate and zero initializes the data for the template type T to
    //! hold the specified number of elements. All the data for the element T and it's referenced 
    //! object has to be initialized complete the data initialization.
    //!
    //! \param[in] elements Specify the number of elements to allocate of type T. The default will
    //! allocate one element of type T.
    explicit Builder(size_t elements= 1);
    
    //! \brief Constructs a Builder using the specified source string to initialize.
    //!
    //! This constructor should only be used if single string is being built. It's been created 
    //! for completeness so the builder can be used for all the (OF) type data. It is also just
    //! as easy to use the MEM_string_copy but the idea is to use the builder for all
    //! (OF) types for consistency.
    //!
    //! \param[in] pSrcString The string to assign to the builder. Can be NULL to initialize with a NULL value.
    //! \param[in] maxLength Specify the maximum string lenght to hold in the builder including the null terminator. 
    //! If not specified, no maximum will be applied.
    explicit Builder(const char *pSrcString, size_t maxLength= ~0);

    //! Gives out the pointer to the allocated data of type T. If the element count is zero for the Builder,
    //! the returned value will be a NULL pointer.
    //! \return Pointer to the allocated data of type T.
    T *get() { return m_pObjectData; }

    //! Gives out the const pointer to the allocated data of type T. If the element count is zero for the Builder,
    //! the returned value will be a NULL pointer.
    //! \return Const pointer to the allocated data of type T.
    const T *get() const { return m_pObjectData; }

    //! \brief Sets the status if string lookup should done when adding strings to the builder. 
    //!
    //! 
    //! If the lookupStringData is enabled (the builder has this default off), any string added to 
    //! the builder using the addString method will be checked for uniqueness. When a string is 
    //! added using the addString method, a string table will be use to search for strings 
    //! already added to the builder. If the same string is found, the addReference method will 
    //! be used to reuse the already allocated data. This can save memory if many of the same 
    //! strings are expected to be added to the builder.
    //!
    //! \param[in] lookupStringData If this parameter is true, a string table will be use to search for reuse.
    void setLookupStringData(bool lookupStringData) { m_lookupStringData= lookupStringData; }

    //! Gets the status if the string lookup is enabled when using the addString method. 
    //! \return The boolean value indicating if string lookup is enabled.
    bool getLookupStringData() const { return m_lookupStringData; }

    //! \brief Adds a referenced string to the specified pDestPtr pointer.
    //!
    //! This method can be used to add string data to the Builder in a more convenient way instead of
    //! using the addData method. If the pSrcString is NULL, the pDestPtr will be initialzed with a 
    //! NULL value. The Builder will allocate storage to hold the specified string and copy pSrcString.
    //!
    //! \param[in] pDestPtr Destination pointer where the string shall be referenced.
    //! \param[in] pSrcString The string to assign to the destination. Can be NULL to initialize with a NULL value.
    //! \return A pointer to the allocated string which has been initialized with the pSrcSting.
    //! \throws IFail BASE_UTILS_invalid_parameter if an invalid pDestPtr has been specified or it has already been initialized.
    template <typename K> 
    inline K addString(K &pDestPtr, const char *pSrcString);

    //! \brief Adds referenced data of type K to the specified pDestPtr pointer.
    //!
    //! This method should be used to add referenced data to the Builder. When building data that has pointers
    //! to other data in the output, this method has to be use to allocate storage for it. If source data is
    //! specified using pSourceData the data will be copied to the allocated storage using memcpy.
    //!
    //! \param[in] pDestPtr Destination pointer where the data shall be referenced.
    //! \param[in] elements number of elements to allocate of the K type parameter. Default is one element.
    //! \param[in] pSourceData If specified, the source data will be copied to the allocated storage.
    //! \return A pointer to the allocated storage.
    //! \throws IFail BASE_UTILS_invalid_parameter if an invalid pDestPtr has been specified or it has already been initialized.
    template <typename K> 
    K addData(K &pDestPtr, size_t elements= 1, const void *pSourceData= NULL);

    //! \brief Adds the already created data of type K to the specified @p pDestPtr pointer.
    //!
    //! This method should be used to add data to the Builder that already has been created using addData or addString. 
    //! When the same datatype is referenced and the caller knows that the same values are already added to the builder,
    //! using addReference will reuse the allocation and the same data will be referenced.
    //!
    //! \param[in] pDestPtr Destination pointer where the same reference data shall be assigned.
    //! \param[in] pReferenceData the data already added to the builder and returned from addData or addString.
    //! \return A pointer to the referenced data. Also the same as pDestPtr value.
    //! \throws IFail BASE_UTILS_invalid_parameter if an invalid pDestPtr has been specified or it has already been initialized.
    //! \throws IFail BASE_UTILS_invalid_reference if the pReferenceData is not allocated using addData or addString.
    template <typename K, typename J>
    K addReference(K &pDestPtr, J &pReferenceData);

    //! \brief Create the packed memory buffer using the data added to the Builder.
    //!
    //! The pDestination has to be a valid output pointer and storage will be allocated using the 
    //! AllocatorFunction template argument. 
    //!
    //! \param[out] pDestination Destination pointer where the packed data should be stored.
    //! \param[in] createMode Default is NoMarking. See the CreateMode for more information.
    //! \throws IFail BASE_UTILS_invalid_parameter if an invalid pDestination has been specified.
    //! \throws IFail BASE_UTILS_no_memory if the memory could not be allocated.
    //! \throws IFail BASE_UTILS_internal_error if an unexpected state of the internal data is found.
    void createBuffer(T **pDestination, CreateMode createMode = NoMarking) const;

    //! Gives out the allocator Alloc reference for this class instance.
    //!
    //! \return The allocator for this class instance.
    Alloc& allocator() { return m_allocator; }

    //! Gives out the allocator const Alloc reference for this class instance.
    //!
    //! \return The allocator for this class instance.
    const Alloc& allocator() const { return m_allocator; }
    
private:
    //! Private copy constructor to prevent accidental copy.
    Builder(const Builder &);
    //! Private assignment operator to prevent accidental assignement.
    Builder & operator = (const Builder &);

    //! \brief Struct to efficiently add a back reference to a mapped pointer.
    //!
    //! We have a need to search within allocated buffers where a pointer might fall. 
    //! This struct are used as value in a map and the key has the pointer address. 
    //! The datasize is telling how big the allocation is made for the key pointer 
    //! and therefor the end of the buffer can be established.  When looking for a 
    //! buffer to a pointer, the upper_bound is used to find the next bigger pointer 
    //! and iterator is then moved back to the previous location. The iterator->first 
    //! is used to see if the pointer is equal or greater and that the 
    //! iterator->first + iterator->second.datasize is less than the pointer. If this
    //! is true, we have found a buffer that encapsulate the pointer and the pData 
    //! member can be used.
    struct BackRefData
    {
        BackRefData() : pData(NULL), datasize(0) {}
        BackRefData(void *_pData, size_t _datasize) : pData(_pData), datasize(_datasize) {}

        void  *pData;               //!< The object data pointer owning the mapped buffer (the key ObjectDataMap).
        size_t datasize;            //!< The datasize of the buffer (the vector size in ObjectDataMap).
    };

    //! \brief Struct to efficiently store unique string we have added to the builder.
    //!
    //! This data structure is used in a std::set to do a lookup of strings we have added
    //! to the builder.If we encounter a new unique string in the addString method, we use
    //! the addData method to allocate storage for this new string.This stable allocation
    //! is then added to a set using this data structure to point to the string data. The
    //! key to this structure is to avoid allocation for the string and to be efficient
    //! in comparison between big collections of strings.
    struct StringData
    {
        StringData() : m_pString(NULL), m_datasize(0) {}
        StringData(const char *pString, size_t datasize) : m_pString(pString), m_datasize(datasize) {}

        bool operator < (const StringData &rhs) const 
        { 
            return m_datasize < rhs.m_datasize || 
                (m_datasize == rhs.m_datasize && m_pString && rhs.m_pString && memcmp(m_pString, rhs.m_pString, m_datasize) < 0);
        }

        const char *m_pString;            //!< The string pointer. This pointer can be NULL.
        size_t      m_datasize;           //!< The datasize of the string. This is strlen(m_pString) + 1.
    };



    typedef std::vector<char> Buffer;                                       //!< Type of buffer used to allocate internal storage.
    typedef std::vector<void *> ReferencedObjectsList;                      //!< Type of vector used for referenced objects.
    typedef std::map<void *, Buffer> ObjectDataMap;                         //!< Type of map used to allocate data for objects.
    typedef std::map<void *, ReferencedObjectsList> ReferencedObjectsMap;   //!< Type of map used to link referenced objects.
    typedef std::map<void *, void *> ObjectPointerMap;                      //!< Type of map used to link object pointer to a destination buffer.
    typedef std::map<const void *, BackRefData> ObjectBackRefDataMap;       //!< Type of map used to link object data to the owning pointer.
    typedef std::set<StringData> StringDataSet;                             //!< A set of StringData object to efficiently store string unique stings.



    ObjectDataMap m_objectDataMap;                  //!< Map of data allocated for the objects in the Builder.
    ObjectBackRefDataMap m_objectBackRefDataMap;    //!< Map of buffer object pointer back to the owning data first -> m_objectDataMap.second, second.pData -> m_objectDataMap.first.
    ReferencedObjectsMap m_referencedObjects;       //!< Map of referenced objects to the owner object.
    ObjectPointerMap m_backReferencedObjects;       //!< Map of data that has been back referenced for reuse.
    StringDataSet m_addedStringData;                //!< The set of StringData is collections of strings we added. Used to so we can add a reference to common strings. 

    T *m_pObjectData;                               //!< The object data for this Builder. This is used by the operators for performance reasons.
    Alloc m_allocator;                              //!< Allocator to use for creating the output buffer.
    bool m_lookupStringData;                        //!< Flag to indicate if the m_addedStringData table should be used when adding strings to the builder.

    //! \brief Checks if the specified pBaseAddress is a valid address and gives out the mapped object. 
    //!
    //! The pBaseAddress has to be something that's been allocated and exists in the m_objectDataMap.
    //! If the pBaseAddress is found to fall within a allocated object, the key mapping that object 
    //! will be assigned to pMappedObject. If the pBaseAddress is not found, false is returned.
    //! 
    //! \param[in] pBaseAddress The base address to search for in the created object buffers.
    //! \param[out] pMappedObject if found, the key mapping that object in the m_objectDataMap.
    //! \param[in] bBaseOnly if only the base address should be search for and not the entire allocated buffer.
    //! \return true if the pBaseAdress falls within a object allocated in m_objectDataMap, false otherwise.
    bool getMappedObject(const void *pBaseAddress, void *&pMappedObject, bool bBaseOnly= false) const;

    //! \brief Creates the output buffer based on the current data in the builder. 
    //!
    //! The Alloc template argument allocator will be user to create the destination buffer. If the pDestination
    //! pointer is invalid, a IFail exception will be thrown. The Destination map will have a mapping of the keys in the 
    //! ReferencedObjectsMap and the corresponding destination address of the object data.
    //!
    //! \param[out] destinationMap the map of referenced objects to the destination data.
    //! \param[out] pDestination The destination output pointer that receives the allocated memory pointer.
    //! \throws IFail BASE_UTILS_invalid_parameter if an invalid pDestination has been specified.
    //! \throws IFail BASE_UTILS_no_memory if the memory could not be allocated.
    //! \throws IFail BASE_UTILS_internal_error if an unexpected state of the internal data is found.
    void createOutputBuffer(ObjectPointerMap &destinationMap, T **pDestination) const;

    //! \brief Creates the output buffer based on the current data in the builder using Packed Marking. 
    //!
    //! The Alloc template argument alloc_packed will be user to create the destination buffer. If the pDestination
    //! pointer is invalid, a IFail exception will be thrown. The Destination map will have a mapping of the keys in the 
    //! ReferencedObjectsMap and the corresponding destination address of the object data.
    //!
    //! \param[out] destinationMap the map of referenced objects to the destination data.
    //! \param[out] pDestination The destination output pointer that receives the allocated memory pointer.
    //! \param[out] allowOverride Specifies if the an override is allowed to generate unpacked data.
    //! \throws IFail BASE_UTILS_invalid_parameter if an invalid pDestination has been specified.
    //! \throws IFail BASE_UTILS_no_memory if the memory could not be allocated.
    //! \throws IFail BASE_UTILS_internal_error if an unexpected state of the internal data is found.
    void createOutputBufferUsingSMPack(ObjectPointerMap &destinationMap, T **pDestination, logical allowOverride) const;
};

template <class T, typename Alloc>
Builder<T, Alloc>::Builder(size_t elements):
    m_pObjectData(NULL), m_lookupStringData(false)
{ 
    // Figure out the size we need.
    size_t datasize= getAlignedSize(sizeof(T) * elements);

    // Create the buffer for the object(s) T we are storing.
    // We use NULL to indicate the base object. This should now always
    // be ordered first in the map wich is important.
    Buffer &buffer= m_objectDataMap[NULL];
    buffer.resize(datasize);

    // Also populate the back reference.
    m_objectBackRefDataMap[&buffer[0]]= BackRefData(NULL, datasize);

    // Assign the data to the buffer for quick access.
    m_pObjectData= datasize ? reinterpret_cast<T *>(&buffer[0]) : NULL;
}

template <class T, typename Alloc>
Builder<T, Alloc>::Builder(const char *pSrcString, size_t maxLength):
    m_pObjectData(NULL), m_lookupStringData(false)
{ 
    // What the size we need for this string.
    const size_t stringSize= pSrcString ? strlen(pSrcString) + 1 : 0;
    const size_t datasize= stringSize < maxLength ? stringSize : maxLength;
    
    // Create the buffer for the object(s) T we are storing.
    // We use NULL to indicate the base object. This should now always
    // be ordered first in the map wich is important.
    Buffer &buffer= m_objectDataMap[NULL];
    buffer.resize(datasize);

    // Also populate the back reference.
    m_objectBackRefDataMap[&buffer[0]]= BackRefData(NULL, datasize);
    
    // Assign the data to the buffer for quick access.
    m_pObjectData= datasize ? reinterpret_cast<T *>(&buffer[0]) : NULL;

    // If we have the data to copy, do it now.
    if (pSrcString && datasize)
    {
        strncpy(&buffer[0], pSrcString, datasize);
        buffer[datasize - 1] = '\0';
    }
}

template <class T, typename Alloc> template <typename K> 
K Builder<T, Alloc>::addString(K &pDestPtr, const char *pSrcString)
{
    // What the size we need for this string.
    const size_t datasize= pSrcString ? strlen(pSrcString) + 1 : 0;

    // If we have a string,
    if (datasize && m_lookupStringData)
    {
        // see if we can find the string in our string reference set.
        typename StringDataSet::iterator searchIter = m_addedStringData.find(StringData(pSrcString, datasize));
        // If we already have seen this string,
        if (searchIter != m_addedStringData.end())
        {
            // start by casting the found string to the proper output type.
            K pReferenceData = reinterpret_cast<K>(const_cast<char *>(searchIter->m_pString));
            // Now we can add the reference to the found string to the destination and return value.
            return addReference(pDestPtr, pReferenceData);
        }
        else
        {
            // We have not seen this before so forward having the pSourceData as the string data.
            K pAddedData = addData(pDestPtr, datasize, pSrcString);
            // Make sure we add the string into the StringReferenceData set (we do handle nulls).
            m_addedStringData.insert(StringData(pAddedData, datasize));

            // Return the string we added.
            return pAddedData;
        }
    }

    // This is either a NULL pointer or we are not doing lookups in the string collection set.
    // Just forward to addData having the pSourceData as the string data.
    return addData(pDestPtr, datasize, pSrcString);
}

template <class T, typename Alloc> template <typename K>
K Builder<T, Alloc>::addData(K &pDestPtr, size_t elements, const void *pSourceData)
{
    // What the size we need for these referenced data.
    const size_t datasize= getAlignedSize(sizeof(*pDestPtr) * elements);

    // The address of the parameter K &pDestPtr.
    void *pParameterAddess= (void *)&pDestPtr;

    // Make sure that we have a pointer address that we have seen before. We can't add data
    // unless we reference some of the external data buffers.
    void *pBaseObject= NULL;
    if (!getMappedObject(pParameterAddess, pBaseObject))
    { 
        throw IFail( BASE_UTILS_invalid_reference ); 
    }

    // See if we have seen this before, this is most likely a coding error.
    ObjectDataMap::const_iterator searchIter= m_objectDataMap.find(pParameterAddess);
    if (searchIter != m_objectDataMap.end())
    {
        throw IFail( BASE_UTILS_already_initialized, "m_objectDataMap" ); 
    }

    // Add new allocation we did for this base object.
    m_referencedObjects[pBaseObject].push_back(pParameterAddess);

    // Create the buffer for the object(s) T we are storing.
    Buffer &buffer= m_objectDataMap[pParameterAddess];
    buffer.resize(datasize);
    
    // Also populate the back reference.
    m_objectBackRefDataMap[&buffer[0]]= BackRefData(pParameterAddess, datasize);

    // If we have source data,
    if (pSourceData && datasize)
    {
        // copy the data to the destination.
        memcpy(&buffer[0], pSourceData, datasize);
    }

    // Set the pointer to the buffer we have allocated.
    pDestPtr= datasize ? reinterpret_cast<K>(&buffer[0]) : NULL;

    // Give out the pointer to the caller.
    return pDestPtr;
}

template <class T, typename Alloc> template <typename K, typename J>
K Builder<T, Alloc>::addReference(K &pDestPtr, J &pReferenceData)
{
    // The address of the parameter K &pDestPtr.
    void *pParameterAddess= (void *)&pDestPtr;

    // Make sure that we have a pointer address that we have seen before. We can't add data
    // unless we reference some of the external data buffers.
    void *pBaseObject= NULL;
    if (!getMappedObject(pParameterAddess, pBaseObject))
    { 
        throw IFail( BASE_UTILS_invalid_reference ); 
    }

    // See if we have seen this before, this is most likely a coding error.
    ObjectDataMap::const_iterator searchIter= m_objectDataMap.find(pParameterAddess);
    if (searchIter != m_objectDataMap.end())
    {
        throw IFail( BASE_UTILS_already_initialized, "m_objectDataMap" ); 
    }

    // Make sure that we have a pointer address that we have seen before. We can't add data
    // unless we reference some of the external data buffers that we have created.
    void *pRefBaseObject= NULL;
    if (!getMappedObject(pReferenceData, pRefBaseObject, true))
    { 
        throw IFail( BASE_UTILS_invalid_reference ); 
    }

    // Add new allocation we did for this base object.
    m_referencedObjects[pBaseObject].push_back(pParameterAddess);

    // Add new back reference to the existing buffer.
    m_backReferencedObjects[pParameterAddess]= pRefBaseObject;
    
    // Create the buffer for the object(s) T we are storing with zero length.
    // This will indicate that we should look in the back reference for the data.
    Buffer &buffer= m_objectDataMap[pParameterAddess];
    buffer.resize(0);

    // Also populate the back reference.
    m_objectBackRefDataMap[&buffer[0]] = BackRefData(pParameterAddess, 0);

    // Set the desination pointer to the reference data.
    pDestPtr= pReferenceData;

    // Give out the pointer to the caller.
    return pDestPtr;
}

template <class T, typename Alloc>
void Builder<T, Alloc>::createBuffer(T **pDestination, CreateMode createMode) const
{
    // Create the output data and give the object mapping back to us.
    ObjectPointerMap destinationMap;
    
    // See how we are creating the buffer.
    if (createMode == NoMarking)
    {
        // This will create a tight packing.
        createOutputBuffer(destinationMap, pDestination);
    }
    else
    {
        // Create a packed buffer with marked allocation (unless override is enabled). 
        createOutputBufferUsingSMPack(destinationMap, pDestination, createMode == AllowOverride);
    }

    // Loop over all the extra reference object we have added.
    ReferencedObjectsMap::const_iterator referencedIter= m_referencedObjects.begin();
    for ( ; referencedIter != m_referencedObjects.end(); ++referencedIter)
    {
        // Find the mapped Object Data for this reference.
        ObjectDataMap::const_iterator objectDataIter= m_objectDataMap.find(referencedIter->first);
        if (objectDataIter == m_objectDataMap.end())
        {
            throw IFail( BASE_UTILS_expected_data_missing, "m_objectDataMap" ); 
        }

        // We also need to find this object in the output buffer.
        ObjectPointerMap::iterator destinationObject= destinationMap.find(referencedIter->first);
        if (destinationObject == destinationMap.end())
        {
            throw IFail( BASE_UTILS_expected_data_missing, "destinationMap" ); 
        }

        // Loop over all the objects we reference to this object and update the desination.
        ReferencedObjectsList::const_iterator childIter= referencedIter->second.begin();
        for ( ; childIter != referencedIter->second.end(); ++childIter)
        {
            // Find this object in the destination map.
            ObjectPointerMap::iterator referencedDestinationObject= destinationMap.find(*childIter);
            if (referencedDestinationObject == destinationMap.end())
            {
                throw IFail( BASE_UTILS_expected_data_missing, "destinationMap" ); 
            }

            // Calculate the offset where this object is located in the referenced data structure.
            size_t offset= size_t(*childIter) - size_t(&objectDataIter->second[0]);

            // Now we can re-point that pointer to the new destination location.
            *reinterpret_cast<void **>(size_t(destinationObject->second) + offset) = referencedDestinationObject->second;
        }
    }
}


template <class T, typename Alloc>
bool Builder<T, Alloc>::getMappedObject(const void *pBaseAddress, void *&pMappedObject, bool bBaseOnly) const
{
    // If we have a base address and we have something mapped,
    if (pBaseAddress != NULL && !m_objectBackRefDataMap.empty())
    {
        // First find the address. Try to find an address that is the next bigger pointer.
        typename ObjectBackRefDataMap::const_iterator searchIter = m_objectBackRefDataMap.upper_bound(pBaseAddress);
        // If we are not at the beginning, move to the prior element. This should be the object for the pBaseAddress.
        if (searchIter != m_objectBackRefDataMap.begin()) { --searchIter; }

        // Figure out the end pointer address for the buffer.
        const void *pObjectEnd = reinterpret_cast<const char *>(searchIter->first) + searchIter->second.datasize;

        // If the address is within the object, the reference is good.
        if ((bBaseOnly && pBaseAddress == searchIter->first) || (searchIter->first <= pBaseAddress && pBaseAddress < pObjectEnd))
        {
            // Assign the pMappedObject with the pData value we mapped and return true.
            pMappedObject = searchIter->second.pData;
            return true;
        }
    }

    // Could not find a object that encapsulate the address. Assign NULL to pMappedObject and return false.
    pMappedObject = NULL;
    return false;
}

template <class T, typename Alloc>
void Builder<T, Alloc>::createOutputBuffer(ObjectPointerMap &destinationMap, T **pDestination) const
{
    // Make sure we have an output pointer.
    if (!pDestination)
    { 
        throw IFail( BASE_UTILS_invalid_parameter ); 
    }

    // Size of the buffer we need.
    size_t totalSize= 0;

    // Count up all of the buffers we have allocated including the our object...
    ObjectDataMap::const_iterator objectDataIter= m_objectDataMap.begin();
    for ( ; objectDataIter != m_objectDataMap.end(); ++objectDataIter)
    {
        totalSize+= objectDataIter->second.size();
    }

    // Allocate the output buffer using the specific Alloc. 
    char *pOutputBuffer= totalSize ? reinterpret_cast<char *>(allocator().alloc(totalSize)) : NULL;
    // Make sure we have a buffer if expected.
    if (totalSize && !pOutputBuffer)
    {
        throw IFail( BASE_UTILS_no_memory ); 
    }

    // We assing this now to the destination so we have a better chance not to leak the memory.
    // Just incase we throw an exception while processing the data to the buffer.    
    *pDestination= totalSize ? reinterpret_cast<T *>(pOutputBuffer) : NULL;

    // This is the end of the buffer where the "pOutputBuffer" should be 
    // after we have processed all the data, if not, we failed.
    char *pOutputBufferEnd= pOutputBuffer + totalSize;

    // Now we can start fixing up all the pointer data.
    objectDataIter= m_objectDataMap.begin();
    for ( ; objectDataIter != m_objectDataMap.end(); ++objectDataIter)
    {
        // If we don't have any data,
        if (objectDataIter->second.empty())
        {
            // just set the destination as NULL.
            destinationMap[objectDataIter->first]= NULL;
        }
        else
        {
            // otherwise copy the data and move the output buffer pointer.
            memcpy(pOutputBuffer, &objectDataIter->second[0], objectDataIter->second.size());
            destinationMap[objectDataIter->first]= pOutputBuffer;
            pOutputBuffer+= objectDataIter->second.size();
        }
    }


    // Also add all the back reference data.
    ObjectPointerMap::const_iterator backRefIter= m_backReferencedObjects.begin();
    for ( ; backRefIter != m_backReferencedObjects.end(); ++backRefIter)
    {
        // Find the back reference data in the destination. This should be NULL.
        ObjectPointerMap::iterator backRefDataIter= destinationMap.find(backRefIter->first);
        if (backRefDataIter == destinationMap.end() || backRefDataIter->second != NULL)
        {
            throw IFail( BASE_UTILS_internal_error );
        }

        // We should also be able to find referenced data in the destination.
        ObjectPointerMap::const_iterator searchIter= destinationMap.find(backRefIter->second);
        if (searchIter == destinationMap.end() || searchIter->second == NULL)
        {
            throw IFail( BASE_UTILS_internal_error );
        }

        // Add another destination mapping to exiting data.
        backRefDataIter->second= searchIter->second;
    }

    // Cross the "T's" :)... We should have moved the pointer as expected and the "NULL"
    // mapped object pointing to the beginning of the buffer.
    if (pOutputBuffer != pOutputBufferEnd || destinationMap[NULL] != *pDestination)
    {
        throw IFail( BASE_UTILS_internal_error );
    }
}

template <class T, typename Alloc>
void Builder<T, Alloc>::createOutputBufferUsingSMPack(ObjectPointerMap &destinationMap, T **pDestination, logical allowOverride) const
{
    // Make sure we have an output pointer.
    if (!pDestination)
    {
        throw IFail(BASE_UTILS_invalid_parameter);
    }

    // This will hold the information we are packing.
    std::vector<MEM_data_to_pack_t> dataToPack;
    dataToPack.reserve(m_objectDataMap.size());

    // Count up all of the buffers we have allocated including the our object...
    ObjectDataMap::const_iterator objectDataIter = m_objectDataMap.begin();
    for (; objectDataIter != m_objectDataMap.end(); ++objectDataIter)
    {
        MEM_data_to_pack_t data;
        data.data = !objectDataIter->second.empty() ? objectDataIter->second.data() : NULL;
        data.data_size = objectDataIter->second.size();

        dataToPack.push_back(data);
    }

    // We assing the allocation to the destination so we have a better chance not to leak the memory.
    // Just incase we throw an exception while processing the data to the buffer.    
    *pDestination = reinterpret_cast<T *>(allocator().alloc_packed(dataToPack.size(), &dataToPack[0], allowOverride));
    // Make sure we have a buffer if expected.
    if (dataToPack.front().data_size && !*pDestination)
    {
        throw IFail(BASE_UTILS_no_memory);
    }

    // Now we can start fixing up all the pointer data.
    objectDataIter = m_objectDataMap.begin();
    for (size_t i = 0; objectDataIter != m_objectDataMap.end(); ++i, ++objectDataIter)
    {
        destinationMap[objectDataIter->first] = (void *)dataToPack[i].data;
    }

    // Also add all the back reference data.
    ObjectPointerMap::const_iterator backRefIter = m_backReferencedObjects.begin();
    for (; backRefIter != m_backReferencedObjects.end(); ++backRefIter)
    {
        // Find the back reference data in the destination. This should be NULL.
        ObjectPointerMap::iterator backRefDataIter = destinationMap.find(backRefIter->first);
        if (backRefDataIter == destinationMap.end() || backRefDataIter->second != NULL)
        {
            throw IFail(BASE_UTILS_internal_error);
        }

        // We should also be able to find referenced data in the destination.
        ObjectPointerMap::const_iterator searchIter = destinationMap.find(backRefIter->second);
        if (searchIter == destinationMap.end() || searchIter->second == NULL)
        {
            throw IFail(BASE_UTILS_internal_error);
        }

        // Add another destination mapping to exiting data.
        backRefDataIter->second = searchIter->second;
    }
}

} // namespace PackedMemory
} // namespace Teamcenter

#endif // PACKED_MEMORY_BUILDER_HXX