diff --git a/SConstruct b/SConstruct index b3427c93d9..2703d6253c 100644 --- a/SConstruct +++ b/SConstruct @@ -495,6 +495,27 @@ o.Add( "/usr/local/appleseed/lib", ) +# Blosc options + +o.Add( + "BLOSC_INCLUDE_PATH", + "The path to the blosc include directory.", + "/usr/local/include", +) + +o.Add( + "BLOSC_LIB_PATH", + "The path to the blosc lib directory.", + "/usr/local/lib", +) + +o.Add( + "BLOSC_LIB_SUFFIX", + "The suffix appended to the names of the blosc libraries. You can modify this " + "to link against libraries installed with non-default names", + "", +) + # Build options o.Add( @@ -1051,7 +1072,8 @@ dependencyIncludes = [ "-isystem", "$PNG_INCLUDE_PATH", "-isystem", "$JPEG_INCLUDE_PATH", "-isystem", "$TIFF_INCLUDE_PATH", - "-isystem", "$FREETYPE_INCLUDE_PATH", + "-isystem", "$FREETYPE_INCLUDE_PATH", + "-isystem", "$BLOSC_INCLUDE_PATH", ] env.Prepend( @@ -1069,9 +1091,11 @@ env.Prepend( "$JPEG_LIB_PATH", "$TIFF_LIB_PATH", "$FREETYPE_LIB_PATH", + "$BLOSC_LIB_PATH", ], LIBS = [ "pthread", + "blosc" + env["BLOSC_LIB_SUFFIX"], ] ) @@ -1149,9 +1173,13 @@ if doConfigure : if not c.CheckLibWithHeader( "tbb" + env["TBB_LIB_SUFFIX"], "tbb/tbb.h", "C++" ) : sys.stderr.write( "ERROR : unable to find the TBB libraries - check TBB_INCLUDE_PATH and TBB_LIB_PATH.\n" ) Exit( 1 ) - + + if c.CheckLibWithHeader( env.subst( "blosc" + env["BLOSC_LIB_SUFFIX"] ), "blosc.h", "CXX" ) : + + env.Append( CPPFLAGS = "-DIECORE_WITH_BLOSC" ) + c.Finish() - + env.Append( LIBS = [ "Half" + env["OPENEXR_LIB_SUFFIX"], "Iex" + env["OPENEXR_LIB_SUFFIX"], diff --git a/config/ie/options b/config/ie/options index 1bfb8d6f30..5f254b731c 100644 --- a/config/ie/options +++ b/config/ie/options @@ -83,6 +83,7 @@ if targetApp : compilerVersion = targetAppReg["compilerVersion"] openEXRVersion = targetAppReg.get( "OpenEXRVersion", cortexReg["OpenEXRVersion"] ) alembicVersion = targetAppReg.get( "AlembicVersion", cortexReg["AlembicVersion"] ) + bloscVersion = targetAppReg.get( "bloscVersion", cortexReg["bloscVersion"] ) hdf5Version = targetAppReg.get( "hdf5Version", cortexReg["hdf5Version"] ) glewVersion = targetAppReg.get( "glewVersion", cortexReg["glewVersion"] ) pythonVersion = targetAppReg["pythonVersion"] @@ -99,6 +100,7 @@ else : pythonVersion = cortexReg["preferredPythonVersion"] openEXRVersion = cortexReg["OpenEXRVersion"] alembicVersion = cortexReg["AlembicVersion"] + bloscVersion = cortexReg["bloscVersion"] hdf5Version = cortexReg["hdf5Version"] glewVersion = cortexReg["glewVersion"] tbbVersion = cortexReg["tbbVersion"] @@ -372,6 +374,10 @@ ALEMBIC_LIB_SUFFIX = "-" + alembicVersion HDF5_INCLUDE_PATH = os.path.join( "/software/apps/hdf5", hdf5Version, platform, compiler, compilerVersion, "include" ) HDF5_LIB_PATH = os.path.join( "/software/apps/hdf5", hdf5Version, platform, compiler, compilerVersion, "lib" ) +# find blosc: +BLOSC_INCLUDE_PATH = os.path.join( "/software/tools/include/blosc/", bloscVersion ) +BLOSC_LIB_SUFFIX = "-" + bloscVersion + # find doxygen DOXYGEN = os.path.join( "/software/apps/doxygen", os.environ["DOXYGEN_VERSION"], platform, "bin", "doxygen" ) diff --git a/include/IECore/IECore.h b/include/IECore/IECore.h index 5847c18814..c334d2483c 100644 --- a/include/IECore/IECore.h +++ b/include/IECore/IECore.h @@ -59,6 +59,8 @@ IECORE_API bool withASIO(); IECORE_API bool withSignals(); /// Returns true if IECore was built with boost::math::factorial support IECORE_API bool withBoostFactorial(); +/// Returns true if IECore was built with blosc support +IECORE_API bool withBlosc(); /// Returns true if IECore was built with TIFF support IECORE_API bool withTIFF(); /// Returns true if IECore was built with JPEG support diff --git a/include/IECore/MemoryIndexedIO.h b/include/IECore/MemoryIndexedIO.h index 27ba3d7e50..f62bba11db 100644 --- a/include/IECore/MemoryIndexedIO.h +++ b/include/IECore/MemoryIndexedIO.h @@ -66,6 +66,9 @@ class IECORE_API MemoryIndexedIO : public StreamIndexedIO IndexedIO *duplicate(StreamIndexedIO::Node &rootNode) const; class StreamFile; + class IStreamFile; + class OStreamFile; + class CharVectorDataDevice; }; IE_CORE_DECLAREPTR( MemoryIndexedIO ) diff --git a/include/IECore/ObjectReader.h b/include/IECore/ObjectReader.h index 9ead89f9ea..831e40a086 100644 --- a/include/IECore/ObjectReader.h +++ b/include/IECore/ObjectReader.h @@ -1,6 +1,6 @@ ////////////////////////////////////////////////////////////////////////// // -// Copyright (c) 2007-2011, Image Engine Design Inc. All rights reserved. +// Copyright (c) 2007-2015, Image Engine Design Inc. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are diff --git a/src/IECore/IECore.cpp b/src/IECore/IECore.cpp index 79d8f3db34..37195c7375 100644 --- a/src/IECore/IECore.cpp +++ b/src/IECore/IECore.cpp @@ -92,6 +92,15 @@ bool withBoostFactorial() #endif } +bool withBlosc() +{ +#ifdef IECORE_WITH_BLOSC + return true; +#else + return false; +#endif +} + bool withTIFF() { #ifdef IECORE_WITH_TIFF diff --git a/src/IECore/MemoryIndexedIO.cpp b/src/IECore/MemoryIndexedIO.cpp index 0a738a915d..309051a1fc 100644 --- a/src/IECore/MemoryIndexedIO.cpp +++ b/src/IECore/MemoryIndexedIO.cpp @@ -36,106 +36,195 @@ #include "IECore/FileIndexedIO.h" #include "IECore/VectorTypedData.h" +#include +#include +#include + using namespace IECore; IE_CORE_DEFINERUNTIMETYPEDDESCRIPTION( MemoryIndexedIO ) -/////////////////////////////////////////////// +////////////////////////////////////////////////////// // -// FileIndexedIO::StreamFile (begin) +// FileIndexedIO::StreamFile subclass implementations // -/////////////////////////////////////////////// +////////////////////////////////////////////////////// -class MemoryIndexedIO::StreamFile : public StreamIndexedIO::StreamFile +class MemoryIndexedIO::CharVectorDataDevice { public: - StreamFile( const char *buf, size_t size, IndexedIO::OpenMode mode ); - CharVectorDataPtr buffer(); + typedef char char_type; + typedef boost::iostreams::seekable_device_tag category; + + CharVectorDataDevice( CharVectorData *container ) + : m_container(container), m_pos(0) + {} + + std::streamsize read(char* s, std::streamsize n) + { + std::streamsize amt = static_cast(m_container->readable().size() - m_pos); + std::streamsize result = std::min(n, amt); + if (result != 0) + { + std::copy( m_container->readable().begin() + m_pos, m_container->readable().begin() + m_pos + result, s ); + m_pos += result; + return result; + } + else + { + return -1; // EOF + } + } - virtual ~StreamFile(); + std::streamsize write(const char* s, std::streamsize n) + { + using namespace std; + streamsize result = 0; + if (m_pos != m_container->writable().size()) + { + streamsize amt = static_cast(m_container->writable().size() - m_pos); + result = (min)(n, amt); + std::copy(s, s + result, m_container->writable().begin() + m_pos); + m_pos += result; + } + if (result < n) + { + m_container->writable().insert(m_container->writable().end(), s, s + n); + m_pos = m_container->writable().size(); + } + return n; + } - void flush( size_t endPosition ); + boost::iostreams::stream_offset seek(boost::iostreams::stream_offset off, std::ios_base::seekdir way ) + { + using namespace std; + // Determine new value of m_pos + boost::iostreams::stream_offset next; + if (way == std::ios_base::beg) + { + next = off; + } + else if (way == std::ios_base::cur) + { + next = m_pos + off; + } + else if (way == std::ios_base::end) + { + next = m_container->readable().size() + off; + } + else + { + throw ios_base::failure("bad seek direction"); + } + + // Check for errors + if (next < 0 || (size_t)next > m_container->readable().size()) + { + throw ios_base::failure("bad seek offset"); + } + + m_pos = next; + return m_pos; + } private: - size_t m_endPosition; + CharVectorData* m_container; + size_t m_pos; }; -MemoryIndexedIO::StreamFile::StreamFile( const char *buf, size_t size, IndexedIO::OpenMode mode ) : StreamIndexedIO::StreamFile(mode), m_endPosition(0) +class MemoryIndexedIO::StreamFile : public StreamIndexedIO::StreamFile { - if (mode & IndexedIO::Write) - { - std::stringstream *f = new std::stringstream( std::ios::trunc | std::ios::binary | std::ios::in | std::ios::out ); - setStream( f, true ); - } - else if (mode & IndexedIO::Append) - { - if ( !buf || !size ) + public: + StreamFile( IndexedIO::OpenMode mode ) : StreamIndexedIO::StreamFile(mode) { - /// Create new file - std::stringstream *f = new std::stringstream( std::ios::trunc | std::ios::binary | std::ios::in | std::ios::out ); - setStream( f, true ); } - else - { - /// Read existing file - assert( buf ); - /// Read existing file - std::stringstream *f = new std::stringstream( std::string(buf, size), std::ios::binary | std::ios::in | std::ios::out ); - setStream( f, false ); + virtual CharVectorDataPtr buffer() = 0; + + virtual ~StreamFile() + { } - } - else - { - assert( buf ); - assert( mode & IndexedIO::Read ); - std::stringstream *f = new std::stringstream( std::string(buf, size), std::ios::binary | std::ios::in | std::ios::out ); - setStream( f, false ); - } -} -void MemoryIndexedIO::StreamFile::flush( size_t endPosition ) -{ - m_endPosition = endPosition; -} +}; -CharVectorDataPtr MemoryIndexedIO::StreamFile::buffer() +class MemoryIndexedIO::IStreamFile : public MemoryIndexedIO::StreamFile { - std::stringstream *s = static_cast< std::stringstream *>( m_stream ); - assert( s ); + public: + IStreamFile( ConstCharVectorDataPtr buffer, IndexedIO::OpenMode mode ) : MemoryIndexedIO::StreamFile(mode), m_buffer( buffer ) + { + setStream( new boost::iostreams::stream( const_cast( buffer.get() ) ), buffer->readable().empty() ); + } + + virtual CharVectorDataPtr buffer() + { + return m_buffer->copy(); + } - CharVectorData::ValueType d; - const std::string &str = s->str(); + virtual ~IStreamFile() + { + } - d.assign( str.begin(), str.end() ); - d.resize( m_endPosition ); - assert( d.size() == m_endPosition ); - return new CharVectorData( d ); -} + private: + ConstCharVectorDataPtr m_buffer; + +}; -MemoryIndexedIO::StreamFile::~StreamFile() +class MemoryIndexedIO::OStreamFile : public MemoryIndexedIO::StreamFile { -} + public: + OStreamFile( CharVectorDataPtr buffer, IndexedIO::OpenMode mode ) : MemoryIndexedIO::StreamFile(mode), m_buffer( buffer ) + { + setStream( new boost::iostreams::stream( buffer.get() ), buffer->readable().empty() ); + } + + virtual CharVectorDataPtr buffer() + { + return m_buffer; + } + + virtual ~OStreamFile() + { + } + + private: + CharVectorDataPtr m_buffer; + +}; -/////////////////////////////////////////////// +////////////////////////////////////////////////////// // -// MemoryIndexedIO::StreamFile (end) +// FileIndexedIO::StreamFile subclasses end // -/////////////////////////////////////////////// +////////////////////////////////////////////////////// MemoryIndexedIO::MemoryIndexedIO( ConstCharVectorDataPtr buf, const IndexedIO::EntryIDList &root, IndexedIO::OpenMode mode) { - const char *bufPtr = 0; - size_t size = 0; - if ( buf ) + if (mode & IndexedIO::Write) + { + open( new OStreamFile( new CharVectorData, mode ), root ); + } + else if (mode & IndexedIO::Append) { - bufPtr = &(buf->readable()[0]); - size = buf->readable().size(); + if ( !buf ) + { + /// Create new file + open( new OStreamFile( new CharVectorData, mode ), root ); + } + else + { + /// Read existing file + open( new OStreamFile( buf->copy(), mode ), root ); + } + } + else + { + assert( buf ); + assert( mode & IndexedIO::Read ); + open( new IStreamFile( buf, mode ), root ); } - open( new StreamFile( bufPtr, size, mode ), root ); } MemoryIndexedIO::MemoryIndexedIO( StreamIndexedIO::Node &rootNode ) : StreamIndexedIO( rootNode ) diff --git a/src/IECore/ObjectReader.cpp b/src/IECore/ObjectReader.cpp index 60547ccde6..d6a2e9255d 100644 --- a/src/IECore/ObjectReader.cpp +++ b/src/IECore/ObjectReader.cpp @@ -1,6 +1,6 @@ ////////////////////////////////////////////////////////////////////////// // -// Copyright (c) 2007-2010, Image Engine Design Inc. All rights reserved. +// Copyright (c) 2007-2015, Image Engine Design Inc. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are @@ -32,13 +32,22 @@ // ////////////////////////////////////////////////////////////////////////// +#include "IECore/ObjectWriter.h" #include "IECore/ObjectReader.h" #include "IECore/FileIndexedIO.h" +#include "IECore/MemoryIndexedIO.h" #include "IECore/FileNameParameter.h" #include "IECore/CompoundData.h" +#include "boost/filesystem/convenience.hpp" +#include "boost/iostreams/filter/gzip.hpp" + #include +#ifdef IECORE_WITH_BLOSC +#include "blosc.h" +#endif // IECORE_WITH_BLOSC + using namespace IECore; using namespace boost; @@ -90,8 +99,73 @@ bool ObjectReader::canRead( const std::string &fileName ) ObjectPtr ObjectReader::doOperation( const CompoundObject * operands ) { + CompoundObjectPtr header = readHeader(); + std::string compressionType = "none"; + StringData* compressionTypeData = header->member( "compressionType" ); + if( compressionTypeData ) + { + compressionType = compressionTypeData->readable(); + } IndexedIOPtr io = open(fileName()); + +#ifdef IECORE_WITH_BLOSC + // is this file compressed using blosc? + if( compressionType == "blosc" ) + { + IndexedIOPtr objectCompressed = io->subdirectory( "objectCompressed" ); + + // we split the file up into 1gb blocks to avoid integer overflow in the + // blosc library. Read the number of compressed blocks: + size_t numBlocks; + objectCompressed->read( "numBlocks", numBlocks ); + + // calculate total decompressed size: + size_t totalDecompressedSize = 0; + for( size_t i=0; i < numBlocks; ++i ) + { + size_t s; + objectCompressed->subdirectory( InternedString(i) )->read( "decompressedSize", s ); + totalDecompressedSize += s; + } + + // create buffer for decompression: + CharVectorDataPtr memBufferData = new CharVectorData; + memBufferData->writable().resize( totalDecompressedSize ); + char* buffer = memBufferData->writable().data(); + + // loop through all the blocks (max size = 1gb), and decompress into memBufferData: + blosc_init(); + for( size_t i=0; i < numBlocks; ++i ) + { + IndexedIOPtr compressedBlock = objectCompressed->subdirectory( InternedString(i) ); + size_t compressedSize, decompressedSize; + compressedBlock->read( "compressedSize", compressedSize ); + compressedBlock->read( "decompressedSize", decompressedSize ); + + // read the actual data + std::vector compressedData( compressedSize ); + char* cd = compressedData.data(); + compressedBlock->read( InternedString( "data" ), cd, compressedSize * sizeof(char) ); + + // decompress into a buffer for a MemoryIndexedIO: + blosc_decompress( compressedData.data(), buffer, decompressedSize ); + + buffer += decompressedSize; + } + blosc_destroy(); + + // create a MemoryIndexedIO from the buffer we've just read: + io = new MemoryIndexedIO( memBufferData, IndexedIO::rootPath, IndexedIO::Read ); + return Object::load( io, "object" ); + } +#endif // IECORE_WITH_BLOSC + if( compressionType != "none" ) + { + throw Exception( "ObjectReader::doOperation(): unsupported compression type '" + compressionType + "'" ); + } + return Object::load( io, "object" ); + } CompoundObjectPtr ObjectReader::readHeader() @@ -99,7 +173,32 @@ CompoundObjectPtr ObjectReader::readHeader() CompoundObjectPtr header = Reader::readHeader(); IndexedIOPtr io = open(fileName()); - CompoundDataPtr objectHeader = runTimeCast( Object::load( io, "header" ) ); + IndexedIOPtr headerIO = io->subdirectory( "header", IndexedIO::NullIfMissing ); + if( !headerIO ) + { + throw Exception( "ObjectReader::readHeader(): couldn't find header io entry" ); + } + + // older .cob files store the header in "header/object" rather than "/header", + // so we need to check for that: + CompoundDataPtr objectHeader; + IndexedIO::EntryIDList names; + headerIO->entryIds( names ); + if( names.size() == 1 && names[0] == "object" ) + { + // load from "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/header/object" + objectHeader = runTimeCast( Object::load( headerIO, "object" ) ); + } + else + { + // load from "/header" + objectHeader = runTimeCast( Object::load( io, "header" ) ); + } + + if( !objectHeader ) + { + throw Exception( "ObjectReader::readHeader(): header was not a CompoundData" ); + } for ( CompoundData::ValueType::const_iterator it = objectHeader->readable().begin(); it != objectHeader->readable().end(); ++it ) { diff --git a/src/IECore/ObjectWriter.cpp b/src/IECore/ObjectWriter.cpp index 453bc11412..c63e1d804b 100644 --- a/src/IECore/ObjectWriter.cpp +++ b/src/IECore/ObjectWriter.cpp @@ -1,6 +1,6 @@ ////////////////////////////////////////////////////////////////////////// // -// Copyright (c) 2007-2010, Image Engine Design Inc. All rights reserved. +// Copyright (c) 2007-2015, Image Engine Design Inc. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are @@ -34,9 +34,14 @@ #include +#include "boost/filesystem/convenience.hpp" +#include "boost/iostreams/filter/gzip.hpp" + #include "IECore/ObjectWriter.h" #include "IECore/FileIndexedIO.h" +#include "IECore/MemoryIndexedIO.h" #include "IECore/FileNameParameter.h" +#include "IECore/NumericParameter.h" #include "IECore/CompoundParameter.h" #include "IECore/ObjectParameter.h" #include "IECore/CompoundData.h" @@ -45,6 +50,10 @@ #include "IECore/HeaderGenerator.h" #include "IECore/VisibleRenderable.h" +#ifdef IECORE_WITH_BLOSC +#include "blosc.h" +#endif // IECORE_WITH_BLOSC + using namespace IECore; using namespace std; using namespace boost; @@ -82,7 +91,9 @@ void ObjectWriter::doWrite( const CompoundObject *operands ) // write the header CompoundDataPtr header = boost::static_pointer_cast( m_headerParameter->getValue()->copy() ); + std::string compressionType = parameters()->parameter("compressionType")->getTypedValue(); header->writable()["typeName"] = new StringData( object()->typeName() ); + header->writable()["compressionType"] = new StringData( compressionType ); if( const VisibleRenderable* visibleRenderable = runTimeCast(object()) ) { @@ -101,8 +112,73 @@ void ObjectWriter::doWrite( const CompoundObject *operands ) ((ObjectPtr)header)->save( io, "header" ); - // write the object - object()->save( io, "object" ); +#ifdef IECORE_WITH_BLOSC + if( compressionType == "blosc" ) + { + // write the object to a memory buffer: + MemoryIndexedIOPtr bodyIO = new MemoryIndexedIO( ConstCharVectorDataPtr(), IndexedIO::rootPath, IndexedIO::Exclusive | IndexedIO::Write ); + object()->save( bodyIO, "object" ); + + IndexedIOPtr objectCompressed = io->createSubdirectory( "objectCompressed" ); + + // blosc_compress() can only accept input buffers up to 2147483631 bytes in size, so we compress the file in 1gb blocks: + size_t bufferPos = 0; + CharVectorDataPtr buffer = bodyIO->buffer(); + size_t bufferEnd = buffer->readable().size(); + size_t numBlocks=0; + + // blosc + const size_t blockSize = 1024l * 1024l * 1024l; + blosc_init(); + for( ; bufferPos < bufferEnd; ++numBlocks, bufferPos += blockSize ) + { + size_t blockEnd = std::min( bufferEnd, bufferPos + blockSize ); + + // compress: + std::vector compressedData( (blockEnd - bufferPos) + BLOSC_MAX_OVERHEAD ); + + int dataSize = blosc_compress( + 9, + true, + 4, + (blockEnd - bufferPos), + buffer->readable().data() + bufferPos, + compressedData.data(), + compressedData.size() + ); + + if( dataSize == 0 ) + { + blosc_destroy(); + throw Exception( ( boost::format( "ObjectWriter::doWrite(): Could not compress %d bytes into %d bytes" ) % (blockEnd - bufferPos) % compressedData.size() ).str() ); + } + if( dataSize < 0 ) + { + blosc_destroy(); + throw Exception( "ObjectWriter::doWrite(): internal error in the blosc compression library" ); + } + compressedData.resize( dataSize ); + std::string blockName = ( boost::format( "%d" ) % numBlocks ).str(); + IndexedIOPtr compressedBlock = objectCompressed->createSubdirectory( blockName ); + compressedBlock->write( "decompressedSize", (blockEnd - bufferPos) ); + compressedBlock->write( "compressedSize", compressedData.size() ); + compressedBlock->write( "data", compressedData.data(), compressedData.size() ); + } + blosc_destroy(); + + objectCompressed->write( "numBlocks", numBlocks ); + return; + + } +#endif // IECORE_WITH_BLOSC + if( compressionType == "none" ) + { + // no compression, just write the object: + object()->save( io, "object" ); + return; + } + + throw Exception( "ObjectWriter::doWrite(): unsupported compression type '" + compressionType + "'" ); } void ObjectWriter::constructParameters() @@ -115,4 +191,17 @@ void ObjectWriter::constructParameters() ); parameters()->addParameter( m_headerParameter ); + + + StringParameter::PresetsContainer compressionTypePresets; + compressionTypePresets.push_back( StringParameter::Preset( "none", "none" ) ); + compressionTypePresets.push_back( StringParameter::Preset( "blosc", "blosc" ) ); + + parameters()->addParameter( new StringParameter( + "compressionType", + "Compression type to use", + "blosc", + compressionTypePresets, + true + ) ); } diff --git a/src/IECorePythonModule/IECore.cpp b/src/IECorePythonModule/IECore.cpp index 6c36ededb2..27bb847f27 100644 --- a/src/IECorePythonModule/IECore.cpp +++ b/src/IECorePythonModule/IECore.cpp @@ -607,6 +607,7 @@ BOOST_PYTHON_MODULE(_IECore) def( "patchVersion", &IECore::patchVersion ); def( "versionString", &IECore::versionString, return_value_policy() ); def( "withASIO", &IECore::withASIO ); + def( "withBlosc", &IECore::withBlosc ); def( "withSignals", &IECore::withSignals ); def( "withTIFF", &IECore::withTIFF ); def( "withJPEG", &IECore::withJPEG ); diff --git a/test/IECore/ObjectWriter.py b/test/IECore/ObjectWriter.py index 96fe9e405a..ab99368015 100644 --- a/test/IECore/ObjectWriter.py +++ b/test/IECore/ObjectWriter.py @@ -41,11 +41,22 @@ class TestObjectWriter( unittest.TestCase ) : def testBasics( self ) : + # read an old file: r = IECore.Reader.create( "test/IECore/data/cobFiles/compoundData.cob" ) p = r.read() + self.assertEqual( + p, + IECore.CompoundData({'banana':IECore.IntData( 2 ),'melon':IECore.FloatData( 2.5 ),'lemon':IECore.IntData( -3 ),'apple':IECore.IntData( 12 )}) + ) w = IECore.Writer.create( p, "test/compoundData.cob" ) + w["compressionType"].setTypedValue("none") w.write() + + # check FileIndexedIO contents: + fio = IECore.FileIndexedIO( "test/compoundData.cob", IECore.IndexedIO.OpenMode.Read ) + fio.subdirectory("header") + fio.subdirectory("object") r = IECore.Reader.create( "test/compoundData.cob" ) p2 = r.read() @@ -57,6 +68,7 @@ def testHeader( self ) : o = IECore.IntData() w = IECore.Writer.create( o, "test/intData.cob" ) + w["compressionType"].setTypedValue("none") w["header"].getValue()["testHeaderData"] = IECore.StringData( "i am part of a header" ) w["header"].getValue()["testHeaderData2"] = IECore.IntData( 100 ) w.write() @@ -69,20 +81,85 @@ def testHeader( self ) : self.assertEqual( h["host"]["nodeName"].value, socket.gethostname() ) self.assertEqual( h["ieCoreVersion"].value, IECore.versionString() ) self.assertEqual( h["typeName"].value, "IntData" ) - + def testBoundInHeader( self ) : o = IECore.SpherePrimitive() w = IECore.Writer.create( o, "test/spherePrimitive.cob" ) + w["compressionType"].setTypedValue("none") w.write() h = IECore.Reader.create( "test/spherePrimitive.cob" ).readHeader() self.assertEqual( h["bound"].value, o.bound() ) + def testCompressed( self ) : + + if IECore.withBlosc(): + + floats = IECore.FloatVectorData( range(0,10000) ) + w = IECore.Writer.create( floats, "test/compressed.cob" ) + w["compressionType"].setTypedValue("blosc") + + w.write() + + # check FileIndexedIO contents: + fio = IECore.FileIndexedIO( "test/compressed.cob", IECore.IndexedIO.OpenMode.Read ) + self.assertEqual( set( fio.entryIds() ), set( [ IECore.InternedString( s ) for s in ["header", "objectCompressed"] ] ) ) + self.assertEqual( set( fio.subdirectory("objectCompressed").entryIds() ), set( [ IECore.InternedString( s ) for s in ["0", "numBlocks"] ] ) ) + self.assertEqual( set( fio.subdirectory("objectCompressed").subdirectory("0").entryIds() ), set( [ IECore.InternedString( s ) for s in ["data", "compressedSize", "decompressedSize"] ] ) ) + self.assertEqual( fio.subdirectory("objectCompressed").read("numBlocks").value, 1 ) + + # read back and check object: + self.assertEqual( IECore.Reader.create( "test/compressed.cob" ).read(), floats ) + + def testGiantCompressed( self ) : + + if IECore.withBlosc(): + + # create a 1.5gb mesh so multiple blocks get written into the file: + stupidMesh = IECore.MeshPrimitive.createPlane( IECore.Box2f( IECore.V2f(-1), IECore.V2f(1) ), IECore.V2i( 5000,5000 ) ) + + w = IECore.Writer.create( stupidMesh, "test/compressed.cob" ) + w["compressionType"].setTypedValue("blosc") + + w.write() + + # check FileIndexedIO contents: + fio = IECore.FileIndexedIO( "test/compressed.cob", IECore.IndexedIO.OpenMode.Read ) + self.assertEqual( set( fio.entryIds() ), set( [ IECore.InternedString( s ) for s in ["header", "objectCompressed"] ] ) ) + self.assertEqual( set( fio.subdirectory("objectCompressed").entryIds() ), set( [ IECore.InternedString( s ) for s in ["0", "1", "numBlocks"] ] ) ) + self.assertEqual( set( fio.subdirectory("objectCompressed").subdirectory("0").entryIds() ), set( [ IECore.InternedString( s ) for s in ["data", "compressedSize", "decompressedSize"] ] ) ) + self.assertEqual( set( fio.subdirectory("objectCompressed").subdirectory("1").entryIds() ), set( [ IECore.InternedString( s ) for s in ["data", "compressedSize", "decompressedSize"] ] ) ) + self.assertEqual( fio.subdirectory("objectCompressed").read("numBlocks").value, 2 ) + + # read back and check object: + self.assertEqual( IECore.Reader.create( "test/compressed.cob" ).read(), stupidMesh ) + + def testCompressedHeader( self ) : + + if IECore.withBlosc(): + + o = IECore.IntData() + + w = IECore.Writer.create( o, "test/compressed.cob" ) + w["compressionType"].setTypedValue("blosc") + w["header"].getValue()["testHeaderData"] = IECore.StringData( "i am part of a header" ) + w["header"].getValue()["testHeaderData2"] = IECore.IntData( 100 ) + w.write() + + h = IECore.Reader.create( "test/compressed.cob" ).readHeader() + + for k in w["header"].getValue().keys() : + self.assertEqual( w["header"].getValue()[k], h[k] ) + + self.assertEqual( h["host"]["nodeName"].value, socket.gethostname() ) + self.assertEqual( h["ieCoreVersion"].value, IECore.versionString() ) + self.assertEqual( h["typeName"].value, "IntData" ) + def tearDown( self ) : - for f in ( "test/compoundData.cob", "test/intData.cob", "test/spherePrimitive.cob" ) : + for f in ( "test/compoundData.cob", "test/intData.cob", "test/compressed.cob" ) : if os.path.isfile( f ) : os.remove( f )