diff --git a/cpp/csp/python/CMakeLists.txt b/cpp/csp/python/CMakeLists.txt index 8064e524b..45f265534 100644 --- a/cpp/csp/python/CMakeLists.txt +++ b/cpp/csp/python/CMakeLists.txt @@ -10,8 +10,10 @@ add_library(csptypesimpl CspTypeFactory.cpp PyCspEnum.cpp PyCspType.cpp - PyStruct.cpp) + PyStruct.cpp + PyStructToJson.cpp) set_target_properties(csptypesimpl PROPERTIES PUBLIC_HEADER "${CSPTYPESIMPL_PUBLIC_HEADERS}") +target_compile_definitions(csptypesimpl PUBLIC RAPIDJSON_HAS_STDSTRING=1) target_link_libraries(csptypesimpl csp_core csp_types) set(CSPIMPL_PUBLIC_HEADERS @@ -36,7 +38,8 @@ set(CSPIMPL_PUBLIC_HEADERS PyObjectPtr.h PyOutputAdapterWrapper.h PyOutputProxy.h - PyConstants.h) + PyConstants.h + PyStructToJson.h) add_library(cspimpl SHARED cspimpl.cpp diff --git a/cpp/csp/python/PyStruct.cpp b/cpp/csp/python/PyStruct.cpp index 9e6e9203b..4e5f031de 100644 --- a/cpp/csp/python/PyStruct.cpp +++ b/cpp/csp/python/PyStruct.cpp @@ -3,6 +3,8 @@ #include #include #include +#include + #include #include @@ -170,9 +172,9 @@ static PyObject * PyStructMeta_new( PyTypeObject *subtype, PyObject *args, PyObj /*back reference to the struct type that will be accessible on the csp struct -> meta() DialectStructMeta will hold a borrowed reference to the type to avoid a circular dep - + This is the layout of references between all these types - StructMeta (shared_ptr) <-------- strong ref + StructMeta (shared_ptr) <-------- strong ref | | DialectStructMeta ---> weak ref to PyStructMeta ( the PyType ) /\ /\ @@ -859,7 +861,7 @@ PyObject * PyStruct_deepcopy( PyStruct * self ) CSP_BEGIN_METHOD; //Note that once tp_alloc is called, the object will get added to GC //deepcopy traversal may kick in a GC collect, so we have to call that first before the PyStruct is created - //of it may traverse a partially consturcted object and crash + //of it may traverse a partially consturcted object and crash auto deepcopy = self -> struct_ -> deepcopy(); PyObject * pyDeepcopy = self -> ob_type -> tp_alloc( self -> ob_type, 0 ); new ( pyDeepcopy ) PyStruct( deepcopy ); @@ -912,6 +914,30 @@ PyObject * PyStruct_all_fields_set( PyStruct * self ) return toPython( self -> struct_ -> allFieldsSet() ); } +PyObject * PyStruct_to_json( PyStruct * self, PyObject * args, PyObject * kwargs ) +{ + CSP_BEGIN_METHOD; + + // NOTE: Consider grouping customization properties into a dictionary + PyObject * callable = nullptr; + + if( PyArg_ParseTuple( args, "O:to_json", &callable ) ) + { + if( !PyCallable_Check( callable ) ) + { + CSP_THROW( TypeError, "Parameter must be callable" ); + } + } + else + { + CSP_THROW( TypeError, "Expected a callable as the argument" ); + } + auto struct_ptr = self -> struct_; + auto buffer = structToJson( struct_ptr, callable ); + return toPython( buffer ); + CSP_RETURN_NULL; +} + static PyMethodDef PyStruct_methods[] = { { "copy", (PyCFunction) PyStruct_copy, METH_NOARGS, "make a shallow copy of the struct" }, { "deepcopy", (PyCFunction) PyStruct_deepcopy, METH_NOARGS, "make a deep copy of the struct" }, @@ -921,6 +947,7 @@ static PyMethodDef PyStruct_methods[] = { { "update_from", (PyCFunction) PyStruct_update_from, METH_O, "update from struct. struct must be same type or a derived type. unset fields will be not be copied" }, { "update", (PyCFunction) PyStruct_update, METH_VARARGS | METH_KEYWORDS, "update from key=val. given fields will be set on struct. other fields will remain as is in struct" }, { "all_fields_set", (PyCFunction) PyStruct_all_fields_set, METH_NOARGS, "return true if all fields on the struct are set" }, + { "to_json", (PyCFunction) PyStruct_to_json, METH_VARARGS | METH_KEYWORDS, "return a json string of the struct by recursively converting struct members into json format" }, { NULL} }; diff --git a/cpp/csp/python/PyStructToJson.cpp b/cpp/csp/python/PyStructToJson.cpp new file mode 100644 index 000000000..1226326cc --- /dev/null +++ b/cpp/csp/python/PyStructToJson.cpp @@ -0,0 +1,396 @@ +#include +#include +#include + +namespace csp::python +{ + +// Helper function to convert csp Structs into json objects recursively +rapidjson::Value toJsonRecursive( const StructPtr& self, rapidjson::Document& doc, PyObject * callable ); + +// Helper function to parse some python objects in cpp, this should not be used extensively. +// instead add support for those python types to csp so that they can be handled more generically +// and in a language agnostic way +rapidjson::Value pyObjectToJson( PyObject * value, rapidjson::Document& doc, PyObject * callable, bool is_recursing ); + +// Helper fallback function to convert any type into json format recursively +template +inline rapidjson::Value toJson( const T& val, const CspType& typ, rapidjson::Document& doc, PyObject * callable ) +{ + // Default handler for any unknown T + return rapidjson::Value( val ); +} + +// Helper function to convert Enums into json format recursively +template<> +inline rapidjson::Value toJson( const CspEnum& val, const CspType& typ, rapidjson::Document& doc, PyObject * callable ) +{ + // NOTE: Assume that passed enum has greater lifetime than the json object + return rapidjson::Value( rapidjson::StringRef( val.name() ) ); +} + +// Helper function to convert strings into json format recursively +template<> +inline rapidjson::Value toJson( const std::string& val, const CspType& typ, rapidjson::Document& doc, PyObject * callable ) +{ + // NOTE: Assume that passed string has greater lifetime than the json object + return rapidjson::Value( rapidjson::StringRef( val ) ); +} + +// Helper function to convert TimeDelta into json format recursively +template<> +inline rapidjson::Value toJson( const TimeDelta& val, const CspType& typ, rapidjson::Document& doc, PyObject * callable ) +{ + // Convert TimeDelta to . + // sign( 1 ) + seconds ( 18 ) + '.'( 1 ) + microseconds( 9 ) + '\0'( 1 ) + char buf[32] = {}; + auto seconds = val.abs().asSeconds(); + auto microseconds = static_cast( val.abs().nanoseconds() / NANOS_PER_MICROSECOND ); + auto len = sprintf( buf, "%c%ld.%06u", ( val.sign() >= 0 ) ? '+' : '-', seconds, microseconds ); + rapidjson::Value res; + res.SetString( buf, len, doc.GetAllocator() ); + return res; +} + +// Helper function to convert Date into json format recursively +template<> +inline rapidjson::Value toJson( const Date& val, const CspType& typ, rapidjson::Document& doc, PyObject * callable ) +{ + // Convert Date to -- + // year( 4 ) + '-'( 1 ) + month( 2 ) + '-'( 1 ) + day( 2 ) + '\0'( 1 ) + char buf[32] = {}; + auto len = sprintf( buf, "%04u-%02u-%02u", val.year(), val.month(), val.day() ); + rapidjson::Value res; + res.SetString( buf, len, doc.GetAllocator() ); + return res; +} + +// Helper function to convert Time into json format recursively +template<> +inline rapidjson::Value toJson( const Time& val, const CspType& typ, rapidjson::Document& doc, PyObject * callable ) +{ + // Convert the Time to ::. + // hours( 2 ) + ':'( 1 ) + minutes( 2 ) + ':'( 1 ) + seconds( 2 ) + '.'( 1 ) + micros( 6 ) + '\0'( 1 ) + char buf[48] = {}; + auto micros = static_cast( val.nanosecond() / NANOS_PER_MICROSECOND ); + auto len = sprintf( buf, "%02u:%02u:%02u.%06u", val.hour(), val.minute(), val.second(), micros ); + rapidjson::Value res; + res.SetString( buf, len, doc.GetAllocator() ); + return res; +} + +// Helper function to convert DateTime into json format recursively +template<> +inline rapidjson::Value toJson( const DateTime& val, const CspType& typ, rapidjson::Document& doc, PyObject * callable ) +{ + // Convert the datetime value into an ISO 8601 formatted string + DateTimeEx dtx( val ); + + char iso_str[80] = {}; + static const std::string utc_offset = "+00:00"; + auto micros = static_cast( dtx.nanoseconds() / NANOS_PER_MICROSECOND ); + // Hardcode UTC since all times in csp are UTC + // NOTE: Python's datetime.fromisoformat() API does not support nanoseconds in the string + // Hence we truncate from nanos to micros to allow easy conversion to python datetime objects + auto len = sprintf( iso_str, "%04u-%02u-%02uT%02u:%02u:%02u.%06u%s", + dtx.year(), dtx.month(), dtx.day(), + dtx.hour(), dtx.minute(), dtx.second(), micros, utc_offset.c_str() ); + rapidjson::Value res; + res.SetString( iso_str, len, doc.GetAllocator() ); + return res; +} + +// Helper function to convert csp Structs into json format recursively +template<> +inline rapidjson::Value toJson( const StructPtr& val, const CspType& typ, rapidjson::Document& doc, PyObject * callable ) +{ + return toJsonRecursive( val, doc, callable ); +} + +// Helper function to convert python objects in csp Structs into json format recursively +template<> +inline rapidjson::Value toJson( const DialectGenericType& val, const CspType& typ, rapidjson::Document& doc, PyObject * callable ) +{ + auto py_obj = toPython( val ); + return pyObjectToJson( py_obj, doc, callable, false ); +} + +// Helper function to convert arrays in csp Structs into json lists recursively +template +inline rapidjson::Value toJson( const std::vector& val, const CspType& typ, rapidjson::Document& doc, PyObject * callable ) +{ + using ElemT = typename CspType::Type::toCArrayElemType::type; + + auto const * arrayType = static_cast( &typ ); + const CspType * elemType = arrayType -> elemType().get(); + rapidjson::Value new_list; + new_list.SetArray(); + + for( size_t idx = 0; idx < val.size(); ++idx ) + { + auto sub_json = toJson( val[idx], *elemType, doc, callable ); + new_list.PushBack( sub_json, doc.GetAllocator() ); + } + return new_list; +} + +rapidjson::Value toJsonRecursive( const StructPtr& self, rapidjson::Document& doc, PyObject * callable ) +{ + auto * struct_ptr = self.get(); + if( struct_ptr == nullptr ) + { + CSP_THROW( ValueError, "Cannot call to_json on NULL struct object" ); + } + auto * meta = static_cast( self -> meta() ); + rapidjson::Value new_dict; + new_dict.SetObject(); + + auto& fields = meta -> fields(); + for( const auto& field: fields ) + { + if( !field -> isSet( struct_ptr ) ) + { + continue; + } + auto& key = field -> fieldname(); + auto sub_json = switchCspType( field -> type(), [field, struct_ptr, callable, &doc]( auto tag ) + { + using CType = typename decltype( tag )::type; + auto *typedField = static_cast::type *>( field.get() ); + return toJson( typedField -> value( struct_ptr ), *field -> type(), doc, callable ); + } ); + new_dict.AddMember( rapidjson::StringRef( key ), sub_json, doc.GetAllocator() ); + } + return new_dict; +} + +rapidjson::Value pyDictKeyToName( PyObject * py_key, rapidjson::Document& doc ) +{ + // Only support strings, ints, and floats as keys + // JSON encoding requires all names to be strings so convert them to strings + rapidjson::Value val; + if( PyUnicode_Check( py_key ) ) + { + Py_ssize_t len; + auto str = PyUnicode_AsUTF8AndSize( py_key, &len ); + val.SetString( str, len, doc.GetAllocator() ); + } + else if( PyLong_Check( py_key ) ) + { + auto key = PyLong_AsLong( py_key ); + val.SetString( std::to_string( key ), doc.GetAllocator() ); + } + else if( PyFloat_Check( py_key ) ) + { + auto key = PyFloat_AsDouble( py_key ); + val.SetString( std::to_string( key ), doc.GetAllocator() ); + } + else + { + CSP_THROW( ValueError, "Cannot serialize key of type: " + std::string( Py_TYPE( py_key ) -> tp_name ) ); + // Never reaches here + } + return val; +} + +// Helper function to parse python lists into json arrays recursively +rapidjson::Value pyListToJson( PyObject * py_list, rapidjson::Document& doc, PyObject * callable ) +{ + size_t size = PyList_GET_SIZE( py_list ); + rapidjson::Value new_list; + new_list.SetArray(); + + for( size_t idx = 0; idx < size; ++idx ) + { + auto * item = PyList_GET_ITEM( py_list, idx ); + auto res = pyObjectToJson( item, doc, callable, false ); + new_list.PushBack( res, doc.GetAllocator() ); + } + return new_list; +} + +// Helper function to parse python tuples into json arrays recursively +rapidjson::Value pyTupleToJson( PyObject * py_tuple, rapidjson::Document& doc, PyObject * callable ) +{ + size_t size = PyTuple_GET_SIZE( py_tuple ); + rapidjson::Value new_list; + new_list.SetArray(); + + for( size_t idx = 0; idx < size; ++idx ) + { + auto * item = PyTuple_GetItem( py_tuple, idx ); + auto res = pyObjectToJson( item, doc, callable, false ); + new_list.PushBack( res, doc.GetAllocator() ); + } + return new_list; +} + +// Helper function to parse python dicts into json objects recursively +rapidjson::Value pyDictToJson( PyObject * py_dict, rapidjson::Document& doc, PyObject * callable ) +{ + PyObject * py_key = NULL; + PyObject * py_value = NULL; + Py_ssize_t pos = 0; + rapidjson::Value new_dict; + new_dict.SetObject(); + + while( PyDict_Next( py_dict, &pos, &py_key, &py_value ) ) + { + auto key = pyDictKeyToName( py_key, doc ); + auto res = pyObjectToJson( py_value, doc, callable, false ); + new_dict.AddMember( key, res, doc.GetAllocator() ); + } + return new_dict; +} + +rapidjson::Value pyObjectToJson( PyObject * value, rapidjson::Document& doc, PyObject * callable, bool is_recursing ) +{ + if( PyBool_Check( value ) ) + { + return rapidjson::Value( fromPython( value ) ); + } + else if( PyLong_Check( value ) ) + { + return rapidjson::Value( fromPython( value ) ); + } + else if( PyFloat_Check( value ) ) + { + return rapidjson::Value( fromPython( value ) ); + } + else if( PyUnicode_Check( value ) ) + { + Py_ssize_t len; + auto str = PyUnicode_AsUTF8AndSize( value , &len ); + rapidjson::Value str_val; + str_val.SetString( str, len, doc.GetAllocator() ); + return str_val; + } + else if( PyBytes_Check( value ) ) + { + Py_ssize_t len = PyBytes_Size( value ); + auto str = PyBytes_AsString( value ); + rapidjson::Value str_val; + str_val.SetString( str, len, doc.GetAllocator() ); + return str_val; + } + else if( PyTime_CheckExact( value ) ) + { + auto v = fromPython