Check deserialization

src/factory/FieldCreateFactory.cpp

@@ -28,6 +28,8 @@
 #include <pv/thread.h>
 #include <pv/pvData.h>
 
+#include "factoryPvt.h"
+
 using std::tr1::static_pointer_cast;
 using std::size_t;
 using std::string;
@@ -36,7 +38,6 @@ namespace epics { namespace pvData {
 
 size_t Field::num_instances;
 
-
 struct Field::Helper {
     static unsigned hash(Field *fld) {
         std::ostringstream key;
@@ -1476,6 +1477,7 @@ static int decodeScalar(int8 code)
 
 FieldConstPtr FieldCreate::deserialize(ByteBuffer* buffer, DeserializableControl* control) const
 {
+    RecurseGuard RG;
     control->ensureData(1);
     int8 code = buffer->getByte();
     if (code == -1)
@@ -1564,7 +1566,9 @@ FieldConstPtr FieldCreate::deserialize(ByteBuffer* buffer, DeserializableControl
                 throw std::invalid_argument("fixed and bounded structure array not supported");
 
             // Type type = Type.structureArray;
-            StructureConstPtr elementStructure = std::tr1::static_pointer_cast<const Structure>(control->cachedDeserialize(buffer));
+            StructureConstPtr elementStructure = std::tr1::dynamic_pointer_cast<const Structure>(control->cachedDeserialize(buffer));
+            if(!elementStructure)
+                throw std::invalid_argument("Expected Structure");
             // TODO use std::make_shared
             std::tr1::shared_ptr<Field> sp(new StructureArray(elementStructure));
             Helper::cache(this, sp);
@@ -1577,7 +1581,9 @@ FieldConstPtr FieldCreate::deserialize(ByteBuffer* buffer, DeserializableControl
                 throw std::invalid_argument("fixed and bounded structure array not supported");
 
             // Type type = Type.unionArray;
-            UnionConstPtr elementUnion = std::tr1::static_pointer_cast<const Union>(control->cachedDeserialize(buffer));
+            UnionConstPtr elementUnion = std::tr1::dynamic_pointer_cast<const Union>(control->cachedDeserialize(buffer));
+            if(!elementUnion)
+                throw std::invalid_argument("Expected Union");
             // TODO use std::make_shared
             std::tr1::shared_ptr<Field> sp(new UnionArray(elementUnion));
             Helper::cache(this, sp);
@@ -1600,6 +1606,7 @@ FieldConstPtr FieldCreate::deserialize(ByteBuffer* buffer, DeserializableControl
 namespace detail {
 struct field_factory {
     FieldCreatePtr fieldCreate;
+    epicsThreadPrivate<unsigned> deserDepth;
     field_factory() :fieldCreate(new FieldCreate()) {
         registerRefCounter("Field", &Field::num_instances);
         registerRefCounter("Thread", &Thread::num_instances);
@@ -1610,6 +1617,23 @@ struct field_factory {
 static detail::field_factory* field_factory_s;
 static epicsThreadOnceId field_factory_once = EPICS_THREAD_ONCE_INIT;
 
+#if defined(__rtems__) || defined(vxWorks)
+// lower limit on embedded targets
+const unsigned maxDepth = 10;
+#else
+const unsigned maxDepth = 20;
+#endif
+RecurseGuard::RecurseGuard() {
+    size_t current = (size_t)field_factory_s->deserDepth.get();
+    if(current>=maxDepth)
+        throw std::logic_error("FieldCreateFactory recursion depth exceeded");
+    field_factory_s->deserDepth.set((unsigned*)(current+1));
+}
+RecurseGuard::~RecurseGuard() {
+    size_t current = (size_t)field_factory_s->deserDepth.get();
+    field_factory_s->deserDepth.set((unsigned*)(current-1));
+}
+
 static void field_factory_init(void*)
 {
     try {

src/factory/PVStructure.cpp

@@ -18,6 +18,8 @@
 #include <pv/factory.h>
 #include <pv/bitSet.h>
 
+#include "factoryPvt.h"
+
 using std::tr1::static_pointer_cast;
 using std::size_t;
 using std::string;
@@ -224,6 +226,7 @@ void PVStructure::serialize(ByteBuffer *pbuffer,
 
 void PVStructure::deserialize(ByteBuffer *pbuffer,
         DeserializableControl *pcontrol) {
+    RecurseGuard G;
     size_t fieldsSize = pvFields.size();
     for(size_t i = 0; i<fieldsSize; i++)
         pvFields[i]->deserialize(pbuffer, pcontrol);

src/factory/PVStructureArray.cpp

@@ -16,6 +16,8 @@
 #include <pv/factory.h>
 #include <pv/serializeHelper.h>
 
+#include "factoryPvt.h"
+
 using std::tr1::static_pointer_cast;
 using std::size_t;
 
@@ -158,6 +160,7 @@ void PVStructureArray::serialize(ByteBuffer *pbuffer,
 
 void PVStructureArray::deserialize(ByteBuffer *pbuffer,
         DeserializableControl *pcontrol) {
+    RecurseGuard G;
     svector data(reuse());
 
     size_t size = this->getArray()->getArraySizeType() == Array::fixed ?

src/factory/PVUnion.cpp

@@ -18,6 +18,8 @@
 #include <pv/factory.h>
 #include <pv/serializeHelper.h>
 
+#include "factoryPvt.h"
+
 using std::tr1::static_pointer_cast;
 using std::size_t;
 using std::string;
@@ -143,6 +145,7 @@ void PVUnion::serialize(ByteBuffer *pbuffer, SerializableControl *pflusher) cons
 
 void PVUnion::deserialize(ByteBuffer *pbuffer, DeserializableControl *pcontrol)
 {
+    RecurseGuard G;
     if (variant)
     {
         FieldConstPtr field = pcontrol->cachedDeserialize(pbuffer);

src/factory/PVUnionArray.cpp

@@ -16,6 +16,8 @@
 #include <pv/factory.h>
 #include <pv/serializeHelper.h>
 
+#include "factoryPvt.h"
+
 using std::tr1::static_pointer_cast;
 using std::size_t;
 
@@ -157,6 +159,7 @@ void PVUnionArray::serialize(ByteBuffer *pbuffer,
 
 void PVUnionArray::deserialize(ByteBuffer *pbuffer,
         DeserializableControl *pcontrol) {
+    RecurseGuard G;
     svector data(reuse());
 
     size_t size = this->getArray()->getArraySizeType() == Array::fixed ?

src/factory/factoryPvt.h

@@ -0,0 +1,14 @@
+#ifndef FACTORYPVT_H
+#define FACTORYPVT_H
+
+namespace epics { namespace pvData {
+
+// use to bound recursion in FieldCreate and PVDataCreator singletons
+struct RecurseGuard {
+    RecurseGuard();
+    ~RecurseGuard();
+};
+
+}} // namespace epics::pvData
+
+#endif // FACTORYPVT_H

testApp/misc/testSerialization.cpp

@@ -849,11 +849,60 @@ void testFromString(int byteOrder)
     testOk1(_data->getSubFieldT<PVString>("Y")->get()=="testing");
 }
 
+void testDeserializeTypeFail(const char *buf, size_t buflen)
+{
+
+    std::vector<char> bytes(buf, buf + buflen);
+    ByteBuffer B(&bytes[0], bytes.size(), EPICS_ENDIAN_BIG);
+
+    try{
+        FieldConstPtr ptr(getFieldCreate()->deserialize(&B, control));
+    }catch(std::exception& e) {
+        testPass("Expected exception: %s", e.what());
+        return;
+    }
+    testFail("Unexpected success? %d", __LINE__);
+}
+
+void testMalformed()
+{
+    testDiag("testMalformed()");
+
+#define CASE(STR) testDeserializeTypeFail(STR, sizeof(STR)-1)
+    // a structure array where the element type is not a structure
+    CASE("\x88\x83\x2b\x01\x01");
+    // a union array where ...
+    CASE("\x89\x83\x2b\x01\x01");
+#undef CASE
+    {
+        std::vector<char> inp(500u);
+        for(size_t i=0; i<inp.size(); i+=5) {
+            inp[i+0] = '\x80'; // Struct
+            inp[i+1] = '\xff'; // null ID String
+            inp[i+2] = '\x01'; // 1 member field
+            inp[i+3] = '\x01'; // member name length 1
+            inp[i+4] = 'x'   ; // member name
+        }
+        testDeserializeTypeFail(&inp[0], inp.size());
+    }
+    {
+        std::vector<char> inp(500u);
+        for(size_t i=0; i<inp.size(); i+=5) {
+            inp[i+0] = '\x81'; // Union
+            inp[i+1] = '\xff'; // null ID String
+            inp[i+2] = '\x01'; // 1 member field
+            inp[i+3] = '\x01'; // member name length 1
+            inp[i+4] = 'x'   ; // member name
+        }
+        testDeserializeTypeFail(&inp[0], inp.size());
+    }
+}
+
 } // end namespace
 
 MAIN(testSerialization) {
 
-    testPlan(234);
+    testPlan(238);
 
     flusher = new SerializableControlImpl();
     control = new DeserializableControlImpl();
@@ -880,6 +929,8 @@ MAIN(testSerialization) {
     testFromString(EPICS_ENDIAN_BIG);
     testFromString(EPICS_ENDIAN_LITTLE);
 
+    testMalformed();
+
     delete buffer;
     delete control;
     delete flusher;