Harden array allocation in BitSet::deserialize()

Move bounds check to before the allocation.
Treat NULL bitset values as empty to avoid overflow.

Add tests for excessive allocation, and Null values.

src/misc/bitSet.cpp

@@ -337,17 +337,15 @@ namespace epics { namespace pvData {
     }
 
     void BitSet::deserialize(ByteBuffer* buffer, DeserializableControl* control) {
+        size_t size = SerializeHelper::readSize(buffer, control);
+        uint32 bytes = (size == (size_t) -1) ? 0 : static_cast<uint32>(size);
 
-        uint32 bytes = static_cast<uint32>(SerializeHelper::readSize(buffer, control)); // in bytes
+        // Validate availability before allocating storage
+        control->ensureData(bytes);
 
         size_t wordsInUse = (bytes + 7) / BYTES_PER_WORD;
         words.resize(wordsInUse);
 
-        if (wordsInUse == 0)
-            return;
-
-        control->ensureData(bytes);
-
         uint32 i = 0;
         uint32 longs = bytes / 8;
         while (i < longs)

testApp/misc/testBitSet.cpp

@@ -303,15 +303,57 @@ static void testSerialize()
 #undef TOFRO
 }
 
+static void testDeserializeOversize()
+{
+    testDiag("testDeserializeOversize");
+
+    // Size field declares INT32_MAX bytes but no payload follows: 0xFE
+    // long-form marker + big-endian int32. deserialize() must reject this
+    // via ensureData() before allocating a ~2GB backing store.
+    const char msg[] = {(char)0xFE, 0x7F, (char)0xFF, (char)0xFF, (char)0xFF};
+    BitSet dut;
+    ByteBuffer buf((char*)msg, sizeof(msg), EPICS_ENDIAN_BIG);
+    try {
+        deserializeFromBuffer(&dut, buf);
+        testFail("oversized BitSet size was not rejected");
+    } catch(std::exception& e) {
+        testPass("oversized BitSet size rejected: %s", e.what());
+    }
+}
+
+static void testDeserializeNull()
+{
+    testDiag("testDeserializeNull");
+
+    // A NULL size is encoded on the wire as the single byte 0xFF. Without
+    // the null check in deserialize() this would be cast to UINT32_MAX bytes
+    // and overflow the allocation size; it must instead yield an empty BitSet.
+    const char msg[] = {(char)0xFF};
+    BitSet dut;
+    dut.set(5); // ensure deserialize actually clears/replaces existing bits
+    ByteBuffer buf((char*)msg, sizeof(msg), EPICS_ENDIAN_BIG);
+    try {
+        deserializeFromBuffer(&dut, buf);
+        testPass("NULL BitSet size accepted");
+    } catch(std::exception& e) {
+        testFail("NULL BitSet size rejected: %s", e.what());
+    }
+    testOk1(dut.isEmpty());
+    testOk(buf.getRemaining()==0, "buffer remaining 0 == %u",
+        (unsigned)buf.getRemaining());
+}
+
 } // namespace
 
 MAIN(testBitSet)
 {
-    testPlan(90);
+    testPlan(94);
     testInitialize();
     testGetSetClearFlip();
     testOperators();
     testLogical();
     testSerialize();
+    testDeserializeOversize();
+    testDeserializeNull();
     return testDone();
 }