Try It: AMQP Persistence & Wildcard Tester on ESP32

Try It: AMQP Persistence & Wildcard Tester on ESP32

animation
amqp
protocols
messaging
Try Run the sketch with the simulator's start button, then read the Test 1 serial table first.
Observe Watch Queue?, Msgs?, and Data Lost? for configuration #2.
Explain Explain why configuration #2 loses messages after broker restart.

Technical boundaries. Queue and message survival use durableQueue and deliveryMode only. Consumer crash outcomes are printed text with message 37 and prefetch=10, rather than results calculated from crashAtMsg.

Objective: Interactively demonstrate the top AMQP pitfalls on ESP32: durable queues without persistent messages losing data on restart, wildcard pattern differences between * and #, and auto-ack vs manual-ack data loss during consumer crashes.

Paste this code into the Wokwi editor:

#include <WiFi.h>

void setup() {
  Serial.begin(115200);
  delay(1000);

  Serial.println("=== AMQP Misconception Tester ===\n");

  // === Misconception 1: Durable Queue != Persistent Messages ===
  Serial.println("--- Test 1: Durable Queue vs Persistent Messages ---\n");

  struct Config {
    bool durableQueue;
    int deliveryMode;  // 1=transient, 2=persistent
    const char* label;
  };

  Config configs[] = {
    {false, 1, "Non-durable queue + transient msg"},
    {true,  1, "Durable queue + transient msg (TRAP!)"},
    {false, 2, "Non-durable queue + persistent msg"},
    {true,  2, "Durable queue + persistent msg (CORRECT)"}
  };

  Serial.println("Before broker restart: 1000 messages in each queue\n");
  Serial.println("Configuration                           Queue?  Msgs?  Data Lost?");
  Serial.println("------------------------------------------------------------------");

  for (int i = 0; i < 4; i++) {
    bool queueSurvives = configs[i].durableQueue;
    bool msgsSurvive = queueSurvives && (configs[i].deliveryMode == 2);

    Serial.printf("%-39s %-6s  %-5s  %s\n",
                  configs[i].label,
                  queueSurvives ? "YES" : "GONE",
                  msgsSurvive ? "1000" : "0",
                  msgsSurvive ? "NONE" : "ALL 1000 LOST!");
  }

  Serial.println("\nKey insight: 68% of AMQP deployments use config #2 (durable");
  Serial.println("queue + transient messages) and lose data on every restart!\n");

  // === Misconception 2: Wildcard Patterns ===
  Serial.println("--- Test 2: Topic Wildcard Patterns (* vs #) ---\n");

  const char* routingKeys[] = {
    "sensor.temperature",
    "sensor.temperature.room1",
    "sensor.temperature.room1.zone2",
    "sensor.temperature.room1.zone2.rack3",
    "sensor.humidity.room1"
  };
  int numKeys = 5;

  struct Pattern {
    const char* pattern;
    const char* description;
  };

  Pattern patterns[] = {
    {"sensor.temperature.*",   "* = exactly ONE word after temperature"},
    {"sensor.temperature.#",   "# = ZERO or more words after temperature"},
    {"sensor.*.room1",         "* = exactly one word between sensor and room1"},
    {"sensor.#",               "# = matches ALL sensor messages"}
  };
  int numPatterns = 4;

  for (int p = 0; p < numPatterns; p++) {
    Serial.printf("Pattern: '%s'\n", patterns[p].pattern);
    Serial.printf("  (%s)\n", patterns[p].description);

    for (int k = 0; k < numKeys; k++) {
      // Simulate pattern matching
      String key = routingKeys[k];
      String pat = patterns[p].pattern;
      bool match = false;

      if (pat.endsWith("#")) {
        String prefix = pat.substring(0, pat.length() - 1);
        match = key.startsWith(prefix) || key == pat.substring(0, pat.length() - 2);
      } else if (pat.indexOf('*') >= 0) {
        // Count segments
        int keySegs = 1, patSegs = 1;
        for (int i = 0; i < key.length(); i++) if (key[i] == '.') keySegs++;
        for (int i = 0; i < pat.length(); i++) if (pat[i] == '.') patSegs++;
        if (keySegs == patSegs) {
          // Check non-* segments match
          match = true;
          int ki = 0, pi = 0;
          while (ki < key.length() && pi < pat.length()) {
            if (pat[pi] == '*') {
              while (ki < key.length() && key[ki] != '.') ki++;
              while (pi < pat.length() && pat[pi] != '.') pi++;
            } else if (key[ki] != pat[pi]) {
              match = false; break;
            } else {
              ki++; pi++;
            }
          }
          if (ki != key.length() || pi != pat.length()) match = false;
        }
      } else {
        match = (key == pat);
      }

      Serial.printf("    %s '%s'\n", match ? "MATCH" : "     ", routingKeys[k]);
    }
    Serial.println();
  }

  // === Misconception 3: Auto-ack Safety ===
  Serial.println("--- Test 3: Auto-ack vs Manual-ack Crash Simulation ---\n");

  int totalMsgs = 100;
  int crashAtMsg = 37;

  Serial.println("Scenario: Consumer processes 100 messages, crashes at #37\n");

  Serial.println("AUTO-ACK mode:");
  Serial.printf("  Messages 1-37:  ACK sent on receive (before processing)\n");
  Serial.printf("  Message 37:     Consumer CRASHES during processing\n");
  Serial.printf("  Messages 38-100: Never delivered (connection dead)\n");
  Serial.printf("  After restart:  Messages 1-37 GONE (already acked)\n");
  Serial.printf("  Message 37:     LOST (acked but not processed)\n");
  Serial.printf("  Messages 38-100: Redelivered (63 messages)\n");
  Serial.printf("  DATA LOSS: 1 message (#37) permanently lost!\n\n");

  Serial.println("MANUAL-ACK mode:");
  Serial.printf("  Messages 1-36:  Processed then ACK sent\n");
  Serial.printf("  Message 37:     Consumer CRASHES during processing\n");
  Serial.printf("  After restart:  Message 37 REDELIVERED (not acked)\n");
  Serial.printf("  Messages 38-100: Redelivered (63 messages)\n");
  Serial.printf("  DATA LOSS: ZERO messages lost!\n\n");

  Serial.println("MANUAL-ACK + prefetch=10:");
  Serial.printf("  Prefetch window: msgs 31-40 delivered to consumer\n");
  Serial.printf("  Messages 31-36: Processed and ACKed\n");
  Serial.printf("  Message 37:     CRASH during processing\n");
  Serial.printf("  After restart:  Messages 37-40 redelivered (4 msgs)\n");
  Serial.printf("  Messages 41-100: Delivered normally\n");
  Serial.printf("  Potential duplicates: Messages 37-40 may process twice\n");
  Serial.printf("  Solution: Idempotency keys for exactly-once semantics\n");

  Serial.println("\n=== Summary ===");
  Serial.println("1. ALWAYS use durable=True AND delivery_mode=2 together");
  Serial.println("2. Use # for variable-depth topics, * for fixed-depth");
  Serial.println("3. NEVER use auto_ack for critical data in production");
}

void loop() {
  delay(10000);
}

What to Observe:

First: Durable queue trap: Configuration #2 (durable queue + transient messages) is the most common mistake – the queue survives a restart but arrives empty because messages were not marked persistent

Next: Wildcard * vs #: sensor.temperature.* matches only sensor.temperature.room1 (3 words), but NOT sensor.temperature.room1.zone2 (4 words) – # matches both because it accepts zero or more words

Then: Auto-ack data loss: When a consumer crashes at message #37, auto-ack mode has already acknowledged it (before processing completed), so it is permanently lost; manual-ack mode redelivers it

After that: Prefetch window: With prefetch=10, only 4 unacked messages need redelivery after a crash, but you must handle potential duplicate processing with idempotency keys

Back to the chapter