table: dead Nonce list
refs #1953
Change-Id: I65eb2346716dd47bcf1850c832e37e5354042fd0
diff --git a/.gitignore b/.gitignore
index b54981e..4629c34 100644
--- a/.gitignore
+++ b/.gitignore
@@ -5,3 +5,4 @@
**/*.pyc
build/
VERSION
+unit-tests.conf
diff --git a/daemon/table/dead-nonce-list.cpp b/daemon/table/dead-nonce-list.cpp
new file mode 100644
index 0000000..71dfcb3
--- /dev/null
+++ b/daemon/table/dead-nonce-list.cpp
@@ -0,0 +1,169 @@
+/* -*- Mode:C++; c-file-style:"gnu"; indent-tabs-mode:nil; -*- */
+/**
+ * Copyright (c) 2014, Regents of the University of California,
+ * Arizona Board of Regents,
+ * Colorado State University,
+ * University Pierre & Marie Curie, Sorbonne University,
+ * Washington University in St. Louis,
+ * Beijing Institute of Technology,
+ * The University of Memphis
+ *
+ * This file is part of NFD (Named Data Networking Forwarding Daemon).
+ * See AUTHORS.md for complete list of NFD authors and contributors.
+ *
+ * NFD is free software: you can redistribute it and/or modify it under the terms
+ * of the GNU General Public License as published by the Free Software Foundation,
+ * either version 3 of the License, or (at your option) any later version.
+ *
+ * NFD is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
+ * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+ * PURPOSE. See the GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * NFD, e.g., in COPYING.md file. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "dead-nonce-list.hpp"
+#include "core/city-hash.hpp"
+#include "core/logger.hpp"
+
+NFD_LOG_INIT("DeadNonceList");
+
+namespace nfd {
+
+const time::nanoseconds DeadNonceList::DEFAULT_LIFETIME = time::seconds(6);
+const time::nanoseconds DeadNonceList::MIN_LIFETIME = time::milliseconds(1);
+const size_t DeadNonceList::INITIAL_CAPACITY = (1 << 7);
+const size_t DeadNonceList::MIN_CAPACITY = (1 << 3);
+const size_t DeadNonceList::MAX_CAPACITY = (1 << 24);
+const DeadNonceList::Entry DeadNonceList::MARK = 0;
+const size_t DeadNonceList::EXPECTED_MARK_COUNT = 5;
+const double DeadNonceList::CAPACITY_UP = 1.2;
+const double DeadNonceList::CAPACITY_DOWN = 0.9;
+const size_t DeadNonceList::EVICT_LIMIT = (1 << 6);
+
+DeadNonceList::DeadNonceList(const time::nanoseconds& lifetime)
+ : m_lifetime(lifetime)
+ , m_queue(m_index.get<0>())
+ , m_ht(m_index.get<1>())
+ , m_capacity(INITIAL_CAPACITY)
+ , m_markInterval(m_lifetime / EXPECTED_MARK_COUNT)
+ , m_adjustCapacityInterval(m_lifetime)
+{
+ if (m_lifetime < MIN_LIFETIME) {
+ throw std::invalid_argument("lifetime is less than MIN_LIFETIME");
+ }
+
+ for (size_t i = 0; i < EXPECTED_MARK_COUNT; ++i) {
+ m_queue.push_back(MARK);
+ }
+
+ m_markEvent = scheduler::schedule(m_markInterval, bind(&DeadNonceList::mark, this));
+ m_adjustCapacityEvent = scheduler::schedule(m_adjustCapacityInterval,
+ bind(&DeadNonceList::adjustCapacity, this));
+}
+
+DeadNonceList::~DeadNonceList()
+{
+ scheduler::cancel(m_markEvent);
+ scheduler::cancel(m_adjustCapacityEvent);
+
+ BOOST_ASSERT(MIN_LIFETIME <= DEFAULT_LIFETIME);
+ BOOST_ASSERT(INITIAL_CAPACITY >= MIN_CAPACITY);
+ BOOST_ASSERT(INITIAL_CAPACITY <= MAX_CAPACITY);
+ BOOST_ASSERT(static_cast<size_t>(MIN_CAPACITY * CAPACITY_UP) > MIN_CAPACITY);
+ BOOST_ASSERT(static_cast<size_t>(MAX_CAPACITY * CAPACITY_DOWN) < MAX_CAPACITY);
+ BOOST_ASSERT(CAPACITY_UP > 1.0);
+ BOOST_ASSERT(CAPACITY_DOWN < 1.0);
+ BOOST_ASSERT(EVICT_LIMIT >= 1);
+}
+
+size_t
+DeadNonceList::size() const
+{
+ return m_queue.size() - this->countMarks();
+}
+
+bool
+DeadNonceList::has(const Name& name, uint32_t nonce) const
+{
+ Entry entry = DeadNonceList::makeEntry(name, nonce);
+ return m_ht.find(entry) != m_ht.end();
+}
+
+void
+DeadNonceList::add(const Name& name, uint32_t nonce)
+{
+ Entry entry = this->makeEntry(name, nonce);
+ m_queue.push_back(entry);
+
+ this->evictEntries();
+}
+
+DeadNonceList::Entry
+DeadNonceList::makeEntry(const Name& name, uint32_t nonce)
+{
+ Block nameWire = name.wireEncode();
+ return CityHash64WithSeed(reinterpret_cast<const char*>(nameWire.wire()), nameWire.size(),
+ static_cast<uint64_t>(nonce));
+}
+
+size_t
+DeadNonceList::countMarks() const
+{
+ return m_ht.count(MARK);
+}
+
+void
+DeadNonceList::mark()
+{
+ m_queue.push_back(MARK);
+ size_t nMarks = this->countMarks();
+ m_actualMarkCounts.insert(nMarks);
+
+ NFD_LOG_DEBUG("mark nMarks=" << nMarks);
+
+ scheduler::schedule(m_markInterval, bind(&DeadNonceList::mark, this));
+}
+
+void
+DeadNonceList::adjustCapacity()
+{
+ std::pair<std::multiset<size_t>::iterator, std::multiset<size_t>::iterator> equalRange =
+ m_actualMarkCounts.equal_range(EXPECTED_MARK_COUNT);
+
+ if (equalRange.second == m_actualMarkCounts.begin()) {
+ // all counts are above expected count, adjust down
+ m_capacity = std::max(MIN_CAPACITY,
+ static_cast<size_t>(m_capacity * CAPACITY_DOWN));
+ NFD_LOG_DEBUG("adjustCapacity DOWN capacity=" << m_capacity);
+ }
+ else if (equalRange.first == m_actualMarkCounts.end()) {
+ // all counts are below expected count, adjust up
+ m_capacity = std::min(MAX_CAPACITY,
+ static_cast<size_t>(m_capacity * CAPACITY_UP));
+ NFD_LOG_DEBUG("adjustCapacity UP capacity=" << m_capacity);
+ }
+
+ m_actualMarkCounts.clear();
+
+ this->evictEntries();
+
+ m_adjustCapacityEvent = scheduler::schedule(m_adjustCapacityInterval,
+ bind(&DeadNonceList::adjustCapacity, this));
+}
+
+void
+DeadNonceList::evictEntries()
+{
+ ssize_t nOverCapacity = m_queue.size() - m_capacity;
+ if (nOverCapacity <= 0) // not over capacity
+ return;
+
+ for (ssize_t nEvict = std::min<ssize_t>(nOverCapacity, EVICT_LIMIT); nEvict > 0; --nEvict) {
+ m_queue.erase(m_queue.begin());
+ }
+ BOOST_ASSERT(m_queue.size() >= m_capacity);
+}
+
+} // namespace nfd
diff --git a/daemon/table/dead-nonce-list.hpp b/daemon/table/dead-nonce-list.hpp
new file mode 100644
index 0000000..ad2e0ad
--- /dev/null
+++ b/daemon/table/dead-nonce-list.hpp
@@ -0,0 +1,211 @@
+/* -*- Mode:C++; c-file-style:"gnu"; indent-tabs-mode:nil; -*- */
+/**
+ * Copyright (c) 2014, Regents of the University of California,
+ * Arizona Board of Regents,
+ * Colorado State University,
+ * University Pierre & Marie Curie, Sorbonne University,
+ * Washington University in St. Louis,
+ * Beijing Institute of Technology,
+ * The University of Memphis
+ *
+ * This file is part of NFD (Named Data Networking Forwarding Daemon).
+ * See AUTHORS.md for complete list of NFD authors and contributors.
+ *
+ * NFD is free software: you can redistribute it and/or modify it under the terms
+ * of the GNU General Public License as published by the Free Software Foundation,
+ * either version 3 of the License, or (at your option) any later version.
+ *
+ * NFD is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
+ * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+ * PURPOSE. See the GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * NFD, e.g., in COPYING.md file. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef NFD_DAEMON_TABLE_DEAD_NONCE_LIST_HPP
+#define NFD_DAEMON_TABLE_DEAD_NONCE_LIST_HPP
+
+#include "common.hpp"
+#include <boost/multi_index_container.hpp>
+#include <boost/multi_index/sequenced_index.hpp>
+#include <boost/multi_index/hashed_index.hpp>
+#include "core/scheduler.hpp"
+
+namespace nfd {
+
+/** \brief represents the Dead Nonce list
+ *
+ * The Dead Nonce List is a global table that supplements PIT for loop detection.
+ * When a Nonce is erased (dead) from PIT entry, the Nonce and the Interest Name is added to
+ * Dead Nonce List, and kept for a duration in which most loops are expected to have occured.
+ *
+ * To reduce memory usage, the Interest Name and Nonce are stored as a 64-bit hash.
+ * There could be false positives (non-looping Interest could be considered looping),
+ * but the probability is small, and the error is recoverable when consumer retransmits
+ * with a different Nonce.
+ *
+ * To reduce memory usage, entries do not have associated timestamps. Instead,
+ * lifetime of entries is controlled by dynamically adjusting the capacity of the container.
+ * At fixed intervals, the MARK, an entry with a special value, is inserted into the container.
+ * The number of MARKs stored in the container reflects the lifetime of entries,
+ * because MARKs are inserted at fixed intervals.
+ */
+class DeadNonceList : noncopyable
+{
+public:
+ /** \brief constructs the Dead Nonce List
+ * \param lifetime duration of the expected lifetime of each nonce,
+ * must be no less than MIN_LIFETIME.
+ * This should be set to the duration in which most loops would have occured.
+ * A loop cannot be detected if delay of the cycle is greater than lifetime.
+ * \throw std::invalid_argument if lifetime is less than MIN_LIFETIME
+ */
+ explicit
+ DeadNonceList(const time::nanoseconds& lifetime = DEFAULT_LIFETIME);
+
+ ~DeadNonceList();
+
+ /** \brief determines if name+nonce exists
+ * \return true if name+nonce exists
+ */
+ bool
+ has(const Name& name, uint32_t nonce) const;
+
+ /** \brief records name+nonce
+ */
+ void
+ add(const Name& name, uint32_t nonce);
+
+ /** \return number of stored Nonces
+ * \note The return value does not contain non-Nonce entries in the index, if any.
+ */
+ size_t
+ size() const;
+
+private: // Entry and Index
+ typedef uint64_t Entry;
+
+ static Entry
+ makeEntry(const Name& name, uint32_t nonce);
+
+ typedef boost::multi_index_container<
+ Entry,
+ boost::multi_index::indexed_by<
+ boost::multi_index::sequenced<>,
+ boost::multi_index::hashed_non_unique<
+ boost::multi_index::identity<Entry>
+ >
+ >
+ > Index;
+
+ typedef Index::nth_index<0>::type Queue;
+ typedef Index::nth_index<1>::type Hashtable;
+
+private: // actual lifetime estimation and capacity control
+ /** \return number of MARKs in the index
+ */
+ size_t
+ countMarks() const;
+
+ /** \brief add a MARK, then record number of MARKs in m_actualMarkCounts
+ */
+ void
+ mark();
+
+ /** \brief adjust capacity according to m_actualMarkCounts
+ *
+ * If all counts are above EXPECTED_MARK_COUNT, reduce capacity to m_capacity * CAPACITY_DOWN.
+ * If all counts are below EXPECTED_MARK_COUNT, increase capacity to m_capacity * CAPACITY_UP.
+ */
+ void
+ adjustCapacity();
+
+ /** \brief evict some entries if index is over capacity
+ */
+ void
+ evictEntries();
+
+public:
+ /// default entry lifetime
+ static const time::nanoseconds DEFAULT_LIFETIME;
+
+ /// minimum entry lifetime
+ static const time::nanoseconds MIN_LIFETIME;
+
+private:
+ time::nanoseconds m_lifetime;
+ Index m_index;
+ Queue& m_queue;
+ Hashtable& m_ht;
+
+PUBLIC_WITH_TESTS_ELSE_PRIVATE: // actual lifetime estimation and capacity control
+
+ // ---- current capacity and hard limits
+
+ /** \brief current capacity of index
+ *
+ * The index size is maintained to be near this capacity.
+ *
+ * The capacity is adjusted so that every Entry is expected to be kept for m_lifetime.
+ * This is achieved by mark() and adjustCapacity().
+ */
+ size_t m_capacity;
+
+ static const size_t INITIAL_CAPACITY;
+
+ /** \brief minimum capacity
+ *
+ * This is to ensure correct algorithm operations.
+ */
+ static const size_t MIN_CAPACITY;
+
+ /** \brief maximum capacity
+ *
+ * This is to limit memory usage.
+ */
+ static const size_t MAX_CAPACITY;
+
+ // ---- actual entry lifetime estimation
+
+ /** \brief the MARK for capacity
+ *
+ * The MARK doesn't have a distinct type.
+ * Entry is a hash. The hash function should have non-invertible property,
+ * so it's unlikely for a usual Entry to have collision with the MARK.
+ */
+ static const Entry MARK;
+
+ /** \brief expected number of MARKs in the index
+ */
+ static const size_t EXPECTED_MARK_COUNT;
+
+ /** \brief number of MARKs in the index after each MARK insertion
+ *
+ * adjustCapacity uses this to determine whether and how to adjust capcity,
+ * and then clears this list.
+ */
+ std::multiset<size_t> m_actualMarkCounts;
+
+ time::nanoseconds m_markInterval;
+
+ scheduler::EventId m_markEvent;
+
+ // ---- capacity adjustments
+
+ static const double CAPACITY_UP;
+
+ static const double CAPACITY_DOWN;
+
+ time::nanoseconds m_adjustCapacityInterval;
+
+ scheduler::EventId m_adjustCapacityEvent;
+
+ /** \brief maximum number of entries to evict at each operation if index is over capacity
+ */
+ static const size_t EVICT_LIMIT;
+};
+
+} // namespace nfd
+
+#endif // NFD_DAEMON_TABLE_DEAD_NONCE_LIST_HPP
diff --git a/tests/daemon/table/dead-nonce-list.cpp b/tests/daemon/table/dead-nonce-list.cpp
new file mode 100644
index 0000000..c8a59bb
--- /dev/null
+++ b/tests/daemon/table/dead-nonce-list.cpp
@@ -0,0 +1,153 @@
+/* -*- Mode:C++; c-file-style:"gnu"; indent-tabs-mode:nil; -*- */
+/**
+ * Copyright (c) 2014, Regents of the University of California,
+ * Arizona Board of Regents,
+ * Colorado State University,
+ * University Pierre & Marie Curie, Sorbonne University,
+ * Washington University in St. Louis,
+ * Beijing Institute of Technology,
+ * The University of Memphis
+ *
+ * This file is part of NFD (Named Data Networking Forwarding Daemon).
+ * See AUTHORS.md for complete list of NFD authors and contributors.
+ *
+ * NFD is free software: you can redistribute it and/or modify it under the terms
+ * of the GNU General Public License as published by the Free Software Foundation,
+ * either version 3 of the License, or (at your option) any later version.
+ *
+ * NFD is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
+ * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+ * PURPOSE. See the GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * NFD, e.g., in COPYING.md file. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "table/dead-nonce-list.hpp"
+
+#include "tests/test-common.hpp"
+#include "tests/limited-io.hpp"
+
+namespace nfd {
+namespace tests {
+
+BOOST_FIXTURE_TEST_SUITE(TableDeadNonceList, BaseFixture)
+
+BOOST_AUTO_TEST_CASE(Basic)
+{
+ Name nameA("ndn:/A");
+ Name nameB("ndn:/B");
+ const uint32_t nonce1 = 0x53b4eaa8;
+ const uint32_t nonce2 = 0x1f46372b;
+
+ DeadNonceList dnl;
+
+ BOOST_CHECK_EQUAL(dnl.has(nameA, nonce1), false);
+ dnl.add(nameA, nonce1);
+ BOOST_CHECK_EQUAL(dnl.has(nameA, nonce1), true);
+ BOOST_CHECK_EQUAL(dnl.has(nameA, nonce2), false);
+ BOOST_CHECK_EQUAL(dnl.has(nameB, nonce1), false);
+}
+
+/// A Fixture that periodically inserts Nonces
+class PeriodicalInsertionFixture : public BaseFixture
+{
+protected:
+ PeriodicalInsertionFixture()
+ : dnl(LIFETIME)
+ , name("ndn:/N")
+ , addNonceBatch(0)
+ , addNonceInterval(LIFETIME / DeadNonceList::EXPECTED_MARK_COUNT)
+ {
+ this->addNonce();
+ }
+
+ ~PeriodicalInsertionFixture()
+ {
+ scheduler::cancel(addNonceEvent);
+ }
+
+ void
+ setRate(size_t nNoncesPerLifetime)
+ {
+ addNonceBatch = nNoncesPerLifetime / DeadNonceList::EXPECTED_MARK_COUNT;
+ }
+
+ void
+ addNonce()
+ {
+ for (size_t i = 0; i < addNonceBatch; ++i) {
+ dnl.add(name, ++lastNonce);
+ }
+
+ scheduler::cancel(addNonceEvent); // avoid double schedules
+ if (addNonceInterval > time::nanoseconds::zero()) {
+ addNonceEvent = scheduler::schedule(addNonceInterval,
+ bind(&PeriodicalInsertionFixture::addNonce, this));
+ }
+ }
+
+protected:
+ static const time::nanoseconds LIFETIME;
+ DeadNonceList dnl;
+ Name name;
+ uint32_t lastNonce;
+ size_t addNonceBatch;
+ time::nanoseconds addNonceInterval;
+ scheduler::EventId addNonceEvent;
+};
+const time::nanoseconds PeriodicalInsertionFixture::LIFETIME = time::milliseconds(200);
+
+BOOST_FIXTURE_TEST_CASE(Lifetime, PeriodicalInsertionFixture)
+{
+ LimitedIo limitedIo;
+
+ const int RATE = DeadNonceList::INITIAL_CAPACITY / 2;
+ this->setRate(RATE);
+ limitedIo.defer(LIFETIME * 10);
+
+ Name nameC("ndn:/C");
+ const uint32_t nonceC = 0x25390656;
+ BOOST_CHECK_EQUAL(dnl.has(nameC, nonceC), false);
+ dnl.add(nameC, nonceC);
+ BOOST_CHECK_EQUAL(dnl.has(nameC, nonceC), true);
+
+ limitedIo.defer(LIFETIME / 2); // -50%, entry should exist
+ BOOST_CHECK_EQUAL(dnl.has(nameC, nonceC), true);
+
+ limitedIo.defer(LIFETIME); // +50%, entry should be gone
+ BOOST_CHECK_EQUAL(dnl.has(nameC, nonceC), false);
+}
+
+BOOST_FIXTURE_TEST_CASE(CapacityDown, PeriodicalInsertionFixture)
+{
+ LimitedIo limitedIo;
+
+ ssize_t cap0 = dnl.m_capacity;
+
+ const int RATE = DeadNonceList::INITIAL_CAPACITY / 3;
+ this->setRate(RATE);
+ limitedIo.defer(LIFETIME * 10);
+
+ ssize_t cap1 = dnl.m_capacity;
+ BOOST_CHECK_LT(std::abs(cap1 - RATE), std::abs(cap0 - RATE));
+}
+
+BOOST_FIXTURE_TEST_CASE(CapacityUp, PeriodicalInsertionFixture)
+{
+ LimitedIo limitedIo;
+
+ ssize_t cap0 = dnl.m_capacity;
+
+ const int RATE = DeadNonceList::INITIAL_CAPACITY * 3;
+ this->setRate(RATE);
+ limitedIo.defer(LIFETIME * 10);
+
+ ssize_t cap1 = dnl.m_capacity;
+ BOOST_CHECK_LT(std::abs(cap1 - RATE), std::abs(cap0 - RATE));
+}
+
+BOOST_AUTO_TEST_SUITE_END()
+
+} // namespace tests
+} // namespace nfd
diff --git a/tests/limited-io.hpp b/tests/limited-io.hpp
index cb81247..92aef02 100644
--- a/tests/limited-io.hpp
+++ b/tests/limited-io.hpp
@@ -1,11 +1,12 @@
/* -*- Mode:C++; c-file-style:"gnu"; indent-tabs-mode:nil; -*- */
/**
- * Copyright (c) 2014 Regents of the University of California,
- * Arizona Board of Regents,
- * Colorado State University,
- * University Pierre & Marie Curie, Sorbonne University,
- * Washington University in St. Louis,
- * Beijing Institute of Technology
+ * Copyright (c) 2014, Regents of the University of California,
+ * Arizona Board of Regents,
+ * Colorado State University,
+ * University Pierre & Marie Curie, Sorbonne University,
+ * Washington University in St. Louis,
+ * Beijing Institute of Technology,
+ * The University of Memphis
*
* This file is part of NFD (Named Data Networking Forwarding Daemon).
* See AUTHORS.md for complete list of NFD authors and contributors.
@@ -20,7 +21,7 @@
*
* You should have received a copy of the GNU General Public License along with
* NFD, e.g., in COPYING.md file. If not, see <http://www.gnu.org/licenses/>.
- **/
+ */
#ifndef NFD_TESTS_LIMITED_IO_HPP
#define NFD_TESTS_LIMITED_IO_HPP
@@ -33,7 +34,7 @@
/** \brief provides IO operations limit and/or time limit for unit testing
*/
-class LimitedIo
+class LimitedIo : noncopyable
{
public:
LimitedIo();
@@ -41,7 +42,7 @@
/// indicates why .run returns
enum StopReason
{
- /// g_io.run() runs normally because there's no work to do
+ /// g_io.run() returns normally because there's no work to do
NO_WORK,
/// .afterOp() has been invoked nOpsLimit times
EXCEED_OPS,
@@ -66,6 +67,16 @@
const std::exception&
getLastException() const;
+ /** \brief defer for specified duration
+ *
+ * equivalent to .run(UNLIMITED_OPS, d)
+ */
+ void
+ defer(const time::nanoseconds& d)
+ {
+ this->run(UNLIMITED_OPS, d);
+ }
+
private:
void
afterTimeout();