fs-watcher uses its own database
add test for fs-watcher
Change-Id: Iac54e5d5c748f80099ef798a6515993ab432c725
diff --git a/cmd/csd.cc b/cmd/csd.cc
index a1d4d41..627f06d 100644
--- a/cmd/csd.cc
+++ b/cmd/csd.cc
@@ -53,8 +53,7 @@
FsWatcher watcher (path.c_str (),
bind (&Dispatcher::Did_LocalFile_AddOrModify, &dispatcher, _1),
- bind (&Dispatcher::Did_LocalFile_Delete, &dispatcher, _1),
- dispatcher.GetFileState ());
+ bind (&Dispatcher::Did_LocalFile_Delete, &dispatcher, _1));
return app.exec ();
}
diff --git a/gui/chronosharegui.cpp b/gui/chronosharegui.cpp
index 5889ede..f1a5cf3 100644
--- a/gui/chronosharegui.cpp
+++ b/gui/chronosharegui.cpp
@@ -119,8 +119,7 @@
// Alex: this **must** be here, otherwise m_dirPath will be uninitialized
m_watcher = new FsWatcher (m_dirPath,
bind (&Dispatcher::Did_LocalFile_AddOrModify, m_dispatcher, _1),
- bind (&Dispatcher::Did_LocalFile_Delete, m_dispatcher, _1),
- m_dispatcher->GetFileState ());
+ bind (&Dispatcher::Did_LocalFile_Delete, m_dispatcher, _1));
}
ChronoShareGui::~ChronoShareGui()
diff --git a/gui/fs-watcher.cc b/gui/fs-watcher.cc
index 18d004d..f646a9c 100644
--- a/gui/fs-watcher.cc
+++ b/gui/fs-watcher.cc
@@ -21,6 +21,7 @@
*/
#include "fs-watcher.h"
+#include "db-helper.h"
#include "logging.h"
#include <boost/bind.hpp>
@@ -35,7 +36,6 @@
FsWatcher::FsWatcher (QString dirPath,
LocalFile_Change_Callback onChange, LocalFile_Change_Callback onDelete,
- FileState *fileState,
QObject* parent)
: QObject(parent)
, m_watcher (new QFileSystemWatcher())
@@ -43,11 +43,12 @@
, m_dirPath (dirPath)
, m_onChange (onChange)
, m_onDelete (onDelete)
- , m_fileState (fileState)
{
_LOG_DEBUG ("Monitor dir: " << m_dirPath.toStdString ());
// add main directory to monitor
+ initFileStateDb();
+
m_watcher->addPath (m_dirPath);
// register signals (callback functions)
@@ -57,17 +58,18 @@
m_scheduler->start ();
Scheduler::scheduleOneTimeTask (m_scheduler, 0,
- bind (&FsWatcher::ScanDirectory_NotifyRemovals_Execute, this, m_dirPath, false/* don't remove incomplete files*/),
+ bind (&FsWatcher::ScanDirectory_NotifyRemovals_Execute, this, m_dirPath),
"rescan-r-" + m_dirPath.toStdString ()); // only one task will be scheduled per directory
Scheduler::scheduleOneTimeTask (m_scheduler, 0,
- bind (&FsWatcher::ScanDirectory_NotifyUpdates_Execute, this, m_dirPath, true),
+ bind (&FsWatcher::ScanDirectory_NotifyUpdates_Execute, this, m_dirPath),
"rescan-" +m_dirPath.toStdString ()); // only one task will be scheduled per directory
}
FsWatcher::~FsWatcher()
{
m_scheduler->shutdown ();
+ sqlite3_close(m_db);
}
void
@@ -79,23 +81,23 @@
if (!filesystem::exists (filesystem::path (absPathTriggeredDir)))
{
Scheduler::scheduleOneTimeTask (m_scheduler, 0.5,
- bind (&FsWatcher::ScanDirectory_NotifyRemovals_Execute, this, dirPath, true/* ignore incomplete file flag. the whole directory got removed*/),
+ bind (&FsWatcher::ScanDirectory_NotifyRemovals_Execute, this, dirPath),
"r-" + dirPath.toStdString ()); // only one task will be scheduled per directory
}
else
{
// m_executor.execute (bind (&FsWatcher::ScanDirectory_NotifyUpdates_Execute, this, dirPath));
Scheduler::scheduleOneTimeTask (m_scheduler, 0.5,
- bind (&FsWatcher::ScanDirectory_NotifyUpdates_Execute, this, dirPath, false),
+ bind (&FsWatcher::ScanDirectory_NotifyUpdates_Execute, this, dirPath),
dirPath.toStdString ()); // only one task will be scheduled per directory
// m_executor.execute (bind (&FsWatcher::ScanDirectory_NotifyUpdates_Execute, this, dirPath));
Scheduler::scheduleOneTimeTask (m_scheduler, 300,
- bind (&FsWatcher::ScanDirectory_NotifyUpdates_Execute, this, dirPath, true),
+ bind (&FsWatcher::ScanDirectory_NotifyUpdates_Execute, this, dirPath),
"rescan-"+dirPath.toStdString ()); // only one task will be scheduled per directory
Scheduler::scheduleOneTimeTask (m_scheduler, 300,
- bind (&FsWatcher::ScanDirectory_NotifyRemovals_Execute, this, m_dirPath, false/* don't remove incomplete files*/),
+ bind (&FsWatcher::ScanDirectory_NotifyRemovals_Execute, this, m_dirPath),
"rescan-r-" + m_dirPath.toStdString ()); // only one task will be scheduled per directory
}
}
@@ -105,7 +107,7 @@
{
if (!filePath.startsWith (m_dirPath))
{
- _LOG_ERROR ("Got notification about a file not from the monitored directory");
+ _LOG_ERROR ("Got notification about a file not from the monitored directory: " << filePath.toStdString());
return;
}
filesystem::path absPathTriggeredFile (filePath.toStdString ());
@@ -133,7 +135,7 @@
}
void
-FsWatcher::ScanDirectory_NotifyUpdates_Execute (QString dirPath, bool notifyCallbacks)
+FsWatcher::ScanDirectory_NotifyUpdates_Execute (QString dirPath)
{
_LOG_TRACE (" >> ScanDirectory_NotifyUpdates_Execute");
@@ -169,9 +171,11 @@
relFile.remove (0, m_dirPath.size ());
filesystem::path aFile (relFile.toStdString ());
- if (notifyCallbacks ||
- !m_fileState->LookupFile (aFile.relative_path ().generic_string ()) /* file does not exist there, but exists locally: added */)
+ if (
+ //!m_fileState->LookupFile (aFile.relative_path ().generic_string ()) /* file does not exist there, but exists locally: added */)
+ !fileExists(aFile.relative_path().c_str()) /*file does not exist in db, but exists in fs: add */)
{
+ addFile(aFile.relative_path().c_str());
DidFileChanged (absFilePath);
}
}
@@ -185,7 +189,7 @@
void
-FsWatcher::ScanDirectory_NotifyRemovals_Execute (QString dirPath, bool removeIncomplete)
+FsWatcher::ScanDirectory_NotifyRemovals_Execute (QString dirPath)
{
_LOG_DEBUG ("Triggered DirPath: " << dirPath.toStdString ());
@@ -194,6 +198,7 @@
filesystem::path triggeredDir (dirPath.toStdString ());
+ /*
FileItemsPtr files = m_fileState->LookupFilesInFolderRecursively (triggeredDir.relative_path ().generic_string ());
for (std::list<FileItem>::iterator file = files->begin (); file != files->end (); file ++)
{
@@ -209,8 +214,116 @@
}
}
}
+ */
+
+ vector<string> files;
+ getFilesInDir(triggeredDir.relative_path().c_str(), files);
+ for (vector<string>::iterator file = files.begin(); file != files.end(); file++)
+ {
+ filesystem::path targetFile = filesystem::path (m_dirPath.toStdString()) / file->c_str();
+ if (!filesystem::exists (targetFile))
+ {
+ deleteFile(file->c_str());
+ m_onDelete(file->c_str());
+ }
+ }
}
+const string INIT_DATABASE = "\
+CREATE TABLE IF NOT EXISTS \n\
+ Files( \n\
+ filename TEXT NOT NULL, \n\
+ PRIMARY KEY (filename) \n\
+); \n\
+CREATE INDEX filename_index ON Files (filename); \n\
+";
+
+void
+FsWatcher::initFileStateDb()
+{
+ filesystem::path dbFolder = filesystem::path (m_dirPath.toStdString().c_str()) / ".chronoshare" / "fs_watcher";
+ filesystem::create_directories(dbFolder);
+
+ int res = sqlite3_open((dbFolder / "filestate.db").c_str(), &m_db);
+ if (res != SQLITE_OK)
+ {
+ BOOST_THROW_EXCEPTION(Error::Db() << errmsg_info_str("Cannot open database: " + (dbFolder / "filestate.db").string()));
+ }
+
+ char *errmsg = 0;
+ res = sqlite3_exec(m_db, INIT_DATABASE.c_str(), NULL, NULL, &errmsg);
+ if (res != SQLITE_OK && errmsg != 0)
+ {
+ // _LOG_TRACE ("Init \"error\": " << errmsg);
+ cout << "FS-Watcher DB error: " << errmsg << endl;
+ sqlite3_free (errmsg);
+ }
+}
+
+bool
+FsWatcher::fileExists(const filesystem::path &filename)
+{
+ sqlite3_stmt *stmt;
+ sqlite3_prepare_v2(m_db, "SELECT * FROM Files WHERE filename = ?;", -1, &stmt, 0);
+ sqlite3_bind_text(stmt, 1, filename.c_str(), -1, SQLITE_STATIC);
+ bool retval = false;
+ if (sqlite3_step (stmt) == SQLITE_ROW)
+ {
+ retval = true;
+ }
+ sqlite3_finalize(stmt);
+
+ return retval;
+}
+
+void
+FsWatcher::addFile(const filesystem::path &filename)
+{
+ sqlite3_stmt *stmt;
+ sqlite3_prepare_v2(m_db, "INSERT OR IGNORE INTO Files (filename) VALUES (?);", -1, &stmt, 0);
+ sqlite3_bind_text(stmt, 1, filename.c_str(), -1, SQLITE_STATIC);
+ sqlite3_step(stmt);
+ sqlite3_finalize(stmt);
+}
+
+void
+FsWatcher::deleteFile(const filesystem::path &filename)
+{
+ sqlite3_stmt *stmt;
+ sqlite3_prepare_v2(m_db, "DELETE FROM Files WHERE filename = ?;", -1, &stmt, 0);
+ sqlite3_bind_text(stmt, 1, filename.c_str(), -1, SQLITE_STATIC);
+ sqlite3_step(stmt);
+ sqlite3_finalize(stmt);
+}
+
+void
+FsWatcher::getFilesInDir(const filesystem::path &dir, vector<string> &files)
+{
+ sqlite3_stmt *stmt;
+ sqlite3_prepare_v2(m_db, "SELECT * FROM Files WHERE filename LIKE ?;", -1, &stmt, 0);
+
+ string dirStr = dir.string();
+ ostringstream escapedDir;
+ for (string::const_iterator ch = dirStr.begin (); ch != dirStr.end (); ch ++)
+ {
+ if (*ch == '%')
+ escapedDir << "\\%";
+ else
+ escapedDir << *ch;
+ }
+ escapedDir << "/" << "%";
+ string escapedDirStr = escapedDir.str ();
+ sqlite3_bind_text (stmt, 1, escapedDirStr.c_str (), escapedDirStr.size (), SQLITE_STATIC);
+
+ while(sqlite3_step(stmt) == SQLITE_ROW)
+ {
+ string filename(reinterpret_cast<const char *>(sqlite3_column_text (stmt, 0)), sqlite3_column_bytes(stmt, 0));
+ files.push_back(filename);
+ }
+
+ sqlite3_finalize (stmt);
+
+}
#if WAF
#include "fs-watcher.moc"
diff --git a/gui/fs-watcher.h b/gui/fs-watcher.h
index f10850c..e3c9305 100644
--- a/gui/fs-watcher.h
+++ b/gui/fs-watcher.h
@@ -25,9 +25,9 @@
#include <vector>
#include <QFileSystemWatcher>
#include <boost/filesystem.hpp>
+#include <sqlite3.h>
#include "scheduler.h"
-#include "file-state.h"
class FsWatcher : public QObject
{
@@ -39,7 +39,6 @@
// constructor
FsWatcher (QString dirPath,
LocalFile_Change_Callback onChange, LocalFile_Change_Callback onDelete,
- FileState *fileState,
QObject* parent = 0);
// destructor
@@ -61,10 +60,25 @@
// handle callback from the watcher
// scan directory and notify callback about any file changes
void
- ScanDirectory_NotifyUpdates_Execute (QString dirPath, bool notifyCallbacks);
+ ScanDirectory_NotifyUpdates_Execute (QString dirPath);
void
- ScanDirectory_NotifyRemovals_Execute (QString dirPath, bool removeIncomplete);
+ ScanDirectory_NotifyRemovals_Execute (QString dirPath);
+
+ void
+ initFileStateDb();
+
+ bool
+ fileExists(const boost::filesystem::path &filename);
+
+ void
+ addFile(const boost::filesystem::path &filename);
+
+ void
+ deleteFile(const boost::filesystem::path &filename);
+
+ void
+ getFilesInDir(const boost::filesystem::path &dir, std::vector<std::string> &files);
private:
QFileSystemWatcher* m_watcher; // filesystem watcher
@@ -75,7 +89,7 @@
LocalFile_Change_Callback m_onChange;
LocalFile_Change_Callback m_onDelete;
- FileState *m_fileState;
+ sqlite3 *m_db;
};
#endif // FILESYSTEMWATCHER_H
diff --git a/test/test-fs-watcher.cc b/test/test-fs-watcher.cc
new file mode 100644
index 0000000..10e7b95
--- /dev/null
+++ b/test/test-fs-watcher.cc
@@ -0,0 +1,179 @@
+#include "fs-watcher.h"
+#include <boost/make_shared.hpp>
+#include <boost/filesystem.hpp>
+#include <boost/test/unit_test.hpp>
+#include <boost/thread/thread.hpp>
+#include <boost/bind.hpp>
+#include <boost/lexical_cast.hpp>
+#include <fstream>
+#include <set>
+#include <QtGui>
+
+using namespace std;
+using namespace boost;
+namespace fs = boost::filesystem;
+
+BOOST_AUTO_TEST_SUITE(FsWatcherTests)
+
+void
+onChange(set<string> &files, const fs::path &file)
+{
+ files.insert(file.string());
+}
+
+void
+onDelete(set<string> &files, const fs::path &file)
+{
+ files.erase(file.string());
+}
+
+void create_file( const fs::path & ph, const std::string & contents )
+{
+ std::ofstream f( ph.string().c_str() );
+ if ( !f )
+ {
+ abort();
+ }
+ if ( !contents.empty() )
+ {
+ f << contents;
+ }
+}
+
+void run(fs::path dir, FsWatcher::LocalFile_Change_Callback c, FsWatcher::LocalFile_Change_Callback d)
+{
+ int x = 0;
+ QCoreApplication app (x, 0);
+ FsWatcher watcher (dir.string().c_str(), c, d);
+ app.exec();
+ sleep(100);
+}
+
+BOOST_AUTO_TEST_CASE (TestFsWatcher)
+{
+ fs::path dir = fs::absolute(fs::path("TestFsWatcher"));
+ if (fs::exists(dir))
+ {
+ fs::remove_all(dir);
+ }
+
+ fs::create_directory(dir);
+
+ set<string> files;
+
+ FsWatcher::LocalFile_Change_Callback fileChange = boost::bind(onChange,ref(files), _1);
+ FsWatcher::LocalFile_Change_Callback fileDelete = boost::bind(onDelete, ref(files), _1);
+
+ thread workThread(run, dir, fileChange, fileDelete);
+ //FsWatcher watcher (dir.string().c_str(), fileChange, fileDelete);
+
+ // ============ check create file detection ================
+ create_file(dir / "test.txt", "hello");
+ // have to at least wait 0.5 seconds
+ usleep(600000);
+ // test.txt
+ BOOST_CHECK_EQUAL(files.size(), 1);
+ BOOST_CHECK(files.find("test.txt") != files.end());
+
+ // =========== check create a bunch of files in sub dir =============
+ fs::path subdir = dir / "sub";
+ fs::create_directory(subdir);
+ for (int i = 0; i < 10; i++)
+ {
+ string filename = boost::lexical_cast<string>(i);
+ create_file(subdir / filename.c_str(), boost::lexical_cast<string>(i));
+ }
+ // have to at least wait 0.5 * 2 seconds
+ usleep(1100000);
+ // test.txt
+ // sub/0..9
+ BOOST_CHECK_EQUAL(files.size(), 11);
+ for (int i = 0; i < 10; i++)
+ {
+ string filename = boost::lexical_cast<string>(i);
+ BOOST_CHECK(files.find("sub/" +filename) != files.end());
+ }
+
+ // ============== check copy directory with files to two levels of sub dirs =================
+ fs::create_directory(dir / "sub1");
+ fs::path subdir1 = dir / "sub1" / "sub2";
+ fs::copy_directory(subdir, subdir1);
+ for (int i = 0; i < 5; i++)
+ {
+ string filename = boost::lexical_cast<string>(i);
+ fs::copy_file(subdir / filename.c_str(), subdir1 / filename.c_str());
+ }
+ // have to at least wait 0.5 * 2 seconds
+ usleep(1100000);
+ // test.txt
+ // sub/0..9
+ // sub1/sub2/0..4
+ BOOST_CHECK_EQUAL(files.size(), 16);
+ for (int i = 0; i < 5; i++)
+ {
+ string filename = boost::lexical_cast<string>(i);
+ BOOST_CHECK(files.find("sub1/sub2/" + filename) != files.end());
+ }
+
+ // =============== check remove files =========================
+ for (int i = 0; i < 7; i++)
+ {
+ string filename = boost::lexical_cast<string>(i);
+ fs::remove(subdir / filename.c_str());
+ }
+ usleep(1100000);
+ // test.txt
+ // sub/7..9
+ // sub1/sub2/0..4
+ BOOST_CHECK_EQUAL(files.size(), 9);
+ for (int i = 0; i < 10; i++)
+ {
+ string filename = boost::lexical_cast<string>(i);
+ if (i < 7)
+ BOOST_CHECK(files.find("sub/" + filename) == files.end());
+ else
+ BOOST_CHECK(files.find("sub/" + filename) != files.end());
+ }
+
+ // =================== check remove files again, remove the whole dir this time ===================
+ // before remove check
+ for (int i = 0; i < 5; i++)
+ {
+ string filename = boost::lexical_cast<string>(i);
+ BOOST_CHECK(files.find("sub1/sub2/" + filename) != files.end());
+ }
+ fs::remove_all(subdir1);
+ usleep(1100000);
+ BOOST_CHECK_EQUAL(files.size(), 4);
+ // test.txt
+ // sub/7..9
+ for (int i = 0; i < 5; i++)
+ {
+ string filename = boost::lexical_cast<string>(i);
+ BOOST_CHECK(files.find("sub1/sub2/" + filename) == files.end());
+ }
+
+ // =================== check rename files =======================
+ for (int i = 7; i < 10; i++)
+ {
+ string filename = boost::lexical_cast<string>(i);
+ fs::rename(subdir / filename.c_str(), dir / filename.c_str());
+ }
+ usleep(1100000);
+ // test.txt
+ // 7
+ // 8
+ // 9
+ // sub
+ BOOST_CHECK_EQUAL(files.size(), 4);
+ for (int i = 7; i < 10; i++)
+ {
+ string filename = boost::lexical_cast<string>(i);
+ BOOST_CHECK(files.find("sub/" + filename) == files.end());
+ BOOST_CHECK(files.find(filename) != files.end());
+ }
+
+
+}
+
+BOOST_AUTO_TEST_SUITE_END()
diff --git a/wscript b/wscript
index d514db8..dc59ba5 100644
--- a/wscript
+++ b/wscript
@@ -103,10 +103,11 @@
if bld.env['TEST']:
unittests = bld.program (
target="unit-tests",
- source = bld.path.ant_glob(['test/*.cc']),
- features=['cxx', 'cxxprogram'],
- use = 'BOOST_TEST BOOST_FILESYSTEM LOG4CXX ccnx database chronoshare',
- includes = "ccnx scheduler src executor",
+ features = "qt4 cxx cxxprogram",
+ defines = "WAF",
+ source = bld.path.ant_glob(['test/*.cc', 'gui/fs-watcher.cc']),
+ use = 'BOOST_TEST BOOST_FILESYSTEM LOG4CXX SQLITE3 QTCORE QTGUI ccnx database chronoshare',
+ includes = "ccnx scheduler src executor gui",
)
app_plist = '''<?xml version="1.0" encoding="UTF-8"?>