Adding is_complete flag to FileState and enabling fs-watcher to use this flag in some (not all) cases
Change-Id: I9b02ef933074ec5313af9a84b5b465e8af6e80ea
diff --git a/gui/fs-watcher.cc b/gui/fs-watcher.cc
index 04e761f..a4b46ef 100644
--- a/gui/fs-watcher.cc
+++ b/gui/fs-watcher.cc
@@ -57,12 +57,12 @@
m_scheduler->start ();
Scheduler::scheduleOneTimeTask (m_scheduler, 0,
- bind (&FsWatcher::ScanDirectory_NotifyRemovals_Execute, this, m_dirPath),
- "r-" + m_dirPath.toStdString ()); // only one task will be scheduled per directory
+ bind (&FsWatcher::ScanDirectory_NotifyRemovals_Execute, this, m_dirPath, false/* don't remove incomplete files*/),
+ "rescan-r-" + m_dirPath.toStdString ()); // only one task will be scheduled per directory
Scheduler::scheduleOneTimeTask (m_scheduler, 0,
bind (&FsWatcher::ScanDirectory_NotifyUpdates_Execute, this, m_dirPath, true),
- m_dirPath.toStdString ()); // only one task will be scheduled per directory
+ "rescan-" +m_dirPath.toStdString ()); // only one task will be scheduled per directory
}
FsWatcher::~FsWatcher()
@@ -79,7 +79,7 @@
if (!filesystem::exists (filesystem::path (absPathTriggeredDir)))
{
Scheduler::scheduleOneTimeTask (m_scheduler, 0.5,
- bind (&FsWatcher::ScanDirectory_NotifyRemovals_Execute, this, dirPath),
+ bind (&FsWatcher::ScanDirectory_NotifyRemovals_Execute, this, dirPath, true/* ignore incomplete file flag. the whole directory got removed*/),
"r-" + dirPath.toStdString ()); // only one task will be scheduled per directory
}
else
@@ -93,6 +93,10 @@
Scheduler::scheduleOneTimeTask (m_scheduler, 300,
bind (&FsWatcher::ScanDirectory_NotifyUpdates_Execute, this, dirPath, true),
"rescan-"+dirPath.toStdString ()); // only one task will be scheduled per directory
+
+ Scheduler::scheduleOneTimeTask (m_scheduler, 300,
+ bind (&FsWatcher::ScanDirectory_NotifyRemovals_Execute, this, m_dirPath, false/* don't remove incomplete files*/),
+ "rescan-r-" + m_dirPath.toStdString ()); // only one task will be scheduled per directory
}
}
@@ -173,7 +177,7 @@
void
-FsWatcher::ScanDirectory_NotifyRemovals_Execute (QString dirPath)
+FsWatcher::ScanDirectory_NotifyRemovals_Execute (QString dirPath, bool removeIncomplete)
{
_LOG_DEBUG ("Triggered DirPath: " << dirPath.toStdString ());
@@ -190,137 +194,15 @@
if (!filesystem::exists (testFile))
{
- _LOG_DEBUG ("Notifying about removed file [" << file->filename () << "]");
- m_onDelete (file->filename ());
+ if (removeIncomplete || file->is_complete ())
+ {
+ _LOG_DEBUG ("Notifying about removed file [" << file->filename () << "]");
+ m_onDelete (file->filename ());
+ }
}
}
}
-// std::vector<sEventInfo> FsWatcher::reconcileDirectory(QHash<QString, qint64> currentState, QString dirPath)
-// {
-// // list of files changed
-// std::vector<sEventInfo> dirChanges;
-
-// // compare result (database/stored snapshot) to fileList (current snapshot)
-// QMutableHashIterator<QString, qint64> i(m_storedState);
-
-// while(i.hasNext())
-// {
-// i.next();
-
-// QString absFilePath = i.key();
-// qint64 storedCreated = i.value();
-
-// // if this file is in a level higher than
-// // this directory, ignore
-// if(!absFilePath.startsWith(dirPath))
-// {
-// continue;
-// }
-
-// // check file existence
-// if(currentState.contains(absFilePath))
-// {
-// qint64 currentCreated = currentState.value(absFilePath);
-
-// if(storedCreated != currentCreated)
-// {
-// // update stored state
-// i.setValue(currentCreated);
-
-// // this file has been modified
-// sEventInfo eventInfo;
-// eventInfo.event = MODIFIED;
-// eventInfo.absFilePath = absFilePath.toStdString();
-// dirChanges.push_back(eventInfo);
-// }
-
-// // delete this file from fileList we have processed it
-// currentState.remove(absFilePath);
-// }
-// else
-// {
-// // delete from stored state
-// i.remove();
-
-// // this file has been deleted
-// sEventInfo eventInfo;
-// eventInfo.event = DELETED;
-// eventInfo.absFilePath = absFilePath.toStdString();
-// dirChanges.push_back(eventInfo);
-// }
-// }
-
-// // any files left in fileList have been added
-// for(QHash<QString, qint64>::iterator i = currentState.begin(); i != currentState.end(); ++i)
-// {
-// QString absFilePath = i.key();
-// qint64 currentCreated = i.value();
-
-// m_storedState.insert(absFilePath, currentCreated);
-
-// // this file has been added
-// sEventInfo eventInfo;
-// eventInfo.event = ADDED;
-// eventInfo.absFilePath = absFilePath.toStdString();
-// dirChanges.push_back(eventInfo);
-// }
-
-// return dirChanges;
-// }
-
-// QByteArray FsWatcher::calcChecksum(QString absFilePath)
-// {
-// // initialize checksum
-// QCryptographicHash crypto(QCryptographicHash::Md5);
-
-// // open file
-// QFile file(absFilePath);
-// file.open(QFile::ReadOnly);
-
-// // calculate checksum
-// while(!file.atEnd())
-// {
-// crypto.addData(file.read(8192));
-// }
-
-// return crypto.result();
-// }
-
-// void FsWatcher::printChanges(std::vector<sEventInfo> dirChanges)
-// {
-// if(!dirChanges.empty())
-// {
-// for(size_t i = 0; i < dirChanges.size(); i++)
-// {
-// QString tempString;
-
-// eEvent event = dirChanges[i].event;
-// QString absFilePath = QString::fromStdString(dirChanges[i].absFilePath);
-
-// switch(event)
-// {
-// case ADDED:
-// tempString.append("ADDED: ");
-// break;
-// case MODIFIED:
-// tempString.append("MODIFIED: ");
-// break;
-// case DELETED:
-// tempString.append("DELETED: ");
-// break;
-// }
-
-// tempString.append(absFilePath);
-
-// _LOG_DEBUG ("\t" << tempString.toStdString ());
-// }
-// }
-// else
-// {
-// _LOG_DEBUG ("\t[EMPTY]");
-// }
-// }
#if WAF
#include "fs-watcher.moc"
diff --git a/gui/fs-watcher.h b/gui/fs-watcher.h
index 6f5e349..f10850c 100644
--- a/gui/fs-watcher.h
+++ b/gui/fs-watcher.h
@@ -64,17 +64,7 @@
ScanDirectory_NotifyUpdates_Execute (QString dirPath, bool notifyCallbacks);
void
- ScanDirectory_NotifyRemovals_Execute (QString dirPath);
-
- // // reconcile directory, find changes
- // std::vector<sEventInfo>
- // reconcileDirectory (QHash<QString, qint64> fileList, QString dirPath);
-
- // // calculate checksum
- // QByteArray calcChecksum(QString absFilePath);
-
- // // print Changes (DEBUG)
- // void printChanges(std::vector<sEventInfo> dirChanges);
+ ScanDirectory_NotifyRemovals_Execute (QString dirPath, bool removeIncomplete);
private:
QFileSystemWatcher* m_watcher; // filesystem watcher
diff --git a/src/action-log.cc b/src/action-log.cc
index 292a68c..5514022 100644
--- a/src/action-log.cc
+++ b/src/action-log.cc
@@ -93,6 +93,7 @@
file_ctime TIMESTAMP, \n\
file_chmod INTEGER, \n\
file_seg_num INTEGER, \n\
+ is_complete INTEGER, \n\
\n\
PRIMARY KEY (type, filename) \n\
); \n\
@@ -146,6 +147,9 @@
sqlite3_exec (m_db, "ALTER TABLE FileState ADD COLUMN directory TEXT", NULL, NULL, NULL);
sqlite3_exec (m_db, "CREATE INDEX FileState_directory ON FileState (directory)", NULL, NULL, NULL);
sqlite3_exec (m_db, "UPDATE FileState SET directory = directory_name(filename) WHERE directory IS NULL", NULL, NULL, NULL);
+
+ // Another "upgrade"
+ sqlite3_exec (m_db, "BEGIN TRANSACTION; ALTER TABLE FileState ADD COLUMN is_complete INTEGER; UPDATE FileState SET is_complete = 1; END TRANSACTION;", NULL, NULL, NULL);
}
tuple<sqlite3_int64 /*version*/, Ccnx::CcnxCharbufPtr /*device name*/, sqlite3_int64 /*seq_no*/>
@@ -687,6 +691,23 @@
sqlite3_result_null (context);
}
+
+void
+ActionLog::SetFileComplete (const std::string &filename)
+{
+ sqlite3_stmt *stmt;
+ sqlite3_prepare_v2 (m_db,
+ "UPDATE FileState SET is_complete=1 WHERE type = 0 AND filename = ?", -1, &stmt, 0);
+ _LOG_DEBUG_COND (sqlite3_errcode (m_db) != SQLITE_OK, sqlite3_errmsg (m_db));
+ sqlite3_bind_text(stmt, 1, filename.c_str(), -1, SQLITE_STATIC);
+
+ sqlite3_step (stmt);
+ _LOG_DEBUG_COND (sqlite3_errcode (m_db) != SQLITE_DONE, sqlite3_errmsg (m_db));
+
+ sqlite3_finalize (stmt);
+}
+
+
/**
* @todo Implement checking modification time and permissions
*/
@@ -695,9 +716,10 @@
{
sqlite3_stmt *stmt;
sqlite3_prepare_v2 (m_db,
- "SELECT filename,device_name,seq_no,file_hash,strftime('%s', file_mtime),file_chmod,file_seg_num "
+ "SELECT filename,device_name,seq_no,file_hash,strftime('%s', file_mtime),file_chmod,file_seg_num,is_complete "
" FROM FileState "
" WHERE type = 0 AND filename = ?", -1, &stmt, 0);
+ _LOG_DEBUG_COND (sqlite3_errcode (m_db) != SQLITE_OK, sqlite3_errmsg (m_db));
sqlite3_bind_text(stmt, 1, filename.c_str(), -1, SQLITE_STATIC);
FileItemPtr retval;
@@ -711,7 +733,9 @@
retval->set_mtime (sqlite3_column_int (stmt, 4));
retval->set_mode (sqlite3_column_int (stmt, 5));
retval->set_seg_num (sqlite3_column_int64 (stmt, 6));
+ retval->set_is_complete (sqlite3_column_int (stmt, 7));
}
+ _LOG_DEBUG_COND (sqlite3_errcode (m_db) != SQLITE_DONE, sqlite3_errmsg (m_db));
sqlite3_finalize (stmt);
return retval;
@@ -722,10 +746,12 @@
{
sqlite3_stmt *stmt;
sqlite3_prepare_v2 (m_db,
- "SELECT filename,device_name,seq_no,file_hash,strftime('%s', file_mtime),file_chmod,file_seg_num "
+ "SELECT filename,device_name,seq_no,file_hash,strftime('%s', file_mtime),file_chmod,file_seg_num,is_complete "
" FROM FileState "
" WHERE type = 0 AND file_hash = ?", -1, &stmt, 0);
+ _LOG_DEBUG_COND (sqlite3_errcode (m_db) != SQLITE_OK, sqlite3_errmsg (m_db));
sqlite3_bind_blob(stmt, 1, hash.GetHash (), hash.GetHashBytes (), SQLITE_STATIC);
+ _LOG_DEBUG_COND (sqlite3_errcode (m_db) != SQLITE_OK, sqlite3_errmsg (m_db));
FileItemsPtr retval = make_shared<FileItems> ();
while (sqlite3_step (stmt) == SQLITE_ROW)
@@ -738,9 +764,11 @@
file.set_mtime (sqlite3_column_int (stmt, 4));
file.set_mode (sqlite3_column_int (stmt, 5));
file.set_seg_num (sqlite3_column_int64 (stmt, 6));
+ file.set_is_complete (sqlite3_column_int (stmt, 7));
retval->push_back (file);
}
+ _LOG_DEBUG_COND (sqlite3_errcode (m_db) != SQLITE_DONE, sqlite3_errmsg (m_db));
sqlite3_finalize (stmt);
@@ -753,7 +781,7 @@
{
sqlite3_stmt *stmt;
sqlite3_prepare_v2 (m_db,
- "SELECT filename,device_name,seq_no,file_hash,strftime('%s', file_mtime),file_chmod,file_seg_num "
+ "SELECT filename,device_name,seq_no,file_hash,strftime('%s', file_mtime),file_chmod,file_seg_num,is_complete "
" FROM FileState "
" WHERE type = 0 AND directory = ?", -1, &stmt, 0);
sqlite3_bind_text (stmt, 1, folder.c_str (), folder.size (), SQLITE_STATIC);
@@ -769,6 +797,7 @@
file.set_mtime (sqlite3_column_int (stmt, 4));
file.set_mode (sqlite3_column_int (stmt, 5));
file.set_seg_num (sqlite3_column_int64 (stmt, 6));
+ file.set_is_complete (sqlite3_column_int (stmt, 7));
retval->push_back (file);
}
@@ -789,7 +818,7 @@
if (folder != "")
{
sqlite3_prepare_v2 (m_db,
- "SELECT filename,device_name,seq_no,file_hash,strftime('%s', file_mtime),file_chmod,file_seg_num "
+ "SELECT filename,device_name,seq_no,file_hash,strftime('%s', file_mtime),file_chmod,file_seg_num,is_complete "
" FROM FileState "
" WHERE type = 0 AND (directory = ? OR directory LIKE ?)", -1, &stmt, 0);
_LOG_DEBUG_COND (sqlite3_errcode (m_db) != SQLITE_OK, sqlite3_errmsg (m_db));
@@ -814,7 +843,7 @@
else
{
sqlite3_prepare_v2 (m_db,
- "SELECT filename,device_name,seq_no,file_hash,strftime('%s', file_mtime),file_chmod,file_seg_num "
+ "SELECT filename,device_name,seq_no,file_hash,strftime('%s', file_mtime),file_chmod,file_seg_num,is_complete "
" FROM FileState "
" WHERE type = 0", -1, &stmt, 0);
}
@@ -832,6 +861,7 @@
file.set_mtime (sqlite3_column_int (stmt, 4));
file.set_mode (sqlite3_column_int (stmt, 5));
file.set_seg_num (sqlite3_column_int64 (stmt, 6));
+ file.set_is_complete (sqlite3_column_int (stmt, 7));
retval->push_back (file);
}
diff --git a/src/action-log.h b/src/action-log.h
index f28a398..9c6fdd6 100644
--- a/src/action-log.h
+++ b/src/action-log.h
@@ -100,6 +100,14 @@
ActionItemPtr
LookupAction (const Ccnx::Name &actionName);
+ /**
+ * @brief Set "complete" flag
+ *
+ * The call will do nothing if FileState does not have a record for the file (e.g., file got subsequently deleted)
+ */
+ void
+ SetFileComplete (const std::string &filename);
+
///////////////////////////
// File state operations //
///////////////////////////
diff --git a/src/dispatcher.cc b/src/dispatcher.cc
index 63b59cb..cbd6b41 100644
--- a/src/dispatcher.cc
+++ b/src/dispatcher.cc
@@ -397,6 +397,8 @@
last_write_time (filePath, file->mtime ());
permissions (filePath, static_cast<filesystem::perms> (file->mode ()));
+
+ m_actionLog->SetFileComplete (file->filename ());
}
}
diff --git a/src/file-item.proto b/src/file-item.proto
index 02ea9e8..f63aa9d 100644
--- a/src/file-item.proto
+++ b/src/file-item.proto
@@ -8,4 +8,6 @@
required uint32 mtime = 6;
required uint32 mode = 7;
required uint64 seg_num = 8;
+
+ required uint32 is_complete = 9;
}