Adding version to FileState database (incompatible DB formats and protobuf FileItem messages!)
Change-Id: I12b785720b5f033e9f631d0b62eb9e7fdeea0270
diff --git a/src/action-log.cc b/src/action-log.cc
index 0e22483..2d8ebc1 100644
--- a/src/action-log.cc
+++ b/src/action-log.cc
@@ -77,7 +77,7 @@
device_name > NEW.device_name) IS NULL \n\
BEGIN \n\
SELECT apply_action (NEW.device_name, NEW.seq_no, \
- NEW.action,NEW.filename,NEW.file_hash, \
+ NEW.action,NEW.filename,NEW.version,NEW.file_hash, \
strftime('%s', NEW.file_atime),strftime('%s', NEW.file_mtime),strftime('%s', NEW.file_ctime), \
NEW.file_chmod, NEW.file_seg_num); /* function that applies action and adds record the FileState */ \n \
END; \n\
@@ -737,7 +737,7 @@
{
ActionLog *the = reinterpret_cast<ActionLog*> (sqlite3_user_data (context));
- if (argc != 10)
+ if (argc != 11)
{
sqlite3_result_error (context, "``apply_action'' expects 10 arguments", -1);
return;
@@ -747,6 +747,7 @@
sqlite3_int64 seq_no = sqlite3_value_int64 (argv[1]);
int action = sqlite3_value_int (argv[2]);
string filename = reinterpret_cast<const char*> (sqlite3_value_text (argv[3]));
+ sqlite3_int64 version = sqlite3_value_int64 (argv[4]);
_LOG_TRACE ("apply_function called with " << argc);
_LOG_TRACE ("device_name: " << Name (device_name)
@@ -755,16 +756,16 @@
if (action == 0) // update
{
- Hash hash (sqlite3_value_blob (argv[4]), sqlite3_value_bytes (argv[4]));
- time_t atime = static_cast<time_t> (sqlite3_value_int64 (argv[5]));
- time_t mtime = static_cast<time_t> (sqlite3_value_int64 (argv[6]));
- time_t ctime = static_cast<time_t> (sqlite3_value_int64 (argv[7]));
- int mode = sqlite3_value_int (argv[8]);
- int seg_num = sqlite3_value_int (argv[9]);
+ Hash hash (sqlite3_value_blob (argv[5]), sqlite3_value_bytes (argv[5]));
+ time_t atime = static_cast<time_t> (sqlite3_value_int64 (argv[6]));
+ time_t mtime = static_cast<time_t> (sqlite3_value_int64 (argv[7]));
+ time_t ctime = static_cast<time_t> (sqlite3_value_int64 (argv[8]));
+ int mode = sqlite3_value_int (argv[9]);
+ int seg_num = sqlite3_value_int (argv[10]);
_LOG_DEBUG ("Update " << filename << " " << atime << " " << mtime << " " << ctime << " " << hash);
- the->m_fileState->UpdateFile (filename, hash, device_name, seq_no, atime, mtime, ctime, mode, seg_num);
+ the->m_fileState->UpdateFile (filename, version, hash, device_name, seq_no, atime, mtime, ctime, mode, seg_num);
// no callback here
}
diff --git a/src/file-item.proto b/src/file-item.proto
index f63aa9d..4840cec 100644
--- a/src/file-item.proto
+++ b/src/file-item.proto
@@ -1,13 +1,14 @@
message FileItem
{
required string filename = 2;
- required bytes device_name = 3;
- required uint64 seq_no = 4;
+ required uint64 version = 3;
+ required bytes device_name = 4;
+ required uint64 seq_no = 5;
- required bytes file_hash = 5;
- required uint32 mtime = 6;
- required uint32 mode = 7;
- required uint64 seg_num = 8;
+ required bytes file_hash = 6;
+ required uint32 mtime = 7;
+ required uint32 mode = 8;
+ required uint64 seg_num = 9;
- required uint32 is_complete = 9;
+ required uint32 is_complete = 10;
}
diff --git a/src/file-state.cc b/src/file-state.cc
index 63647c4..6ace814 100644
--- a/src/file-state.cc
+++ b/src/file-state.cc
@@ -33,6 +33,7 @@
CREATE TABLE FileState ( \n\
type INTEGER NOT NULL, /* 0 - newest, 1 - oldest */ \n\
filename TEXT NOT NULL, \n\
+ version INTEGER, \n\
directory TEXT, \n\
device_name BLOB NOT NULL, \n\
seq_no INTEGER NOT NULL, \n\
@@ -63,13 +64,15 @@
}
void
-FileState::UpdateFile (const std::string &filename, const Hash &hash, const Ccnx::CcnxCharbuf &device_name, sqlite3_int64 seq_no,
+FileState::UpdateFile (const std::string &filename, sqlite3_int64 version,
+ const Hash &hash, const Ccnx::CcnxCharbuf &device_name, sqlite3_int64 seq_no,
time_t atime, time_t mtime, time_t ctime, int mode, int seg_num)
{
sqlite3_stmt *stmt;
sqlite3_prepare_v2 (m_db, "UPDATE FileState "
"SET "
"device_name=?, seq_no=?, "
+ "version=?,"
"file_hash=?,"
"file_atime=datetime(?, 'unixepoch'),"
"file_mtime=datetime(?, 'unixepoch'),"
@@ -80,13 +83,14 @@
sqlite3_bind_blob (stmt, 1, device_name.buf (), device_name.length (), SQLITE_STATIC);
sqlite3_bind_int64 (stmt, 2, seq_no);
- sqlite3_bind_blob (stmt, 3, hash.GetHash (), hash.GetHashBytes (), SQLITE_STATIC);
- sqlite3_bind_int64 (stmt, 4, atime);
- sqlite3_bind_int64 (stmt, 5, mtime);
- sqlite3_bind_int64 (stmt, 6, ctime);
- sqlite3_bind_int (stmt, 7, mode);
- sqlite3_bind_int (stmt, 8, seg_num);
- sqlite3_bind_text (stmt, 9, filename.c_str (), -1, SQLITE_STATIC);
+ sqlite3_bind_int64 (stmt, 3, version);
+ sqlite3_bind_blob (stmt, 4, hash.GetHash (), hash.GetHashBytes (), SQLITE_STATIC);
+ sqlite3_bind_int64 (stmt, 5, atime);
+ sqlite3_bind_int64 (stmt, 6, mtime);
+ sqlite3_bind_int64 (stmt, 7, ctime);
+ sqlite3_bind_int (stmt, 8, mode);
+ sqlite3_bind_int (stmt, 9, seg_num);
+ sqlite3_bind_text (stmt, 10, filename.c_str (), -1, SQLITE_STATIC);
sqlite3_step (stmt);
@@ -100,22 +104,22 @@
{
sqlite3_stmt *stmt;
sqlite3_prepare_v2 (m_db, "INSERT INTO FileState "
- "(type,filename,device_name,seq_no,file_hash,file_atime,file_mtime,file_ctime,file_chmod,file_seg_num) "
- "VALUES (0, ?, ?, ?, ?, "
+ "(type,filename,version,device_name,seq_no,file_hash,file_atime,file_mtime,file_ctime,file_chmod,file_seg_num) "
+ "VALUES (0, ?, ?, ?, ?, ?, "
"datetime(?, 'unixepoch'), datetime(?, 'unixepoch'), datetime(?, 'unixepoch'), ?, ?)", -1, &stmt, 0);
_LOG_DEBUG_COND (sqlite3_errcode (m_db) != SQLITE_OK, sqlite3_errmsg (m_db));
sqlite3_bind_text (stmt, 1, filename.c_str (), -1, SQLITE_STATIC);
- sqlite3_bind_blob (stmt, 2, device_name.buf (), device_name.length (), SQLITE_STATIC);
- sqlite3_bind_int64 (stmt, 3, seq_no);
- sqlite3_bind_blob (stmt, 4, hash.GetHash (), hash.GetHashBytes (), SQLITE_STATIC);
- sqlite3_bind_int64 (stmt, 5, atime);
- sqlite3_bind_int64 (stmt, 6, mtime);
- sqlite3_bind_int64 (stmt, 7, ctime);
- sqlite3_bind_int (stmt, 8, mode);
- sqlite3_bind_int (stmt, 9, seg_num);
- // sqlite3_bind_text (stmt, 10, filename.c_str (), -1, SQLITE_STATIC);
+ sqlite3_bind_int64 (stmt, 2, version);
+ sqlite3_bind_blob (stmt, 3, device_name.buf (), device_name.length (), SQLITE_STATIC);
+ sqlite3_bind_int64 (stmt, 4, seq_no);
+ sqlite3_bind_blob (stmt, 5, hash.GetHash (), hash.GetHashBytes (), SQLITE_STATIC);
+ sqlite3_bind_int64 (stmt, 6, atime);
+ sqlite3_bind_int64 (stmt, 7, mtime);
+ sqlite3_bind_int64 (stmt, 8, ctime);
+ sqlite3_bind_int (stmt, 9, mode);
+ sqlite3_bind_int (stmt, 10, seg_num);
sqlite3_step (stmt);
_LOG_DEBUG_COND (sqlite3_errcode (m_db) != SQLITE_DONE,
@@ -175,7 +179,7 @@
{
sqlite3_stmt *stmt;
sqlite3_prepare_v2 (m_db,
- "SELECT filename,device_name,seq_no,file_hash,strftime('%s', file_mtime),file_chmod,file_seg_num,is_complete "
+ "SELECT filename,version,device_name,seq_no,file_hash,strftime('%s', file_mtime),file_chmod,file_seg_num,is_complete "
" FROM FileState "
" WHERE type = 0 AND filename = ?", -1, &stmt, 0);
_LOG_DEBUG_COND (sqlite3_errcode (m_db) != SQLITE_OK, sqlite3_errmsg (m_db));
@@ -186,13 +190,14 @@
{
retval = make_shared<FileItem> ();
retval->set_filename (reinterpret_cast<const char *> (sqlite3_column_text (stmt, 0)), sqlite3_column_bytes (stmt, 0));
- retval->set_device_name (sqlite3_column_blob (stmt, 1), sqlite3_column_bytes (stmt, 1));
- retval->set_seq_no (sqlite3_column_int64 (stmt, 2));
- retval->set_file_hash (sqlite3_column_blob (stmt, 3), sqlite3_column_bytes (stmt, 3));
- retval->set_mtime (sqlite3_column_int (stmt, 4));
- retval->set_mode (sqlite3_column_int (stmt, 5));
- retval->set_seg_num (sqlite3_column_int64 (stmt, 6));
- retval->set_is_complete (sqlite3_column_int (stmt, 7));
+ retval->set_version (sqlite3_column_int64 (stmt, 1));
+ retval->set_device_name (sqlite3_column_blob (stmt, 2), sqlite3_column_bytes (stmt, 2));
+ retval->set_seq_no (sqlite3_column_int64 (stmt, 3));
+ retval->set_file_hash (sqlite3_column_blob (stmt, 4), sqlite3_column_bytes (stmt, 4));
+ retval->set_mtime (sqlite3_column_int (stmt, 5));
+ retval->set_mode (sqlite3_column_int (stmt, 6));
+ retval->set_seg_num (sqlite3_column_int64 (stmt, 7));
+ retval->set_is_complete (sqlite3_column_int (stmt, 8));
}
_LOG_DEBUG_COND (sqlite3_errcode (m_db) != SQLITE_DONE, sqlite3_errmsg (m_db));
sqlite3_finalize (stmt);
@@ -205,7 +210,7 @@
{
sqlite3_stmt *stmt;
sqlite3_prepare_v2 (m_db,
- "SELECT filename,device_name,seq_no,file_hash,strftime('%s', file_mtime),file_chmod,file_seg_num,is_complete "
+ "SELECT filename,version,device_name,seq_no,file_hash,strftime('%s', file_mtime),file_chmod,file_seg_num,is_complete "
" FROM FileState "
" WHERE type = 0 AND file_hash = ?", -1, &stmt, 0);
_LOG_DEBUG_COND (sqlite3_errcode (m_db) != SQLITE_OK, sqlite3_errmsg (m_db));
@@ -217,13 +222,14 @@
{
FileItem file;
file.set_filename (reinterpret_cast<const char *> (sqlite3_column_text (stmt, 0)), sqlite3_column_bytes (stmt, 0));
- file.set_device_name (sqlite3_column_blob (stmt, 1), sqlite3_column_bytes (stmt, 1));
- file.set_seq_no (sqlite3_column_int64 (stmt, 2));
- file.set_file_hash (sqlite3_column_blob (stmt, 3), sqlite3_column_bytes (stmt, 3));
- file.set_mtime (sqlite3_column_int (stmt, 4));
- file.set_mode (sqlite3_column_int (stmt, 5));
- file.set_seg_num (sqlite3_column_int64 (stmt, 6));
- file.set_is_complete (sqlite3_column_int (stmt, 7));
+ file.set_version (sqlite3_column_int64 (stmt, 1));
+ file.set_device_name (sqlite3_column_blob (stmt, 2), sqlite3_column_bytes (stmt, 2));
+ file.set_seq_no (sqlite3_column_int64 (stmt, 3));
+ file.set_file_hash (sqlite3_column_blob (stmt, 4), sqlite3_column_bytes (stmt, 4));
+ file.set_mtime (sqlite3_column_int (stmt, 5));
+ file.set_mode (sqlite3_column_int (stmt, 6));
+ file.set_seg_num (sqlite3_column_int64 (stmt, 7));
+ file.set_is_complete (sqlite3_column_int (stmt, 8));
retval->push_back (file);
}
@@ -239,7 +245,7 @@
{
sqlite3_stmt *stmt;
sqlite3_prepare_v2 (m_db,
- "SELECT filename,device_name,seq_no,file_hash,strftime('%s', file_mtime),file_chmod,file_seg_num,is_complete "
+ "SELECT filename,version,device_name,seq_no,file_hash,strftime('%s', file_mtime),file_chmod,file_seg_num,is_complete "
" FROM FileState "
" WHERE type = 0 AND directory = ?"
" LIMIT ? OFFSET ?", -1, &stmt, 0);
@@ -255,13 +261,14 @@
{
FileItem file;
file.set_filename (reinterpret_cast<const char *> (sqlite3_column_text (stmt, 0)), sqlite3_column_bytes (stmt, 0));
- file.set_device_name (sqlite3_column_blob (stmt, 1), sqlite3_column_bytes (stmt, 1));
- file.set_seq_no (sqlite3_column_int64 (stmt, 2));
- file.set_file_hash (sqlite3_column_blob (stmt, 3), sqlite3_column_bytes (stmt, 3));
- file.set_mtime (sqlite3_column_int (stmt, 4));
- file.set_mode (sqlite3_column_int (stmt, 5));
- file.set_seg_num (sqlite3_column_int64 (stmt, 6));
- file.set_is_complete (sqlite3_column_int (stmt, 7));
+ file.set_version (sqlite3_column_int64 (stmt, 1));
+ file.set_device_name (sqlite3_column_blob (stmt, 2), sqlite3_column_bytes (stmt, 2));
+ file.set_seq_no (sqlite3_column_int64 (stmt, 3));
+ file.set_file_hash (sqlite3_column_blob (stmt, 4), sqlite3_column_bytes (stmt, 4));
+ file.set_mtime (sqlite3_column_int (stmt, 5));
+ file.set_mode (sqlite3_column_int (stmt, 6));
+ file.set_seg_num (sqlite3_column_int64 (stmt, 7));
+ file.set_is_complete (sqlite3_column_int (stmt, 8));
visitor (file);
}
@@ -280,20 +287,24 @@
return retval;
}
-FileItemsPtr
+bool
FileState::LookupFilesInFolderRecursively (const boost::function<void (const FileItem&)> &visitor, const std::string &folder, int offset/*=0*/, int limit/*=-1*/)
{
_LOG_DEBUG ("LookupFilesInFolderRecursively: [" << folder << "]");
+ if (limit >= 0)
+ limit ++;
+
sqlite3_stmt *stmt;
if (folder != "")
{
/// @todo Do something to improve efficiency of this query. Right now it is basically scanning the whole database
sqlite3_prepare_v2 (m_db,
- "SELECT filename,device_name,seq_no,file_hash,strftime('%s', file_mtime),file_chmod,file_seg_num,is_complete "
+ "SELECT filename,version,device_name,seq_no,file_hash,strftime('%s', file_mtime),file_chmod,file_seg_num,is_complete "
" FROM FileState "
" WHERE type = 0 AND is_prefix (?, directory)=1 "
+ " ORDER BY filename "
" LIMIT ? OFFSET ?", -1, &stmt, 0); // there is a small ambiguity with is_prefix matching, but should be ok for now
_LOG_DEBUG_COND (sqlite3_errcode (m_db) != SQLITE_OK, sqlite3_errmsg (m_db));
@@ -306,9 +317,10 @@
else
{
sqlite3_prepare_v2 (m_db,
- "SELECT filename,device_name,seq_no,file_hash,strftime('%s', file_mtime),file_chmod,file_seg_num,is_complete "
+ "SELECT filename,version,device_name,seq_no,file_hash,strftime('%s', file_mtime),file_chmod,file_seg_num,is_complete "
" FROM FileState "
" WHERE type = 0"
+ " ORDER BY filename "
" LIMIT ? OFFSET ?", -1, &stmt, 0);
sqlite3_bind_int (stmt, 1, limit);
sqlite3_bind_int (stmt, 2, offset);
@@ -318,22 +330,29 @@
while (sqlite3_step (stmt) == SQLITE_ROW)
{
+ if (limit == 1)
+ break;
+
FileItem file;
file.set_filename (reinterpret_cast<const char *> (sqlite3_column_text (stmt, 0)), sqlite3_column_bytes (stmt, 0));
- file.set_device_name (sqlite3_column_blob (stmt, 1), sqlite3_column_bytes (stmt, 1));
- file.set_seq_no (sqlite3_column_int64 (stmt, 2));
- file.set_file_hash (sqlite3_column_blob (stmt, 3), sqlite3_column_bytes (stmt, 3));
- file.set_mtime (sqlite3_column_int (stmt, 4));
- file.set_mode (sqlite3_column_int (stmt, 5));
- file.set_seg_num (sqlite3_column_int64 (stmt, 6));
- file.set_is_complete (sqlite3_column_int (stmt, 7));
+ file.set_version (sqlite3_column_int64 (stmt, 1));
+ file.set_device_name (sqlite3_column_blob (stmt, 2), sqlite3_column_bytes (stmt, 2));
+ file.set_seq_no (sqlite3_column_int64 (stmt, 3));
+ file.set_file_hash (sqlite3_column_blob (stmt, 4), sqlite3_column_bytes (stmt, 4));
+ file.set_mtime (sqlite3_column_int (stmt, 5));
+ file.set_mode (sqlite3_column_int (stmt, 6));
+ file.set_seg_num (sqlite3_column_int64 (stmt, 7));
+ file.set_is_complete (sqlite3_column_int (stmt, 8));
visitor (file);
+ limit --;
}
_LOG_DEBUG_COND (sqlite3_errcode (m_db) != SQLITE_DONE, sqlite3_errmsg (m_db));
sqlite3_finalize (stmt);
+
+ return (limit == 1);
}
FileItemsPtr
diff --git a/src/file-state.h b/src/file-state.h
index 6c4ace2..0a8a7b2 100644
--- a/src/file-state.h
+++ b/src/file-state.h
@@ -48,7 +48,8 @@
* @brief Update or add a file
*/
void
- UpdateFile (const std::string &filename, const Hash &hash, const Ccnx::CcnxCharbuf &device_name, sqlite3_int64 seqno,
+ UpdateFile (const std::string &filename, sqlite3_int64 version,
+ const Hash &hash, const Ccnx::CcnxCharbuf &device_name, sqlite3_int64 seqno,
time_t atime, time_t mtime, time_t ctime, int mode, int seg_num);
/**
@@ -92,7 +93,7 @@
/**
* @brief Recursively lookup all files in the specified folder and call visitor(file) for each file
*/
- FileItemsPtr
+ bool
LookupFilesInFolderRecursively (const boost::function<void (const FileItem&)> &visitor, const std::string &folder, int offset=0, int limit=-1);
/**