17#include <QJsonDocument>
19#include <QMutexLocker>
20#include <QStandardPaths>
32 using private_ns::CValuePage;
35 class CDataCacheRevision::LockGuard
38 LockGuard(
const LockGuard &) =
delete;
39 LockGuard &operator=(
const LockGuard &) =
delete;
40 LockGuard(LockGuard &&other) noexcept : m_movedFrom(
true) { *
this = std::move(other); }
41 LockGuard &operator=(LockGuard &&other)
noexcept
43 auto tuple = std::tie(other.m_movedFrom, other.m_keepPromises, other.m_rev);
44 std::tie(m_movedFrom, m_keepPromises, m_rev).swap(tuple);
52 m_rev->finishUpdate(m_keepPromises);
56 operator bool()
const {
return !m_movedFrom; }
59 LockGuard() : m_movedFrom(true) {}
60 LockGuard(CDataCacheRevision *rev) : m_movedFrom(!rev), m_rev(rev) {}
61 LockGuard &keepPromises()
63 m_keepPromises =
true;
66 friend class CDataCacheRevision;
68 bool m_movedFrom =
false;
69 bool m_keepPromises =
false;
70 CDataCacheRevision *m_rev =
nullptr;
77 CLogMessage(
this).error(u
"Failed to create directory '%1'") << persistentStore();
80 connect(
this, &CValueCache::valuesChangedByLocal,
this, &CDataCache::saveToStoreAsync);
81 connect(
this, &CValueCache::valuesChangedByLocal,
this, [=,
this](CValueCachePacket values) {
83 changeValuesFromRemote(values, CIdentifier());
86 connect(m_serializer, &CDataCacheSerializer::valuesLoadedFromStore,
this, &CDataCache::changeValuesFromRemote,
94 m_serializer->loadFromStore({},
false,
true);
98 m_serializer->start();
99 m_watcher.addPath(revisionFileName());
100 loadFromStoreAsync();
104 CDataCache::~CDataCache() { m_serializer->quitAndWait(); }
106 CDataCache *CDataCache::instance()
108 static std::unique_ptr<CDataCache> cache(
new CDataCache);
114 const QString &CDataCache::persistentStore()
116 static const QString dir = CFileUtils::appendFilePaths(getCacheRootDirectory(), relativeFilePath());
120 const QString &CDataCache::revisionFileName()
122 static const QString rev = CFileUtils::appendFilePaths(persistentStore(),
".rev");
128 return CFileUtils::appendFilePaths(persistentStore(), instance()->CValueCache::filenameForKey(key));
131 QStringList CDataCache::enumerateStore()
const {
return enumerateFiles(persistentStore()); }
133 bool CDataCache::synchronize(
const QString &key)
135 constexpr auto timeout = std::chrono::seconds(1);
136 constexpr auto ready = std::future_status::ready;
137 constexpr auto zero = std::chrono::seconds::zero();
139 std::future<void> future = m_revision.promiseLoadedValue(key, getTimestampSync(key));
142 std::future_status s {};
145 s = future.wait_for(timeout);
147 while (s != ready && m_revision.isNewerValueAvailable(key, getTimestampSync(key)));
150 s = future.wait_for(zero);
165 catch (
const std::future_error &)
174 void CDataCache::setTimeToLive(
const QString &key,
int ttl)
176 singleShot(0, m_serializer, [
this, key, ttl] { m_revision.setTimeToLive(key, ttl); });
179 void CDataCache::renewTimestamp(
const QString &key, qint64 timestamp)
181 singleShot(0, m_serializer, [
this, key, timestamp] { m_revision.overrideTimestamp(key, timestamp); });
184 qint64 CDataCache::getTimestampOnDisk(
const QString &key) {
return m_revision.getTimestampOnDisk(key); }
186 void CDataCache::pinValue(
const QString &key)
188 singleShot(0, m_serializer, [
this, key] { m_revision.pinValue(key); });
191 void CDataCache::deferValue(
const QString &key)
193 singleShot(0, m_serializer, [
this, key] { m_revision.deferValue(key); });
196 void CDataCache::admitValue(
const QString &key,
bool triggerLoad)
198 m_revision.admitValue(key);
201 loadFromStoreAsync();
205 void CDataCache::sessionValue(
const QString &key)
207 singleShot(0, m_serializer, [
this, key] { m_revision.sessionValue(key); });
210 const QString &CDataCache::relativeFilePath()
212 static const QString p(
"/data/cache/core");
219 [
this, values] { m_serializer->saveToStore(values.
toVariantMap(), getAllValuesWithTimestamps()); });
222 void CDataCache::loadFromStoreAsync()
224 singleShot(0, m_serializer, [
this] { m_serializer->loadFromStore(getAllValuesWithTimestamps()); });
227 void CDataCache::connectPage(CValuePage *page)
229 auto *queue =
new CDataPageQueue(page);
230 connect(page, &CValuePage::valuesWantToCache,
this, &CDataCache::changeValues);
234 void CDataPageQueue::queueValuesFromCache(
const CValueCachePacket &values,
QObject *changedBy)
237 if (m_queue.isEmpty())
239 singleShot(0,
this, [
this] { setQueuedValuesFromCache(); });
241 m_queue.push_back(std::make_pair(values, changedBy));
244 void CDataPageQueue::setQueuedValuesFromCache()
247 decltype(m_queue) queue;
248 std::swap(m_queue, queue);
251 for (
const auto &pair : std::as_const(queue))
253 m_page->setValuesFromCache(pair.first, pair.second);
257 void CDataPageQueue::setQueuedValueFromCache(
const QString &key)
261 decltype(m_queue) filtered;
262 for (
auto &pair : m_queue)
264 if (pair.first.contains(key))
266 filtered.push_back({ pair.first.takeByKey(key), pair.second });
270 for (
const auto &pair : filtered)
272 m_page->setValuesFromCache(pair.first, pair.second);
276 const QStringList &CDataCacheSerializer::getLogCategories()
282 CDataCacheSerializer::CDataCacheSerializer(CDataCache *owner,
const QString &revisionFileName)
283 : CContinuousWorker(owner, QStringLiteral(
"CDataCacheSerializer '%1'").arg(revisionFileName)), m_cache(owner),
284 m_revisionFileName(revisionFileName)
287 const QString &CDataCacheSerializer::persistentStore()
const {
return CDataCache::persistentStore(); }
292 m_cache->m_revision.notifyPendingWrite();
294 loadFromStore(baseline,
true);
295 for (
const auto &key : values.
keys())
297 m_deferredChanges.remove(key);
307 msg.setCategories(
this);
308 CLogMessage::preformatted(msg);
310 applyDeferredChanges();
313 CDataCacheRevision::LockGuard CDataCacheSerializer::loadFromStore(
const CValueCachePacket &baseline,
bool defer,
316 auto lock = m_cache->m_revision.beginUpdate(baseline.toTimestampMap(), !pinsOnly, pinsOnly);
317 if (lock && m_cache->m_revision.isPendingRead())
319 CValueCachePacket newValues;
320 if (!m_cache->m_revision.isFound())
322 m_cache->loadFromFiles(persistentStore(), {}, {}, newValues, {},
true);
323 m_cache->m_revision.regenerate(newValues);
327 m_cache->loadFromFiles(persistentStore(), m_cache->m_revision.keysWithNewerTimestamps(),
328 baseline.toVariantMap(), newValues, m_cache->m_revision.timestampsAsString());
329 newValues.setTimestamps(m_cache->m_revision.newerTimestamps());
331 auto missingKeys = m_cache->m_revision.keysWithNewerTimestamps() - newValues.keys();
332 if (!missingKeys.isEmpty())
334 m_cache->m_revision.writeNewRevision({}, missingKeys);
337 msg.setCategories(
this);
338 CLogMessage::preformatted(msg);
339 m_deferredChanges.insert(newValues);
344 applyDeferredChanges();
349 void CDataCacheSerializer::applyDeferredChanges()
351 if (!m_deferredChanges.isEmpty())
353 m_deferredChanges.setSaved();
354 emit valuesLoadedFromStore(m_deferredChanges, CIdentifier::null());
355 deliverPromises(m_cache->m_revision.loadedValuePromises());
356 m_deferredChanges.clear();
360 void CDataCacheSerializer::deliverPromises(std::vector<std::promise<void>> i_promises)
363 [promises = std::make_shared<
decltype(i_promises)>(std::move(i_promises))]() {
364 for (
auto &promise : *promises)
375 Session(
const QString &filename) : m_filename(filename) {}
376 void updateSession();
377 const QUuid &uuid()
const {
return m_uuid; }
380 const QString m_filename;
384 CDataCacheRevision::CDataCacheRevision(
const QString &basename)
385 : m_basename(basename), m_session(std::make_unique<Session>(m_basename +
"/.session"))
388 CDataCacheRevision::~CDataCacheRevision() =
default;
390 CDataCacheRevision::LockGuard CDataCacheRevision::beginUpdate(
const QMap<QString, qint64> ×tamps,
391 bool updateUuid,
bool pinsOnly)
395 Q_ASSERT(!m_updateInProgress);
396 Q_ASSERT(!m_lockFile.isLocked());
398 if (!m_lockFile.lock())
400 CLogMessage(
this).error(u
"Failed to lock %1: %2") << m_basename << CFileUtils::lockFileError(m_lockFile);
403 m_updateInProgress =
true;
404 LockGuard guard(
this);
406 m_timestamps.clear();
407 m_originalTimestamps.clear();
409 QFile revisionFile(CFileUtils::appendFilePaths(m_basename,
".rev"));
410 if (m_found = revisionFile.exists(); m_found)
414 CLogMessage(
this).error(u
"Failed to open %1: %2")
415 << revisionFile.fileName() << revisionFile.errorString();
420 if (json.contains(
"uuid") && json.contains(
"timestamps"))
422 m_originalTimestamps = fromJson(json.value(
"timestamps").toObject());
424 QUuid id(json.value(
"uuid").toString());
425 if (
id == m_uuid && m_admittedQueue.isEmpty())
438 auto timesToLive = fromJson(json.value(
"ttl").toObject());
439 for (
auto it = m_originalTimestamps.cbegin(); it != m_originalTimestamps.cend(); ++it)
441 auto current = timestamps.
value(it.key(), -1);
442 auto ttl = timesToLive.value(it.key(), -1);
445 m_timestamps.insert(it.key(), it.value());
448 if (m_timestamps.isEmpty())
459 auto pins = fromJson(json.value(
"pins").toArray());
460 for (
const auto &key : m_timestamps.keys())
462 if (!pins.contains(key))
464 m_timestamps.remove(key);
469 auto deferrals = fromJson(json.value(
"deferrals").toArray());
470 m_admittedValues.unite(m_admittedQueue);
473 m_admittedQueue.clear();
475 else if (!m_admittedQueue.isEmpty())
477 m_admittedQueue.intersect(
QSet<QString>(m_timestamps.keyBegin(), m_timestamps.keyEnd()));
480 for (
const auto &key : m_timestamps.keys())
482 if (deferrals.contains(key) && !m_admittedValues.contains(key))
484 m_timestamps.remove(key);
488 m_session->updateSession();
489 auto sessionIds = sessionFromJson(json.value(
"session").toObject());
490 for (
auto it = sessionIds.cbegin(); it != sessionIds.cend(); ++it)
492 m_sessionValues[it.key()] = it.value();
493 if (it.value() != m_session->uuid())
495 m_timestamps.remove(it.key());
496 m_originalTimestamps.remove(it.key());
500 else if (revisionFile.size() > 0)
502 CLogMessage(
this).error(u
"Invalid format of %1") << revisionFile.fileName();
516 m_pendingRead =
true;
525 Q_ASSERT(m_updateInProgress);
526 Q_ASSERT(m_lockFile.isLocked());
528 CAtomicFile revisionFile(CFileUtils::appendFilePaths(m_basename,
".rev"));
531 CLogMessage(
this).error(u
"Failed to open %1: %2") << revisionFile.fileName() << revisionFile.errorString();
535 m_uuid = CIdentifier().toUuid();
536 auto timestamps = m_originalTimestamps;
537 for (
auto it = i_timestamps.
cbegin(); it != i_timestamps.
cend(); ++it)
541 timestamps.
insert(it.key(), it.value());
544 for (
const auto &key : excludeKeys)
549 for (
auto it = timestamps.
cbegin(); it != timestamps.
cend(); ++it)
551 if (m_sessionValues.contains(it.key()))
553 m_sessionValues[it.key()] = m_session->uuid();
558 json.
insert(
"uuid", m_uuid.toString());
559 json.
insert(
"timestamps", toJson(timestamps));
560 json.
insert(
"ttl", toJson(m_timesToLive));
561 json.
insert(
"pins", toJson(m_pinnedValues));
562 json.
insert(
"deferrals", toJson(m_deferredValues));
563 json.
insert(
"session", toJson(m_sessionValues));
566 if (!revisionFile.checkedClose())
569 QStringLiteral(
"If this error persists, try restarting your computer or delete the file manually.");
570 CLogMessage(
this).error(u
"Failed to replace %1: %2 (%3)")
571 << revisionFile.fileName() << revisionFile.errorString() << advice;
575 void CDataCacheRevision::regenerate(
const CValueCachePacket &keys)
579 Q_ASSERT(m_updateInProgress);
580 Q_ASSERT(m_lockFile.isLocked());
582 writeNewRevision(m_originalTimestamps = keys.toTimestampMap());
585 void CDataCacheRevision::finishUpdate(
bool keepPromises)
589 Q_ASSERT(m_updateInProgress);
590 Q_ASSERT(m_lockFile.isLocked());
592 m_updateInProgress =
false;
593 m_pendingRead =
false;
594 m_pendingWrite =
false;
602 bool CDataCacheRevision::isFound()
const
606 Q_ASSERT(m_updateInProgress);
610 bool CDataCacheRevision::isPendingRead()
const
614 Q_ASSERT(m_updateInProgress);
615 return !m_timestamps.isEmpty() || !m_found;
618 void CDataCacheRevision::notifyPendingWrite()
622 m_pendingWrite =
true;
625 QSet<QString> CDataCacheRevision::keysWithNewerTimestamps()
const
629 Q_ASSERT(m_updateInProgress);
630 return { m_timestamps.keyBegin(), m_timestamps.keyEnd() };
637 Q_ASSERT(m_updateInProgress);
641 bool CDataCacheRevision::isNewerValueAvailable(
const QString &key, qint64 timestamp)
648 return (m_updateInProgress || m_pendingWrite || beginUpdate({ { key, timestamp } },
false).keepPromises()) &&
649 (m_timestamps.contains(key) || m_admittedQueue.contains(key));
652 std::future<void> CDataCacheRevision::promiseLoadedValue(
const QString &key, qint64 currentTimestamp)
656 if (isNewerValueAvailable(key, currentTimestamp))
658 std::promise<void> promise;
659 auto future = promise.get_future();
660 m_promises.push_back(std::move(promise));
666 std::vector<std::promise<void>> CDataCacheRevision::loadedValuePromises()
670 Q_ASSERT(m_updateInProgress);
671 return std::move(m_promises);
674 void CDataCacheRevision::breakPromises()
678 if (!m_promises.empty())
680 CLogMessage(
this).debug() <<
"Breaking" << m_promises.size() <<
"promises";
685 QString CDataCacheRevision::timestampsAsString()
const
690 for (
auto it = m_timestamps.cbegin(); it != m_timestamps.cend(); ++it)
695 return result.
join(
",");
698 void CDataCacheRevision::setTimeToLive(
const QString &key,
int ttl)
702 Q_ASSERT(!m_updateInProgress);
703 m_timesToLive.insert(key, ttl);
706 void CDataCacheRevision::overrideTimestamp(
const QString &key, qint64 timestamp)
710 Q_ASSERT(!m_updateInProgress);
711 Q_ASSERT(!m_lockFile.isLocked());
713 if (!m_lockFile.lock())
715 CLogMessage(
this).error(u
"Failed to lock %1: %2") << m_basename << CFileUtils::lockFileError(m_lockFile);
720 CAtomicFile revisionFile(CFileUtils::appendFilePaths(m_basename,
".rev"));
721 if (revisionFile.exists())
725 CLogMessage(
this).error(u
"Failed to open %1: %2")
726 << revisionFile.fileName() << revisionFile.errorString();
733 timestamps.
insert(key, timestamp);
734 json.
insert(
"timestamps", timestamps);
736 if (revisionFile.seek(0) && revisionFile.resize(0) && revisionFile.write(
QJsonDocument(json).toJson()))
738 if (!revisionFile.checkedClose())
740 static const QString advice = QStringLiteral(
741 "If this error persists, try restarting your computer or delete the file manually.");
742 CLogMessage(
this).error(u
"Failed to replace %1: %2 (%3)")
743 << revisionFile.fileName() << revisionFile.errorString() << advice;
748 CLogMessage(
this).error(u
"Failed to write to %1: %2")
749 << revisionFile.fileName() << revisionFile.errorString();
755 qint64 CDataCacheRevision::getTimestampOnDisk(
const QString &key)
759 if (m_lockFile.isLocked())
761 return m_originalTimestamps.value(key);
764 if (!m_lockFile.lock())
766 CLogMessage(
this).error(u
"Failed to lock %1: %2") << m_basename << CFileUtils::lockFileError(m_lockFile);
772 QFile revisionFile(CFileUtils::appendFilePaths(m_basename,
".rev"));
773 if (revisionFile.exists())
778 result =
static_cast<qint64
>(json.
value(
"timestamps").toObject().value(key).toDouble());
782 CLogMessage(
this).error(u
"Failed to open %1: %2")
783 << revisionFile.fileName() << revisionFile.errorString();
790 void CDataCacheRevision::pinValue(
const QString &key)
794 Q_ASSERT(!m_updateInProgress);
795 m_pinnedValues.insert(key);
798 void CDataCacheRevision::deferValue(
const QString &key)
802 Q_ASSERT(!m_updateInProgress);
803 m_deferredValues.insert(key);
806 void CDataCacheRevision::admitValue(
const QString &key)
810 m_admittedQueue.insert(key);
813 void CDataCacheRevision::sessionValue(
const QString &key)
817 Q_ASSERT(!m_updateInProgress);
818 m_sessionValues[key];
824 for (
auto it = timestamps.
begin(); it != timestamps.
end(); ++it)
826 result.
insert(it.key(), it.value());
834 for (
auto it = timestamps.
begin(); it != timestamps.
end(); ++it)
836 result.
insert(it.key(),
static_cast<qint64
>(it.value().toDouble()));
844 for (
const auto &pin : pins)
854 for (
auto pin : pins)
856 result.
insert(pin.toString());
864 for (
auto it = timestamps.
begin(); it != timestamps.
end(); ++it)
866 result.
insert(it.key(), it.value().toString());
874 for (
auto it = session.
begin(); it != session.
end(); ++it)
876 result.
insert(it.key(),
QUuid(it.value().toString()));
881 void CDataCacheRevision::Session::updateSession()
883 CAtomicFile file(m_filename);
887 CLogMessage(
this).error(u
"Failed to open session file %1: %2") << m_filename << file.errorString();
892 CSequence<CProcessInfo> apps;
893 auto status = apps.convertFromJsonNoThrow(json.
value(
"apps").
toObject(),
this,
894 QStringLiteral(
"Error in %1 apps object").arg(m_filename));
895 apps.removeIf([](
const CProcessInfo &pi) {
return !pi.exists(); });
899 id = CIdentifier().toUuid();
903 CProcessInfo currentProcess = CProcessInfo::currentProcess();
904 Q_ASSERT(currentProcess.exists());
905 apps.replaceOrAdd(currentProcess);
906 json.
insert(
"apps", apps.toJson());
907 json.
insert(
"uuid", m_uuid.toString());
908 if (file.seek(0) && file.resize(0) && file.write(
QJsonDocument(json).toJson()))
910 if (!file.checkedClose())
913 QStringLiteral(
"If this error persists, try restarting your computer or delete the file manually.");
914 CLogMessage(
this).error(u
"Failed to replace %1: %2 (%3)")
915 << file.fileName() << file.errorString() << advice;
920 CLogMessage(
this).error(u
"Failed to write to %1: %2") << file.fileName() << file.errorString();
Worker which performs (de)serialization on behalf of CDataCache, in a separate thread so that the mai...
auto keys() const
Return a range of all keys (does not allocate a temporary container).
static const QString & cache()
Cache.
Manages a map of { QString, CVariant } pairs, which can be distributed among multiple processes.
Value class used for signalling changed values in the cache.
QString toTimestampMapString(const QStringList &keys) const
Return map of timestamps converted to string.
QMap< QString, qint64 > toTimestampMap() const
Discard values and return as map of timestamps.
CVariantMap toVariantMap() const
Discard timestamps and return as variant map.
Map of { QString, CVariant } pairs.
Decorator for CValuePage which allows incoming remote changes to be queued to allow for more flexibil...
Free functions in swift::misc.
auto singleShot(int msec, QObject *target, F &&task)
Starts a single-shot timer which will call a task in the thread of the given object when it times out...
qint64 currentMSecsSinceEpoch()
QDateTime fromMSecsSinceEpoch(qint64 msecs)
bool exists(const QString &fileName)
bool open(FILE *fh, QIODeviceBase::OpenMode mode, QFileDevice::FileHandleFlags handleFlags)
void fileChanged(const QString &path)
void push_back(const QJsonValue &value)
QJsonDocument fromJson(const QByteArray &json, QJsonParseError *error)
QJsonObject object() const const
QJsonObject::iterator begin()
QJsonObject::iterator end()
QJsonObject::iterator insert(QLatin1StringView key, const QJsonValue &value)
QJsonValue value(QLatin1StringView key) const const
QJsonObject toObject() const const
QString toString() const const
void push_back(QList< T >::parameter_type value)
QMap< Key, T >::iterator begin()
QMap< Key, T >::const_iterator cbegin() const const
QMap< Key, T >::const_iterator cend() const const
QMap< Key, T >::iterator end()
QMap< Key, T >::iterator insert(QMap< Key, T >::const_iterator pos, const Key &key, const T &value)
QMap< Key, T >::size_type remove(const Key &key)
T value(const Key &key, const T &defaultValue) const const
void destroyed(QObject *obj)
QSet< T >::iterator insert(QSet< T >::const_iterator it, const T &value)
QString join(QChar separator) const const
QFuture< typename qValueType< Iterator >::value_type > filtered(Iterator begin, Iterator end, KeepFunctor &&filterFunction)
QFuture< QtFuture::ArgsType< Signal > > connect(Sender *sender, Signal signal)
#define SWIFT_MISC_EXPORT
Export a class or function from the library.
#define SWIFT_VERIFY_X(COND, WHERE, WHAT)
A weaker kind of assert.