Diff
Modified: trunk/Source/WebKit/ChangeLog (283193 => 283194)
--- trunk/Source/WebKit/ChangeLog 2021-09-28 21:10:21 UTC (rev 283193)
+++ trunk/Source/WebKit/ChangeLog 2021-09-28 21:13:01 UTC (rev 283194)
@@ -1,3 +1,39 @@
+2021-09-28 Kate Cheney <[email protected]>
+
+ PCM: different bundleID entries will override each other
+ https://bugs.webkit.org/show_bug.cgi?id=230839
+
+ Reviewed by Alex Christensen.
+
+ We recently added a bundleID column to PCM tables. We want to make
+ sure entries with different bundleIDs do not override each other,
+ so we should make it a part of the unique constraint on both PCM
+ tables that contain it. This requires creating new tables and
+ migrating existing data to them. Luckily this code already exists
+ in the ITP database, and we can just move it to the shared
+ DatabaseUtilities class.
+
+ * NetworkProcess/Classifier/ResourceLoadStatisticsDatabaseStore.cpp:
+ (WebKit::ResourceLoadStatisticsDatabaseStore::expectedTableAndIndexQueries):
+ (WebKit::stripIndexQueryToMatchStoredValue): Deleted.
+ (WebKit::expectedTableAndIndexQueries): Deleted.
+ (WebKit::ResourceLoadStatisticsDatabaseStore::currentTableAndIndexQueries): Deleted.
+ (WebKit::insertDistinctValuesInTableStatement): Deleted.
+ (WebKit::ResourceLoadStatisticsDatabaseStore::migrateDataToNewTablesIfNecessary): Deleted.
+ * NetworkProcess/Classifier/ResourceLoadStatisticsDatabaseStore.h:
+ * NetworkProcess/DatabaseUtilities.cpp:
+ (WebKit::DatabaseUtilities::stripIndexQueryToMatchStoredValue):
+ (WebKit::DatabaseUtilities::currentTableAndIndexQueries):
+ (WebKit::insertDistinctValuesInTableStatement):
+ (WebKit::DatabaseUtilities::migrateDataToNewTablesIfNecessary):
+ * NetworkProcess/DatabaseUtilities.h:
+ * NetworkProcess/PrivateClickMeasurement/PrivateClickMeasurementDatabase.cpp:
+ (WebKit::PCM::Database::Database):
+ (WebKit::PCM::Database::expectedTableAndIndexQueries):
+ (WebKit::PCM::Database::createUniqueIndices):
+ (WebKit::PCM::Database::needsUpdatedSchema):
+ * NetworkProcess/PrivateClickMeasurement/PrivateClickMeasurementDatabase.h:
+
2021-09-28 Brent Fulgham <[email protected]>
Explicitly deny 'system-privilege' in the sandbox profile as a hardening measure
Modified: trunk/Source/WebKit/NetworkProcess/Classifier/ResourceLoadStatisticsDatabaseStore.cpp (283193 => 283194)
--- trunk/Source/WebKit/NetworkProcess/Classifier/ResourceLoadStatisticsDatabaseStore.cpp 2021-09-28 21:10:21 UTC (rev 283193)
+++ trunk/Source/WebKit/NetworkProcess/Classifier/ResourceLoadStatisticsDatabaseStore.cpp 2021-09-28 21:13:01 UTC (rev 283194)
@@ -50,7 +50,6 @@
#include <wtf/CrossThreadCopier.h>
#include <wtf/DateMath.h>
#include <wtf/MathExtras.h>
-#include <wtf/RobinHoodHashMap.h>
#include <wtf/Scope.h>
#include <wtf/StdSet.h>
#include <wtf/SuspendableWorkQueue.h>
@@ -253,13 +252,8 @@
return schema.contains("REFERENCES TopLevelDomains");
}
-static String stripIndexQueryToMatchStoredValue(const char* originalQuery)
+const MemoryCompactLookupOnlyRobinHoodHashMap<String, TableAndIndexPair>& ResourceLoadStatisticsDatabaseStore::expectedTableAndIndexQueries()
{
- return String(originalQuery).replace("CREATE UNIQUE INDEX IF NOT EXISTS", "CREATE UNIQUE INDEX");
-}
-
-static const MemoryCompactLookupOnlyRobinHoodHashMap<String, TableAndIndexPair>& expectedTableAndIndexQueries()
-{
static auto expectedTableAndIndexQueries = makeNeverDestroyed(MemoryCompactLookupOnlyRobinHoodHashMap<String, TableAndIndexPair> {
{ "ObservedDomains"_s, std::make_pair<String, std::optional<String>>(createObservedDomain, std::nullopt) },
{ "TopLevelDomains"_s, std::make_pair<String, std::optional<String>>(createTopLevelDomains, std::nullopt) },
@@ -279,6 +273,27 @@
return expectedTableAndIndexQueries;
}
+const Vector<String>& ResourceLoadStatisticsDatabaseStore::sortedTables()
+{
+ static auto sortedTables = makeNeverDestroyed(Vector<String> {
+ "ObservedDomains"_s,
+ "TopLevelDomains"_s,
+ "StorageAccessUnderTopFrameDomains"_s,
+ "TopFrameUniqueRedirectsTo"_s,
+ "TopFrameUniqueRedirectsToSinceSameSiteStrictEnforcement"_s,
+ "TopFrameUniqueRedirectsFrom"_s,
+ "TopFrameLinkDecorationsFrom"_s,
+ "TopFrameLoadedThirdPartyScripts"_s,
+ "SubframeUnderTopFrameDomains"_s,
+ "SubresourceUnderTopFrameDomains"_s,
+ "SubresourceUniqueRedirectsTo"_s,
+ "SubresourceUniqueRedirectsFrom"_s,
+ "OperatingDates"_s
+ });
+
+ return sortedTables;
+}
+
template <typename ContainerType>
static String buildList(const ContainerType& values)
{
@@ -367,52 +382,6 @@
}
}
-TableAndIndexPair ResourceLoadStatisticsDatabaseStore::currentTableAndIndexQueries(const String& tableName)
-{
- auto getTableStatement = m_database.prepareStatement("SELECT sql FROM sqlite_master WHERE tbl_name=? AND type = 'table'"_s);
- if (!getTableStatement) {
- RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::currentTableAndIndexQueries Unable to prepare statement to fetch schema for the table, error message: %" PRIVATE_LOG_STRING, this, m_database.lastErrorMsg());
- ASSERT_NOT_REACHED();
- return { };
- }
-
- if (getTableStatement->bindText(1, tableName) != SQLITE_OK) {
- RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::currentTableAndIndexQueries Unable to bind statement to fetch schema for the table, error message: %" PRIVATE_LOG_STRING, this, m_database.lastErrorMsg());
- ASSERT_NOT_REACHED();
- return { };
- }
-
- if (getTableStatement->step() != SQLITE_ROW) {
- RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::currentTableAndIndexQueries error executing statement to fetch table schema, error message: %" PRIVATE_LOG_STRING, this, m_database.lastErrorMsg());
- ASSERT_NOT_REACHED();
- return { };
- }
-
- String createTableQuery = getTableStatement->columnText(0);
-
- auto getIndexStatement = m_database.prepareStatement("SELECT sql FROM sqlite_master WHERE tbl_name=? AND type = 'index'"_s);
- if (!getIndexStatement) {
- RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::currentTableAndIndexQueries Unable to prepare statement to fetch index for the table, error message: %" PRIVATE_LOG_STRING, this, m_database.lastErrorMsg());
- ASSERT_NOT_REACHED();
- return { };
- }
-
- if (getIndexStatement->bindText(1, tableName) != SQLITE_OK) {
- RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::currentTableAndIndexQueries Unable to bind statement to fetch index for the table, error message: %" PRIVATE_LOG_STRING, this, m_database.lastErrorMsg());
- ASSERT_NOT_REACHED();
- return { };
- }
-
- std::optional<String> index;
- if (getIndexStatement->step() == SQLITE_ROW) {
- auto rawIndex = String(getIndexStatement->columnText(0));
- if (!rawIndex.isEmpty())
- index = rawIndex;
- }
-
- return std::make_pair<String, std::optional<String>>(WTFMove(createTableQuery), WTFMove(index));
-}
-
bool ResourceLoadStatisticsDatabaseStore::missingUniqueIndices()
{
auto statement = m_database.prepareStatement("SELECT COUNT(*) FROM sqlite_master WHERE type = 'index'"_s);
@@ -456,68 +425,6 @@
return false;
}
-static Expected<SQLiteStatement, int> insertDistinctValuesInTableStatement(SQLiteDatabase& database, const String& table)
-{
- if (table == "SubframeUnderTopFrameDomains")
- return database.prepareStatement("INSERT INTO SubframeUnderTopFrameDomains SELECT subFrameDomainID, MAX(lastUpdated), topFrameDomainID FROM _SubframeUnderTopFrameDomains GROUP BY subFrameDomainID, topFrameDomainID"_s);
-
- if (table == "SubresourceUnderTopFrameDomains")
- return database.prepareStatement("INSERT INTO SubresourceUnderTopFrameDomains SELECT subresourceDomainID, MAX(lastUpdated), topFrameDomainID FROM _SubresourceUnderTopFrameDomains GROUP BY subresourceDomainID, topFrameDomainID"_s);
-
- if (table == "SubresourceUniqueRedirectsTo")
- return database.prepareStatement("INSERT INTO SubresourceUniqueRedirectsTo SELECT subresourceDomainID, MAX(lastUpdated), toDomainID FROM _SubresourceUniqueRedirectsTo GROUP BY subresourceDomainID, toDomainID"_s);
-
- if (table == "TopFrameLinkDecorationsFrom")
- return database.prepareStatement("INSERT INTO TopFrameLinkDecorationsFrom SELECT toDomainID, MAX(lastUpdated), fromDomainID FROM _TopFrameLinkDecorationsFrom GROUP BY toDomainID, fromDomainID"_s);
-
- return database.prepareStatementSlow(makeString("INSERT INTO ", table, " SELECT DISTINCT * FROM _", table));
-}
-
-void ResourceLoadStatisticsDatabaseStore::migrateDataToNewTablesIfNecessary()
-{
- if (!needsUpdatedSchema())
- return;
-
- auto transactionScope = beginTransactionIfNecessary();
-
- for (auto& table : expectedTableAndIndexQueries().keys()) {
- auto alterTable = m_database.prepareStatementSlow(makeString("ALTER TABLE ", table, " RENAME TO _", table));
- if (!alterTable || alterTable->step() != SQLITE_DONE) {
- RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::migrateDataToNewTablesIfNecessary failed to rename table, error message: %s", this, m_database.lastErrorMsg());
- ASSERT_NOT_REACHED();
- return;
- }
- }
-
- if (!createSchema()) {
- RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::migrateDataToNewTablesIfNecessary failed to create schema, error message: %s", this, m_database.lastErrorMsg());
- ASSERT_NOT_REACHED();
- return;
- }
-
- for (auto& table : expectedTableAndIndexQueries().keys()) {
- auto migrateTableData = insertDistinctValuesInTableStatement(m_database, table);
- if (!migrateTableData || migrateTableData->step() != SQLITE_DONE) {
- RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::migrateDataToNewTablesIfNecessary failed to migrate schema, error message: %s", this, m_database.lastErrorMsg());
- ASSERT_NOT_REACHED();
- return;
- }
-
- auto dropTableQuery = m_database.prepareStatementSlow(makeString("DROP TABLE _", table));
- if (!dropTableQuery || dropTableQuery->step() != SQLITE_DONE) {
- RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::migrateDataToNewTablesIfNecessary failed to drop temporary tables, error message: %s", this, m_database.lastErrorMsg());
- ASSERT_NOT_REACHED();
- return;
- }
- }
-
- if (!createUniqueIndices()) {
- RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::migrateDataToNewTablesIfNecessary failed to create unique indices, error message: %s", this, m_database.lastErrorMsg());
- ASSERT_NOT_REACHED();
- return;
- }
-}
-
void ResourceLoadStatisticsDatabaseStore::migrateDataToPCMDatabaseIfNecessary()
{
if (!tableExists("UnattributedPrivateClickMeasurement")
Modified: trunk/Source/WebKit/NetworkProcess/Classifier/ResourceLoadStatisticsDatabaseStore.h (283193 => 283194)
--- trunk/Source/WebKit/NetworkProcess/Classifier/ResourceLoadStatisticsDatabaseStore.h 2021-09-28 21:10:21 UTC (rev 283193)
+++ trunk/Source/WebKit/NetworkProcess/Classifier/ResourceLoadStatisticsDatabaseStore.h 2021-09-28 21:13:01 UTC (rev 283194)
@@ -48,8 +48,6 @@
static constexpr size_t numberOfStatistics = 7;
static constexpr std::array<unsigned, numberOfBucketsPerStatistic> bucketSizes {{ 1, 3, 10, 50, 100 }};
-typedef std::pair<String, std::optional<String>> TableAndIndexPair;
-
// This is always constructed / used / destroyed on the WebResourceLoadStatisticsStore's statistics queue.
class ResourceLoadStatisticsDatabaseStore final : public ResourceLoadStatisticsStore, public DatabaseUtilities {
public:
@@ -118,6 +116,8 @@
static void interruptAllDatabases();
private:
+ const MemoryCompactLookupOnlyRobinHoodHashMap<String, TableAndIndexPair>& expectedTableAndIndexQueries() final;
+ const Vector<String>& sortedTables() final;
void includeTodayAsOperatingDateIfNecessary() override;
void clearOperatingDates() override { }
bool hasStatisticsExpired(WallTime mostRecentUserInteractionTime, OperatingDatesWindow) const override;
@@ -126,10 +126,8 @@
void openITPDatabase();
void addMissingTablesIfNecessary();
bool missingUniqueIndices();
- bool needsUpdatedSchema();
- TableAndIndexPair currentTableAndIndexQueries(const String&);
+ bool needsUpdatedSchema() final;
bool missingReferenceToObservedDomains();
- void migrateDataToNewTablesIfNecessary();
void migrateDataToPCMDatabaseIfNecessary();
bool tableExists(StringView);
void deleteTable(StringView);
@@ -206,7 +204,7 @@
RegistrableDomainsToDeleteOrRestrictWebsiteDataFor registrableDomainsToDeleteOrRestrictWebsiteDataFor() override;
bool isDatabaseStore() const final { return true; }
- bool createUniqueIndices();
+ bool createUniqueIndices() final;
bool createSchema() final;
String ensureAndMakeDomainList(const HashSet<RegistrableDomain>&);
std::optional<WallTime> mostRecentUserInteractionTime(const DomainData&);
Modified: trunk/Source/WebKit/NetworkProcess/DatabaseUtilities.cpp (283193 => 283194)
--- trunk/Source/WebKit/NetworkProcess/DatabaseUtilities.cpp 2021-09-28 21:10:21 UTC (rev 283193)
+++ trunk/Source/WebKit/NetworkProcess/DatabaseUtilities.cpp 2021-09-28 21:13:01 UTC (rev 283194)
@@ -187,4 +187,119 @@
return attribution;
}
+String DatabaseUtilities::stripIndexQueryToMatchStoredValue(const char* originalQuery)
+{
+ return String(originalQuery).replace("CREATE UNIQUE INDEX IF NOT EXISTS", "CREATE UNIQUE INDEX");
+}
+
+TableAndIndexPair DatabaseUtilities::currentTableAndIndexQueries(const String& tableName)
+{
+ auto getTableStatement = m_database.prepareStatement("SELECT sql FROM sqlite_master WHERE tbl_name=? AND type = 'table'"_s);
+ if (!getTableStatement) {
+ RELEASE_LOG_ERROR(PrivateClickMeasurement, "%p - DatabaseUtilities::currentTableAndIndexQueries Unable to prepare statement to fetch schema for the table, error message: %" PRIVATE_LOG_STRING, this, m_database.lastErrorMsg());
+ ASSERT_NOT_REACHED();
+ return { };
+ }
+
+ if (getTableStatement->bindText(1, tableName) != SQLITE_OK) {
+ RELEASE_LOG_ERROR(PrivateClickMeasurement, "%p - DatabaseUtilities::currentTableAndIndexQueries Unable to bind statement to fetch schema for the table, error message: %" PRIVATE_LOG_STRING, this, m_database.lastErrorMsg());
+ ASSERT_NOT_REACHED();
+ return { };
+ }
+
+ if (getTableStatement->step() != SQLITE_ROW) {
+ RELEASE_LOG_ERROR(PrivateClickMeasurement, "%p - DatabaseUtilities::currentTableAndIndexQueries error executing statement to fetch table schema, error message: %" PRIVATE_LOG_STRING, this, m_database.lastErrorMsg());
+ ASSERT_NOT_REACHED();
+ return { };
+ }
+
+ String createTableQuery = getTableStatement->columnText(0);
+
+ auto getIndexStatement = m_database.prepareStatement("SELECT sql FROM sqlite_master WHERE tbl_name=? AND type = 'index'"_s);
+ if (!getIndexStatement) {
+ RELEASE_LOG_ERROR(PrivateClickMeasurement, "%p - DatabaseUtilities::currentTableAndIndexQueries Unable to prepare statement to fetch index for the table, error message: %" PRIVATE_LOG_STRING, this, m_database.lastErrorMsg());
+ ASSERT_NOT_REACHED();
+ return { };
+ }
+
+ if (getIndexStatement->bindText(1, tableName) != SQLITE_OK) {
+ RELEASE_LOG_ERROR(PrivateClickMeasurement, "%p - DatabaseUtilities::currentTableAndIndexQueries Unable to bind statement to fetch index for the table, error message: %" PRIVATE_LOG_STRING, this, m_database.lastErrorMsg());
+ ASSERT_NOT_REACHED();
+ return { };
+ }
+
+ std::optional<String> index;
+ if (getIndexStatement->step() == SQLITE_ROW) {
+ auto rawIndex = String(getIndexStatement->columnText(0));
+ if (!rawIndex.isEmpty())
+ index = rawIndex;
+ }
+
+ return std::make_pair<String, std::optional<String>>(WTFMove(createTableQuery), WTFMove(index));
+}
+
+static Expected<WebCore::SQLiteStatement, int> insertDistinctValuesInTableStatement(WebCore::SQLiteDatabase& database, const String& table)
+{
+ if (table == "SubframeUnderTopFrameDomains")
+ return database.prepareStatement("INSERT INTO SubframeUnderTopFrameDomains SELECT subFrameDomainID, MAX(lastUpdated), topFrameDomainID FROM _SubframeUnderTopFrameDomains GROUP BY subFrameDomainID, topFrameDomainID"_s);
+
+ if (table == "SubresourceUnderTopFrameDomains")
+ return database.prepareStatement("INSERT INTO SubresourceUnderTopFrameDomains SELECT subresourceDomainID, MAX(lastUpdated), topFrameDomainID FROM _SubresourceUnderTopFrameDomains GROUP BY subresourceDomainID, topFrameDomainID"_s);
+
+ if (table == "SubresourceUniqueRedirectsTo")
+ return database.prepareStatement("INSERT INTO SubresourceUniqueRedirectsTo SELECT subresourceDomainID, MAX(lastUpdated), toDomainID FROM _SubresourceUniqueRedirectsTo GROUP BY subresourceDomainID, toDomainID"_s);
+
+ if (table == "TopFrameLinkDecorationsFrom")
+ return database.prepareStatement("INSERT INTO TopFrameLinkDecorationsFrom SELECT toDomainID, MAX(lastUpdated), fromDomainID FROM _TopFrameLinkDecorationsFrom GROUP BY toDomainID, fromDomainID"_s);
+
+ return database.prepareStatementSlow(makeString("INSERT INTO ", table, " SELECT DISTINCT * FROM _", table));
+}
+
+void DatabaseUtilities::migrateDataToNewTablesIfNecessary()
+{
+ if (!needsUpdatedSchema())
+ return;
+
+ auto transactionScope = beginTransactionIfNecessary();
+
+ for (auto& table : expectedTableAndIndexQueries().keys()) {
+ auto alterTable = m_database.prepareStatementSlow(makeString("ALTER TABLE ", table, " RENAME TO _", table));
+ if (!alterTable || alterTable->step() != SQLITE_DONE) {
+ RELEASE_LOG_ERROR(PrivateClickMeasurement, "%p - DatabaseUtilities::migrateDataToNewTablesIfNecessary failed to rename table, error message: %s", this, m_database.lastErrorMsg());
+ ASSERT_NOT_REACHED();
+ return;
+ }
+ }
+
+ if (!createSchema()) {
+ ASSERT_NOT_REACHED();
+ return;
+ }
+
+ // Maintain the order of tables to make sure the ObservedDomains table is created first. Other tables have foreign key constraints referencing it.
+ for (auto& table : sortedTables()) {
+ auto migrateTableData = insertDistinctValuesInTableStatement(m_database, table);
+ if (!migrateTableData || migrateTableData->step() != SQLITE_DONE) {
+ RELEASE_LOG_ERROR(PrivateClickMeasurement, "%p - DatabaseUtilities::migrateDataToNewTablesIfNecessary (table %s) failed to migrate schema, error message: %s", this, table.utf8().data(), m_database.lastErrorMsg());
+ ASSERT_NOT_REACHED();
+ return;
+ }
+ }
+
+ // Drop all tables at the end to avoid trashing data that references data in other tables.
+ for (auto& table : sortedTables()) {
+ auto dropTableQuery = m_database.prepareStatementSlow(makeString("DROP TABLE _", table));
+ if (!dropTableQuery || dropTableQuery->step() != SQLITE_DONE) {
+ RELEASE_LOG_ERROR(PrivateClickMeasurement, "%p - DatabaseUtilities::migrateDataToNewTablesIfNecessary failed to drop temporary tables, error message: %s", this, m_database.lastErrorMsg());
+ ASSERT_NOT_REACHED();
+ return;
+ }
+ }
+
+ if (!createUniqueIndices()) {
+ RELEASE_LOG_ERROR(PrivateClickMeasurement, "%p - DatabaseUtilities::migrateDataToNewTablesIfNecessary failed to create unique indices, error message: %s", this, m_database.lastErrorMsg());
+ ASSERT_NOT_REACHED();
+ }
+}
+
} // namespace WebKit
Modified: trunk/Source/WebKit/NetworkProcess/DatabaseUtilities.h (283193 => 283194)
--- trunk/Source/WebKit/NetworkProcess/DatabaseUtilities.h 2021-09-28 21:10:21 UTC (rev 283193)
+++ trunk/Source/WebKit/NetworkProcess/DatabaseUtilities.h 2021-09-28 21:13:01 UTC (rev 283194)
@@ -27,6 +27,7 @@
#include <WebCore/SQLiteDatabase.h>
#include <WebCore/SQLiteTransaction.h>
+#include <wtf/RobinHoodHashMap.h>
#include <wtf/Scope.h>
namespace WebCore {
@@ -39,6 +40,8 @@
enum class PrivateClickMeasurementAttributionType : bool;
+using TableAndIndexPair = std::pair<String, std::optional<String>>;
+
class DatabaseUtilities {
protected:
DatabaseUtilities(String&& storageFilePath);
@@ -52,8 +55,15 @@
void close();
void interrupt();
virtual bool createSchema() = 0;
+ virtual bool createUniqueIndices() = 0;
virtual void destroyStatements() = 0;
virtual String getDomainStringFromDomainID(unsigned) const = 0;
+ virtual bool needsUpdatedSchema() = 0;
+ virtual const MemoryCompactLookupOnlyRobinHoodHashMap<String, TableAndIndexPair>& expectedTableAndIndexQueries() = 0;
+ virtual const Vector<String>& sortedTables() = 0;
+ TableAndIndexPair currentTableAndIndexQueries(const String&);
+ String stripIndexQueryToMatchStoredValue(const char* originalQuery);
+ void migrateDataToNewTablesIfNecessary();
WebCore::PrivateClickMeasurement buildPrivateClickMeasurementFromDatabase(WebCore::SQLiteStatement&, PrivateClickMeasurementAttributionType) const;
Modified: trunk/Source/WebKit/NetworkProcess/PrivateClickMeasurement/PrivateClickMeasurementDatabase.cpp (283193 => 283194)
--- trunk/Source/WebKit/NetworkProcess/PrivateClickMeasurement/PrivateClickMeasurementDatabase.cpp 2021-09-28 21:10:21 UTC (rev 283193)
+++ trunk/Source/WebKit/NetworkProcess/PrivateClickMeasurement/PrivateClickMeasurementDatabase.cpp 2021-09-28 21:13:01 UTC (rev 283194)
@@ -68,8 +68,8 @@
"earliestTimeToSendToSource REAL, token TEXT, signature TEXT, keyID TEXT, earliestTimeToSendToDestination REAL, sourceApplicationBundleID TEXT, "
"FOREIGN KEY(sourceSiteDomainID) REFERENCES PCMObservedDomains(domainID) ON DELETE CASCADE, FOREIGN KEY(destinationSiteDomainID) REFERENCES "
"PCMObservedDomains(domainID) ON DELETE CASCADE)"_s;
-constexpr auto createUniqueIndexUnattributedPrivateClickMeasurement = "CREATE UNIQUE INDEX IF NOT EXISTS UnattributedPrivateClickMeasurement_sourceSiteDomainID_destinationSiteDomainID on UnattributedPrivateClickMeasurement ( sourceSiteDomainID, destinationSiteDomainID )"_s;
-constexpr auto createUniqueIndexAttributedPrivateClickMeasurement = "CREATE UNIQUE INDEX IF NOT EXISTS AttributedPrivateClickMeasurement_sourceSiteDomainID_destinationSiteDomainID on AttributedPrivateClickMeasurement ( sourceSiteDomainID, destinationSiteDomainID )"_s;
+constexpr auto createUniqueIndexUnattributedPrivateClickMeasurement = "CREATE UNIQUE INDEX IF NOT EXISTS UnattributedPrivateClickMeasurement_sourceSiteDomainID_destinationSiteDomainID_sourceApplicationBundleID on UnattributedPrivateClickMeasurement ( sourceSiteDomainID, destinationSiteDomainID, sourceApplicationBundleID )"_s;
+constexpr auto createUniqueIndexAttributedPrivateClickMeasurement = "CREATE UNIQUE INDEX IF NOT EXISTS AttributedPrivateClickMeasurement_sourceSiteDomainID_destinationSiteDomainID_sourceApplicationBundleID on AttributedPrivateClickMeasurement ( sourceSiteDomainID, destinationSiteDomainID, sourceApplicationBundleID )"_s;
constexpr auto createPCMObservedDomain = "CREATE TABLE PCMObservedDomains ("
"domainID INTEGER PRIMARY KEY, registrableDomain TEXT NOT NULL UNIQUE ON CONFLICT FAIL)"_s;
constexpr auto insertObservedDomainQuery = "INSERT INTO PCMObservedDomains (registrableDomain) VALUES (?)"_s;
@@ -89,6 +89,7 @@
openDatabaseAndCreateSchemaIfNecessary();
enableForeignKeys();
addBundleIDColumnIfNecessary();
+ migrateDataToNewTablesIfNecessary();
allDatabases().add(this);
}
@@ -99,6 +100,28 @@
allDatabases().remove(this);
}
+const MemoryCompactLookupOnlyRobinHoodHashMap<String, TableAndIndexPair>& Database::expectedTableAndIndexQueries()
+{
+ static auto expectedTableAndIndexQueries = makeNeverDestroyed(MemoryCompactLookupOnlyRobinHoodHashMap<String, TableAndIndexPair> {
+ { "PCMObservedDomains"_s, std::make_pair<String, std::optional<String>>(createPCMObservedDomain, std::nullopt) },
+ { "UnattributedPrivateClickMeasurement"_s, std::make_pair<String, std::optional<String>>(createUnattributedPrivateClickMeasurement, stripIndexQueryToMatchStoredValue(createUniqueIndexUnattributedPrivateClickMeasurement)) },
+ { "AttributedPrivateClickMeasurement"_s, std::make_pair<String, std::optional<String>>(createAttributedPrivateClickMeasurement, stripIndexQueryToMatchStoredValue(createUniqueIndexAttributedPrivateClickMeasurement)) },
+ });
+
+ return expectedTableAndIndexQueries;
+}
+
+const Vector<String>& Database::sortedTables()
+{
+ static auto sortedTables = makeNeverDestroyed(Vector<String> {
+ "PCMObservedDomains"_s,
+ "UnattributedPrivateClickMeasurement"_s,
+ "AttributedPrivateClickMeasurement"_s
+ });
+
+ return sortedTables;
+}
+
void Database::interruptAllDatabases()
{
ASSERT(!RunLoop::isMain());
@@ -106,6 +129,16 @@
database->interrupt();
}
+bool Database::createUniqueIndices()
+{
+ if (!m_database.executeCommand(createUniqueIndexUnattributedPrivateClickMeasurement)
+ || !m_database.executeCommand(createUniqueIndexAttributedPrivateClickMeasurement)) {
+ LOG_ERROR("Error creating indexes");
+ return false;
+ }
+ return true;
+}
+
bool Database::createSchema()
{
ASSERT(!RunLoop::isMain());
@@ -674,7 +707,17 @@
checkColumns(attributedTableName);
checkColumns(unattributedTableName);
}
+bool Database::needsUpdatedSchema()
+{
+ // FIXME: Remove this at the end of 2021. No public release was made with the schema missing sourceApplicationBundleID, so this is only needed to migrate internal users who updated in September 2021.
+ for (auto& table : expectedTableAndIndexQueries().keys()) {
+ if (currentTableAndIndexQueries(table) != expectedTableAndIndexQueries().get(table))
+ return true;
+ }
+ return false;
+}
+
Vector<String> Database::columnsForTable(const String& tableName)
{
auto statement = m_database.prepareStatementSlow(makeString("PRAGMA table_info(", tableName, ")"));
Modified: trunk/Source/WebKit/NetworkProcess/PrivateClickMeasurement/PrivateClickMeasurementDatabase.h (283193 => 283194)
--- trunk/Source/WebKit/NetworkProcess/PrivateClickMeasurement/PrivateClickMeasurementDatabase.h 2021-09-28 21:10:21 UTC (rev 283193)
+++ trunk/Source/WebKit/NetworkProcess/PrivateClickMeasurement/PrivateClickMeasurementDatabase.h 2021-09-28 21:13:01 UTC (rev 283194)
@@ -77,6 +77,11 @@
String getDomainStringFromDomainID(DomainID) const final;
void addBundleIDColumnIfNecessary();
+ bool needsUpdatedSchema() final;
+ bool createUniqueIndices() final;
+ const MemoryCompactLookupOnlyRobinHoodHashMap<String, TableAndIndexPair>& expectedTableAndIndexQueries() final;
+ const Vector<String>& sortedTables() final;
+
Vector<String> columnsForTable(const String& tableName);
void addMissingColumnToTable(const String& tableName, const String& columnName);
Modified: trunk/Tools/ChangeLog (283193 => 283194)
--- trunk/Tools/ChangeLog 2021-09-28 21:10:21 UTC (rev 283193)
+++ trunk/Tools/ChangeLog 2021-09-28 21:13:01 UTC (rev 283194)
@@ -1,3 +1,25 @@
+2021-09-28 Kate Cheney <[email protected]>
+
+ PCM: different bundleID entries will override each other
+ https://bugs.webkit.org/show_bug.cgi?id=230839
+
+ Reviewed by Alex Christensen.
+
+ API test coverage for the case of existing PCM data with a bundleID
+ column but an expired unique index.
+
+ * TestWebKitAPI/Tests/WebKitCocoa/PrivateClickMeasurement.mm:
+ (addUnattributedPCMv4):
+ (addAttributedPCMv4):
+ (dumpedPCM):
+ (pollUntilPCMIsMigrated):
+ (emptyPcmDBPath):
+ (createAndPopulatePCMObservedDomainTable):
+ (setUpFromResourceLoadStatisticsDatabase):
+ (setUpFromPCMDatabase):
+ (TEST):
+ (setUp): Deleted.
+
2021-09-28 Eddy Wong <[email protected]>
Added GlobalSign R3/R5 Root CA cert to webkitcorepy to resolve certain pip module download SSL error.
Modified: trunk/Tools/TestWebKitAPI/Tests/WebKitCocoa/PrivateClickMeasurement.mm (283193 => 283194)
--- trunk/Tools/TestWebKitAPI/Tests/WebKitCocoa/PrivateClickMeasurement.mm 2021-09-28 21:10:21 UTC (rev 283193)
+++ trunk/Tools/TestWebKitAPI/Tests/WebKitCocoa/PrivateClickMeasurement.mm 2021-09-28 21:13:01 UTC (rev 283194)
@@ -164,6 +164,49 @@
addValuesToTable<7>(database, insertUnattributedPrivateClickMeasurementQueryV3, { 2, 3, 43, 1.0, "test token", "test signature", "test key id" });
}
+static void addUnattributedPCMv4(WebCore::SQLiteDatabase& database)
+{
+ constexpr auto createUnattributedPrivateClickMeasurementV4 = "CREATE TABLE UnattributedPrivateClickMeasurement ("
+ "sourceSiteDomainID INTEGER NOT NULL, destinationSiteDomainID INTEGER NOT NULL, sourceID INTEGER NOT NULL, "
+ "timeOfAdClick REAL NOT NULL, token TEXT, signature TEXT, keyID TEXT, sourceApplicationBundleID TEXT, FOREIGN KEY(sourceSiteDomainID) "
+ "REFERENCES PCMObservedDomains(domainID) ON DELETE CASCADE, FOREIGN KEY(destinationSiteDomainID) REFERENCES "
+ "PCMObservedDomains(domainID) ON DELETE CASCADE)"_s;
+
+ EXPECT_TRUE(database.executeCommand(createUnattributedPrivateClickMeasurementV4));
+
+ constexpr auto insertUnattributedPrivateClickMeasurementQueryV4 = "INSERT OR REPLACE INTO UnattributedPrivateClickMeasurement (sourceSiteDomainID, destinationSiteDomainID, "
+ "sourceID, timeOfAdClick, token, signature, keyID, sourceApplicationBundleID) VALUES (?, ?, ?, ?, ?, ?, ?, ?)"_s;
+
+#if PLATFORM(MAC)
+ auto bundleID = "com.apple.Safari";
+#else
+ auto bundleID = "com.apple.mobilesafari";
+#endif
+ addValuesToTable<8>(database, insertUnattributedPrivateClickMeasurementQueryV4, { 2, 3, 43, 1.0, "test token", "test signature", "test key id", bundleID });
+}
+
+static void addAttributedPCMv4(WebCore::SQLiteDatabase& database)
+{
+ constexpr auto createAttributedPrivateClickMeasurementV4 = "CREATE TABLE AttributedPrivateClickMeasurement ("
+ "sourceSiteDomainID INTEGER NOT NULL, destinationSiteDomainID INTEGER NOT NULL, sourceID INTEGER NOT NULL, "
+ "attributionTriggerData INTEGER NOT NULL, priority INTEGER NOT NULL, timeOfAdClick REAL NOT NULL, "
+ "earliestTimeToSendToSource REAL, token TEXT, signature TEXT, keyID TEXT, earliestTimeToSendToDestination REAL, sourceApplicationBundleID TEXT,"
+ "FOREIGN KEY(sourceSiteDomainID) REFERENCES PCMObservedDomains(domainID) ON DELETE CASCADE, FOREIGN KEY(destinationSiteDomainID) REFERENCES "
+ "PCMObservedDomains(domainID) ON DELETE CASCADE)"_s;
+
+ EXPECT_TRUE(database.executeCommand(createAttributedPrivateClickMeasurementV4));
+
+ constexpr auto insertAttributedPrivateClickMeasurementQueryV4 = "INSERT OR REPLACE INTO AttributedPrivateClickMeasurement (sourceSiteDomainID, destinationSiteDomainID, "
+ "sourceID, attributionTriggerData, priority, timeOfAdClick, earliestTimeToSendToSource, token, signature, keyID, earliestTimeToSendToDestination, sourceApplicationBundleID) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"_s;
+
+#if PLATFORM(MAC)
+ auto bundleID = "com.apple.Safari";
+#else
+ auto bundleID = "com.apple.mobilesafari";
+#endif
+ addValuesToTable<12>(database, insertAttributedPrivateClickMeasurementQueryV4, { 1, 2, 42, 14, 7, 1.0, earliestTimeToSend(), "test token", "test signature", "test key id", earliestTimeToSend(), bundleID });
+}
+
static RetainPtr<NSString> dumpedPCM(WKWebView *webView)
{
__block RetainPtr<NSString> pcm;
@@ -172,14 +215,18 @@
}];
while (!pcm)
TestWebKitAPI::Util::spinRunLoop();
+
return pcm;
}
-static void pollUntilPCMIsMigrated(WKWebView *webView)
+enum class MigratingFromResourceLoadStatistics : bool { No, Yes };
+static void pollUntilPCMIsMigrated(WKWebView *webView, MigratingFromResourceLoadStatistics migratingFromResourceLoadStatistics)
{
- // This query is the first thing to open the old database, so migration has not happened yet.
- const char* emptyPCMDatabase = "\nNo stored Private Click Measurement data.\n";
- EXPECT_WK_STREQ(dumpedPCM(webView).get(), emptyPCMDatabase);
+ if (migratingFromResourceLoadStatistics == MigratingFromResourceLoadStatistics::Yes) {
+ // This query is the first thing to open the old database, so migration has not happened yet.
+ const char* emptyPCMDatabase = "\nNo stored Private Click Measurement data.\n";
+ EXPECT_WK_STREQ(dumpedPCM(webView).get(), emptyPCMDatabase);
+ }
NSString *expectedMigratedPCMDatabase = @""
"Unattributed Private Click Measurements:\n"
@@ -224,6 +271,17 @@
return fileURL.path;
}
+static NSString *emptyPcmDBPath()
+{
+ NSFileManager *defaultFileManager = NSFileManager.defaultManager;
+ NSURL *itpRootURL = WKWebsiteDataStore.defaultDataStore._configuration._resourceLoadStatisticsDirectory;
+ NSURL *fileURL = [itpRootURL URLByAppendingPathComponent:@"pcm.db"];
+ [defaultFileManager removeItemAtPath:itpRootURL.path error:nil];
+ EXPECT_FALSE([defaultFileManager fileExistsAtPath:itpRootURL.path]);
+ [defaultFileManager createDirectoryAtURL:itpRootURL withIntermediateDirectories:YES attributes:nil error:nil];
+ return fileURL.path;
+}
+
static void cleanUp()
{
NSFileManager *defaultFileManager = NSFileManager.defaultManager;
@@ -254,8 +312,24 @@
addObservedDomain("www.webkit.org");
}
-void setUp(void(*addUnattributedPCM)(WebCore::SQLiteDatabase&), void(*addAttributedPCM)(WebCore::SQLiteDatabase&))
+static void createAndPopulatePCMObservedDomainTable(WebCore::SQLiteDatabase& database)
{
+ auto addObservedDomain = [&](const char* domain) {
+ constexpr auto insertObservedDomainQuery = "INSERT INTO PCMObservedDomains (registrableDomain) VALUES (?)"_s;
+ addValuesToTable<1>(database, insertObservedDomainQuery, { domain });
+ };
+
+ constexpr auto createPCMObservedDomain = "CREATE TABLE PCMObservedDomains ("
+ "domainID INTEGER PRIMARY KEY, registrableDomain TEXT NOT NULL UNIQUE ON CONFLICT FAIL)"_s;
+
+ EXPECT_TRUE(database.executeCommand(createPCMObservedDomain));
+ addObservedDomain("example.com");
+ addObservedDomain("webkit.org");
+ addObservedDomain("www.webkit.org");
+}
+
+void setUpFromResourceLoadStatisticsDatabase(void(*addUnattributedPCM)(WebCore::SQLiteDatabase&), void(*addAttributedPCM)(WebCore::SQLiteDatabase&))
+{
WebCore::SQLiteDatabase database;
EXPECT_TRUE(database.open(emptyObservationsDBPath()));
createAndPopulateObservedDomainTable(database);
@@ -264,26 +338,44 @@
database.close();
}
+void setUpFromPCMDatabase(void(*addUnattributedPCM)(WebCore::SQLiteDatabase&), void(*addAttributedPCM)(WebCore::SQLiteDatabase&))
+{
+ WebCore::SQLiteDatabase database;
+ EXPECT_TRUE(database.open(emptyPcmDBPath()));
+ createAndPopulatePCMObservedDomainTable(database);
+ addUnattributedPCM(database);
+ addAttributedPCM(database);
+ database.close();
+}
+
TEST(PrivateClickMeasurement, MigrateFromResourceLoadStatistics1)
{
- setUp(addUnattributedPCMv1, addAttributedPCMv1);
+ setUpFromResourceLoadStatisticsDatabase(addUnattributedPCMv1, addAttributedPCMv1);
auto webView = webViewWithResourceLoadStatisticsEnabledInNetworkProcess();
- pollUntilPCMIsMigrated(webView.get());
+ pollUntilPCMIsMigrated(webView.get(), MigratingFromResourceLoadStatistics::Yes);
cleanUp();
}
TEST(PrivateClickMeasurement, MigrateFromResourceLoadStatistics2)
{
- setUp(addUnattributedPCMv2, addAttributedPCMv2);
+ setUpFromResourceLoadStatisticsDatabase(addUnattributedPCMv2, addAttributedPCMv2);
auto webView = webViewWithResourceLoadStatisticsEnabledInNetworkProcess();
- pollUntilPCMIsMigrated(webView.get());
+ pollUntilPCMIsMigrated(webView.get(), MigratingFromResourceLoadStatistics::Yes);
cleanUp();
}
TEST(PrivateClickMeasurement, MigrateFromResourceLoadStatistics3)
{
- setUp(addUnattributedPCMv3, addAttributedPCMv3);
+ setUpFromResourceLoadStatisticsDatabase(addUnattributedPCMv3, addAttributedPCMv3);
auto webView = webViewWithResourceLoadStatisticsEnabledInNetworkProcess();
- pollUntilPCMIsMigrated(webView.get());
+ pollUntilPCMIsMigrated(webView.get(), MigratingFromResourceLoadStatistics::Yes);
cleanUp();
}
+
+TEST(PrivateClickMeasurement, MigrateFromPCM1)
+{
+ setUpFromPCMDatabase(addUnattributedPCMv4, addAttributedPCMv4);
+ auto webView = webViewWithResourceLoadStatisticsEnabledInNetworkProcess();
+ pollUntilPCMIsMigrated(webView.get(), MigratingFromResourceLoadStatistics::No);
+ cleanUp();
+}
