Library: improve logged scan summary, also log new and missing tracks

src/library/dao/trackdao.cpp

@@ -277,6 +277,26 @@ QSet<QString> TrackDAO::getAllTrackLocations() const {
     return locations;
 }
 
+QSet<QString> TrackDAO::getAllExistingTrackLocations() const {
+    QSet<QString> locations;
+    QSqlQuery query(m_database);
+    query.prepare(
+            "SELECT track_locations.location "
+            "FROM library INNER JOIN track_locations "
+            "ON library.location = track_locations.id "
+            "WHERE fs_deleted=0");
+    if (!query.exec()) {
+        LOG_FAILED_QUERY(query);
+        DEBUG_ASSERT(!"Failed query");
+    }
+
+    int locationColumn = query.record().indexOf("location");
+    while (query.next()) {
+        locations.insert(query.value(locationColumn).toString());
+    }
+    return locations;
+}
+
 // Some code (eg. drag and drop) needs to just get a track's location, and it's
 // not worth retrieving a whole Track.
 QString TrackDAO::getTrackLocation(TrackId trackId) const {
@@ -1768,27 +1788,27 @@ void TrackDAO::markUnverifiedTracksAsDeleted() {
 }
 
 namespace {
-    // Computed the longest match from the right of both strings
-    int matchStringSuffix(const QString& str1, const QString& str2) {
-        int matchLength = 0;
-        int minLength = math_min(str1.length(), str2.length());
-        while (matchLength < minLength) {
-            if (str1[str1.length() - matchLength - 1] != str2[str2.length() - matchLength - 1]) {
-                // first mismatch
-                break;
-            }
-            ++matchLength;
+// Computed the longest match from the right of both strings
+int matchStringSuffix(const QString& str1, const QString& str2) {
+    int matchLength = 0;
+    int minLength = math_min(str1.length(), str2.length());
+    while (matchLength < minLength) {
+        if (str1[str1.length() - matchLength - 1] != str2[str2.length() - matchLength - 1]) {
+            // first mismatch
+            break;
         }
-        return matchLength;
+        ++matchLength;
     }
-    } // namespace
+    return matchLength;
+}
+} // namespace
 
 // Look for moved files. Look for files that have been marked as
-// "deleted on disk" and see if another "file" with the same name and
-// files size exists in the track_locations table. That means the file has
+// 'deleted on disk' and see if another track with the same file name and
+// duration exists in the track_locations table. That means the file has been
 // moved instead of being deleted outright, and so we can salvage your
 // existing metadata that you have in your DB (like cue points, etc.).
-// returns falls if canceled
+// Returns false if canceled.
 bool TrackDAO::detectMovedTracks(
         QList<RelocatedTrack> *pRelocatedTracks,
         const QStringList& addedTracks,
@@ -1880,6 +1900,7 @@ bool TrackDAO::detectMovedTracks(
             const auto nextSuffixMatch =
                     matchStringSuffix(nextTrackLocation, oldTrackLocation);
             DEBUG_ASSERT(nextSuffixMatch >= filename.length());
+            // document this
             if (newTrackLocationSuffixMatch < nextSuffixMatch) {
                 newTrackLocationSuffixMatch = nextSuffixMatch;
                 newTrackId = TrackId(newTrackQuery.value(newTrackIdColumn));
@@ -2031,8 +2052,7 @@ bool TrackDAO::verifyRemainingTracks(
         int fs_deleted = 0;
         for (const auto& rootDir : libraryRootDirs) {
             if (trackLocation.startsWith(rootDir.location())) {
-                // Track is under the library root,
-                // but was not verified.
+                // Track is under the library root, but was not verified.
                 // This happens if the track was deleted
                 // a symlink duplicate or on a non normalized
                 // path like on non case sensitive file systems.

src/library/dao/trackdao.h

@@ -59,8 +59,11 @@ class TrackDAO : public QObject, public virtual DAO, public virtual GlobalTrackC
     TrackPointer getTrackByRef(
             const TrackRef& trackRef) const;
 
-    // Returns a set of all track locations in the library.
+    // Returns a set of all track locations in the library,
+    // incl. locations of tracks currently marked as missing.
     QSet<QString> getAllTrackLocations() const;
+    // Return only tracks that are reported to exist during last scan.
+    QSet<QString> getAllExistingTrackLocations() const;
     QString getTrackLocation(TrackId trackId) const;
 
     // Only used by friend class LibraryScanner, but public for testing!

src/library/scanner/libraryscanner.cpp

@@ -199,6 +199,9 @@ void LibraryScanner::slotStartScan() {
     changeScannerState(SCANNING);
 
     QSet<QString> trackLocations = m_trackDao.getAllTrackLocations();
+    // Store number of existing tracks so we can calculat the number
+    // of missing tracks in slotFinishUnhashedScan().
+    m_numPreviousExistingTrackLocations = m_trackDao.getAllExistingTrackLocations().size();
     QHash<QString, mixxx::cache_key_t> directoryHashes = m_libraryHashDao.getDirectoryHashes();
     QRegularExpression extensionFilter(SoundSourceProxy::getSupportedFileNamesRegex());
     QRegularExpression coverExtensionFilter =
@@ -301,7 +304,8 @@ void LibraryScanner::slotFinishHashedScan() {
     pWatcher->taskDone();
 }
 
-void LibraryScanner::cleanUpScan() {
+// Quick hack: return number of relocated tracks
+int LibraryScanner::cleanUpScan() {
     // At the end of a scan, mark all tracks and directories that weren't
     // "verified" as "deleted" (as long as the scan wasn't canceled half way
     // through). This condition is important because our rescanning algorithm
@@ -335,7 +339,7 @@ void LibraryScanner::cleanUpScan() {
             m_libraryRootDirs,
             m_scannerGlobal->shouldCancelPointer())) {
         // canceled
-        return;
+        return 0;
     }
 
     kLogger.debug() << "Marking unverified tracks as deleted";
@@ -346,6 +350,7 @@ void LibraryScanner::cleanUpScan() {
 
     // Check to see if the "deleted" tracks showed up in another location,
     // and if so, do some magic to update all our tables.
+    int numRelocatedTracks = 0;
     kLogger.debug() << "Detecting moved files";
     {
         QList<RelocatedTrack> relocatedTracks;
@@ -355,14 +360,16 @@ void LibraryScanner::cleanUpScan() {
                 m_scannerGlobal->shouldCancelPointer())) {
             kLogger.info()
                     << "Detecting moved files has been canceled or aborted";
-            return;
+            return 0;
         }
         if (!relocatedTracks.isEmpty()) {
+            numRelocatedTracks = relocatedTracks.size();
             kLogger.info()
                     << "Found"
-                    << relocatedTracks.size()
+                    << numRelocatedTracks
                     << "moved track(s)";
-            emit tracksRelocated(relocatedTracks);
+            m_scannerGlobal->addedTracks(),
+                    emit tracksRelocated(relocatedTracks);
         }
     }
 
@@ -383,8 +390,9 @@ void LibraryScanner::cleanUpScan() {
     if (!coverArtTracksChanged.isEmpty()) {
         emit tracksChanged(coverArtTracksChanged);
     }
-}
 
+    return numRelocatedTracks;
+}
 
 // is called when all tasks of the second stage are done (threads are finished)
 void LibraryScanner::slotFinishUnhashedScan() {
@@ -407,8 +415,9 @@ void LibraryScanner::slotFinishUnhashedScan() {
     m_trackDao.addTracksFinish(!m_scannerGlobal->shouldCancel() &&
                                !bScanFinishedCleanly);
 
+    int numMovedTracks = 0;
     if (!m_scannerGlobal->shouldCancel() && bScanFinishedCleanly) {
-        cleanUpScan();
+        numMovedTracks = cleanUpScan();
     }
 
     if (!m_scannerGlobal->shouldCancel() && bScanFinishedCleanly) {
@@ -422,17 +431,24 @@ void LibraryScanner::slotFinishUnhashedScan() {
         kLogger.debug() << "Scan cancelled";
     }
 
-    // TODO(XXX) doesn't take into account verifyRemainingTracks.
-    qDebug("Scan took: %s. "
-           "%d unchanged directories. "
-           "%d changed/added directories. "
-           "%d tracks verified from changed/added directories. "
-           "%d new tracks.",
-            m_scannerGlobal->timerElapsed().formatNanosWithUnit().toLocal8Bit().constData(),
-            static_cast<int>(m_scannerGlobal->verifiedDirectories().size()),
-            m_scannerGlobal->numScannedDirectories(),
-            static_cast<int>(m_scannerGlobal->verifiedTracks().size()),
-            static_cast<int>(m_scannerGlobal->addedTracks().size()));
+    const QString durationString = m_scannerGlobal->timerElapsed().formatMillisWithUnit();
+    const int numUnchangedDirs = static_cast<int>(m_scannerGlobal->verifiedDirectories().size());
+    const int numChangedAddedDirs = m_scannerGlobal->numScannedDirectories();
+    const int numVerifiedTracks = static_cast<int>(m_scannerGlobal->verifiedTracks().size());
+    const int numNewTracks = m_scannerGlobal->addedTracks().size() - numMovedTracks;
+    const int numMissingTracks = m_numPreviousExistingTrackLocations +
+            numNewTracks -
+            m_trackDao.getAllExistingTrackLocations().size();
+    // TODO Use this information to display a scan summary popup.
+    qInfo() << "----------------------------------------------";
+    qInfo("Library scan finished after %s", durationString.toLocal8Bit().constData());
+    qInfo(" %d unchanged directories", numUnchangedDirs);
+    qInfo(" %d changed/added directories", numChangedAddedDirs);
+    qInfo(" %d tracks verified from changed/added directories", numVerifiedTracks);
+    qInfo(" %d new tracks", numNewTracks);
+    qInfo(" %d moved tracks", numMovedTracks);
+    qInfo(" %d missing tracks", numMissingTracks);
+    qInfo() << "----------------------------------------------";
 
     m_scannerGlobal.clear();
     changeScannerState(FINISHED);
@@ -566,6 +582,7 @@ void LibraryScanner::slotTrackExists(const QString& trackPath) {
     }
 }
 
+// triggered by ScannerTask::addNewTrack / in ImportFilesTask::run()
 void LibraryScanner::slotAddNewTrack(const QString& trackPath) {
     //kLogger.debug() << "slotAddNewTrack" << trackPath;
     ScopedTimer timer(QStringLiteral("LibraryScanner::addNewTrack"));
@@ -575,16 +592,20 @@ void LibraryScanner::slotAddNewTrack(const QString& trackPath) {
             false);
     if (pTrack) {
         DEBUG_ASSERT(!pTrack->isDirty());
-        // The track's actual location might differ from the
-        // given trackPath
+        // The track's actual location might differ from the given trackPath.
+        // TODO why?
         const QString trackLocation(pTrack->getLocation());
         // Acknowledge successful track addition
         if (m_scannerGlobal) {
+            // TODO emit pTrack, or use a new signal to store the track id
+            // elsewhere, so we can create a 'New' trackset
             m_scannerGlobal->trackAdded(trackLocation);
         }
-        // Signal the main instance of TrackDAO, that there is
-        // a new track in the database.
+        // Signal for TrackCollectionManager::afterTrackAdded() that there is
+        // a new track in the database. (will add save track to external collections?)
+        // Note: new tracks are marked clean  = !ptrack->isDirty()
         emit trackAdded(pTrack);
+        // Update the scanner dialog
         emit progressLoading(trackLocation);
     } else {
         // Acknowledge failed track addition

src/library/scanner/libraryscanner.h

@@ -93,7 +93,7 @@ class LibraryScanner : public QThread {
     // CANCELING -> IDLE
     bool changeScannerState(LibraryScanner::ScannerState newState);
 
-    void cleanUpScan();
+    int cleanUpScan();
 
     mixxx::DbConnectionPoolPtr m_pDbConnectionPool;
 
@@ -118,6 +118,8 @@ class LibraryScanner : public QThread {
     // this is accessed main and LibraryScanner thread
     volatile ScannerState m_state;
 
+    int m_numPreviousExistingTrackLocations;
+
     QList<mixxx::FileInfo> m_libraryRootDirs;
     QScopedPointer<LibraryScannerDlg> m_pProgressDlg;
 };

src/library/scanner/scannerglobal.h

@@ -144,6 +144,9 @@ class ScannerGlobal {
     }
     void trackAdded(const QString& trackLocation) {
         m_addedTracks << trackLocation;
+        // TODO Collect TrackIds?
+        // If Track is clean / !isDirty() it is new and we may add it
+        // to some sort of 'New Tracks' list (special crate?)
     }
 
     int numScannedDirectories() const {