From 1918535783947b1114b7b5d165eceebb0a4b0267 Mon Sep 17 00:00:00 2001
From: Varun Patil <radialapps@gmail.com>
Date: Wed, 3 Apr 2024 01:14:08 -0700
Subject: [PATCH] timeline: allow de-dup with AUID (close #1112)

Signed-off-by: Varun Patil <radialapps@gmail.com>
---
 CHANGELOG.md                    |  4 ++++
 lib/Db/TimelineQuery.php        |  1 +
 lib/Db/TimelineQueryNativeX.php |  2 +-
 src/components/Settings.vue     | 12 ++++++++++++
 src/components/Timeline.vue     |  7 +++++++
 src/services/static-config.ts   |  1 +
 src/typings/config.d.ts         |  1 +
 7 files changed, 27 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index f4d5d742e..063c12776 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,10 @@
 
 All notable changes to this project will be documented in this file.
 
+## [Unreleased]
+
+- **Feature**: Add option to de-duplicate identical files ([#1112](https://github.com/pulsejet/memories/issues/1112)).
+
 ## [v7.1.0] - 2024-04-01
 
 - **Notice**: This release significantly overhauls the core querying infrastructure for better performance. If you run into any regressions (including performance), please [file a bug report](https://github.com/pulsejet/memories/issues)
diff --git a/lib/Db/TimelineQuery.php b/lib/Db/TimelineQuery.php
index da33d965a..c3e719679 100644
--- a/lib/Db/TimelineQuery.php
+++ b/lib/Db/TimelineQuery.php
@@ -23,6 +23,7 @@ class TimelineQuery
         'm.w', 'm.h', 'm.liveid',
         'm.isvideo', 'm.video_duration',
         'f.etag', 'f.name AS basename',
+        'f.size', 'm.epoch', // auid
         'mimetypes.mimetype',
     ];
 
diff --git a/lib/Db/TimelineQueryNativeX.php b/lib/Db/TimelineQueryNativeX.php
index 4eab3db9a..918e515f3 100644
--- a/lib/Db/TimelineQueryNativeX.php
+++ b/lib/Db/TimelineQueryNativeX.php
@@ -11,7 +11,7 @@ trait TimelineQueryNativeX
     public function transformNativeQuery(IQueryBuilder &$query, bool $aggregate): void
     {
         if (!$aggregate) {
-            $query->addSelect('m.epoch', 'f.size', 'm.buid');
+            $query->addSelect('m.buid');
         }
     }
 }
diff --git a/src/components/Settings.vue b/src/components/Settings.vue
index b5f17b6fa..cc9786e32 100644
--- a/src/components/Settings.vue
+++ b/src/components/Settings.vue
@@ -36,6 +36,14 @@
         >
           {{ t('memories', 'Stack RAW files with same name') }}
         </NcCheckboxRadioSwitch>
+
+        <NcCheckboxRadioSwitch
+          :checked.sync="config.dedup_identical"
+          @update:checked="updateDedupIdentical"
+          type="switch"
+        >
+          {{ t('memories', 'De-duplicate identical files') }}
+        </NcCheckboxRadioSwitch>
       </NcAppSettingsSection>
 
       <NcAppSettingsSection id="viewer-settings" :name="names.viewer">
@@ -314,6 +322,10 @@ export default defineComponent({
       await this.updateSetting('stack_raw_files', 'stackRawFiles');
     },
 
+    async updateDedupIdentical() {
+      await this.updateSetting('dedup_identical', 'dedupIdentical');
+    },
+
     // Viewer settings
     async updateHighResCond(val: IConfig['high_res_cond']) {
       this.config.high_res_cond = val;
diff --git a/src/components/Timeline.vue b/src/components/Timeline.vue
index 3d1e77cf2..3af9f549f 100644
--- a/src/components/Timeline.vue
+++ b/src/components/Timeline.vue
@@ -1071,6 +1071,7 @@ export default defineComponent({
       // Set of basenames without extension
       const res1: IPhoto[] = [];
       const toStack = new Map<string, IPhoto[]>();
+      const auids = new Set<string>();
 
       // First pass -- remove hidden and prepare
       for (const photo of data) {
@@ -1078,6 +1079,12 @@ export default defineComponent({
         if (photo.ishidden) continue;
         if (photo.basename?.startsWith('.')) continue;
 
+        // Skip identical duplicates
+        if (this.config.dedup_identical && photo.auid) {
+          if (auids.has(photo.auid)) continue;
+          auids.add(photo.auid);
+        }
+
         // Add to first pass result
         res1.push(photo);
 
diff --git a/src/services/static-config.ts b/src/services/static-config.ts
index 2a2630050..897fb447f 100644
--- a/src/services/static-config.ts
+++ b/src/services/static-config.ts
@@ -134,6 +134,7 @@ class StaticConfig {
       timeline_path: '_unknown_',
       enable_top_memories: true,
       stack_raw_files: true,
+      dedup_identical: false,
 
       // viewer settings
       high_res_cond_default: 'zoom',
diff --git a/src/typings/config.d.ts b/src/typings/config.d.ts
index dee9cdb7c..9849f16e9 100644
--- a/src/typings/config.d.ts
+++ b/src/typings/config.d.ts
@@ -21,6 +21,7 @@ declare module '@typings' {
     timeline_path: string;
     enable_top_memories: boolean;
     stack_raw_files: boolean;
+    dedup_identical: boolean;
 
     // viewer settings
     high_res_cond_default: HighResCond;