Mark person in preview (fix #79)

lib/Controller/ApiController.php

@@ -99,6 +99,11 @@ private function getTransformations() {
             if ($face) {
                 $transforms[] = array($this->timelineQuery, 'transformFaceFilter', $face);
             }
+
+            $faceRect = $this->request->getParam('facerect');
+            if ($faceRect) {
+                $transforms[] = array($this->timelineQuery, 'transformFaceRect', $face);
+            }
         }
 
         // Filter only for one tag

lib/Db/TimelineQueryDays.php

@@ -18,6 +18,9 @@ private function processDays(&$days) {
         foreach($days as &$row) {
             $row["dayid"] = intval($row["dayid"]);
             $row["count"] = intval($row["count"]);
+
+            // All transform processing
+            $this->processFace($row, true);
         }
         return $days;
     }
@@ -44,6 +47,9 @@ private function processDay(&$day) {
                 $row["isfavorite"] = 1;
             }
             unset($row["categoryid"]);
+
+            // All transform processing
+            $this->processFace($row);
         }
         return $day;
     }

lib/Db/TimelineQueryFaces.php

@@ -17,25 +17,49 @@ public function transformFaceFilter(IQueryBuilder &$query, string $userId, strin
         $faceUid = $faceNames[0];
         $faceName = $faceNames[1];
 
-        // Get cluster ID
-        $sq = $query->getConnection()->getQueryBuilder();
-        $idQuery = $sq->select('id')->from('recognize_face_clusters')
-            ->where($query->expr()->eq('user_id', $sq->createNamedParameter($faceUid)));
-
-        // If name is a number then it is an ID
-        $nameField = is_numeric($faceName) ? 'id' : 'title';
-        $idQuery->andWhere($query->expr()->eq($nameField, $sq->createNamedParameter($faceName)));
-
-        $id = $idQuery->executeQuery()->fetchOne();
-        if (!$id) throw new \Exception("Unknown person: $faceStr");
-
         // Join with cluster
+        $nameField = is_numeric($faceName) ? 'rfc.id' : 'rfc.title';
+        $query->innerJoin('m', 'recognize_face_clusters', 'rfc', $query->expr()->andX(
+            $query->expr()->eq('user_id', $query->createNamedParameter($faceUid)),
+            $query->expr()->eq($nameField, $query->createNamedParameter($faceName)),
+        ));
+
+        // Join with detections
         $query->innerJoin('m', 'recognize_face_detections', 'rfd', $query->expr()->andX(
             $query->expr()->eq('rfd.file_id', 'm.fileid'),
-            $query->expr()->eq('rfd.cluster_id', $query->createNamedParameter($id)),
+            $query->expr()->eq('rfd.cluster_id', 'rfc.id'),
         ));
     }
 
+    public function transformFaceRect(IQueryBuilder &$query, string $userId) {
+        // Include detection params in response
+        $query->addSelect(
+            'rfd.width AS face_w',
+            'rfd.height AS face_h',
+            'rfd.x AS face_x',
+            'rfd.y AS face_y',
+        );
+    }
+
+    /** Convert face fields to object */
+    private function processFace(&$row, $days=false) {
+        if (!isset($row) || !isset($row['face_w'])) return;
+
+        if (!$days) {
+            $row["facerect"] = [
+                "w" => floatval($row["face_w"]),
+                "h" => floatval($row["face_h"]),
+                "x" => floatval($row["face_x"]),
+                "y" => floatval($row["face_y"]),
+            ];
+        }
+
+        unset($row["face_w"]);
+        unset($row["face_h"]);
+        unset($row["face_x"]);
+        unset($row["face_y"]);
+    }
+
     public function getFaces(Folder $folder) {
         $query = $this->connection->getQueryBuilder();
 

src/components/Timeline.vue

@@ -106,6 +106,7 @@ import { generateUrl } from '@nextcloud/router'
 import { showError } from '@nextcloud/dialogs'
 import { NcEmptyContent } from '@nextcloud/vue';
 import GlobalMixin from '../mixins/GlobalMixin';
+import UserConfig from "../mixins/UserConfig";
 
 import { ViewerManager } from "../services/Viewer";
 import { getLayout } from "../services/Layout";
@@ -119,7 +120,6 @@ import TopMatter from "./top-matter/TopMatter.vue";
 import OnThisDay from "./top-matter/OnThisDay.vue";
 import SelectionManager from './SelectionManager.vue';
 import ScrollerManager from './ScrollerManager.vue';
-import UserConfig from "../mixins/UserConfig";
 
 import ArchiveIcon from 'vue-material-design-icons/PackageDown.vue';
 import CheckCircle from 'vue-material-design-icons/CheckCircle.vue';
@@ -436,6 +436,11 @@ export default class Timeline extends Mixins(GlobalMixin, UserConfig) {
         // People
         if (this.$route.name === 'people' && this.$route.params.user && this.$route.params.name) {
             query.set('face', `${this.$route.params.user}/${this.$route.params.name}`);
+
+            // Face rect
+            if (this.config_showFaceRect) {
+                query.set('facerect', '1');
+            }
         }
 
         // Tags
@@ -492,12 +497,11 @@ export default class Timeline extends Mixins(GlobalMixin, UserConfig) {
 
     /** Fetch timeline main call */
     async fetchDays(noCache=false) {
-        let url = '/apps/memories/api/days';
         let params: any = {};
+        let url = generateUrl(this.appendQuery('/apps/memories/api/days'), params);
 
         // Try cache first
         let cache: IDay[];
-        const cacheUrl = window.location.pathname + 'api/days';
 
         // Make sure to refresh scroll later
         this.currentEnd = -1;
@@ -515,18 +519,18 @@ export default class Timeline extends Mixins(GlobalMixin, UserConfig) {
                 data = await dav.getPeopleData();
             } else {
                 // Try the cache
-                cache = noCache ? null : (await utils.getCachedData(cacheUrl));
+                cache = noCache ? null : (await utils.getCachedData(url));
                 if (cache) {
                     await this.processDays(cache);
                     this.loading--;
                 }
 
                 // Get from network
-                data = (await axios.get<IDay[]>(generateUrl(this.appendQuery(url), params))).data;
+                data = (await axios.get<IDay[]>(url)).data;
             }
 
             // Put back into cache
-            utils.cacheData(cacheUrl, data);
+            utils.cacheData(url, data);
 
             // Make sure we're still on the same page
             if (this.state !== startState) return;

src/components/frame/Photo.vue

@@ -22,16 +22,18 @@
             @touchend="touchend"
             @touchcancel="touchend" >
             <img
+                ref="img"
                 class="fill-block"
-                :src="src()"
+                :src="src"
                 :key="data.fileid"
+                @load="load"
                 @error="error" />
         </div>
     </div>
 </template>
 
 <script lang="ts">
-import { Component, Prop, Emit, Mixins } from 'vue-property-decorator';
+import { Component, Prop, Emit, Mixins, Watch } from 'vue-property-decorator';
 import { IDay, IPhoto } from "../../types";
 
 import { getPreviewUrl } from "../../services/FileUtils";
@@ -51,17 +53,36 @@ import Star from 'vue-material-design-icons/Star.vue';
 })
 export default class Photo extends Mixins(GlobalMixin) {
     private touchTimer = 0;
+    private src = null;
+    private hasFaceRect = false;
 
     @Prop() data: IPhoto;
     @Prop() day: IDay;
 
     @Emit('select') emitSelect(data: IPhoto) {}
     @Emit('click') emitClick() {}
 
+    @Watch('data')
+    onDataChange(newData: IPhoto, oldData: IPhoto) {
+        // Copy flags relevant to this component
+        if (oldData && newData) {
+            newData.flag |= oldData.flag & (this.c.FLAG_SELECTED | this.c.FLAG_LOAD_FAIL);
+        }
+    }
+
+    mounted() {
+        this.hasFaceRect = false;
+        this.refresh();
+    }
+
+    async refresh() {
+        this.src = await this.getSrc();
+    }
+
     /** Get src for image to show */
-    src() {
+    async getSrc() {
         if (this.data.flag & this.c.FLAG_PLACEHOLDER) {
-            return undefined;
+            return null;
         } else if (this.data.flag & this.c.FLAG_LOAD_FAIL) {
             return errorsvg;
         } else {
@@ -78,9 +99,41 @@ export default class Photo extends Mixins(GlobalMixin) {
         return getPreviewUrl(this.data.fileid, this.data.etag, false, size)
     }
 
+    /** Set src with overlay face rect */
+    async addFaceRect() {
+        if (!this.data.facerect || this.hasFaceRect) return;
+        this.hasFaceRect = true;
+
+        const canvas = document.createElement('canvas');
+        const context = canvas.getContext('2d');
+        const img = this.$refs.img as HTMLImageElement;
+
+        canvas.width = img.naturalWidth;
+        canvas.height = img.naturalHeight;
+        context.drawImage(img, 0, 0);
+        context.strokeStyle = '#00ff00';
+        context.lineWidth = 2;
+        context.strokeRect(
+            this.data.facerect.x * img.naturalWidth,
+            this.data.facerect.y * img.naturalHeight,
+            this.data.facerect.w * img.naturalWidth,
+            this.data.facerect.h * img.naturalHeight,
+        );
+
+        canvas.toBlob((blob) => {
+            this.src = URL.createObjectURL(blob);
+        }, 'image/jpeg', 0.95)
+    }
+
+    /** Post load tasks */
+    load() {
+        this.addFaceRect();
+    }
+
     /** Error in loading image */
     error(e: any) {
         this.data.flag |= this.c.FLAG_LOAD_FAIL;
+        this.refresh();
     }
 
     /** Clear timers */

src/components/top-matter/FaceTopMatter.vue

@@ -19,6 +19,10 @@
                     {{ t('memories', 'Merge with different person') }}
                     <template #icon> <MergeIcon :size="20" /> </template>
                 </NcActionButton>
+                <NcActionCheckbox :aria-label="t('memories', 'Mark person in preview')" :checked.sync="config_showFaceRect" @change="changeShowFaceRect">
+                    {{ t('memories', 'Mark person in preview') }}
+                    <template #icon> <MergeIcon :size="20" /> </template>
+                </NcActionCheckbox>
                 <NcActionButton :aria-label="t('memories', 'Remove person')" @click="showDeleteModal=true" close-after-click>
                     {{ t('memories', 'Remove person') }}
                     <template #icon> <DeleteIcon :size="20" /> </template>
@@ -35,8 +39,9 @@
 <script lang="ts">
 import { Component, Mixins, Watch } from 'vue-property-decorator';
 import GlobalMixin from '../../mixins/GlobalMixin';
+import UserConfig from "../../mixins/UserConfig";
 
-import { NcActions, NcActionButton } from '@nextcloud/vue';
+import { NcActions, NcActionButton, NcActionCheckbox } from '@nextcloud/vue';
 import FaceEditModal from '../modal/FaceEditModal.vue';
 import FaceDeleteModal from '../modal/FaceDeleteModal.vue';
 import FaceMergeModal from '../modal/FaceMergeModal.vue';
@@ -49,6 +54,7 @@ import MergeIcon from 'vue-material-design-icons/Merge.vue';
     components: {
         NcActions,
         NcActionButton,
+        NcActionCheckbox,
         FaceEditModal,
         FaceDeleteModal,
         FaceMergeModal,
@@ -58,7 +64,7 @@ import MergeIcon from 'vue-material-design-icons/Merge.vue';
         MergeIcon,
     },
 })
-export default class FaceTopMatter extends Mixins(GlobalMixin) {
+export default class FaceTopMatter extends Mixins(GlobalMixin, UserConfig) {
     private name: string = '';
     private showEditModal: boolean = false;
     private showDeleteModal: boolean = false;
@@ -80,6 +86,13 @@ export default class FaceTopMatter extends Mixins(GlobalMixin) {
     back() {
         this.$router.push({ name: 'people' });
     }
+
+    changeShowFaceRect() {
+        localStorage.setItem('memories_showFaceRect', this.config_showFaceRect ? '1' : '0');
+        setTimeout(() => {
+            this.$router.go(0); // refresh page
+        }, 500);
+    }
 }
 </script>
 

src/mixins/UserConfig.ts

@@ -34,7 +34,9 @@ export default class UserConfig extends Vue {
     config_showHidden = loadState('memories', 'showHidden') === "true";
     config_tagsEnabled = loadState('memories', 'systemtags');
     config_recognizeEnabled = loadState('memories', 'recognize');
+
     config_squareThumbs = localStorage.getItem('memories_squareThumbs') === '1';
+    config_showFaceRect = localStorage.getItem('memories_showFaceRect') === '1';
 
     created() {
         subscribe(eventName, this.updateLocalSetting)

src/types.ts

@@ -39,6 +39,7 @@ export type IPhoto = {
     w?: number;
     /** Height of full image */
     h?: number;
+
     /** Grid display width percentage */
     dispWp?: number;
     /** Grid display height (forced) */
@@ -49,8 +50,13 @@ export type IPhoto = {
     dispY?: number;
     /** Grid display row id (relative to head) */
     dispRowNum?: number;
+
     /** Reference to day object */
     d?: IDay;
+
+    /** Face dimensions */
+    facerect?: IFaceRect;
+
     /** Video flag from server */
     isvideo?: boolean;
     /** Favorite flag from server */
@@ -85,6 +91,13 @@ export interface ITag extends IPhoto {
     previews?: IPhoto[];
 }
 
+export interface IFaceRect {
+    w: number;
+    h: number;
+    x: number;
+    y: number;
+}
+
 export type IRow = {
     /** Vue Recycler identifier */
     id?: string;