LLM custom image fixes (#1297)

* Implement dithering & fix ordering of pixels in raw data

* Fix image data tester (reconstructing following the correct pixel ordering)

* Data verif screen better layout

* Prevent selecting another contrast while the selected one is still loading

* Fix Step1Crop UX: if coming from gallery, back buttons brings back there

* lint


lint

* changeset

.changeset/selfish-schools-end.md

@@ -0,0 +1,5 @@
+---
+"live-mobile": patch
+---
+
+UX and functional fixes in custom image flow

apps/ledger-live-mobile/src/components/CustomImage/ContrastChoice.tsx

@@ -1,10 +1,11 @@
-import { Box, Icons } from "@ledgerhq/native-ui";
+import { Box, Icons, InfiniteLoader } from "@ledgerhq/native-ui";
 import React from "react";
 import styled from "styled-components/native";
 
 type Props = {
   color: string;
   selected: boolean;
+  loading?: boolean;
 };
 
 const Container = styled(Box).attrs((p: { selected: boolean }) => ({
@@ -48,10 +49,14 @@ const CheckPill: React.FC<Record<string, never>> = () => (
   </CheckContainer>
 );
 
-const ContrastChoice: React.FC<Props> = ({ selected, color }) => (
+const ContrastChoice: React.FC<Props> = ({ loading, selected, color }) => (
   <Container selected={selected}>
-    <Round backgroundColor={color} />
-    {selected && <CheckPill />}
+    {selected && loading ? (
+      <InfiniteLoader size={28} />
+    ) : (
+      <Round backgroundColor={color} />
+    )}
+    {selected ? <CheckPill /> : null}
   </Container>
 );
 

apps/ledger-live-mobile/src/components/CustomImage/CustomImageBottomModal.tsx

@@ -24,7 +24,7 @@ const CustomImageBottomModal: React.FC<Props> = props => {
       if (importResult !== null) {
         navigation.navigate(NavigatorName.CustomImage, {
           screen: ScreenName.CustomImageStep1Crop,
-          params: importResult,
+          params: { ...importResult, isPictureFromGallery: true },
         });
       }
     } catch (error) {

apps/ledger-live-mobile/src/components/CustomImage/imageUtils.ts

@@ -12,8 +12,8 @@ import {
 /**
  * Call this to prompt the user to pick an image from its phone.
  *
- * @returns (a promise) null if the user cancelled, otherwise an containing
- * the chosen image file URI as well as the image dimensions
+ * @returns (a promise) null if the user cancelled, otherwise an object
+ * containing the chosen image file URI as well as the image dimensions
  */
 export async function importImageFromPhoneGallery(): Promise<ImageFileUri | null> {
   try {

apps/ledger-live-mobile/src/components/CustomImage/injectedCode/imageProcessing.ts

@@ -34,8 +34,8 @@ function codeToInject() {
 
   "show source";
 
-  function clampRGB(val: number) {
-    return Math.min(255, Math.max(val, 0));
+  function clamp(val: number, min: number, max: number) {
+    return Math.min(max, Math.max(min, val));
   }
 
   function contrastRGB(rgbVal: number, contrastVal: number) {
@@ -44,42 +44,97 @@ function codeToInject() {
 
   // simutaneously apply grayscale and contrast to the image
   function applyFilter(
-    imageData: Uint8ClampedArray,
+    imageData: ImageData,
     contrastAmount: number,
+    dither = true,
   ): { imageDataResult: Uint8ClampedArray; hexRawResult: string } {
     let hexRawResult = "";
     const filteredImageData = [];
 
+    const data = imageData.data;
+
     const numLevelsOfGray = 16;
     const rgbStep = 255 / (numLevelsOfGray - 1);
 
-    for (let i = 0; i < imageData.length; i += 4) {
-      /** gray rgb value for the pixel, in [0, 255] */
-      const gray256 =
-        0.299 * imageData[i] +
-        0.587 * imageData[i + 1] +
-        0.114 * imageData[i + 2];
+    const { width, height } = imageData;
 
-      /** gray rgb value after applying the contrast, in [0, 15] */
-      const contrastedGray16 = Math.floor(
-        clampRGB(contrastRGB(gray256, contrastAmount)) / rgbStep,
-      );
+    const pixels256: number[][] = Array.from(Array(height), () => Array(width));
+    const pixels16: number[][] = Array.from(Array(height), () => Array(width));
 
-      /** gray rgb value after applying the contrast, in [0,255] */
-      const contrastedGray256 = contrastedGray16 * rgbStep;
+    for (let pxIndex = 0; pxIndex < data.length / 4; pxIndex += 1) {
+      const x = pxIndex % width;
+      const y = (pxIndex - x) / width;
 
-      const grayHex = contrastedGray16.toString(16);
+      const [redIndex, greenIndex, blueIndex] = [
+        4 * pxIndex,
+        4 * pxIndex + 1,
+        4 * pxIndex + 2,
+      ];
+      const gray256 =
+        0.299 * data[redIndex] +
+        0.587 * data[greenIndex] +
+        0.114 * data[blueIndex];
 
-      hexRawResult = hexRawResult.concat(grayHex);
-      // adding hexadecimal value of this pixel
+      /** gray rgb value after applying the contrast */
+      pixels256[y][x] = clamp(contrastRGB(gray256, contrastAmount), 0, 255);
+    }
 
-      filteredImageData.push(contrastedGray256);
-      filteredImageData.push(contrastedGray256);
-      filteredImageData.push(contrastedGray256);
-      // push 3 bytes for color (all the same == gray)
+    for (let y = 0; y < height; y++) {
+      for (let x = 0; x < width; x++) {
+        const oldpixel = pixels256[y][x];
+        const posterizedGray16 = Math.floor(oldpixel / rgbStep);
+        const posterizedGray256 = posterizedGray16 * rgbStep;
+        /**
+         * Floyd-Steinberg dithering
+         * https://en.wikipedia.org/wiki/Floyd%E2%80%93Steinberg_dithering
+         *         x - 1  |   x    |  x + 1
+         *   y            |   *    | 7 / 16
+         * y + 1  3 / 16  | 5 / 16 | 1 / 16
+         */
+        const newpixel = posterizedGray256;
+        pixels256[y][x] = newpixel;
+        if (dither) {
+          const quantError = oldpixel - newpixel;
+          if (x < width - 1) {
+            pixels256[y][x + 1] = Math.floor(
+              pixels256[y][x + 1] + quantError * (7 / 16),
+            );
+          }
+          if (x > 0 && y < height - 1) {
+            pixels256[y + 1][x - 1] = Math.floor(
+              pixels256[y + 1][x - 1] + quantError * (3 / 16),
+            );
+          }
+          if (y < height - 1) {
+            pixels256[y + 1][x] = Math.floor(
+              pixels256[y + 1][x] + quantError * (5 / 16),
+            );
+          }
+          if (x < width - 1 && y < height - 1) {
+            pixels256[y + 1][x + 1] = Math.floor(
+              pixels256[y + 1][x + 1] + quantError * (1 / 16),
+            );
+          }
+        }
+
+        const val16 = clamp(Math.floor(pixels256[y][x] / rgbStep), 0, 16 - 1);
+        pixels16[y][x] = val16;
+        /** gray rgb value after applying the contrast, in [0,255] */
+        const val256 = val16 * rgbStep;
+        filteredImageData.push(val256); // R
+        filteredImageData.push(val256); // G
+        filteredImageData.push(val256); // B
+        filteredImageData.push(255); // alpha
+      }
+    }
 
-      filteredImageData.push(255);
-      // push alpha = max = 255
+    // Raw data -> by column, from right to left, from top to bottom
+    for (let x = width - 1; x >= 0; x--) {
+      for (let y = 0; y < height; y++) {
+        const val16 = pixels16[y][x];
+        const grayHex = val16.toString(16);
+        hexRawResult = hexRawResult.concat(grayHex);
+      }
     }
 
     return {
@@ -111,7 +166,7 @@ function codeToInject() {
 
     // 2. applying filter to the image data
     const { imageDataResult: grayData, hexRawResult } = applyFilter(
-      imageData.data,
+      imageData,
       contrastAmount,
     );
 

apps/ledger-live-mobile/src/components/CustomImage/injectedCode/resultDataTesting.ts

@@ -31,6 +31,11 @@ function codeToInject() {
     postDataToWebView({ type: "ERROR", payload: error.toString() });
   };
 
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  const log = (...args: any[]) => {
+    postDataToWebView({ type: "LOG", payload: JSON.stringify(args) });
+  };
+
   /**
    * store functions as a property of window so we can access them easily after minification
    * */
@@ -47,15 +52,28 @@ function codeToInject() {
 
       const numLevelsOfGray = 16;
       const rgbStep = 255 / (numLevelsOfGray - 1);
-      hexData.split("").forEach(char => {
+
+      const pixels256 = Array.from(Array(height), () => Array(width));
+
+      hexData.split("").forEach((char, index) => {
+        /** running from top right to bottom left, column after column */
+        const y = index % height;
+        const x = width - 1 - (index - y) / height;
         const numericVal16 = Number.parseInt(char, 16);
         const numericVal256 = numericVal16 * rgbStep;
-        imageData.push(numericVal256); // R
-        imageData.push(numericVal256); // G
-        imageData.push(numericVal256); // B
-        imageData.push(255);
+        pixels256[y][x] = numericVal256;
       });
 
+      for (let y = 0; y < height; y++) {
+        for (let x = 0; x < width; x++) {
+          const val = pixels256[y][x];
+          imageData.push(val); // R
+          imageData.push(val); // G
+          imageData.push(val); // B
+          imageData.push(255);
+        }
+      }
+
       context.putImageData(
         new ImageData(Uint8ClampedArray.from(imageData), width, height), // eslint-disable-line no-undef
         0,

apps/ledger-live-mobile/src/screens/CustomImage/Step1Crop.tsx

@@ -2,7 +2,15 @@ import React, { useCallback, useEffect, useRef, useState } from "react";
 import { Flex, Icons, InfiniteLoader } from "@ledgerhq/native-ui";
 import { CropView } from "react-native-image-crop-tools";
 import { useTranslation } from "react-i18next";
-import { StackScreenProps } from "@react-navigation/stack";
+import {
+  StackNavigationEventMap,
+  StackScreenProps,
+} from "@react-navigation/stack";
+import {
+  EventListenerCallback,
+  EventMapCore,
+  StackNavigationState,
+} from "@react-navigation/native";
 import { SafeAreaView } from "react-native-safe-area-context";
 import ImageCropper, {
   Props as ImageCropperProps,
@@ -12,7 +20,10 @@ import {
   ImageDimensions,
   ImageFileUri,
 } from "../../components/CustomImage/types";
-import { downloadImageToFile } from "../../components/CustomImage/imageUtils";
+import {
+  downloadImageToFile,
+  importImageFromPhoneGallery,
+} from "../../components/CustomImage/imageUtils";
 import { targetDimensions } from "./shared";
 import Button from "../../components/Button";
 import { ScreenName } from "../../const";
@@ -37,6 +48,8 @@ const Step1Cropping: React.FC<
 
   const { params } = route;
 
+  const { isPictureFromGallery } = params;
+
   const handleError = useCallback(
     (error: Error) => {
       console.error(error);
@@ -48,6 +61,39 @@ const Step1Cropping: React.FC<
     [navigation],
   );
 
+  useEffect(() => {
+    let dead = false;
+    const listener: EventListenerCallback<
+      StackNavigationEventMap & EventMapCore<StackNavigationState<ParamList>>,
+      "beforeRemove"
+    > = e => {
+      if (!isPictureFromGallery) {
+        navigation.dispatch(e.data.action);
+        return;
+      }
+      e.preventDefault();
+      setImageToCrop(null);
+      importImageFromPhoneGallery()
+        .then(importResult => {
+          if (dead) return;
+          if (importResult !== null) {
+            setImageToCrop(importResult);
+          } else {
+            navigation.dispatch(e.data.action);
+          }
+        })
+        .catch(e => {
+          if (dead) return;
+          handleError(e);
+        });
+    };
+    navigation.addListener("beforeRemove", listener);
+    return () => {
+      dead = true;
+      navigation.removeListener("beforeRemove", listener);
+    };
+  }, [navigation, handleError, isPictureFromGallery]);
+
   /** LOAD SOURCE IMAGE FROM PARAMS */
   useEffect(() => {
     let dead = false;

apps/ledger-live-mobile/src/screens/CustomImage/Step2Preview.tsx

@@ -31,7 +31,6 @@ export const PreviewImage = styled.Image.attrs({
   resizeMode: "contain",
 })`
   align-self: center;
-  margin: 16px;
   width: 200px;
   height: 200px;
 `;
@@ -59,6 +58,7 @@ const Step2Preview: React.FC<
   StackScreenProps<ParamList, "CustomImageStep2Preview">
 > = ({ navigation, route }) => {
   const imageProcessorRef = useRef<ImageProcessor>(null);
+  const [loading, setLoading] = useState(true);
   const [resizedImage, setResizedImage] = useState<ResizeResult | null>(null);
   const [contrast, setContrast] = useState(1);
   const [processorPreviewImage, setProcessorPreviewImage] =
@@ -104,6 +104,7 @@ const Step2Preview: React.FC<
     useCallback(
       data => {
         setProcessorPreviewImage(data);
+        setLoading(false);
       },
       [setProcessorPreviewImage],
     );
@@ -193,8 +194,21 @@ const Step2Preview: React.FC<
         {resizedImage?.imageBase64DataUri && (
           <Flex flexDirection="row" my={6} justifyContent="space-between">
             {contrasts.map(({ val, color }) => (
-              <Pressable key={val} onPress={() => setContrast(val)}>
-                <ContrastChoice selected={contrast === val} color={color} />
+              <Pressable
+                disabled={loading}
+                key={val}
+                onPress={() => {
+                  if (contrast !== val) {
+                    setLoading(true);
+                    setContrast(val);
+                  }
+                }}
+              >
+                <ContrastChoice
+                  selected={contrast === val}
+                  loading={loading}
+                  color={color}
+                />
               </Pressable>
             ))}
           </Flex>