improve handling of SmartCollections data

smart-collections/CollectionItem.js

@@ -200,6 +200,9 @@ class CollectionItem {
    * Retrieves string representation of the item, including its key and data.
    * @returns {string} A string representing the item.
    */
-  get ajson() { return `${JSON.stringify(this.key)}: ${JSON.stringify(this.data)}`; }
+  get ajson() { return `${JSON.stringify(this.ajson_key)}: ${(this.deleted) ? null : JSON.stringify(this.data)}`; }
+
+  get ajson_key() { return this.constructor.name + ":" + this.key; }
 }
+
 exports.CollectionItem = CollectionItem;

smart-collections/adapters/fs.js

@@ -1,6 +1,6 @@
 const fs = require('fs').promises;
 const { SmartCollectionsAdapter } = require('./adapter.js');
-const { deep_merge } = require('../utils/ajson_merge.js');
+const { ajson_merge } = require('../utils/ajson_merge.js');
 /**
  * Adapter for file system that handles multiple .ajson files.
  */
@@ -49,7 +49,7 @@ class FsAdapter extends SmartCollectionsAdapter {
             .split('\n')
             .reduce((acc, line) => {
               const parsed = JSON.parse(`{${line}}`);
-              return deep_merge(acc, parsed);
+              return ajson_merge(acc, parsed);
             }, {});
           let main_item = null;
           let updated_content = '';

smart-collections/adapters/obsidian.js

@@ -1,5 +1,5 @@
 const { SmartCollectionsAdapter } = require('./adapter.js');
-const { deep_merge } = require('../utils/ajson_merge.js');
+const { ajson_merge } = require('../utils/ajson_merge.js');
 /**
  * Adapter for Obsidian that handles multiple .ajson files.
  */
@@ -29,40 +29,56 @@ class ObsidianAdapter extends SmartCollectionsAdapter {
     const start = Date.now();
     if(!(await this.exists(this.data_path))) await this.mkdir(this.data_path);
     const files = (await this.list(this.data_path)).files; // List all files in the directory
+    const vault_paths = this.main.env.all_files.reduce((acc, file) => {
+      acc[file.path] = file;
+      return acc;
+    }, {});
+    const item_types = Object.keys(this.env.item_types);
     for (const file_path of files) {
+      let source_is_deleted = false;
       try {
         if (file_path.endsWith('.ajson')) { // Ensure it's an .ajson file
           const content = (await this.read(file_path)).trim();
           const data = content
             .split('\n')
             .reduce((acc, line) => {
-              // if(line.endsWith(',')) line = line.slice(0, -1); // DEPRECATED: should not be necessary
               const parsed = JSON.parse(`{${line}}`);
-              // future: parse key to allow dot notation
-              return deep_merge(acc, parsed);
+              if(Object.values(parsed)[0] === null){
+                if(acc[Object.keys(parsed)[0]]) delete acc[Object.keys(parsed)[0]];
+                return acc;
+              }
+              return ajson_merge(acc, parsed);
             }, {})
           ;
           // const data = JSON.parse(`{${content.startsWith(',\n') ? content.slice(1) : content}}`);
-          let main_item = null;
-          let updated_content = '';
-          Object.entries(data).forEach(([key, value]) => {
-            if(!value) return; // handle null values (deleted)
+          let main_entity;
+          Object.entries(data).forEach(([ajson_key, value]) => {
+            let is_main_entity = false;
+            if(ajson_key.includes("AI computer")) console.log(ajson_key, value); // TEMP
+            if(!value || source_is_deleted) return; // handle null values (deleted)
+            let entity_key;
             let class_name = value.class_name; // DEPRECATED (moved to key so that multiple entities from different classes can have the same key)
-            if(key.includes(":") && key.split(":")[0] in this.env.item_types){
-              class_name = key.split(":").shift();
-              key = key.split(":").slice(1).join(":");
+            if(ajson_key.includes(":") && item_types.includes(ajson_key.split(":")[0])){
+              class_name = ajson_key.split(":").shift();
+              entity_key = ajson_key.split(":").slice(1).join(":"); // key is file path
+            }else entity_key = ajson_key; // DEPRECATED: remove this
+            if(!entity_key.includes("#")){ // if no #, it's a source item (i.e. Note, not block)
+              is_main_entity = true;
+              if(!vault_paths[entity_key]){ // if not in vault path, it's a deleted item
+                source_is_deleted = true;
+                return;
+              }
             }
-            updated_content += `${JSON.stringify(class_name + ":" + key)}: ${JSON.stringify(value)}\n`;
             const entity = new (this.env.item_types[class_name])(this.env, value);
-            this.env[entity.collection_name].items[key] = entity;
-            if(!key.includes("#")) main_item = entity;
+            this.env[entity.collection_name].items[entity_key] = entity;
+            if(is_main_entity) main_entity = entity;
           });
-          updated_content = updated_content.trim();
-          if(!main_item) await this.remove(file_path);
-          else if(updated_content !== content) {
-            // console.log("data: ", data);
-            await this.write(file_path, updated_content);
-            // console.log("Updated file: " + file_path);
+          if(source_is_deleted) await this.remove(file_path);
+          else{
+            if(main_entity.ajson !== content) {
+              await this.write(file_path, main_entity.ajson);
+              // console.log("Updated file: " + file_path);
+            }
           }
         }
       } catch (err) {
@@ -101,8 +117,10 @@ class ObsidianAdapter extends SmartCollectionsAdapter {
       if(!ajson && (await this.exists(item_file_path))){
         await this.remove(item_file_path);
         delete this.main.items[key];
+        console.log("Deleted item: " + key);
+      } else {
+        await this.append(item_file_path, '\n' + ajson);
       }
-      else await this.append(item_file_path, '\n' + ajson);
     } catch (err) {
       if(err.message.includes("ENOENT")) return; // already deleted
       console.warn("Error saving collection item: ", key);

smart-collections/utils/ajson_merge.js

@@ -1,5 +1,6 @@
 // merge two objects, overwriting existing properties with new_obj properties
-function deep_merge(existing, new_obj) {
+function ajson_merge(existing, new_obj) {
+  if(new_obj === null) return null;
   for (const key in new_obj) {
     if (Array.isArray(existing[key]) && Array.isArray(new_obj[key])) {
       // // Check if the first element of the array is also an array, indicating nested arrays
@@ -27,7 +28,7 @@ function deep_merge(existing, new_obj) {
       existing[key] = new_obj[key];
     } else if (isObject(existing[key]) && isObject(new_obj[key])) {
       // Recursively merge objects
-      existing[key] = deep_merge(existing[key], new_obj[key]);
+      existing[key] = ajson_merge(existing[key], new_obj[key]);
     } else {
       // Directly set the value for non-object and non-array types
       existing[key] = new_obj[key];
@@ -39,5 +40,5 @@ function deep_merge(existing, new_obj) {
 function isObject(obj) {
   return obj && typeof obj === 'object' && !Array.isArray(obj);
 }
-exports.deep_merge = deep_merge;
+exports.ajson_merge = ajson_merge;
 

smart-collections/utils/ajson_merge.test.js

@@ -1,12 +1,12 @@
 const test = require('ava');
-const { deep_merge } = require('./ajson_merge');
+const { ajson_merge } = require('./ajson_merge');
 
 test('should correctly merge two objects', t => {
   const obj1 = { a: 1, b: { c: 2 } };
   const obj2 = { b: { d: 3 }, e: 4 };
   const expected = { a: 1, b: { c: 2, d: 3 }, e: 4 };
 
-  const result = deep_merge(obj1, obj2);
+  const result = ajson_merge(obj1, obj2);
 
   t.deepEqual(result, expected);
 });
@@ -16,7 +16,7 @@ test('should overwrite existing properties', t => {
   const obj2 = { a: 2, b: { c: 3 } };
   const expected = { a: 2, b: { c: 3, d: 3 } };
 
-  const result = deep_merge(obj1, obj2);
+  const result = ajson_merge(obj1, obj2);
 
   t.deepEqual(result, expected);
 });
@@ -26,7 +26,7 @@ test('should handle nested objects correctly', t => {
   const obj2 = { a: { b: { d: 2 }, e: 3 }, f: 4 };
   const expected = { a: { b: { c: 1, d: 2 }, e: 3 }, f: 4 };
 
-  const result = deep_merge(obj1, obj2);
+  const result = ajson_merge(obj1, obj2);
 
   t.deepEqual(result, expected);
 });
@@ -36,7 +36,7 @@ test('should not modify the existing object structure when new_obj is empty', t
   const obj2 = {};
   const expected = { a: 1, b: { c: 2 } };
 
-  const result = deep_merge(obj1, obj2);
+  const result = ajson_merge(obj1, obj2);
 
   t.deepEqual(result, expected);
 });
@@ -46,7 +46,7 @@ test("should deep merge nested object a.b.c.d.e", t => {
   const obj2 = { a: { b: { c: { d: { e: 2 } } } } };
   const expected = { a: { b: { c: { d: { e: 2 } } } }, f: 1 };
 
-  const result = deep_merge(obj1, obj2);
+  const result = ajson_merge(obj1, obj2);
 
   t.deepEqual(result, expected);
 })
@@ -56,7 +56,7 @@ test("should deep merge nested objects with arrays", t => {
   const obj2 = { a: { b: { d: {vec: [4,5,6]} } } };
   const expected = { a: { b: { c: {vec: [1,2,3]}, d: {vec: [4,5,6]} } } };
 
-  const result = deep_merge(obj1, obj2);
+  const result = ajson_merge(obj1, obj2);
 
   t.deepEqual(result, expected);
 })
@@ -66,7 +66,7 @@ test("empty object should not overwrite existing object", t => {
   const obj2 = {a: {b: {c: {} } } };
   const expected = { a: { b: { c: {vec: [1,2,3]} } } };
 
-  const result = deep_merge(obj1, obj2);
+  const result = ajson_merge(obj1, obj2);
 
   t.deepEqual(result, expected);
 })
@@ -76,7 +76,26 @@ test("new array should not overwrite existing array", t => {
   const obj2 = {a: {b: {c: {vec: [4,5,6]} } } };
   const expected = { a: { b: { c: {vec: [4,5,6]} } } };
 
-  const result = deep_merge(obj1, obj2);
+  const result = ajson_merge(obj1, obj2);
+
+  t.deepEqual(result, expected);
+})
+
+test("null should overwrite existing object", t => {
+  const obj1 = { a: { b: { c: {vec: [1,2,3]} } } };
+  const obj2 = null;
+  const expected = null;
+
+  const result = ajson_merge(obj1, obj2);
+
+  t.deepEqual(result, expected);
+})
+test("undefined should not overwrite existing object", t => {
+  const obj1 = { a: { b: { c: {vec: [1,2,3]} } } };
+  const obj2 = undefined;
+  const expected = { a: { b: { c: {vec: [1,2,3]} } } };
+
+  const result = ajson_merge(obj1, obj2);
 
   t.deepEqual(result, expected);
 })

smart-entities/smart_entities.js

@@ -552,7 +552,7 @@ class SmartBlocks extends SmartEntities {
     console.log(`Pruning: Found ${remove.length} SmartBlocks in ${Date.now() - start}ms`);
     if((override && (remove_ratio < 0.5)) || confirm(`Are you sure you want to delete ${remove.length} (${Math.floor(remove_ratio*100)}%) Block-level embeddings?`)){
       this.delete_many(remove);
-      if(!override) this.adapter._save_queue();
+      this.adapter._save_queue();
     }
   }
 }