From e9331c582a6e3374a4ad4b820dda5803898ba416 Mon Sep 17 00:00:00 2001 From: WFH Brian Date: Thu, 13 Jun 2024 18:23:06 -0400 Subject: [PATCH] improve handling of SmartCollections data --- smart-collections/CollectionItem.js | 5 +- smart-collections/adapters/fs.js | 4 +- smart-collections/adapters/obsidian.js | 60 +++++++++++++-------- smart-collections/utils/ajson_merge.js | 7 +-- smart-collections/utils/ajson_merge.test.js | 37 +++++++++---- smart-entities/smart_entities.js | 2 +- 6 files changed, 78 insertions(+), 37 deletions(-) diff --git a/smart-collections/CollectionItem.js b/smart-collections/CollectionItem.js index 0f53d589..49d5c882 100644 --- a/smart-collections/CollectionItem.js +++ b/smart-collections/CollectionItem.js @@ -200,6 +200,9 @@ class CollectionItem { * Retrieves string representation of the item, including its key and data. * @returns {string} A string representing the item. */ - get ajson() { return `${JSON.stringify(this.key)}: ${JSON.stringify(this.data)}`; } + get ajson() { return `${JSON.stringify(this.ajson_key)}: ${(this.deleted) ? null : JSON.stringify(this.data)}`; } + + get ajson_key() { return this.constructor.name + ":" + this.key; } } + exports.CollectionItem = CollectionItem; \ No newline at end of file diff --git a/smart-collections/adapters/fs.js b/smart-collections/adapters/fs.js index 0f07ba57..891cdc3f 100644 --- a/smart-collections/adapters/fs.js +++ b/smart-collections/adapters/fs.js @@ -1,6 +1,6 @@ const fs = require('fs').promises; const { SmartCollectionsAdapter } = require('./adapter.js'); -const { deep_merge } = require('../utils/ajson_merge.js'); +const { ajson_merge } = require('../utils/ajson_merge.js'); /** * Adapter for file system that handles multiple .ajson files. */ @@ -49,7 +49,7 @@ class FsAdapter extends SmartCollectionsAdapter { .split('\n') .reduce((acc, line) => { const parsed = JSON.parse(`{${line}}`); - return deep_merge(acc, parsed); + return ajson_merge(acc, parsed); }, {}); let main_item = null; let updated_content = ''; diff --git a/smart-collections/adapters/obsidian.js b/smart-collections/adapters/obsidian.js index 520e1b27..7071a2e4 100644 --- a/smart-collections/adapters/obsidian.js +++ b/smart-collections/adapters/obsidian.js @@ -1,5 +1,5 @@ const { SmartCollectionsAdapter } = require('./adapter.js'); -const { deep_merge } = require('../utils/ajson_merge.js'); +const { ajson_merge } = require('../utils/ajson_merge.js'); /** * Adapter for Obsidian that handles multiple .ajson files. */ @@ -29,40 +29,56 @@ class ObsidianAdapter extends SmartCollectionsAdapter { const start = Date.now(); if(!(await this.exists(this.data_path))) await this.mkdir(this.data_path); const files = (await this.list(this.data_path)).files; // List all files in the directory + const vault_paths = this.main.env.all_files.reduce((acc, file) => { + acc[file.path] = file; + return acc; + }, {}); + const item_types = Object.keys(this.env.item_types); for (const file_path of files) { + let source_is_deleted = false; try { if (file_path.endsWith('.ajson')) { // Ensure it's an .ajson file const content = (await this.read(file_path)).trim(); const data = content .split('\n') .reduce((acc, line) => { - // if(line.endsWith(',')) line = line.slice(0, -1); // DEPRECATED: should not be necessary const parsed = JSON.parse(`{${line}}`); - // future: parse key to allow dot notation - return deep_merge(acc, parsed); + if(Object.values(parsed)[0] === null){ + if(acc[Object.keys(parsed)[0]]) delete acc[Object.keys(parsed)[0]]; + return acc; + } + return ajson_merge(acc, parsed); }, {}) ; // const data = JSON.parse(`{${content.startsWith(',\n') ? content.slice(1) : content}}`); - let main_item = null; - let updated_content = ''; - Object.entries(data).forEach(([key, value]) => { - if(!value) return; // handle null values (deleted) + let main_entity; + Object.entries(data).forEach(([ajson_key, value]) => { + let is_main_entity = false; + if(ajson_key.includes("AI computer")) console.log(ajson_key, value); // TEMP + if(!value || source_is_deleted) return; // handle null values (deleted) + let entity_key; let class_name = value.class_name; // DEPRECATED (moved to key so that multiple entities from different classes can have the same key) - if(key.includes(":") && key.split(":")[0] in this.env.item_types){ - class_name = key.split(":").shift(); - key = key.split(":").slice(1).join(":"); + if(ajson_key.includes(":") && item_types.includes(ajson_key.split(":")[0])){ + class_name = ajson_key.split(":").shift(); + entity_key = ajson_key.split(":").slice(1).join(":"); // key is file path + }else entity_key = ajson_key; // DEPRECATED: remove this + if(!entity_key.includes("#")){ // if no #, it's a source item (i.e. Note, not block) + is_main_entity = true; + if(!vault_paths[entity_key]){ // if not in vault path, it's a deleted item + source_is_deleted = true; + return; + } } - updated_content += `${JSON.stringify(class_name + ":" + key)}: ${JSON.stringify(value)}\n`; const entity = new (this.env.item_types[class_name])(this.env, value); - this.env[entity.collection_name].items[key] = entity; - if(!key.includes("#")) main_item = entity; + this.env[entity.collection_name].items[entity_key] = entity; + if(is_main_entity) main_entity = entity; }); - updated_content = updated_content.trim(); - if(!main_item) await this.remove(file_path); - else if(updated_content !== content) { - // console.log("data: ", data); - await this.write(file_path, updated_content); - // console.log("Updated file: " + file_path); + if(source_is_deleted) await this.remove(file_path); + else{ + if(main_entity.ajson !== content) { + await this.write(file_path, main_entity.ajson); + // console.log("Updated file: " + file_path); + } } } } catch (err) { @@ -101,8 +117,10 @@ class ObsidianAdapter extends SmartCollectionsAdapter { if(!ajson && (await this.exists(item_file_path))){ await this.remove(item_file_path); delete this.main.items[key]; + console.log("Deleted item: " + key); + } else { + await this.append(item_file_path, '\n' + ajson); } - else await this.append(item_file_path, '\n' + ajson); } catch (err) { if(err.message.includes("ENOENT")) return; // already deleted console.warn("Error saving collection item: ", key); diff --git a/smart-collections/utils/ajson_merge.js b/smart-collections/utils/ajson_merge.js index a58058ad..2f2ea25c 100644 --- a/smart-collections/utils/ajson_merge.js +++ b/smart-collections/utils/ajson_merge.js @@ -1,5 +1,6 @@ // merge two objects, overwriting existing properties with new_obj properties -function deep_merge(existing, new_obj) { +function ajson_merge(existing, new_obj) { + if(new_obj === null) return null; for (const key in new_obj) { if (Array.isArray(existing[key]) && Array.isArray(new_obj[key])) { // // Check if the first element of the array is also an array, indicating nested arrays @@ -27,7 +28,7 @@ function deep_merge(existing, new_obj) { existing[key] = new_obj[key]; } else if (isObject(existing[key]) && isObject(new_obj[key])) { // Recursively merge objects - existing[key] = deep_merge(existing[key], new_obj[key]); + existing[key] = ajson_merge(existing[key], new_obj[key]); } else { // Directly set the value for non-object and non-array types existing[key] = new_obj[key]; @@ -39,5 +40,5 @@ function deep_merge(existing, new_obj) { function isObject(obj) { return obj && typeof obj === 'object' && !Array.isArray(obj); } -exports.deep_merge = deep_merge; +exports.ajson_merge = ajson_merge; diff --git a/smart-collections/utils/ajson_merge.test.js b/smart-collections/utils/ajson_merge.test.js index d9cd322e..d00f77b4 100644 --- a/smart-collections/utils/ajson_merge.test.js +++ b/smart-collections/utils/ajson_merge.test.js @@ -1,12 +1,12 @@ const test = require('ava'); -const { deep_merge } = require('./ajson_merge'); +const { ajson_merge } = require('./ajson_merge'); test('should correctly merge two objects', t => { const obj1 = { a: 1, b: { c: 2 } }; const obj2 = { b: { d: 3 }, e: 4 }; const expected = { a: 1, b: { c: 2, d: 3 }, e: 4 }; - const result = deep_merge(obj1, obj2); + const result = ajson_merge(obj1, obj2); t.deepEqual(result, expected); }); @@ -16,7 +16,7 @@ test('should overwrite existing properties', t => { const obj2 = { a: 2, b: { c: 3 } }; const expected = { a: 2, b: { c: 3, d: 3 } }; - const result = deep_merge(obj1, obj2); + const result = ajson_merge(obj1, obj2); t.deepEqual(result, expected); }); @@ -26,7 +26,7 @@ test('should handle nested objects correctly', t => { const obj2 = { a: { b: { d: 2 }, e: 3 }, f: 4 }; const expected = { a: { b: { c: 1, d: 2 }, e: 3 }, f: 4 }; - const result = deep_merge(obj1, obj2); + const result = ajson_merge(obj1, obj2); t.deepEqual(result, expected); }); @@ -36,7 +36,7 @@ test('should not modify the existing object structure when new_obj is empty', t const obj2 = {}; const expected = { a: 1, b: { c: 2 } }; - const result = deep_merge(obj1, obj2); + const result = ajson_merge(obj1, obj2); t.deepEqual(result, expected); }); @@ -46,7 +46,7 @@ test("should deep merge nested object a.b.c.d.e", t => { const obj2 = { a: { b: { c: { d: { e: 2 } } } } }; const expected = { a: { b: { c: { d: { e: 2 } } } }, f: 1 }; - const result = deep_merge(obj1, obj2); + const result = ajson_merge(obj1, obj2); t.deepEqual(result, expected); }) @@ -56,7 +56,7 @@ test("should deep merge nested objects with arrays", t => { const obj2 = { a: { b: { d: {vec: [4,5,6]} } } }; const expected = { a: { b: { c: {vec: [1,2,3]}, d: {vec: [4,5,6]} } } }; - const result = deep_merge(obj1, obj2); + const result = ajson_merge(obj1, obj2); t.deepEqual(result, expected); }) @@ -66,7 +66,7 @@ test("empty object should not overwrite existing object", t => { const obj2 = {a: {b: {c: {} } } }; const expected = { a: { b: { c: {vec: [1,2,3]} } } }; - const result = deep_merge(obj1, obj2); + const result = ajson_merge(obj1, obj2); t.deepEqual(result, expected); }) @@ -76,7 +76,26 @@ test("new array should not overwrite existing array", t => { const obj2 = {a: {b: {c: {vec: [4,5,6]} } } }; const expected = { a: { b: { c: {vec: [4,5,6]} } } }; - const result = deep_merge(obj1, obj2); + const result = ajson_merge(obj1, obj2); + + t.deepEqual(result, expected); +}) + +test("null should overwrite existing object", t => { + const obj1 = { a: { b: { c: {vec: [1,2,3]} } } }; + const obj2 = null; + const expected = null; + + const result = ajson_merge(obj1, obj2); + + t.deepEqual(result, expected); +}) +test("undefined should not overwrite existing object", t => { + const obj1 = { a: { b: { c: {vec: [1,2,3]} } } }; + const obj2 = undefined; + const expected = { a: { b: { c: {vec: [1,2,3]} } } }; + + const result = ajson_merge(obj1, obj2); t.deepEqual(result, expected); }) \ No newline at end of file diff --git a/smart-entities/smart_entities.js b/smart-entities/smart_entities.js index 5a657f24..6ee4455e 100644 --- a/smart-entities/smart_entities.js +++ b/smart-entities/smart_entities.js @@ -552,7 +552,7 @@ class SmartBlocks extends SmartEntities { console.log(`Pruning: Found ${remove.length} SmartBlocks in ${Date.now() - start}ms`); if((override && (remove_ratio < 0.5)) || confirm(`Are you sure you want to delete ${remove.length} (${Math.floor(remove_ratio*100)}%) Block-level embeddings?`)){ this.delete_many(remove); - if(!override) this.adapter._save_queue(); + this.adapter._save_queue(); } } }