Add IEnumerable and more 'data management' docs

Author: crpietschmann

CHANGELOG.md

@@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 Add:
 
 - Added `VectorTextResultItem.Id` property so it's easy to get the database ID for search results if necessary.
+- `IVectorDatabase` now inherits from `IEnumerable` so you can easily look through the texts documents that have been added to the database.
 
 Fixed:
 

docs/docs/get-started/data-management/index.md

@@ -6,7 +6,7 @@ title: Data Management
 
 Since `Build5Nines.SharpVector` is a database, it also has data management methods available. These methods enable you to add, remove, and update the text documents that are vectorized and indexed within the semantic database.
 
-## Get Text ID
+## Get Text Item IDs
 
 Every text item within a `Build5Nines.SharpVector` database is assigned a unique identifier (ID). There are a few ways to get access to the ID of the text items.
 
@@ -22,10 +22,86 @@ Every text item within a `Build5Nines.SharpVector` database is assigned a unique
 
 === ".Search()"
 
-    
+    When you perform a semantic search, the search results will contain the list of texts; each have an ID property.
 
-## Update Text and Metadata
+    ```csharp
+    var results = vdb.Search("query text");
+
+    foreach(var text in results.Texts) {
+        var id = text.Id;
+        var text = text.Text;
+        var metadata = text.Metadata;
+        // do something here    
+    }
+    ```
+
+=== "Enumerator"
+
+    The `IVectorDatabase` classes implement `IEnumerable` so you can easily loop through all the text items that have been added to the database.
+
+    ```csharp
+    foreach(var item in vdb) {
+        var id = item.Id;
+        var text = item.Text;
+        var metadata = item.Metadata;
+        var vector = item.Vector;
+
+        // do something here
+    }
+    ```
+
+## Get
+
+If you know the `id` of a Text item in the database, you can retrieve it directly.
+
+### Get By Id
+
+The `.GetText` method can be used to retrieve a text item from the vector database directly.
+
+```csharp
+vdb.GetText(id);
+```
+
+## Update
+
+Once text items have been added to the database "Update" methods can be used to modify them.
+
+### Update Text
+
+The `.UpdateText` method can be used to update the `Text` value, and associated vectors will be updated.
+
+```csharp
+vdb.UpdateText(id, newTxt);
+```
+
+When the `Text` is updated, new vector embeddings are generated for the new text.
+
+### Update Metadata
+
+The `.UpdateTextMetadata` method can be used to update the `Metadata` for a given text item by `Id`.
+
+```csharp
+vdb.UpdateTextMetadata(id, newTxt);
+```
+
+When `Metadata` is updated, the vector embeddings are not updated.
+
+### Update Text and Metadata
+
+The `.UpdateTextAndMetadata` method can be used to update the `Text` and `Metadata` for a text item in the database for the given text item `Id`.
 
 ```csharp
 vdb.UpdateTextAndMetadata(id, newTxt, newMetadata);
 ```
+
+## Delete
+
+The vector database supports the ability to delete text items.
+
+### Delete Text
+
+The `.DeleteText` method can be used to delete a text item form the database for the given `Id'.
+
+```csharp
+vdb.DeleteText(id);
+```

docs/mkdocs.yml

@@ -139,8 +139,10 @@ nav:
       - Basic Example: get-started/#basic-example
       - Data Management:
           - get-started/data-management/index.md
-          - Get Text ID: get-started/data-management/#get-text-id
-          - Update Text and Metadata: get-started/data-management/#update-text-and-metadata
+          - Get Text Item IDs: get-started/data-management/#get-text-item-ids
+          - Get By Id: get-started/data-management/#get-by-id
+          - Update: get-started/data-management/#update
+          - Delete: get-started/data-management/#delete
   - Concepts:
       - concepts/index.md
       - What is a Vector Database?: concepts/#what-is-a-vector-database

src/Build5Nines.SharpVector/IVectorDatabase.cs

@@ -9,6 +9,7 @@ namespace Build5Nines.SharpVector;
 /// <typeparam name="TMetadata"></typeparam>
 /// <typeparam name="TDocument"></typeparam>
 public interface IVectorDatabase<TId, TMetadata, TDocument>
+    : IEnumerable<IVectorTextDatabaseItem<TId, TDocument, TMetadata>>
     where TId : notnull
 {
     /// <summary>

src/Build5Nines.SharpVector/MemoryVectorDatabaseBase.cs

@@ -10,6 +10,7 @@
 using System.Text.Json;
 using Build5Nines.SharpVector.Embeddings;
 using System.Runtime.ExceptionServices;
+using System.Collections;
 
 namespace Build5Nines.SharpVector;
 
@@ -145,8 +146,14 @@ public void UpdateTextMetadata(TId id, TMetadata metadata) {
         if (VectorStore.ContainsKey(id))
         {
             var existing = VectorStore.Get(id);
-            existing.Metadata = metadata;
-            VectorStore.Set(id, existing);
+
+            var item = new VectorTextItem<TVocabularyKey, TMetadata>(
+                existing.Text,
+                metadata,
+                existing.Vector
+            );
+            
+            VectorStore.Set(id, item);
         }
         else
         {
@@ -340,6 +347,16 @@ public virtual void DeserializeFromBinaryStream(Stream stream)
         }
         DeserializeFromBinaryStreamAsync(stream).Wait();
     }
+
+    public IEnumerator<IVectorTextDatabaseItem<TId, TVocabularyKey, TMetadata>> GetEnumerator()
+    {
+        return VectorStore.Select(kvp => new VectorTextDatabaseItem<TId, TVocabularyKey, TMetadata>(kvp.Key, kvp.Value.Text, kvp.Value.Metadata, kvp.Value.Vector)).GetEnumerator();
+    }
+
+    IEnumerator IEnumerable.GetEnumerator()
+    {
+        return GetEnumerator();
+    }
 }
 
 
@@ -472,8 +489,14 @@ public void UpdateTextMetadata(TId id, TMetadata metadata) {
         if (VectorStore.ContainsKey(id))
         {
             var existing = VectorStore.Get(id);
-            existing.Metadata = metadata;
-            VectorStore.Set(id, existing);
+
+            var item = new VectorTextItem<string, TMetadata>(
+                existing.Text,
+                metadata,
+                existing.Vector
+            );
+            
+            VectorStore.Set(id, item);
         }
         else
         {
@@ -660,4 +683,13 @@ public virtual void DeserializeFromBinaryStream(Stream stream)
         DeserializeFromBinaryStreamAsync(stream).Wait();
     }
 
+    public IEnumerator<IVectorTextDatabaseItem<TId, string, TMetadata>> GetEnumerator()
+    {
+        return VectorStore.Select(kvp => new VectorTextDatabaseItem<TId, string, TMetadata>(kvp.Key, kvp.Value.Text, kvp.Value.Metadata, kvp.Value.Vector)).GetEnumerator();
+    }
+
+    System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator()
+    {
+        return this.GetEnumerator();
+    }
 }

src/Build5Nines.SharpVector/VectorStore/IVectorStore.cs

@@ -21,7 +21,7 @@ public interface IVectorStore<TId, TMetadata, TDocument>
     /// <param name="id"></param>
     /// <returns></returns>
     /// <exception cref="KeyNotFoundException"></exception>
-    VectorTextItem<TDocument, TMetadata> Get(TId id);
+    IVectorTextItem<TDocument, TMetadata> Get(TId id);
 
     /// <summary>
     /// Gets all the Ids for every text.
@@ -51,7 +51,7 @@ public interface IVectorStore<TId, TMetadata, TDocument>
     /// <param name="id"></param>
     /// <returns>The removed text item</returns>
     /// <exception cref="KeyNotFoundException"></exception>
-    VectorTextItem<TDocument, TMetadata> Delete(TId id);
+    IVectorTextItem<TDocument, TMetadata> Delete(TId id);
 
     /// <summary>
     /// Checks if the database contains a key

src/Build5Nines.SharpVector/VectorStore/MemoryDictionaryVectorStore.cs

@@ -62,7 +62,7 @@ public async Task SetAsync(TId id, VectorTextItem<TDocument, TMetadata> item)
     /// <param name="id"></param>
     /// <returns></returns>
     /// <exception cref="KeyNotFoundException"></exception>
-    public VectorTextItem<TDocument, TMetadata> Get(TId id)
+    public IVectorTextItem<TDocument, TMetadata> Get(TId id)
     {
         if (_database.TryGetValue(id, out var entry))
         {
@@ -77,7 +77,7 @@ public VectorTextItem<TDocument, TMetadata> Get(TId id)
     /// <param name="id"></param>
     /// <returns>The removed text item</returns>
     /// <exception cref="KeyNotFoundException"></exception>
-    public VectorTextItem<TDocument, TMetadata> Delete(TId id)
+    public IVectorTextItem<TDocument, TMetadata> Delete(TId id)
     {
         if (_database.ContainsKey(id))
         {

src/Build5Nines.SharpVector/VectorTextDatabaseItem.cs

@@ -0,0 +1,26 @@
+namespace Build5Nines.SharpVector;
+
+public interface IVectorTextDatabaseItem<TId, TDocument, TMetadata>
+{
+    TId Id { get; }
+    TDocument Text { get; }
+    TMetadata? Metadata { get; }
+    float[] Vector { get; }
+}
+
+public class VectorTextDatabaseItem<TId, TDocument, TMetadata>
+    : IVectorTextDatabaseItem<TId, TDocument, TMetadata>
+{
+    public VectorTextDatabaseItem(TId id, TDocument text, TMetadata? metadata, float[] vector)
+    {
+        Id = id;
+        Text = text;
+        Metadata = metadata;
+        Vector = vector;
+    }
+
+    public TId Id { get; private set; }
+    public TDocument Text { get; private set; }
+    public TMetadata? Metadata { get; private set; }
+    public float[] Vector { get; private set; }
+}

src/SharpVectorTest/Preprocessing/BasicTextPreprocessorTests.cs

@@ -18,8 +18,10 @@ public class VectorDatabaseTests
     public void TokenizeAndPreprocess_Null()
     {
         var preprocessor = new BasicTextPreprocessor();
+#pragma warning disable CS8625 // Cannot convert null literal to non-nullable reference type.
         var tokens = preprocessor.TokenizeAndPreprocess(null);
-        
+#pragma warning restore CS8625 // Cannot convert null literal to non-nullable reference type.
+
         Assert.AreEqual(0, tokens.Count());
     }
 

src/SharpVectorTest/VectorDatabaseTests.cs

@@ -1002,6 +1002,28 @@ public void EmbeddingGeneratorMemoryVectorDatabase_001()
         var db = new EmbeddingGeneratorMemoryVectorDatabase();
         db.AddText("Test string", "metadata");
     }
+
+
+    [TestMethod]
+    public void BasicMemoryVectorDatabase_LoopThroughAllTexts_01()
+    {
+        var vdb = new BasicMemoryVectorDatabase();
+        
+        // // Load Vector Database with some sample text
+        vdb.AddText("The 👑 King", "metadata1");
+        vdb.AddText("It's 🔥 Fire.", "metadata2");
+        vdb.AddText("No emoji", "metadata3");
+
+        foreach(var item in vdb)
+        {
+            var id = item.Id;
+            var text = item.Text;
+            var metadata = item.Metadata;
+            var vector = item.Vector;
+            //Console.WriteLine($"ID: {item.Id}, Text: {item.Text}, Metadata: {item.Metadata}");
+            vdb.UpdateText(item.Id, item.Text + " - Updated");
+        }
+    }
 }
 
 public class MockMemoryVectorDatabase