Merge pull request #70 from Build5Nines/dev

v2.1.0

Author: crpietschmann

CHANGELOG.md

@@ -5,6 +5,23 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## v2.1.0
+
+Add:
+
+- Added `VectorTextResultItem.Id` property so it's easy to get the database ID for search results if necessary.
+- `IVectorDatabase` now inherits from `IEnumerable` so you can easily look through the texts documents that have been added to the database.
+
+Fixed:
+
+- Fixed text tokenization to correctly remove special characters
+- Update `BasicTextPreprocessor` to support Emoji characters too
+- Refactorings for more Clean Code
+
+Breaking Changes:
+
+- The `.Search` and `.SearchAsync` methods now return a `IVectorTextResultItem<TId, TDocument, TMetadata>` instead of `VectorTextResultItem<TDocument, TMetadata>`. If you're using things like the documentation shows, then you wont see any changes or have any issues with this update.
+
 ## v2.0.4 (2025-04-16)
 
 Add:

docs/docs/get-started/data-management/index.md

@@ -0,0 +1,107 @@
+---
+title: Data Management
+
+---
+# Data Management
+
+Since `Build5Nines.SharpVector` is a database, it also has data management methods available. These methods enable you to add, remove, and update the text documents that are vectorized and indexed within the semantic database.
+
+## Get Text Item ID
+
+Every text item within a `Build5Nines.SharpVector` database is assigned a unique identifier (ID). There are a few ways to get access to the ID of the text items.
+
+=== ".AddText()"
+
+    When adding an individual text item to the vector database, the ID value will be returned:
+
+    ```csharp
+    var id = vdb.AddText(txt, metadata);
+
+    var id = await vdb.AddTextAsync(txt, metadata);
+    ```
+
+=== ".Search()"
+
+    When you perform a semantic search, the search results will contain the list of texts; each have an ID property.
+
+    ```csharp
+    var results = vdb.Search("query text");
+
+    foreach(var text in results.Texts) {
+        var id = text.Id;
+        var text = text.Text;
+        var metadata = text.Metadata;
+        // do something here    
+    }
+    ```
+
+=== "Enumerator"
+
+    The `IVectorDatabase` classes implement `IEnumerable` so you can easily loop through all the text items that have been added to the database.
+
+    ```csharp
+    foreach(var item in vdb) {
+        var id = item.Id;
+        var text = item.Text;
+        var metadata = item.Metadata;
+        var vector = item.Vector;
+
+        // do something here
+    }
+    ```
+
+## Get
+
+If you know the `id` of a Text item in the database, you can retrieve it directly.
+
+### Get By Id
+
+The `.GetText` method can be used to retrieve a text item from the vector database directly.
+
+```csharp
+vdb.GetText(id);
+```
+
+## Update
+
+Once text items have been added to the database "Update" methods can be used to modify them.
+
+### Update Text
+
+The `.UpdateText` method can be used to update the `Text` value, and associated vectors will be updated.
+
+```csharp
+vdb.UpdateText(id, newTxt);
+```
+
+When the `Text` is updated, new vector embeddings are generated for the new text.
+
+### Update Metadata
+
+The `.UpdateTextMetadata` method can be used to update the `Metadata` for a given text item by `Id`.
+
+```csharp
+vdb.UpdateTextMetadata(id, newTxt);
+```
+
+When `Metadata` is updated, the vector embeddings are not updated.
+
+### Update Text and Metadata
+
+The `.UpdateTextAndMetadata` method can be used to update the `Text` and `Metadata` for a text item in the database for the given text item `Id`.
+
+```csharp
+vdb.UpdateTextAndMetadata(id, newTxt, newMetadata);
+```
+
+## Delete
+
+The vector database supports the ability to delete text items.
+
+### Delete Text
+
+The `.DeleteText` method can be used to delete a text item form the database for the given `Id'.
+
+```csharp
+vdb.DeleteText(id);
+```

docs/mkdocs.yml

@@ -137,6 +137,12 @@ nav:
       - Prerequisites: get-started/#prerequisites
       - Install Nuget Package: get-started/#install-nuget-package
       - Basic Example: get-started/#basic-example
+      - Data Management:
+          - get-started/data-management/index.md
+          - Get Text Item Id: get-started/data-management/#get-text-item-id
+          - Get Item By Id: get-started/data-management/#get
+          - Update Item: get-started/data-management/#update
+          - Delete Item: get-started/data-management/#delete
   - Concepts:
       - concepts/index.md
       - What is a Vector Database?: concepts/#what-is-a-vector-database

src/Build5Nines.SharpVector/Build5Nines.SharpVector.csproj

@@ -9,7 +9,7 @@
     <PackageId>Build5Nines.SharpVector</PackageId>
     <PackageProjectUrl>https://sharpvector.build5nines.com</PackageProjectUrl>
     <RepositoryUrl>https://github.com/Build5Nines/SharpVector</RepositoryUrl>
-    <Version>2.0.4</Version>
+    <Version>2.1.0</Version>
     <Description>Lightweight In-memory Vector Database to embed in any .NET Applications</Description>
     <Copyright>Copyright (c) 2025 Build5Nines LLC</Copyright>
     <PackageReadmeFile>README.md</PackageReadmeFile>

src/Build5Nines.SharpVector/DatabaseFile.cs

@@ -11,9 +11,9 @@ namespace Build5Nines.SharpVector;
 /// </summary>
 public static class DatabaseFile
 {
-    private const string databaseInfoFilename = "database.json";
-    private const string vectorStoreFilename = "vectorstore.json";
-    private const string vocabularyStoreFilename = "vocabularystore.json";
+    internal const string databaseInfoFilename = "database.json";
+    internal const string vectorStoreFilename = "vectorstore.json";
+    internal const string vocabularyStoreFilename = "vocabularystore.json";
 
     /// <summary>
     /// Load the vector database from a stream
@@ -210,6 +210,41 @@ public static async Task LoadVocabularyStoreAsync<TVocabularyKey, TVocabularyVal
         }
     }
 
+    public static async Task SaveDatabaseToZipArchiveAsync(
+        Stream stream,
+        DatabaseInfo databaseInfo,
+        Func<ZipArchive, Task> saveVectorStore
+    )
+    {
+        if (stream == null)
+        {
+            throw new ArgumentNullException(nameof(stream));
+        }
+
+        using (var archive = new ZipArchive(stream, ZipArchiveMode.Create, true))
+        {
+            var entryDatabaseType = archive.CreateEntry(databaseInfoFilename);
+            using(var entryStream = entryDatabaseType.Open())
+            {
+                // Save the database info
+                var databaseInfoJson = JsonSerializer.Serialize(databaseInfo);
+                if (databaseInfoJson != null)
+                {
+                    var databaseTypeBytes = System.Text.Encoding.UTF8.GetBytes(databaseInfoJson);
+                    await entryStream.WriteAsync(databaseTypeBytes);
+                    await entryStream.FlushAsync();
+                }
+                else
+                {
+                    throw new InvalidOperationException("Type name cannot be null.");
+                }
+            }
+
+            await saveVectorStore(archive);
+        }
+    }
+        
+
     public static async Task<DatabaseInfo> LoadDatabaseFromZipArchiveAsync(
         Stream stream, string? dbClassType,
         Func<ZipArchive, Task> loadVectorStore

src/Build5Nines.SharpVector/IVectorDatabase.cs

@@ -9,6 +9,7 @@ namespace Build5Nines.SharpVector;
 /// <typeparam name="TMetadata"></typeparam>
 /// <typeparam name="TDocument"></typeparam>
 public interface IVectorDatabase<TId, TMetadata, TDocument>
+    : IEnumerable<IVectorTextDatabaseItem<TId, TDocument, TMetadata>>
     where TId : notnull
 {
     /// <summary>
@@ -80,7 +81,7 @@ public interface IVectorDatabase<TId, TMetadata, TDocument>
     /// <param name="pageIndex">The page index of the search results. Default is 0.</param>
     /// <param name="pageCount">The number of search results per page. Default is Null and returns all results.</param>
     /// <returns></returns>
-    IVectorTextResult<TDocument, TMetadata> Search(TDocument queryText, float? threshold = null, int pageIndex = 0, int? pageCount = null);
+    IVectorTextResult<TId, TDocument, TMetadata> Search(TDocument queryText, float? threshold = null, int pageIndex = 0, int? pageCount = null);
 
     /// <summary>
     /// Performs an asynchronous search vector search to find the top N most similar texts to the given text
@@ -90,7 +91,7 @@ public interface IVectorDatabase<TId, TMetadata, TDocument>
     /// <param name="pageIndex">The page index of the search results. Default is 0.</param>
     /// <param name="pageCount">The number of search results per page. Default is Null and returns all results.</param>
     /// <returns></returns>
-    Task<IVectorTextResult<TDocument, TMetadata>> SearchAsync(TDocument queryText, float? threshold = null, int pageIndex = 0, int? pageCount = null);
+    Task<IVectorTextResult<TId, TDocument, TMetadata>> SearchAsync(TDocument queryText, float? threshold = null, int pageIndex = 0, int? pageCount = null);
 
 
     [Obsolete("Use SerializeToBinaryStreamAsync Instead")]