Merge pull request #344 from POPWorldMedia/es-8-#335

Es 8 #335

.gitignore

@@ -22,3 +22,4 @@ src/PopForums.Mvc/node_modules
 
 src/PopForums.Web/Areas/Forums
 /src/PopForums.Web/appsettings.development.json
+/src/PopForums.AzureKit.Functions/local.settings.dev.json

docs/elastickitlibrary.md

@@ -5,9 +5,9 @@ nav_order: 7
 ---
 # Using ElasticKit Library
 The `PopForums.ElasticKit` library makes it possible to wire up the following scenarios:
-* Use ElasticSearch for search instead of the built-in search indexing. _Important: The client library referenced in v15.x is designed to work against v6.x of ElasticSearch, while v16.x,v17.x and v18.x, v19.x uses v7.x of ElasticSearch._
+* Use ElasticSearch for search instead of the built-in search indexing. _Important: The client library referenced in v15.x is designed to work against v6.x of ElasticSearch, while v16.x,v17.x and v18.x, v19.x uses v7.x of ElasticSearch. v20.x uses v8.x of ElasticSearch._
 
-ElasticSearch can run quite literally anywhere in a docker container or straight up in a VM, if that's your thing. Also keep in mind that the implementation that AWS uses is actually a fork, so there are some differences about how the managed service is, uh, managed. In the commercial hosted version of POP Forums, we use Elastic's managed service running in Azure. Elastic runs in all of the major clouds and is generally reasonably priced.
+ElasticSearch can run quite literally anywhere in a docker container or straight up in a VM, if that's your thing. Also keep in mind that the implementation that AWS uses is actually a fork, so there are some differences about how the managed service is, uh, managed. In the commercial hosted version of POP Forums, we use Elastic's managed service running in Azure. Elastic runs in _all_ of the major clouds and is generally reasonably priced.
 
 ## Configuration with Azure App Services and Azure Functions
 
@@ -29,7 +29,7 @@ namespace YourWebApp;
 services.AddPopForumsElasticSearch();
 ```
 
-For use in the Azure functions, you'll need to set the `PopForums:Search:Provider` (or `PopForums__Search__Provider` on a Linux instance) setting in the portal blade for the functions to `elasticsearch`.
+For use in the Azure functions, you'll need to set the `PopForums:Search:Provider` (or `PopForums__Search__Provider` on a Linux instance) setting in the portal blade for the functions to `elasticsearch` or `elasticcloud` (see `Provider` config below).
 
 You'll also need to setup the right configuration values if you're running web in-process:
 
@@ -42,8 +42,9 @@ You'll also need to setup the right configuration values if you're running web i
       "Provider": ""
     },
 ```
-* `Url`: The base URL for the ElasticSearch endpoints
-* `Key`: After v16.x, this will optionally set an API key, using the format `id|key` (that's a pipe separating the ID and API key). Make sure that you use the ID, not the name.
-* `Provider`: This is optional in the web app and not actually implemented anywhere other than in our Azure Functions example project, where it's used to switch between `elasticsearch`, `azuresearch` and the default bits in the `PopForums.Sql` library.
+* `Url`: The base URL for the ElasticSearch endpoints. If you're using managed ES from Elastic, this is the "ElasticSearch Copy endpoint" result in the portal.
+* `Key` (non-cloud ElasticSearch): After v16.x, when `Provider` is set to `elasticsearch`, this will optionally set an API key, using the format `id|key` (that's a pipe separating the ID and API key). Make sure that you use the ID, not the name. This only works with a single-node cluster, ideal for local dev.
+* `Key` (Elastic managed cloud): Starting in v20.x, if you specify `elasticcloud` as the provider, you must supply `cloudID|APIkey`. The cloud ID is the big string in given in the portal with the deployment, and the API key is generated via the API console in the portal.
+* `Provider`: This is optional in the web app and not actually implemented anywhere other than in our Azure Functions example project, where it's used to switch between `elasticsearch`, `elasticcloud`, `azuresearch` and the default bits in the `PopForums.Sql` library.
 
 Configuring ElasticSearch and setting up security rules for it are beyond the scope of this wiki.

src/PopForums.AzureKit.Functions/PopForums.AzureKit.Functions.csproj

@@ -34,5 +34,8 @@
 		<None Update="local.settings.json">
 			<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
 		</None>
+		<None Update="local.settings.dev.json" Condition="Exists('local.settings.dev.json')">
+			<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+		</None>
 	</ItemGroup>
 </Project>

src/PopForums.AzureKit.Functions/Program.cs

@@ -16,6 +16,7 @@
 var configuration = new ConfigurationBuilder()
 	.SetBasePath(Environment.CurrentDirectory)
 	.AddJsonFile("local.settings.json", true)
+	.AddJsonFile("local.settings.dev.json", true)
 	.AddEnvironmentVariables()
 	.Build();
 var config = new Config(configuration);
@@ -45,6 +46,7 @@
 		switch (config.SearchProvider.ToLower())
 		{
 			case "elasticsearch":
+			case "elasticcloud":
 				s.AddPopForumsElasticSearch();
 				break;
 			case "azuresearch":

src/PopForums.AzureKit.Functions/local.settings.json

@@ -5,14 +5,14 @@
     "FUNCTIONS_WORKER_RUNTIME": "dotnet-isolated"
   },
   "PopForums": {
-    "WebAppUrlAndArea": "http://localhost:5090/Forums",
+    "WebAppUrlAndArea": "https://localhost:5091/Forums",
     "Database": {
       "ConnectionString": "server=localhost;Database=popforums19;Trusted_Connection=True;TrustServerCertificate=True;"
     },
     "Search": {
-      "Url": "http://localhost:9200",
-      "Key": "99011A70D3D50D251B0A6141A97B40E7",
-      "Provider": ""
+      "Url": "https://localhost:9200",
+      "Key": "elastic|1+GhSZd-+ActqOHZLkAL",
+      "Provider": "elasticsearch"
     },
     "Queue": {
       "ConnectionString": "UseDevelopmentStorage=true"

src/PopForums.ElasticKit/PopForums.ElasticKit.csproj

@@ -15,7 +15,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="NEST" Version="7.17.5" />
+    <PackageReference Include="Elastic.Clients.Elasticsearch" Version="8.9.2" />
     <PackageReference Include="Polly" Version="7.2.3" />
   </ItemGroup>
 

src/PopForums.ElasticKit/Search/ElasticSearchClientWrapper.cs

@@ -1,10 +1,13 @@
 using System;
 using System.Collections.Generic;
 using System.Linq;
-using Nest;
+using Elastic.Clients.Elasticsearch;
+using Elastic.Clients.Elasticsearch.QueryDsl;
+using Elastic.Transport;
 using PopForums.Configuration;
 using PopForums.Models;
 using PopForums.Services;
+using SearchType = PopForums.Models.SearchType;
 
 namespace PopForums.ElasticKit.Search;
 
@@ -20,24 +23,36 @@ public class ElasticSearchClientWrapper : IElasticSearchClientWrapper
 {
 	private readonly IErrorLog _errorLog;
 	private readonly ITenantService _tenantService;
-	private readonly ElasticClient _client;
+	private readonly ElasticsearchClient _client;
 
 	private const string IndexName = "topicindex";
 
 	public ElasticSearchClientWrapper(IConfig config, IErrorLog errorLog, ITenantService tenantService)
 	{
 		_errorLog = errorLog;
 		_tenantService = tenantService;
-		var node = new Uri(config.SearchUrl);
-		var settings = new ConnectionSettings(node)
-			.DefaultIndex(IndexName).DisableDirectStreaming();
-		if (!string.IsNullOrEmpty(config.SearchKey))
+		ElasticsearchClientSettings settings;
+		var pair = config.SearchKey.Split("|");
+		if (pair.Length != 2)
+			throw new NotSupportedException($"{nameof(ElasticSearchClientWrapper)} requires that the PopForums.Search.Key configuration value has two values separated by a pipe (\"|\" character. For search provider 'elasticsearch', this is basic authentication, an API ID followed by a key. For 'elasticcloud', this is the cloud ID followed by the API key.");
+		switch (config.SearchProvider.ToLower())
 		{
-			var pair = config.SearchKey.Split("|");
-			if (pair.Length == 2)
-				settings.ApiKeyAuthentication(pair[0], pair[1]);
+			case "elasticsearch":
+				settings = new ElasticsearchClientSettings(new Uri(config.SearchUrl))
+					.DefaultIndex(IndexName).DisableDirectStreaming();
+				settings.Authentication(new BasicAuthentication(pair[0], pair[1]));
+				break;
+			case "elasticcloud":
+				settings = new ElasticsearchClientSettings(pair[0], new ApiKey(pair[1]))
+						.DefaultIndex(IndexName).DisableDirectStreaming();
+				break;
+			default:
+				settings = new ElasticsearchClientSettings()
+					.DefaultIndex(IndexName).DisableDirectStreaming();
+				break;
 		}
-		_client = new ElasticClient(settings);
+
+		_client = new ElasticsearchClient(settings);
 	}
 
 	public IndexResponse IndexTopic(SearchTopic searchTopic)
@@ -46,7 +61,7 @@ public IndexResponse IndexTopic(SearchTopic searchTopic)
 		if (string.IsNullOrWhiteSpace(tenantID))
 			tenantID = "-";
 		searchTopic.TenantID = tenantID;
-		var indexResult = _client.IndexDocument(searchTopic);
+		var indexResult = _client.Index(searchTopic);
 		return indexResult;
 	}
 
@@ -59,51 +74,60 @@ public DeleteResponse RemoveTopic(string id)
 
 	public Response<IEnumerable<int>> SearchTopicsWithIDs(string searchTerm, List<int> hiddenForums, SearchType searchType, int startRow, int pageSize, out int topicCount)
 	{
-		Func<SortDescriptor<SearchTopic>, IPromise<IList<ISort>>> sortSelector;
+		var sortSelector = new SortOptionsDescriptor<SearchTopic>();
 		switch (searchType)
 		{
 			case SearchType.Date:
-				sortSelector = sort => sort.Descending(d => d.LastPostTime);
+				sortSelector.Field(sort => sort.LastPostTime, config => config.Order(SortOrder.Desc));
 				break;
 			case SearchType.Name:
-				sortSelector = sort => sort.Ascending(d => d.StartedByName).Descending(SortSpecialField.Score);
+				sortSelector.Field(sort => sort.StartedByName, config => config.Order(SortOrder.Asc));
 				break;
 			case SearchType.Replies:
-				sortSelector = sort => sort.Descending(d => d.Replies);
+				sortSelector.Field(sort => sort.Replies, config => config.Order(SortOrder.Desc));
 				break;
 			case SearchType.Title:
-				sortSelector = sort => sort.Ascending(d => d.Title);
+				sortSelector.Field(sort => sort.Title, config => config.Order(SortOrder.Asc));
 				break;
 			default:
-				sortSelector = sort => sort.Descending(SortSpecialField.Score);
+				sortSelector.Score(config => config.Order(SortOrder.Desc));
 				break;
 		}
 
 		var tenantID = _tenantService.GetTenant();
 		if (string.IsNullOrWhiteSpace(tenantID))
 			tenantID = "-";
 		startRow--;
-		var filters = new List<Func<QueryContainerDescriptor<SearchTopic>, QueryContainer>>();
-		filters.Add(tt => tt.Term(ff => ff.TenantID, tenantID));
 		var searchResponse = _client.Search<SearchTopic>(s => s
-			.Source(sf => sf.Includes(i => i.Fields(f => f.TopicID)))
-			.Query(q => 
-				!q.Terms(set => set.Field(field => field.ForumID).Terms(hiddenForums)) &&
-				+q.Bool(bb => bb.Filter(filters)) &&
-				q.MultiMatch(m => m.Query(searchTerm)
-					.Fields(f => f
-						.Field(x => x.Title, boost: 10)
-						.Field(x => x.FirstPost, boost: 5)
-						.Field(x => x.Posts))
-					.Fuzziness(Fuzziness.Auto)))
+			.Query(q => q
+				.Bool(bb => bb
+					.Must(ff => ff
+						.MultiMatch(m => m
+							.Query(searchTerm)
+							.Fields(new [] { "title^10", "firstPost^5", "posts" })
+							.Fuzziness(new Fuzziness("auto")))
+					)
+					.MustNot(ff => ff
+						.Terms(m => m
+							.Field(f => f.ForumID)
+								.Terms(new TermsQueryField(hiddenForums.Select(s => (FieldValue)s).ToArray())))
+					)
+					 .Filter(ff => ff
+					 	.Term(t => t
+					 		.Field(f => f.TenantID).Value(tenantID))
+					)
+				)
+			)
+			.SourceIncludes(new []{"topicID"})
 			.Sort(sortSelector)
-			.Take(pageSize)
-			.Skip(startRow));
+			.From(startRow)
+			.Size(pageSize));
 		Response<IEnumerable<int>> result;
-		if (!searchResponse.IsValid)
+		if (!searchResponse.IsValidResponse)
 		{
-			_errorLog.Log(searchResponse.OriginalException, ErrorSeverity.Error, $"Debugging info: {searchResponse.DebugInformation}");
-			result = new Response<IEnumerable<int>>(null, false, searchResponse.OriginalException, searchResponse.DebugInformation);
+			searchResponse.TryGetOriginalException(out var exception);
+			_errorLog.Log(exception, ErrorSeverity.Error, $"Debugging info: {searchResponse.DebugInformation}");
+			result = new Response<IEnumerable<int>>(null, false, exception, searchResponse.DebugInformation);
 			topicCount = 0;
 			return result;
 		}
@@ -115,46 +139,32 @@ public Response<IEnumerable<int>> SearchTopicsWithIDs(string searchTerm, List<in
 
 	public void VerifyIndexCreate()
 	{
-		var isExists = _client.Indices.Exists(new IndexExistsRequest(IndexName)).Exists;
+		var isExists = _client.Indices.Exists(Indices.Index(IndexName)).Exists;
 		if (isExists)
 			return;
 		var createIndexResponse = _client.Indices.Create(IndexName, c => c
 			.Settings(s => s
 				.Analysis(a => a
 					.Analyzers(aa => aa
 						.Standard("standard_english", sa => sa
-							.StopWords("_english_")
+							.Stopwords(new List<string> { "_english_" })
 						)
 					)
 				)
 			)
-			.Map<SearchTopic>(mm => mm
-				.Properties(p => p
-					.Text(t => t
-						.Name(n => n.Posts)
-						.Analyzer("standard_english")
-					)
-					.Text(t => t
-						.Name(n => n.FirstPost)
-						.Analyzer("standard_english")
-					)
-					.Text(t => t
-						.Name(n => n.Title)
-						.Analyzer("standard_english")
-						.Fielddata(true)
-					)
-					.Text(t => t
-						.Name(n => n.StartedByName)
-						.Fielddata(true)
-					)
-					.Keyword(t => t.Name(n => n.TenantID))
-				)
+			.Mappings(mm => mm
+				.Properties<SearchTopic>(p => p
+					.Text(t => t.Posts)
+					.Text(t => t.FirstPost)
+					.Text(t => t.Title, tp => tp.Fielddata())
+					.Text(t => t.StartedByName, tp => tp.Fielddata())
+					.Keyword(t => t.TenantID))
 			)
 		);
-		if (!createIndexResponse.IsValid)
+		if (!createIndexResponse.IsValidResponse)
 		{
-			_errorLog.Log(createIndexResponse.OriginalException, ErrorSeverity.Error,
-				createIndexResponse.DebugInformation);
+			createIndexResponse.TryGetOriginalException(out var exception);
+			_errorLog.Log(exception, ErrorSeverity.Error, createIndexResponse.DebugInformation);
 		}
 	}
 }

src/PopForums.ElasticKit/Search/SearchIndexSubsystem.cs

@@ -1,6 +1,6 @@
 using System;
 using System.Linq;
-using Nest;
+using Elastic.Clients.Elasticsearch;
 using Polly;
 using PopForums.Configuration;
 using PopForums.Services;
@@ -71,14 +71,16 @@ public void DoIndex(int topicID, string tenantID, bool isForRemoval)
 
 		try
 		{
-			var policy = Polly.Policy.HandleResult<IndexResponse>(x => !x.IsValid)
+			var policy = Polly.Policy.HandleResult<IndexResponse>(x => !x.IsValidResponse)
 				.WaitAndRetry(new[]
 				{
 					TimeSpan.FromSeconds(1),
 					TimeSpan.FromSeconds(5),
 					TimeSpan.FromSeconds(30)
-				}, (exception, timeSpan) => {
-					_errorLog.Log(exception.Result.OriginalException, ErrorSeverity.Error, $"Retry after {timeSpan.Seconds}: {exception.Result.DebugInformation}");
+				}, (result, timeSpan) =>
+				{
+					result.Result.TryGetOriginalException(out var exc);
+					_errorLog.Log(exc, ErrorSeverity.Error, $"Retry after {timeSpan.Seconds}: {result.Result.DebugInformation}");
 				});
 			policy.Execute(() =>
 			{
@@ -101,7 +103,8 @@ public void RemoveIndex(int topicID, string tenantID)
 			var result = _elasticSearchClientWrapper.RemoveTopic(id);
 			if (result.Result != Result.Deleted)
 			{
-				_errorLog.Log(result.OriginalException, ErrorSeverity.Error, $"Debug information: {result.DebugInformation}");
+				result.TryGetOriginalException(out var exc);
+				_errorLog.Log(exc, ErrorSeverity.Error, $"Debug information: {result.DebugInformation}");
 			}
 		}
 		catch (Exception exc)

src/PopForums.ElasticKit/Search/SearchTopic.cs

@@ -1,5 +1,4 @@
 using System;
-using Nest;
 
 namespace PopForums.ElasticKit.Search;
 

src/PopForums.Web/appsettings.json

@@ -21,7 +21,7 @@
       "ForceLocalOnly": false
     },
     "Search": {
-      "Url": "http://localhost:9200",
+      "Url": "https://localhost:9200",
       "Key": "99011A70D3D50D251B0A6141A97B40E7"
     },
     "Queue": {