Skip to content

DiskCache supports custom serializers #450

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Feb 2, 2023
Merged

DiskCache supports custom serializers #450

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
27f5d0a
feat: add serializer on disk cache
Memoyu Jan 30, 2023
973de0d
feat: add system text json support and remove DiskCacheValue msgpack …
Memoyu Jan 31, 2023
6ece3f0
feat：enable disk cache serializer
Memoyu Feb 1, 2023
981040d
fix: fix overwrite file data length is wrong on disk cache
Memoyu Feb 2, 2023
3295169
fix: rollback test changes
Memoyu Feb 2, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions sample/EasyCaching.Demo.ConsoleApp/EasyCaching.Demo.ConsoleApp.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
<ProjectReference Include="..\..\bus\EasyCaching.Bus.CSRedis\EasyCaching.Bus.CSRedis.csproj" />
<ProjectReference Include="..\..\bus\EasyCaching.Bus.RabbitMQ\EasyCaching.Bus.RabbitMQ.csproj" />
<ProjectReference Include="..\..\bus\EasyCaching.Bus.Redis\EasyCaching.Bus.Redis.csproj" />
<ProjectReference Include="..\..\serialization\EasyCaching.Serialization.SystemTextJson\EasyCaching.Serialization.SystemTextJson.csproj" />
<ProjectReference Include="..\..\src\EasyCaching.Core\EasyCaching.Core.csproj" />
<ProjectReference Include="..\..\src\EasyCaching.CSRedis\EasyCaching.CSRedis.csproj" />
<ProjectReference Include="..\..\src\EasyCaching.Disk\EasyCaching.Disk.csproj" />
Expand Down
44 changes: 24 additions & 20 deletions sample/EasyCaching.Demo.ConsoleApp/Program.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,14 @@
namespace EasyCaching.Demo.ConsoleApp
{
using EasyCaching.Core;
using EasyCaching.Disk;
using EasyCaching.Serialization.SystemTextJson.Configurations;
using EasyCaching.SQLite;
using MemoryPack;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Options;
using System;
using System.IO;

class Program
{
Expand Down Expand Up @@ -43,6 +47,15 @@ static void Main(string[] args)
// {
// x.TypeNameHandling = Newtonsoft.Json.TypeNameHandling.None;
// }, "json");

option.UseDisk(cfg =>
{
cfg.DBConfig = new DiskDbOptions { BasePath = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "Cache") };
cfg.SerializerName = "msgpack";
}, "disk")
.WithJson("json")
.WithSystemTextJson("sysjson")
.WithMessagePack("msgpack");
});

IServiceProvider serviceProvider = services.BuildServiceProvider();
Expand All @@ -58,10 +71,6 @@ static void Main(string[] args)
//
// Console.WriteLine($"redis cache get value, {redisVal.HasValue} {redisVal.IsNull} {redisVal.Value}");



var rCache = factory.GetCachingProvider("r1");

var prod = new Product()
{
Name = "Name1",
Expand All @@ -72,32 +81,29 @@ static void Main(string[] args)
Lastname = "Lastname2"
}
};

prod.Inner.Inner = prod;
rCache.Set<Product>("mkey1", prod, TimeSpan.FromSeconds(20));

// redis cache
var rCache = factory.GetCachingProvider("r1");
rCache.Set<Product>("mkey1", prod, TimeSpan.FromSeconds(20));
var mVal1 = rCache.Get<Product>("mkey1");

rCache.Set<string>("mkey", "mvalue", TimeSpan.FromSeconds(20));

var mVal = rCache.Get<string>("mkey");

var mAllKey = rCache.GetAllKeysByPrefix("mk");
Console.WriteLine($"redis cache get value, {mVal.HasValue} {mVal.IsNull} {mVal.Value} ");

Console.WriteLine($"in-memory cache get value, {mVal.HasValue} {mVal.IsNull} {mVal.Value} ");


// sqllite cache
var sCache = factory.GetCachingProvider("s1");


sCache.Set<string>("skey", "svalue", TimeSpan.FromSeconds(20));


var sVal = sCache.Get<string>("skey");


Console.WriteLine($"sqlite cache get value, {sVal.HasValue} {sVal.IsNull} {sVal.Value} ");

// disk cache
var diskCache = factory.GetCachingProvider("disk");
diskCache.Set<string>("diskkey", "diskvalue", TimeSpan.FromSeconds(20));
var diskVal = diskCache.Get<string>("diskkey");
Console.WriteLine($"disk cache get value, {diskVal.HasValue} {diskVal.IsNull} {diskVal.Value} ");

Console.ReadKey();
}
}
Expand All @@ -110,11 +116,9 @@ public partial class Product
public string Name { get; set; }

[MemoryPackOrder(1)]

public string Lastname { get; set; }

[MemoryPackOrder(2)]

public Product Inner { set; get; }
}
}
Expand Down
4 changes: 3 additions & 1 deletion src/EasyCaching.Disk/Configurations/DiskOptionsExtension.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
using EasyCaching.Core;
using EasyCaching.Core.Configurations;
using EasyCaching.Core.DistributedLock;
using EasyCaching.Core.Serialization;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.DependencyInjection.Extensions;
using System;
Expand Down Expand Up @@ -35,10 +36,11 @@ public void AddServices(IServiceCollection services)
{
var optionsMon = x.GetRequiredService<Microsoft.Extensions.Options.IOptionsMonitor<DiskOptions>>();
var options = optionsMon.Get(_name);
var serializers = x.GetServices<IEasyCachingSerializer>();
var dlf = x.GetService<IDistributedLockFactory>();
// ILoggerFactory can be null
var factory = x.GetService<Microsoft.Extensions.Logging.ILoggerFactory>();
return new DefaultDiskCachingProvider(_name, options, dlf, factory);
return new DefaultDiskCachingProvider(_name, serializers, options, dlf, factory);
});
}
}
Expand Down
112 changes: 56 additions & 56 deletions src/EasyCaching.Disk/Configurations/EasyCachingOptionsExtensions.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,61 +1,61 @@
namespace Microsoft.Extensions.DependencyInjection
{
using System;
using EasyCaching.Core;
namespace Microsoft.Extensions.DependencyInjection
{
using System;
using EasyCaching.Core;
using EasyCaching.Core.Configurations;
using EasyCaching.Disk;
using Microsoft.Extensions.Configuration;

public static class EasyCachingOptionsExtensions
{
/// <summary>
/// Uses the disk caching provider (specify the config via hard code).
/// </summary>
/// <param name="options">Options.</param>
/// <param name="configure">Configure provider settings.</param>
/// <param name="name">The name of this provider instance.</param>
public static EasyCachingOptions UseDisk(
this EasyCachingOptions options
, Action<DiskOptions> configure
, string name = EasyCachingConstValue.DefaultDiskName
)
{
ArgumentCheck.NotNull(configure, nameof(configure));

options.RegisterExtension(new DiskOptionsExtension(name, configure));

return options;
}

using Microsoft.Extensions.Configuration;
public static class EasyCachingOptionsExtensions
{
/// <summary>
/// Uses the disk caching provider (specify the config via hard code).
/// </summary>
/// <param name="options">Options.</param>
/// <param name="configure">Configure provider settings.</param>
/// <param name="name">The name of this provider instance.</param>
public static EasyCachingOptions UseDisk(
this EasyCachingOptions options
, Action<DiskOptions> configure
, string name = EasyCachingConstValue.DefaultDiskName
)
{
ArgumentCheck.NotNull(configure, nameof(configure));
options.RegisterExtension(new DiskOptionsExtension(name, configure));
return options;
}
/// <summary>
/// Uses the disk caching provider (read config from configuration file).
/// </summary>
/// <param name="options">Options.</param>
/// <param name="configuration">The configuration.</param>
/// <param name="name">The name of this provider instance.</param>
/// <param name="sectionName">The section name in the configuration file.</param>
public static EasyCachingOptions UseDisk(
this EasyCachingOptions options
, IConfiguration configuration
, string name = EasyCachingConstValue.DefaultDiskName
, string sectionName = EasyCachingConstValue.DiskSection
)
{
var dbConfig = configuration.GetSection(sectionName);
var diskOptions = new DiskOptions();
dbConfig.Bind(diskOptions);

void configure(DiskOptions x)
{
x.EnableLogging = diskOptions.EnableLogging;
x.MaxRdSecond = diskOptions.MaxRdSecond;
x.LockMs = diskOptions.LockMs;
x.SleepMs = diskOptions.SleepMs;
x.SerializerName = diskOptions.SerializerName;
x.CacheNulls = diskOptions.CacheNulls;
x.DBConfig = diskOptions.DBConfig;
}
return options.UseDisk(configure, name);
}
}
}
/// <param name="options">Options.</param>
/// <param name="configuration">The configuration.</param>
/// <param name="name">The name of this provider instance.</param>
/// <param name="sectionName">The section name in the configuration file.</param>
public static EasyCachingOptions UseDisk(
this EasyCachingOptions options
, IConfiguration configuration
, string name = EasyCachingConstValue.DefaultDiskName
, string sectionName = EasyCachingConstValue.DiskSection
)
{
var dbConfig = configuration.GetSection(sectionName);
var diskOptions = new DiskOptions();
dbConfig.Bind(diskOptions);
void configure(DiskOptions x)
{
x.EnableLogging = diskOptions.EnableLogging;
x.MaxRdSecond = diskOptions.MaxRdSecond;
x.LockMs = diskOptions.LockMs;
x.SleepMs = diskOptions.SleepMs;
x.SerializerName = diskOptions.SerializerName;
x.CacheNulls = diskOptions.CacheNulls;
x.DBConfig = diskOptions.DBConfig;
}
return options.UseDisk(configure, name);
}
}
}
38 changes: 20 additions & 18 deletions src/EasyCaching.Disk/DefaultDiskCachingProvider.Async.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,6 @@
using System.Threading;
using System.Threading.Tasks;
using EasyCaching.Core;
using MessagePack;
using MessagePack.Resolvers;
using Microsoft.Extensions.Logging;

public partial class DefaultDiskCachingProvider : EasyCachingAbstractProvider
Expand All @@ -23,7 +21,7 @@ public override async Task<bool> BaseExistsAsync(string cacheKey, CancellationTo

var val = await GetDiskCacheValueAsync(path, cancellationToken);

return val.Expiration > DateTimeOffset.UtcNow;
return val.Expiration > DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
}

public override Task BaseFlushAsync(CancellationToken cancellationToken = default)
Expand Down Expand Up @@ -63,9 +61,9 @@ public override async Task<IDictionary<string, CacheValue<T>>> BaseGetAllAsync<T
{
var cached = await GetDiskCacheValueAsync(path, cancellationToken);

if (cached.Expiration > DateTimeOffset.UtcNow)
if (cached.Expiration > DateTimeOffset.UtcNow.ToUnixTimeMilliseconds())
{
var t = MessagePackSerializer.Deserialize<T>(cached.Value, MessagePackSerializerOptions.Standard.WithResolver(ContractlessStandardResolver.Instance), cancellationToken);
var t = _serializer.Deserialize<T>(cached.Value);

if (!dict.ContainsKey(item))
{
Expand Down Expand Up @@ -110,9 +108,9 @@ public override async Task<CacheValue<T>> BaseGetAsync<T>(string cacheKey, Func<
//var cached = GetDiskCacheValueAsync(path).ConfigureAwait(false).GetAwaiter().GetResult();
//var cached = GetDiskCacheValue(path);

if (cached.Expiration > DateTimeOffset.UtcNow)
if (cached.Expiration > DateTimeOffset.UtcNow.ToUnixTimeMilliseconds())
{
var t = MessagePackSerializer.Deserialize<T>(cached.Value, MessagePackSerializerOptions.Standard.WithResolver(ContractlessStandardResolver.Instance), cancellationToken);
var t = _serializer.Deserialize<T>(cached.Value);

if (_options.EnableLogging)
_logger?.LogInformation($"Cache Hit : cachekey = {cacheKey}");
Expand Down Expand Up @@ -182,14 +180,14 @@ public override async Task<object> BaseGetAsync(string cacheKey, Type type, Canc

var cached = await GetDiskCacheValueAsync(path, cancellationToken);

if (cached.Expiration > DateTimeOffset.UtcNow)
if (cached.Expiration > DateTimeOffset.UtcNow.ToUnixTimeMilliseconds())
{
if (_options.EnableLogging)
_logger?.LogInformation($"Cache Hit : cachekey = {cacheKey}");

CacheStats.OnHit();

var t = MessagePackSerializer.Deserialize(type, cached.Value, MessagePackSerializerOptions.Standard.WithResolver(ContractlessStandardResolver.Instance), cancellationToken);
var t = _serializer.Deserialize(cached.Value, type);
return t;
}
else
Expand Down Expand Up @@ -221,14 +219,14 @@ public override async Task<CacheValue<T>> BaseGetAsync<T>(string cacheKey, Cance

var cached = await GetDiskCacheValueAsync(path, cancellationToken);

if (cached.Expiration > DateTimeOffset.UtcNow)
if (cached.Expiration > DateTimeOffset.UtcNow.ToUnixTimeMilliseconds())
{
if (_options.EnableLogging)
_logger?.LogInformation($"Cache Hit : cachekey = {cacheKey}");

CacheStats.OnHit();

var t = MessagePackSerializer.Deserialize<T>(cached.Value, MessagePackSerializerOptions.Standard.WithResolver(ContractlessStandardResolver.Instance), cancellationToken);
var t = _serializer.Deserialize<T>(cached.Value);
return new CacheValue<T>(t, true);
}
else
Expand Down Expand Up @@ -267,9 +265,9 @@ public override async Task<IDictionary<string, CacheValue<T>>> BaseGetByPrefixAs
{
var cached = await GetDiskCacheValueAsync(path, cancellationToken);

if (cached.Expiration > DateTimeOffset.UtcNow)
if (cached.Expiration > DateTimeOffset.UtcNow.ToUnixTimeMilliseconds())
{
var t = MessagePackSerializer.Deserialize<T>(cached.Value, MessagePackSerializerOptions.Standard.WithResolver(ContractlessStandardResolver.Instance), cancellationToken);
var t = _serializer.Deserialize<T>(cached.Value);

if (!dict.ContainsKey(item))
{
Expand Down Expand Up @@ -302,7 +300,7 @@ public override async Task<TimeSpan> BaseGetExpirationAsync(string cacheKey, Can

var cached = await GetDiskCacheValueAsync(path, cancellationToken);

return cached.Expiration.Subtract(DateTimeOffset.UtcNow);
return DateTimeOffset.FromUnixTimeMilliseconds((long)cached.Expiration).Subtract(DateTimeOffset.UtcNow);
}

public override Task BaseRemoveAllAsync(IEnumerable<string> cacheKeys, CancellationToken cancellationToken = default)
Expand Down Expand Up @@ -449,7 +447,7 @@ public override async Task<bool> BaseTrySetAsync<T>(string cacheKey, T cacheValu
{
var cached = await GetDiskCacheValueAsync(path, cancellationToken);

if (cached.Expiration.Subtract(DateTimeOffset.UtcNow) > TimeSpan.Zero)
if (DateTimeOffset.FromUnixTimeMilliseconds((long)cached.Expiration).Subtract(DateTimeOffset.UtcNow) > TimeSpan.Zero)
{
return false;
}
Expand All @@ -469,9 +467,13 @@ private async Task<DiskCacheValue> GetDiskCacheValueAsync(string path, Cancellat
{
using (FileStream stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
{
var cached = await MessagePackSerializer.DeserializeAsync<DiskCacheValue>(stream, MessagePackSerializerOptions.Standard.WithResolver(ContractlessStandardResolver.Instance), cancellationToken);

return cached;
using (var mStream = new MemoryStream())
{
stream.CopyTo(mStream);
mStream.Seek(0, SeekOrigin.Begin);
var cached = _serializer.Deserialize<DiskCacheValue>(mStream.ToArray());
return cached;
}
}
}
}
Expand Down
Loading